From e6195c53041f5a8aa375b9715f90beb5188d33e5 Mon Sep 17 00:00:00 2001 From: Nevo Hed Date: Wed, 4 Jan 2023 20:57:59 -0500 Subject: [PATCH] Makefile: fix use of many br2-external trees The top level Makefile in buildroot has a recursive rule which causes the appearance of a hang as the number of directories in BR2_EXTERNAL increases. When the number of directories in BR2_EXTERNAL is small, the recursion occurs, but make detects the recursion and determines the target does not have to be remade. This allows make to progress. This is the failing rule: define percent_defconfig # Override the BR2_DEFCONFIG from COMMON_CONFIG_ENV with the new defconfig %_defconfig: $(BUILD_DIR)/buildroot-config/conf $(1)/configs/%_defconfig outputmakefile @$$(COMMON_CONFIG_ENV) BR2_DEFCONFIG=$(1)/configs/$$@ \ $$< --defconfig=$(1)/configs/$$@ $$(CONFIG_CONFIG_IN) endef $(eval $(foreach d,$(call reverse,$(TOPDIR) $(BR2_EXTERNAL_DIRS)),$(call percent_defconfig,$(d))$(sep))) The rule for %defconfig is created for each directory in BR2_EXTERNAL. When the rule is matched, the stem is 'defconfig_name'. The second prerequisite is expanded to $(1)/configs/defconfig_name_defconfig. The rule, and all of the other rules defined by this macro, are invoked again, but the stem is now $(1)/configs/defconfig_name_defconfig. The second prerequisite is now expanded to $(1)/configs/($1)/configs/defconfig_name_defconfig. This expansion continues until make detects the infinite recursion. With up to 5 br2-external trees, the time is very small, so that it is not noticeable. But starting with 6 br2-external trees, the time is insanely big (so much so that we did not even let it finish after it ran for hours); see timings toward the end of the commit log. We fix that by adding a single %_defconfig rule, which is now rsponsible to find the actual defconfig file that triggered the rule, by iterating on the reverse list of br2-external trees and then in main tree. Of course, now, there is no way for make to warn that there is no such defconfig, as it is no longer part of the prerequisites of the rule. So, we delegate to the recipe the responsibility to check for that. Timing (seconds) of `make pc_x86_64_bios_defconfig` with 1..1000 external trees, with make 4.2.1 (* with make 4.3), on a Core i7-7700HQ: #trees Before After 1 0.312 0.319 2 0.319 0.323 3 0.325 0.327 4 0.353 0.339 5 0.993 0.349 6 1.26* 0.347 7 9.10* 0.362 8 85.93* 0.360 9 n/a 0.373 10 n/a 0.374 50 n/a 0.738 100 n/a 1.228 500 n/a 7.483 1000 n/a 16.076 How to reproduce: #!/usr/bin/env bash N="${1:-1000}" for i in $(seq 1 1000); do [ -d "br2-external/${i}/configs" ] && break mkdir -p br2-external/${i}/configs touch br2-external/${i}/{Config.in,external.mk} echo "name: BR_TEST_${i}" >br2-external/${i}/external.desc touch br2-external/${i}/configs/foo{,_${i}}_defconfig done time make \ BR2_EXTERNAL="$( for i in $(seq 1 ${N}); do printf '%s\n' "$(pwd)/br2-external/${i}" done )" \ foo_1_defconfig Notes: the timings are very dependent on how much the CPU is otherwise loaded, but having a multi-core CPU slightly loaded helps maintain a high frequency on the siblings, and that can reduce the above timings in half! Best to try on an otherwise-idle system. Fixes: #14996 Reported-by: David Lawson Signed-off-by: Nevo Hed [yann.morin.1998@free.fr: - split long foreach - drastically extend the commit log - provide reproducer script and redo timings ] Signed-off-by: Yann E. MORIN --- Makefile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 88f90cd2fa..efa0b2a7a0 100644 --- a/Makefile +++ b/Makefile @@ -1013,13 +1013,18 @@ oldconfig syncconfig olddefconfig: $(BUILD_DIR)/buildroot-config/conf outputmake defconfig: $(BUILD_DIR)/buildroot-config/conf outputmakefile @$(COMMON_CONFIG_ENV) $< --defconfig$(if $(DEFCONFIG),=$(DEFCONFIG)) $(CONFIG_CONFIG_IN) -define percent_defconfig -# Override the BR2_DEFCONFIG from COMMON_CONFIG_ENV with the new defconfig -%_defconfig: $(BUILD_DIR)/buildroot-config/conf $(1)/configs/%_defconfig outputmakefile - @$$(COMMON_CONFIG_ENV) BR2_DEFCONFIG=$(1)/configs/$$@ \ - $$< --defconfig=$(1)/configs/$$@ $$(CONFIG_CONFIG_IN) -endef -$(eval $(foreach d,$(call reverse,$(TOPDIR) $(BR2_EXTERNAL_DIRS)),$(call percent_defconfig,$(d))$(sep))) +%_defconfig: $(BUILD_DIR)/buildroot-config/conf outputmakefile + @defconfig=$(or \ + $(firstword \ + $(foreach d, \ + $(call reverse,$(TOPDIR) $(BR2_EXTERNAL_DIRS)), \ + $(wildcard $(d)/configs/$@) \ + ) \ + ), \ + $(error "Can't find $@") \ + ); \ + $(COMMON_CONFIG_ENV) BR2_DEFCONFIG=$${defconfig} \ + $< --defconfig=$${defconfig} $(CONFIG_CONFIG_IN) update-defconfig: savedefconfig