From 02bb0ecefe2b0b2eec21843cd386ad81d24b87a6 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Thu, 5 May 2022 19:19:39 +0200 Subject: [PATCH] package/pkg-download: do not try to vendor _EXTRA_DOWNLOADS For golang- or cargo-based packages, we apply a vendoring pass after the package's "main" download is done. Whether to vendor or not is based on the heuristic that a specific directory exists or not; for golang packages, we look for '/vendor', while for cargo, we look for '/VENDOR'. This is fine for the "main" (by lack of a better term) download, but this falls flat on its face for extra downloads. Indeed, some packages may need to download data sets, or assets, as _EXTRA_DOWNLOADS. Those are usually just data blobs, and are not actual golang or cargo packages; as such they do not need to be vendored, but worse, if we try to actually vendor them, this fails because the required files for vendoring are missing from the archives in such data sets. We fix that by decoupling the download for the extra download, from the download for the main archive. We pass the post-processing option only to the main download. This makes the hard assumption that extra downloads will never need to be post-processed for vendoring, of course; we hope this will always be correct in practice. If not, we can fix it later. Signed-off-by: Yann E. MORIN Cc: Thomas Petazzoni [Arnout: - no loop needed for MAIN_DOWNLOAD, it can have only one; - remove superfluous backslash in the definition of MAIN_DOWNLOAD; - introduce _ADDITIONAL_DOWNLOADS to avoid filter-out.] Signed-off-by: Arnout Vandecappelle (Essensium/Mind) (cherry picked from commit f0c7cb01a960c6892d6082c74b7c07320ddaef0a) Signed-off-by: Peter Korsgaard --- package/pkg-download.mk | 3 ++- package/pkg-generic.mk | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/package/pkg-download.mk b/package/pkg-download.mk index a15e21e110..0718f21aad 100644 --- a/package/pkg-download.mk +++ b/package/pkg-download.mk @@ -103,6 +103,7 @@ endif # # Argument 1 is the source location # Argument 2 is the upper-case package name +# Argument 3 is a space-separated list of optional arguments # ################################################################################ @@ -118,10 +119,10 @@ define DOWNLOAD -n '$($(2)_BASENAME_RAW)' \ -N '$($(2)_RAWNAME)' \ -o '$($(2)_DL_DIR)/$(notdir $(1))' \ - $(if $($(2)_DOWNLOAD_POST_PROCESS),-p '$($(2)_DOWNLOAD_POST_PROCESS)') \ $(if $($(2)_GIT_SUBMODULES),-r) \ $(if $($(2)_GIT_LFS),-l) \ $(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \ + $(3) \ $(QUIET) \ -- \ $($(2)_DL_OPTS) diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk index 79fcf603d3..b03f1991b8 100644 --- a/package/pkg-generic.mk +++ b/package/pkg-generic.mk @@ -192,7 +192,8 @@ $(BUILD_DIR)/%/.stamp_downloaded: break ; \ fi ; \ done - $(foreach p,$($(PKG)_ALL_DOWNLOADS),$(call DOWNLOAD,$(p),$(PKG))$(sep)) + $(call DOWNLOAD,$($(PKG)_MAIN_DOWNLOAD),$(PKG),$(patsubst %,-p '%',$($(PKG)_DOWNLOAD_POST_PROCESS))) + $(foreach p,$($(PKG)_ADDITIONAL_DOWNLOADS),$(call DOWNLOAD,$(p),$(PKG))$(sep)) $(foreach hook,$($(PKG)_POST_DOWNLOAD_HOOKS),$(call $(hook))$(sep)) $(Q)mkdir -p $(@D) @$(call step_end,download) @@ -601,12 +602,16 @@ ifndef $(2)_PATCH endif endif -$(2)_ALL_DOWNLOADS = \ - $$(if $$($(2)_SOURCE),$$($(2)_SITE_METHOD)+$$($(2)_SITE)/$$($(2)_SOURCE)) \ +$(2)_MAIN_DOWNLOAD = \ + $$(if $$($(2)_SOURCE),$$($(2)_SITE_METHOD)+$$($(2)_SITE)/$$($(2)_SOURCE)) + +$(2)_ADDITIONAL_DOWNLOADS = \ $$(foreach p,$$($(2)_PATCH) $$($(2)_EXTRA_DOWNLOADS),\ $$(if $$(findstring ://,$$(p)),$$(p),\ $$($(2)_SITE_METHOD)+$$($(2)_SITE)/$$(p))) +$(2)_ALL_DOWNLOADS = $$($(2)_MAIN_DOWNLOAD) $$($(2)_ADDITIONAL_DOWNLOADS) + ifndef $(2)_SITE ifdef $(3)_SITE $(2)_SITE = $$($(3)_SITE)