From ad971dbd6673df90fec9aa9e9a9a0d5daaa2a0d2 Mon Sep 17 00:00:00 2001 From: Joe Schmitt Date: Tue, 29 Sep 2020 09:29:19 -0700 Subject: [PATCH 1/3] Replace SRPM_URL* with SRPM_URL_LIST --- toolkit/Makefile | 3 +-- toolkit/docs/building/building.md | 24 ++++++++---------------- toolkit/scripts/srpm_pack.mk | 4 +--- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/toolkit/Makefile b/toolkit/Makefile index 345a0fe07dd..aed8d2ca5e4 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -87,8 +87,7 @@ endif SOURCE_URL ?= PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms -SRPM_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms REPO_LIST ?= CA_CERT ?= TLS_CERT ?= diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index a23a50d7bc5..3416c3e09ab 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -30,8 +30,7 @@ - [`SOURCE_URL=...`](#source_url) - [`PACKAGE_URL=...`](#package_url) - [`PACKAGE_UPDATE_URL=...`](#package_update_url) - - [`SRPM_URL=...`](#srpm_url) - - [`SRPM_UPDATE_URL=...`](#srpm_update_url) + - [`SRPM_URL_LIST=...`](#srpm_url_list) - [`REPO_LIST=...`](#repo_list) - [Build Enable/Disable Flags](#build-enabledisable-flags) - [`REBUILD_TOOLCHAIN=...`](#rebuild_toolchain) @@ -292,8 +291,7 @@ Direct file downloads are by default pulled from: SOURCE_URL ?= PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms -SRPM_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms ``` While `tdnf` uses a list of repo files: @@ -323,8 +321,7 @@ The build system can operate without using pre-built components if desired. Ther SOURCE_URL ?= PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms -SRPM_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms REPO_LIST ?= ``` @@ -355,7 +352,7 @@ sudo make go-tools REBUILD_TOOLS=y # then rebuild the toolchain properly using the provided sources # NOTE: Source files must made available via one of: # - `SOURCE_URL=` -# - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL=... and SRPM_UPDATE_URL=...) +# - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL_LIST=...) # - manually placing the correct sources in each /SPECS/* package folder # (SRPM_FILE_SIGNATURE_HANDLING=update must be used if the new sources files to not match the existing hashes) sudo make toolchain PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_TOOLS=y @@ -365,7 +362,7 @@ sudo make toolchain PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UP # Complete rebuild of all tool, package, and image files from source. # NOTE: Source files must made available via one of: # - `SOURCE_URL=` -# - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL=... and SRPM_UPDATE_URL=...) +# - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL_LIST=...) # - manually placing the correct sources in each /SPECS/* package folder # (SRPM_FILE_SIGNATURE_HANDLING=update must be used if the new sources files to not match the existing hashes) sudo make image PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_PACKAGES=y REBUILD_TOOLS=y @@ -391,13 +388,9 @@ If that is not desired all remote sources can be disabled by clearing the follow > URL to download RPM packages from if not found under `$(PACKAGE_URL)` and `$(USE_UPDATE_REPO)` is set to `y`, used to populate the toolchain packages if they are missing. -#### `SRPM_URL=...` +#### `SRPM_URL_LIST=...` -> URL to download packed SRPM packages from prior to build if `$(DOWNLOAD_SRPMS)` is set to `y`. - -#### `SRPM_UPDATE_URL=...` - -> URL to download updated versions of packed SRPM packages from prior to build if `$(DOWNLOAD_SRPMS)` is set to `y`. +> Space seperated list of URLs to download packed SRPM packages from prior to build if `$(DOWNLOAD_SRPMS)` is set to `y`. #### `REPO_LIST=...` @@ -616,8 +609,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | Variable | Default | Description |:------------------------------|:---------------------------------------------------------------------------------------------------------|:--- | SOURCE_URL | | URL to request package sources from -| SRPM_URL | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms` | URL to request packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y` -| SRPM_UPDATE_URL | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms` | URL to request updated versions of packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y` +| SRPM_URL_LIST | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms` | Space seperated list of URLs to request packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y` | PACKAGE_URL | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms` | URL to request full toolchain packages from | PACKAGE_UPDATE_URL | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms` | URL to request full toolchain packages from if not found under `$(PACKAGE_URL)` and `$(USE_UPDATE_REPO)` is set to `y` | REPO_LIST | | Space separated list of repo files for tdnf to pull packages form diff --git a/toolkit/scripts/srpm_pack.mk b/toolkit/scripts/srpm_pack.mk index b273eb2e032..e9720b5debd 100644 --- a/toolkit/scripts/srpm_pack.mk +++ b/toolkit/scripts/srpm_pack.mk @@ -38,13 +38,11 @@ $(BUILD_SRPMS_DIR): $(STATUS_FLAGS_DIR)/build_srpms.flag @echo Finished updating $@ ifeq ($(DOWNLOAD_SRPMS),y) -srpm_download_urls := $(SRPM_URL) $(SRPM_UPDATE_URL) - $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_DIR) for spec in $(local_specs); do \ spec_file=$${spec} && \ srpm_file=$$(rpmspec -q $${spec_file} --srpm --define='with_check 1' --define='dist $(DIST_TAG)' --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) && \ - for url in $(srpm_download_urls); do \ + for url in $(SRPM_URL_LIST); do \ wget $${url}/$${srpm_file} \ -O $(BUILD_SRPMS_DIR)/$${srpm_file} \ --no-verbose \ From a268af23ce3e610292d6078772de70cd4c38ec45 Mon Sep 17 00:00:00 2001 From: Joe Schmitt Date: Wed, 30 Sep 2020 08:36:04 -0700 Subject: [PATCH 2/3] Split urls onto new lines --- toolkit/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/toolkit/Makefile b/toolkit/Makefile index aed8d2ca5e4..267be4018e5 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -87,7 +87,9 @@ endif SOURCE_URL ?= PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms \ + https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms \ + https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms REPO_LIST ?= CA_CERT ?= TLS_CERT ?= From a81ff8a083cb74c991a63f434ddd6f606c99ff60 Mon Sep 17 00:00:00 2001 From: Joe Schmitt Date: Wed, 30 Sep 2020 10:16:24 -0700 Subject: [PATCH 3/3] Fix srpm url priorities --- toolkit/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolkit/Makefile b/toolkit/Makefile index 267be4018e5..8c9eff180ff 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -87,9 +87,9 @@ endif SOURCE_URL ?= PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms \ +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms \ https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms \ - https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms + https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms REPO_LIST ?= CA_CERT ?= TLS_CERT ?=