diff --git a/toolkit/Makefile b/toolkit/Makefile index 8c9eff180ff..7021400568c 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -27,6 +27,7 @@ DOWNLOAD_SRPMS ?= n REBUILD_TOOLS ?= n RUN_CHECK ?= n USE_UPDATE_REPO ?= y +USE_PREVIEW_REPO ?= n DISABLE_UPSTREAM_REPOS ?= n TOOLCHAIN_CONTAINER_ARCHIVE ?= TOOLCHAIN_ARCHIVE ?= @@ -84,12 +85,21 @@ toolchain_rpms_dir := $(CACHED_RPMS_DIR)/cache/ endif # External source server -SOURCE_URL ?= -PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms -PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms \ - https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms \ - https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms +SOURCE_URL ?= + +PACKAGE_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms + +ifeq ($(USE_UPDATE_REPO),y) +PACKAGE_URL_LIST += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms +SRPM_URL_LIST += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms +endif + +ifeq ($(USE_PREVIEW_REPO),y) +PACKAGE_URL_LIST += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/$(build_arch)/rpms +SRPM_URL_LIST += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms +endif + REPO_LIST ?= CA_CERT ?= TLS_CERT ?= diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 05f9486c511..a88936bc18b 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -28,8 +28,7 @@ - [Local Build Variables](#local-build-variables) - [URLS and Repos](#urls-and-repos) - [`SOURCE_URL=...`](#source_url) - - [`PACKAGE_URL=...`](#package_url) - - [`PACKAGE_UPDATE_URL=...`](#package_update_url) + - [`PACKAGE_URL_LIST=...`](#package_url_list) - [`SRPM_URL_LIST=...`](#srpm_url_list) - [`REPO_LIST=...`](#repo_list) - [Build Enable/Disable Flags](#build-enabledisable-flags) @@ -42,6 +41,9 @@ - [`USE_UPDATE_REPO=...`](#use_update_repo) - [`USE_UPDATE_REPO=`**`y`** *(default)*](#use_update_repoy-default) - [`USE_UPDATE_REPO=`**`n`**](#use_update_repon) + - [`USE_PREVIEW_REPO=...`](#use_preview_repo) + - [`USE_PREVIEW_REPO=`**`n`** *(default)*](#use_preview_repon-default) + - [`USE_PREVIEW_REPO=`**`y`**](#use_preview_repoy) - [`DISABLE_UPSTREAM_REPOS=...`](#disable_upstream_repos) - [`DISABLE_UPSTREAM_REPOS=`**`n`** *(default)*](#disable_upstream_reposn-default) - [`DISABLE_UPSTREAM_REPOS=`**`y`**](#disable_upstream_reposy) @@ -289,9 +291,8 @@ Direct file downloads are by default pulled from: ```makefile SOURCE_URL ?= -PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms -PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms +PACKAGE_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms ``` While `tdnf` uses a list of repo files: @@ -301,7 +302,7 @@ REPO_LIST ?= ``` The `REPO_LIST` variable supports multiple repo files, and they are prioritized in the order they appear in the list. -The CBL-Mariner base repo is implicitly provided, and an optional update repo is available by setting `USE_UPDATE_REPO=y`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled. +The CBL-Mariner base repo is implicitly provided, an optional update repo is available by setting `USE_UPDATE_REPO=y` and an optional preview repo is available by setting `USE_PREVIEW_REPO=y`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled. ### Authentication @@ -319,9 +320,8 @@ The build system can operate without using pre-built components if desired. Ther ```makefile SOURCE_URL ?= -PACKAGE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms -PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms -SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms +PACKAGE_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms +SRPM_URL_LIST ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms REPO_LIST ?= ``` @@ -355,7 +355,7 @@ sudo make go-tools REBUILD_TOOLS=y # - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL_LIST=...) # - manually placing the correct sources in each /SPECS/* package folder # (SRPM_FILE_SIGNATURE_HANDLING=update must be used if the new sources files to not match the existing hashes) -sudo make toolchain PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_TOOLS=y +sudo make toolchain PACKAGE_URL_LIST="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_TOOLS=y ``` ```bash @@ -365,7 +365,7 @@ sudo make toolchain PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UP # - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL_LIST=...) # - manually placing the correct sources in each /SPECS/* package folder # (SRPM_FILE_SIGNATURE_HANDLING=update must be used if the new sources files to not match the existing hashes) -sudo make image PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_PACKAGES=y REBUILD_TOOLS=y +sudo make image PACKAGE_URL_LIST="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_PACKAGES=y REBUILD_TOOLS=y ``` ### Local Build Variables @@ -380,13 +380,9 @@ If that is not desired all remote sources can be disabled by clearing the follow > URL to download unavailable source files from when creating `*.src.rpm` files prior to build. -#### `PACKAGE_URL=...` +#### `PACKAGE_URL_LIST=...` -> URL to download RPM packages from, used to populate the toolchain packages if they are missing. - -#### `PACKAGE_UPDATE_URL=...` - -> URL to download RPM packages from if not found under `$(PACKAGE_URL)` and `$(USE_UPDATE_REPO)` is set to `y`, used to populate the toolchain packages if they are missing. +> Space seperated list of URLs to download toolchain RPM packages from, used to populate the toolchain packages if `$(REBUILD_TOOLCHAIN)` is set to `y`. #### `SRPM_URL_LIST=...` @@ -402,7 +398,7 @@ If that is not desired all remote sources can be disabled by clearing the follow ##### `REBUILD_TOOLCHAIN=`**`n`** *(default)* -> Use pre-existing toolchain packages from another source. If `TOOLCHAIN_ARCHIVE=my_toolchain.tar.gz` is also set the build system will extract the required packages from that archive. If `TOOLCHAIN_ARCHIVE` is not set the build system will download the required toolchain packages from `$(PACKAGE_URL)` and `$(PACKAGE_UPDATE_URL)`. +> Use pre-existing toolchain packages from another source. If `TOOLCHAIN_ARCHIVE=my_toolchain.tar.gz` is also set the build system will extract the required packages from that archive. If `TOOLCHAIN_ARCHIVE` is not set the build system will download the required toolchain packages from `$(PACKAGE_URL_LIST)`. ##### `REBUILD_TOOLCHAIN=`**`y`** @@ -426,7 +422,17 @@ If that is not desired all remote sources can be disabled by clearing the follow ##### `USE_UPDATE_REPO=`**`n`** -> Only pull missing packages from the upstream base repository. +> Do not pull missing packages from the upstream update repository. + +#### `USE_PREVIEW_REPO=...` + +##### `USE_PREVIEW_REPO=`**`n`** *(default)* + +> Do not pull missing packages from the upstream preview repository. + +##### `USE_PREVIEW_REPO=`**`y`** + +> Pull missing packages from the upstream preview repository in addition to the base repository. #### `DISABLE_UPSTREAM_REPOS=...` @@ -436,7 +442,7 @@ If that is not desired all remote sources can be disabled by clearing the follow ##### `DISABLE_UPSTREAM_REPOS=`**`y`** -> Only pull missing packages from local repositories. This does not affect hydrating the toolchain from `$(PACKAGE_URL)` and `$(PACKAGE_UPDATE_URL)`. +> Only pull missing packages from local repositories. This does not affect hydrating the toolchain from `$(PACKAGE_URL_LIST)`. #### `REBUILD_PACKAGES=...` @@ -579,7 +585,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | Variable | Default | Description |:------------------------------|:-------------------------------------------------------------------------------------------------------|:--- | CONFIG_FILE | `$(RESOURCES_DIR)`/imageconfigs/core-efi/core-efi.json | Image config file to build -| CONFIG_BASE_DIR | `$(dir $(CONFIG_FILE))` | Base directory to search for image files in (see [image_config.md](../images/image_config.md)) +| CONFIG_BASE_DIR | `$(dir $(CONFIG_FILE))` | Base directory to search for image files in (see [image_config.md](../images/image_config.md)) | UNATTENDED_INSTALLER | | Create unattended ISO installer if set. Overrides all other installer options. | PACKAGE_BUILD_LIST | | Additional packages to build | PACKAGE_REBUILD_LIST | | Always rebuild this package, even if it is up-to-date. Base package name, will match all virtual packages produced as well. @@ -599,7 +605,8 @@ To reproduce an ISO build, run the same make invocation as before, but set: | PACKAGE_ARCHIVE | | Use with `make hydrate-rpms` to populate a set of rpms from an archive. | DOWNLOAD_SRPMS | n | Pack SRPMs from local SPECs or download published ones? | USE_UPDATE_REPO | y | Pull missing packages from the upstream update repository in addition to the base repository? -| DISABLE_UPSTREAM_REPOS | n | Only pull missing packages from local repositories? This does not affect hydrating the toolchain from `$(PACKAGE_URL)` and `$(PACKAGE_UPDATE_URL)`. +| USE_PREVIEW_REPO | n | Pull missing packages from the upstream preview repository in addition to the base repository? +| DISABLE_UPSTREAM_REPOS | n | Only pull missing packages from local repositories? This does not affect hydrating the toolchain from `$(PACKAGE_URL_LIST)`. --- @@ -607,10 +614,9 @@ To reproduce an ISO build, run the same make invocation as before, but set: | Variable | Default | Description |:------------------------------|:---------------------------------------------------------------------------------------------------------|:--- -| SOURCE_URL | | URL to request package sources from -| SRPM_URL_LIST | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms` | Space seperated list of URLs to request packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y` -| PACKAGE_URL | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms` | URL to request full toolchain packages from -| PACKAGE_UPDATE_URL | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms` | URL to request full toolchain packages from if not found under `$(PACKAGE_URL)` and `$(USE_UPDATE_REPO)` is set to `y` +| SOURCE_URL | | URL to request package sources from +| SRPM_URL_LIST | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms` | Space seperated list of URLs to request packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y` +| PACKAGE_URL_LIST | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms` | Space seperated list of URLs to download toolchain RPM packages from, used to populate the toolchain packages if `$(REBUILD_TOOLCHAIN)` is set to `y`. | REPO_LIST | | Space separated list of repo files for tdnf to pull packages form | CA_CERT | | CA cert to access the above resources | TLS_CERT | | TLS cert to access the above resources diff --git a/toolkit/docs/how_it_works/3_package_building.md b/toolkit/docs/how_it_works/3_package_building.md index 4233431bde0..bc114881ad9 100644 --- a/toolkit/docs/how_it_works/3_package_building.md +++ b/toolkit/docs/how_it_works/3_package_building.md @@ -171,7 +171,7 @@ Sometimes packages fail to build correctly, or are otherwise not suitable for bu ### Stage 3: Graphpkgfetcher The `graphpkgfetcher` tool's job is to resolve unresolved remote nodes. Unresolved nodes occur when a local package has `Requires` or `BuildRequires` which are not available from another local package. -The tool uses the `worker_chroot` (see [Chroot Worker](1_initial_prep.md#chroot_worker)) to locate packages. The worker will search in five locations: 1) the local chroot environment, 2) already build RPMs in `./../out/RPMS/`, 3) the upstream base repository 4) the upstream update repository if `$(USE_UPDATE_REPO)` is set to `y` 5) any remote repo listed in `REPO_LIST ?=`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled. +The tool uses the `worker_chroot` (see [Chroot Worker](1_initial_prep.md#chroot_worker)) to locate packages. The worker will search in six locations: 1) the local chroot environment, 2) already build RPMs in `./../out/RPMS/`, 3) the upstream base repository 4) the upstream update repository if `$(USE_UPDATE_REPO)` is set to `y` 5) the upstream preview repository if `$(USE_PREVIEW_REPO)` is set to `y` 6) any remote repo listed in `REPO_LIST ?=`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled. The worker will run the `tdnf` command to search for each missing package. `tdnf` will prioritize local packages over pulling them from a remote location. diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index 25fc8ae720f..42c40cb14c9 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -156,6 +156,7 @@ pinentry-1.1.0-3.cm1.aarch64.rpm gnupg2-2.2.20-3.cm1.aarch64.rpm gpgme-1.13.1-5.cm1.aarch64.rpm mariner-repos-1.0-11.cm1.noarch.rpm +mariner-repos-preview-1.0-11.cm1.noarch.rpm libffi-3.2.1-11.cm1.aarch64.rpm libtasn1-4.14-2.cm1.aarch64.rpm p11-kit-0.23.16.1-2.cm1.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index f0fae29b2fc..ff0859fcf60 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -156,6 +156,7 @@ pinentry-1.1.0-3.cm1.x86_64.rpm gnupg2-2.2.20-3.cm1.x86_64.rpm gpgme-1.13.1-5.cm1.x86_64.rpm mariner-repos-1.0-11.cm1.noarch.rpm +mariner-repos-preview-1.0-11.cm1.noarch.rpm libffi-3.2.1-11.cm1.x86_64.rpm libtasn1-4.14-2.cm1.x86_64.rpm p11-kit-0.23.16.1-2.cm1.x86_64.rpm diff --git a/toolkit/resources/manifests/package/update_manifests.sh b/toolkit/resources/manifests/package/update_manifests.sh index 6b5a3ee3d2e..c80f327ff3d 100755 --- a/toolkit/resources/manifests/package/update_manifests.sh +++ b/toolkit/resources/manifests/package/update_manifests.sh @@ -77,7 +77,6 @@ remove_packages_for_pkggen_core () { sed -i '/libxslt/d' $TmpPkgGen sed -i '/Linux-PAM/d' $TmpPkgGen sed -i '/lua-devel/d' $TmpPkgGen - sed -i '/mariner-repos-preview/d' $TmpPkgGen sed -i '/npth-[[:alpha:]]/d' $TmpPkgGen sed -i '/pcre-[0-9]/d' $TmpPkgGen sed -i '/pcre-devel/d' $TmpPkgGen diff --git a/toolkit/scripts/imggen.mk b/toolkit/scripts/imggen.mk index 753038862a9..213d6b5cc4f 100644 --- a/toolkit/scripts/imggen.mk +++ b/toolkit/scripts/imggen.mk @@ -68,17 +68,19 @@ $(STATUS_FLAGS_DIR)/validate-image-config%.flag: $(go-imageconfigvalidator) $(de touch $@ +imagepkgfetcher_extra_flags := ifeq ($(DISABLE_UPSTREAM_REPOS),y) -imagepkgfetcher_disable_upstream_repos_flag := --disable-upstream-repos -else -imagepkgfetcher_disable_upstream_repos_flag := +imagepkgfetcher_extra_flags += --disable-upstream-repos endif ifeq ($(USE_UPDATE_REPO),y) -imagepkgfetcher_update_repo_flag := --use-update-repo -else -imagepkgfetcher_update_repo_flag := +imagepkgfetcher_extra_flags += --use-update-repo endif + +ifeq ($(USE_PREVIEW_REPO),y) +imagepkgfetcher_extra_flags += --use-preview-repo +endif + $(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(packagelist_files) $(RPMS_DIR) $(imggen_rpms) $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) $(go-imagepkgfetcher) \ @@ -92,8 +94,7 @@ $(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_ --tls-cert=$(TLS_CERT) \ --tls-key=$(TLS_KEY) \ $(foreach repo, $(imagefetcher_local_repo) $(imagefetcher_cloned_repo) $(REPO_LIST),--repo-file="$(repo)" ) \ - $(imagepkgfetcher_update_repo_flag) \ - $(imagepkgfetcher_disable_upstream_repos_flag) \ + $(imagepkgfetcher_extra_flags) \ --input-summary-file=$(IMAGE_CACHE_SUMMARY) \ --output-summary-file=$@ \ --output-dir=$(local_and_external_rpm_cache) diff --git a/toolkit/scripts/pkggen.mk b/toolkit/scripts/pkggen.mk index c75ffd0f5bc..6573acc19bd 100644 --- a/toolkit/scripts/pkggen.mk +++ b/toolkit/scripts/pkggen.mk @@ -92,17 +92,19 @@ $(optimized_file): $(graph_file) $(go-graphoptimizer) $(depend_PACKAGE_BUILD_LIS # We want to detect changes in the RPM cache, but we are not responsible for directly rebuilding any missing files. $(CACHED_RPMS_DIR)/%: ; +graphpkgfetcher_extra_flags := ifeq ($(DISABLE_UPSTREAM_REPOS),y) -graphpkgfetcher_disable_upstream_repos_flag := --disable-upstream-repos -else -graphpkgfetcher_disable_upstream_repos_flag := +graphpkgfetcher_extra_flags += --disable-upstream-repos endif ifeq ($(USE_UPDATE_REPO),y) -graphpkgfetcher_update_repo_flag := --use-update-repo -else -graphpkgfetcher_update_repo_flag := +graphpkgfetcher_extra_flags += --use-update-repo endif + +ifeq ($(USE_PREVIEW_REPO),y) +graphpkgfetcher_extra_flags += --use-preview-repo +endif + # Compare files via checksum (-c) instead of timestamp so unchanged RPMs are left intact without updating the timestamp of the directories $(cached_file): $(optimized_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(shell find $(CACHED_RPMS_DIR)/) $(pkggen_rpms) mkdir -p $(CACHED_RPMS_DIR)/cache && \ @@ -115,8 +117,7 @@ $(cached_file): $(optimized_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkgge --tls-cert=$(TLS_CERT) \ --tls-key=$(TLS_KEY) \ $(foreach repo, $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(REPO_LIST),--repo-file=$(repo) ) \ - $(graphpkgfetcher_update_repo_flag) \ - $(graphpkgfetcher_disable_upstream_repos_flag) \ + $(graphpkgfetcher_extra_flags) \ $(logging_command) \ --input-summary-file=$(PACKAGE_CACHE_SUMMARY) \ --output-summary-file=$(PKGBUILD_DIR)/graph_external_deps.json \ diff --git a/toolkit/scripts/toolchain.mk b/toolkit/scripts/toolchain.mk index bbee26f1d30..81a018acc9a 100644 --- a/toolkit/scripts/toolchain.mk +++ b/toolkit/scripts/toolchain.mk @@ -189,15 +189,10 @@ $(toolchain_rpms): $(toolchain_manifest) $(toolchain_local_temp) touch $@ else # Download from online package server -ifeq ($(USE_UPDATE_REPO),y) -toolchain_package_urls := $(PACKAGE_URL) $(PACKAGE_UPDATE_URL) -else -toolchain_package_urls := $(PACKAGE_URL) -endif $(toolchain_rpms): mkdir -p $(dir $@) && \ cd $(dir $@) && \ - for url in $(toolchain_package_urls); do \ + for url in $(PACKAGE_URL_LIST); do \ wget $${url}/$(notdir $@) \ --no-verbose \ $(if $(TLS_CERT),--certificate=$(TLS_CERT)) \ diff --git a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go index f523ff2f9a0..dbd68437008 100644 --- a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go +++ b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go @@ -31,6 +31,7 @@ var ( workertar = app.Flag("tdnf-worker", "Full path to worker_chroot.tar.gz").Required().ExistingFile() repoFiles = app.Flag("repo-file", "Full path to a repo file").Required().ExistingFiles() useUpdateRepo = app.Flag("use-update-repo", "Pull packages from the upstream update repo").Bool() + usePreviewRepo = app.Flag("use-preview-repo", "Pull packages from the upstream preview repo").Bool() disableUpstreamRepos = app.Flag("disable-upstream-repos", "Disables pulling packages from upstream repos").Bool() tlsClientCert = app.Flag("tls-cert", "TLS client certificate to use when downloading files.").String() @@ -85,7 +86,7 @@ func hasUnresolvedNodes(graph *pkggraph.PkgGraph) bool { func resolveGraphNodes(dependencyGraph *pkggraph.PkgGraph, inputSummaryFile, outputSummaryFile string, disableUpstreamRepos bool) (err error) { // Create the worker environment cloner := rpmrepocloner.New() - err = cloner.Initialize(*outDir, *tmpDir, *workertar, *existingRpmDir, *useUpdateRepo, *repoFiles) + err = cloner.Initialize(*outDir, *tmpDir, *workertar, *existingRpmDir, *useUpdateRepo, *usePreviewRepo, *repoFiles) if err != nil { logger.Log.Errorf("Failed to initialize RPM repo cloner. Error: %s", err) return diff --git a/toolkit/tools/imagepkgfetcher/imagepkgfetcher.go b/toolkit/tools/imagepkgfetcher/imagepkgfetcher.go index a6c48a18cd4..470cebd964f 100644 --- a/toolkit/tools/imagepkgfetcher/imagepkgfetcher.go +++ b/toolkit/tools/imagepkgfetcher/imagepkgfetcher.go @@ -32,6 +32,7 @@ var ( workertar = app.Flag("tdnf-worker", "Full path to worker_chroot.tar.gz").Required().ExistingFile() repoFiles = app.Flag("repo-file", "Full path to a repo file").Required().ExistingFiles() useUpdateRepo = app.Flag("use-update-repo", "Pull packages from the upstream update repo").Bool() + usePreviewRepo = app.Flag("use-preview-repo", "Pull packages from the upstream preview repo").Bool() disableUpstreamRepos = app.Flag("disable-upstream-repos", "Disables pulling packages from upstream repos").Bool() tlsClientCert = app.Flag("tls-cert", "TLS client certificate to use when downloading files.").String() @@ -57,7 +58,7 @@ func main() { } cloner := rpmrepocloner.New() - err := cloner.Initialize(*outDir, *tmpDir, *workertar, *existingRpmDir, *useUpdateRepo, *repoFiles) + err := cloner.Initialize(*outDir, *tmpDir, *workertar, *existingRpmDir, *useUpdateRepo, *usePreviewRepo, *repoFiles) if err != nil { logger.Log.Panicf("Failed to initialize RPM repo cloner. Error: %s", err) } diff --git a/toolkit/tools/internal/packagerepo/repocloner/repocloner.go b/toolkit/tools/internal/packagerepo/repocloner/repocloner.go index 4f49e5f7a6d..593aa5bed40 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/repocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/repocloner.go @@ -24,7 +24,7 @@ type RepoPackage struct { // It is capable of generate a local repository consisting of a set of request packages // and their dependencies. type RepoCloner interface { - Initialize(destinationDir, tmpDir, workerTar, existingRpmsDir string, useUpdateRepo bool, repoDefinitions []string) error + Initialize(destinationDir, tmpDir, workerTar, existingRpmsDir string, useUpdateRepo, usePreviewRepo bool, repoDefinitions []string) error AddNetworkFiles(tlsClientCert, tlsClientKey string) error Clone(cloneDeps bool, packagesToClone ...*pkgjson.PackageVer) error SearchAndClone(cloneDeps bool, singlePackageToClone *pkgjson.PackageVer) error diff --git a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go index 9c5fa71709c..964e045048e 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go @@ -25,6 +25,7 @@ const ( chrootDownloadDir = "/outputrpms" leaveChrootFilesOnDisk = false updateRepoID = "mariner-official-update" + previewRepoID = "mariner-preview" fetcherRepoID = "fetcher-cloned-repo" cacheRepoDir = "/upstream-cached-rpms" ) @@ -48,9 +49,10 @@ const ( // RpmRepoCloner represents an RPM repository cloner. type RpmRepoCloner struct { - chroot *safechroot.Chroot - useUpdateRepo bool - cloneDir string + chroot *safechroot.Chroot + useUpdateRepo bool + usePreviewRepo bool + cloneDir string } // New creates a new RpmRepoCloner @@ -64,8 +66,9 @@ func New() *RpmRepoCloner { // - workerTar is the path to the worker tar used to seed the chroot // - existingRpmsDir is the directory with prebuilt RPMs // - useUpdateRepo if set, the upstream update repository will be used. +// - usePreviewRepo if set, the upstream preview repository will be used. // - repoDefinitions is a list of repo files to use when cloning RPMs -func (r *RpmRepoCloner) Initialize(destinationDir, tmpDir, workerTar, existingRpmsDir string, useUpdateRepo bool, repoDefinitions []string) (err error) { +func (r *RpmRepoCloner) Initialize(destinationDir, tmpDir, workerTar, existingRpmsDir string, useUpdateRepo, usePreviewRepo bool, repoDefinitions []string) (err error) { const ( isExistingDir = false @@ -80,8 +83,13 @@ func (r *RpmRepoCloner) Initialize(destinationDir, tmpDir, workerTar, existingRp ) r.useUpdateRepo = useUpdateRepo - if !useUpdateRepo { - logger.Log.Warnf("Disabling update repo") + if useUpdateRepo { + logger.Log.Info("Enabling update repo") + } + + r.usePreviewRepo = usePreviewRepo + if usePreviewRepo { + logger.Log.Info("Enabling preview repo") } // Ensure that if initialization fails, the chroot is closed @@ -300,6 +308,10 @@ func (r *RpmRepoCloner) SearchAndClone(cloneDeps bool, singlePackageToClone *pkg args = append(args, fmt.Sprintf("--disablerepo=%s", updateRepoID)) } + if !r.usePreviewRepo { + args = append(args, fmt.Sprintf("--disablerepo=%s", previewRepoID)) + } + stdout, stderr, err := shell.Execute("tdnf", args...) logger.Log.Debugf("tdnf search for dependency '%s':\n%s", singlePackageToClone.Name, stdout) @@ -447,6 +459,10 @@ func (r *RpmRepoCloner) clonePackage(baseArgs []string, enabledRepoOrder ...stri args = append(args, fmt.Sprintf("--disablerepo=%s", updateRepoID)) } + if !r.usePreviewRepo { + args = append(args, fmt.Sprintf("--disablerepo=%s", previewRepoID)) + } + var ( stdout string stderr string