Natively support pulling from the preview repo

toolkit/Makefile

@@ -27,6 +27,7 @@ DOWNLOAD_SRPMS                  ?= n
 REBUILD_TOOLS                   ?= n
 RUN_CHECK                       ?= n
 USE_UPDATE_REPO                 ?= y
+USE_PREVIEW_REPO                ?= n
 DISABLE_UPSTREAM_REPOS          ?= n
 TOOLCHAIN_CONTAINER_ARCHIVE     ?=
 TOOLCHAIN_ARCHIVE               ?=
@@ -84,12 +85,21 @@ toolchain_rpms_dir := $(CACHED_RPMS_DIR)/cache/
 endif
 
 # External source server
-SOURCE_URL         ?= 
-PACKAGE_URL        ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms
-PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms
-SRPM_URL_LIST      ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms \
-                      https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms \
-                      https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms
+SOURCE_URL         ?=
+
+PACKAGE_URL_LIST   ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms
+SRPM_URL_LIST      ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms
+
+ifeq ($(USE_UPDATE_REPO),y)
+PACKAGE_URL_LIST   += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms
+SRPM_URL_LIST      += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms
+endif
+
+ifeq ($(USE_PREVIEW_REPO),y)
+PACKAGE_URL_LIST   += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/$(build_arch)/rpms
+SRPM_URL_LIST      += https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms
+endif
+
 REPO_LIST   ?=
 CA_CERT     ?=
 TLS_CERT    ?=

toolkit/docs/building/building.md

@@ -28,8 +28,7 @@
     - [Local Build Variables](#local-build-variables)
       - [URLS and Repos](#urls-and-repos)
       - [`SOURCE_URL=...`](#source_url)
-      - [`PACKAGE_URL=...`](#package_url)
-      - [`PACKAGE_UPDATE_URL=...`](#package_update_url)
+      - [`PACKAGE_URL_LIST=...`](#package_url_list)
       - [`SRPM_URL_LIST=...`](#srpm_url_list)
       - [`REPO_LIST=...`](#repo_list)
       - [Build Enable/Disable Flags](#build-enabledisable-flags)
@@ -42,6 +41,9 @@
       - [`USE_UPDATE_REPO=...`](#use_update_repo)
         - [`USE_UPDATE_REPO=`**`y`** *(default)*](#use_update_repoy-default)
         - [`USE_UPDATE_REPO=`**`n`**](#use_update_repon)
+      - [`USE_PREVIEW_REPO=...`](#use_preview_repo)
+        - [`USE_PREVIEW_REPO=`**`n`** *(default)*](#use_preview_repon-default)
+        - [`USE_PREVIEW_REPO=`**`y`**](#use_preview_repoy)
       - [`DISABLE_UPSTREAM_REPOS=...`](#disable_upstream_repos)
         - [`DISABLE_UPSTREAM_REPOS=`**`n`** *(default)*](#disable_upstream_reposn-default)
         - [`DISABLE_UPSTREAM_REPOS=`**`y`**](#disable_upstream_reposy)
@@ -289,9 +291,8 @@ Direct file downloads are by default pulled from:
 
 ```makefile
 SOURCE_URL         ?=
-PACKAGE_URL        ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms
-PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms
-SRPM_URL_LIST      ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms
+PACKAGE_URL_LIST   ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms
+SRPM_URL_LIST      ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms
 ```
 
 While `tdnf` uses a list of repo files:
@@ -301,7 +302,7 @@ REPO_LIST ?=
 ```
 
 The `REPO_LIST` variable supports multiple repo files, and they are prioritized in the order they appear in the list.
-The CBL-Mariner base repo is implicitly provided, and an optional update repo is available by setting `USE_UPDATE_REPO=y`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled.
+The CBL-Mariner base repo is implicitly provided, an optional update repo is available by setting `USE_UPDATE_REPO=y` and an optional preview repo is available by setting `USE_PREVIEW_REPO=y`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled.
 
 ### Authentication
 
@@ -319,9 +320,8 @@ The build system can operate without using pre-built components if desired. Ther
 
 ```makefile
 SOURCE_URL         ?=
-PACKAGE_URL        ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms
-PACKAGE_UPDATE_URL ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms
-SRPM_URL_LIST      ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms
+PACKAGE_URL_LIST   ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms
+SRPM_URL_LIST      ?= https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms
 REPO_LIST          ?=
 ```
 
@@ -355,7 +355,7 @@ sudo make go-tools REBUILD_TOOLS=y
 # - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL_LIST=...)
 # - manually placing the correct sources in each /SPECS/* package folder
 #     (SRPM_FILE_SIGNATURE_HANDLING=update must be used if the new sources files to not match the existing hashes)
-sudo make toolchain PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_TOOLS=y
+sudo make toolchain PACKAGE_URL_LIST="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_TOOLS=y
 ```
 
 ```bash
@@ -365,7 +365,7 @@ sudo make toolchain PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UP
 # - DOWNLOAD_SRPMS=y (will download pre-packages sources from SRPM_URL_LIST=...)
 # - manually placing the correct sources in each /SPECS/* package folder
 #     (SRPM_FILE_SIGNATURE_HANDLING=update must be used if the new sources files to not match the existing hashes)
-sudo make image PACKAGE_URL="" PACKAGE_UPDATE_URL="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_PACKAGES=y REBUILD_TOOLS=y
+sudo make image PACKAGE_URL_LIST="" REPO_LIST="" DISABLE_UPSTREAM_REPOS=y REBUILD_TOOLCHAIN=y REBUILD_PACKAGES=y REBUILD_TOOLS=y
 ```
 
 ### Local Build Variables
@@ -380,13 +380,9 @@ If that is not desired all remote sources can be disabled by clearing the follow
 
 > URL to download unavailable source files from when creating `*.src.rpm` files prior to build.
 
-#### `PACKAGE_URL=...`
+#### `PACKAGE_URL_LIST=...`
 
-> URL to download RPM packages from, used to populate the toolchain packages if they are missing.
-
-#### `PACKAGE_UPDATE_URL=...`
-
-> URL to download RPM packages from if not found under `$(PACKAGE_URL)` and `$(USE_UPDATE_REPO)` is set to `y`, used to populate the toolchain packages if they are missing.
+> Space seperated list of URLs to download toolchain RPM packages from, used to populate the toolchain packages if `$(REBUILD_TOOLCHAIN)` is set to `y`.
 
 #### `SRPM_URL_LIST=...`
 
@@ -402,7 +398,7 @@ If that is not desired all remote sources can be disabled by clearing the follow
 
 ##### `REBUILD_TOOLCHAIN=`**`n`** *(default)*
 
-> Use pre-existing toolchain packages from another source. If `TOOLCHAIN_ARCHIVE=my_toolchain.tar.gz` is also set the build system will extract the required packages from that archive. If `TOOLCHAIN_ARCHIVE` is not set the build system will download the required toolchain packages from `$(PACKAGE_URL)` and `$(PACKAGE_UPDATE_URL)`.
+> Use pre-existing toolchain packages from another source. If `TOOLCHAIN_ARCHIVE=my_toolchain.tar.gz` is also set the build system will extract the required packages from that archive. If `TOOLCHAIN_ARCHIVE` is not set the build system will download the required toolchain packages from `$(PACKAGE_URL_LIST)`.
 
 ##### `REBUILD_TOOLCHAIN=`**`y`**
 
@@ -426,7 +422,17 @@ If that is not desired all remote sources can be disabled by clearing the follow
 
 ##### `USE_UPDATE_REPO=`**`n`**
 
-> Only pull missing packages from the upstream base repository.
+> Do not pull missing packages from the upstream update repository.
+
+#### `USE_PREVIEW_REPO=...`
+
+##### `USE_PREVIEW_REPO=`**`n`** *(default)*
+
+> Do not pull missing packages from the upstream preview repository.
+
+##### `USE_PREVIEW_REPO=`**`y`**
+
+> Pull missing packages from the upstream preview repository in addition to the base repository.
 
 #### `DISABLE_UPSTREAM_REPOS=...`
 
@@ -436,7 +442,7 @@ If that is not desired all remote sources can be disabled by clearing the follow
 
 ##### `DISABLE_UPSTREAM_REPOS=`**`y`**
 
-> Only pull missing packages from local repositories. This does not affect hydrating the toolchain from `$(PACKAGE_URL)` and `$(PACKAGE_UPDATE_URL)`.
+> Only pull missing packages from local repositories. This does not affect hydrating the toolchain from `$(PACKAGE_URL_LIST)`.
 
 #### `REBUILD_PACKAGES=...`
 
@@ -579,7 +585,7 @@ To reproduce an ISO build, run the same make invocation as before, but set:
 | Variable                      | Default                                                                                                | Description
 |:------------------------------|:-------------------------------------------------------------------------------------------------------|:---
 | CONFIG_FILE                   | `$(RESOURCES_DIR)`/imageconfigs/core-efi/core-efi.json                                                 | Image config file to build
-| CONFIG_BASE_DIR               | `$(dir $(CONFIG_FILE))`                                                                               | Base directory to search for image files in (see [image_config.md](../images/image_config.md))
+| CONFIG_BASE_DIR               | `$(dir $(CONFIG_FILE))`                                                                                | Base directory to search for image files in (see [image_config.md](../images/image_config.md))
 | UNATTENDED_INSTALLER          |                                                                                                        | Create unattended ISO installer if set. Overrides all other installer options.
 | PACKAGE_BUILD_LIST            |                                                                                                        | Additional packages to build
 | PACKAGE_REBUILD_LIST          |                                                                                                        | Always rebuild this package, even if it is up-to-date. Base package name, will match all virtual packages produced as well.
@@ -599,18 +605,18 @@ To reproduce an ISO build, run the same make invocation as before, but set:
 | PACKAGE_ARCHIVE               |                                                                                                        | Use with `make hydrate-rpms` to populate a set of rpms from an archive.
 | DOWNLOAD_SRPMS                | n                                                                                                      | Pack SRPMs from local SPECs or download published ones?
 | USE_UPDATE_REPO               | y                                                                                                      | Pull missing packages from the upstream update repository in addition to the base repository?
-| DISABLE_UPSTREAM_REPOS        | n                                                                                                      | Only pull missing packages from local repositories? This does not affect hydrating the toolchain from `$(PACKAGE_URL)` and `$(PACKAGE_UPDATE_URL)`.
+| USE_PREVIEW_REPO              | n                                                                                                      | Pull missing packages from the upstream preview repository in addition to the base repository?
+| DISABLE_UPSTREAM_REPOS        | n                                                                                                      | Only pull missing packages from local repositories? This does not affect hydrating the toolchain from `$(PACKAGE_URL_LIST)`.
 
 ---
 
 ### Remote Connections
 
 | Variable                      | Default                                                                                                  | Description
 |:------------------------------|:---------------------------------------------------------------------------------------------------------|:---
-| SOURCE_URL                    |                                             | URL to request package sources from
-| SRPM_URL_LIST                 | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/srpms https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/preview/srpms` | Space seperated list of URLs to request packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y`
-| PACKAGE_URL                   | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms`              | URL to request full toolchain packages from
-| PACKAGE_UPDATE_URL            | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/update/$(build_arch)/rpms`            | URL to request full toolchain packages from if not found under `$(PACKAGE_URL)` and `$(USE_UPDATE_REPO)` is set to `y`
+| SOURCE_URL                    |                                                                                                          | URL to request package sources from
+| SRPM_URL_LIST                 | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/srpms`                         | Space seperated list of URLs to request packed SRPMs from if `$(DOWNLOAD_SRPMS)` is set to `y`
+| PACKAGE_URL_LIST              | `https://packages.microsoft.com/cbl-mariner/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch)/rpms`            | Space seperated list of URLs to download toolchain RPM packages from, used to populate the toolchain packages if `$(REBUILD_TOOLCHAIN)` is set to `y`.
 | REPO_LIST                     |                                                                                                          | Space separated list of repo files for tdnf to pull packages form
 | CA_CERT                       |                                                                                                          | CA cert to access the above resources
 | TLS_CERT                      |                                                                                                          | TLS cert to access the above resources

toolkit/docs/how_it_works/3_package_building.md

@@ -171,7 +171,7 @@ Sometimes packages fail to build correctly, or are otherwise not suitable for bu
 ### Stage 3: Graphpkgfetcher
 The `graphpkgfetcher` tool's job is to resolve unresolved remote nodes. Unresolved nodes occur when a local package has `Requires` or `BuildRequires` which are not available from another local package.
 
-The tool uses the `worker_chroot` (see [Chroot Worker](1_initial_prep.md#chroot_worker)) to locate packages. The worker will search in five locations: 1) the local chroot environment, 2) already build RPMs in `./../out/RPMS/`, 3) the upstream base repository 4) the upstream update repository if `$(USE_UPDATE_REPO)` is set to `y` 5) any remote repo listed in `REPO_LIST ?=`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled.
+The tool uses the `worker_chroot` (see [Chroot Worker](1_initial_prep.md#chroot_worker)) to locate packages. The worker will search in six locations: 1) the local chroot environment, 2) already build RPMs in `./../out/RPMS/`, 3) the upstream base repository 4) the upstream update repository if `$(USE_UPDATE_REPO)` is set to `y` 5) the upstream preview repository if `$(USE_PREVIEW_REPO)` is set to `y` 6) any remote repo listed in `REPO_LIST ?=`. If `$(DISABLE_UPSTREAM_REPOS)` is set to `y`, any repo that is accessed through the network is disabled.
 
 The worker will run the `tdnf` command to search for each missing package. `tdnf` will prioritize local packages over pulling them from a remote location.
 

toolkit/resources/manifests/package/pkggen_core_aarch64.txt

@@ -156,6 +156,7 @@ pinentry-1.1.0-3.cm1.aarch64.rpm
 gnupg2-2.2.20-3.cm1.aarch64.rpm
 gpgme-1.13.1-5.cm1.aarch64.rpm
 mariner-repos-1.0-11.cm1.noarch.rpm
+mariner-repos-preview-1.0-11.cm1.noarch.rpm
 libffi-3.2.1-11.cm1.aarch64.rpm
 libtasn1-4.14-2.cm1.aarch64.rpm
 p11-kit-0.23.16.1-2.cm1.aarch64.rpm

toolkit/resources/manifests/package/pkggen_core_x86_64.txt

@@ -156,6 +156,7 @@ pinentry-1.1.0-3.cm1.x86_64.rpm
 gnupg2-2.2.20-3.cm1.x86_64.rpm
 gpgme-1.13.1-5.cm1.x86_64.rpm
 mariner-repos-1.0-11.cm1.noarch.rpm
+mariner-repos-preview-1.0-11.cm1.noarch.rpm
 libffi-3.2.1-11.cm1.x86_64.rpm
 libtasn1-4.14-2.cm1.x86_64.rpm
 p11-kit-0.23.16.1-2.cm1.x86_64.rpm

toolkit/resources/manifests/package/update_manifests.sh

@@ -77,7 +77,6 @@ remove_packages_for_pkggen_core () {
     sed -i '/libxslt/d' $TmpPkgGen
     sed -i '/Linux-PAM/d' $TmpPkgGen
     sed -i '/lua-devel/d' $TmpPkgGen
-    sed -i '/mariner-repos-preview/d' $TmpPkgGen
     sed -i '/npth-[[:alpha:]]/d' $TmpPkgGen
     sed -i '/pcre-[0-9]/d' $TmpPkgGen
     sed -i '/pcre-devel/d' $TmpPkgGen

toolkit/scripts/imggen.mk

@@ -68,17 +68,19 @@ $(STATUS_FLAGS_DIR)/validate-image-config%.flag: $(go-imageconfigvalidator) $(de
 	touch $@
 
 
+imagepkgfetcher_extra_flags :=
 ifeq ($(DISABLE_UPSTREAM_REPOS),y)
-imagepkgfetcher_disable_upstream_repos_flag := --disable-upstream-repos
-else
-imagepkgfetcher_disable_upstream_repos_flag :=
+imagepkgfetcher_extra_flags += --disable-upstream-repos
 endif
 
 ifeq ($(USE_UPDATE_REPO),y)
-imagepkgfetcher_update_repo_flag := --use-update-repo
-else
-imagepkgfetcher_update_repo_flag :=
+imagepkgfetcher_extra_flags += --use-update-repo
 endif
+
+ifeq ($(USE_PREVIEW_REPO),y)
+imagepkgfetcher_extra_flags += --use-preview-repo
+endif
+
 $(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(packagelist_files) $(RPMS_DIR) $(imggen_rpms)
 	$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
 	$(go-imagepkgfetcher) \
@@ -92,8 +94,7 @@ $(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_
 		--tls-cert=$(TLS_CERT) \
 		--tls-key=$(TLS_KEY) \
 		$(foreach repo, $(imagefetcher_local_repo) $(imagefetcher_cloned_repo) $(REPO_LIST),--repo-file="$(repo)" ) \
-		$(imagepkgfetcher_update_repo_flag) \
-		$(imagepkgfetcher_disable_upstream_repos_flag) \
+		$(imagepkgfetcher_extra_flags) \
 		--input-summary-file=$(IMAGE_CACHE_SUMMARY) \
 		--output-summary-file=$@ \
 		--output-dir=$(local_and_external_rpm_cache)