Add and use stream metadata for RHCOS, add openshift-install coreos print-stream-json

cmd/openshift-install/coreos.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/openshift/installer/pkg/coreoscli"
+)
+
+func newCoreOSCmd() *cobra.Command {
+	return coreoscli.NewCmd()
+}

cmd/openshift-install/main.go

@@ -60,6 +60,7 @@ func installerMain() {
 		newGatherCmd(),
 		newVersionCmd(),
 		newGraphCmd(),
+		newCoreOSCmd(),
 		newCompletionCmd(),
 		newMigrateCmd(),
 		newExplainCmd(),

docs/dev/pinned-coreos.md

@@ -0,0 +1,52 @@
+# CoreOS and the installer
+
+A key decision made before the release of OpenShift 4 is to pin the CoreOS bootimage in
+the installer: https://github.com/openshift/installer/commit/e080f0494708b2674fe37af02f670c8030c32bf6
+
+That is still the case today; when one gets an `openshift-install` binary, that
+binary contains the 2-tuple `(CoreOS, release image)`, meaning the result of an
+install will be the same thing each time.
+
+More background:
+
+ - https://github.com/openshift/enhancements/pull/201
+ - https://github.com/openshift/machine-config-operator/blob/master/docs/OSUpgrades.md
+
+## Stream metadata
+
+As of 4.8 the [stream metadata enhancement](https://github.com/openshift/enhancements/blobmaster/enhancements/coreos-bootimages.md)
+is in progress which provides a standardized JSON format and injects
+that data into the cluster as well.
+
+### Updating pinned stream metadata
+
+
+To update the bootimage for one or more architectures, use e.g.
+
+```
+$ plume cosa2stream --target data/data/rhcos-stream.json --distro rhcos  x86_64=48.83.202102230316-0 s390x=47.83.202102090311-0 ppc64le=47.83.202102091015-0
+```
+
+For more information on this command, see:
+
+- https://github.com/coreos/coreos-assembler/pull/2000 
+- https://github.com/coreos/coreos-assembler/pull/2052
+
+### Updating pinned legacy metadata
+
+To update the legacy metadata, use:
+
+```
+./hack/update-rhcos-bootimage.py https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.6/46.82.202008260918-0/x86_64/meta.json amd64
+```
+
+This will hopefully be removed soon.
+
+### Origin of stream metadata
+
+
+For historical reference, the initial file `data/data/rhcos-stream.json` was generated this way:
+
+```
+$ plume cosa2stream --name rhcos-4.8 --distro rhcos  x86_64=48.83.202102230316-0 s390x=47.83.202102090311-0 ppc64le=47.83.202102091015-0 > data/data/rhcos-stream.json
+```

docs/user/aws/install_upi.md

@@ -21,8 +21,8 @@ $ openshift-install create install-config
 ### Optional: Create Encrypted AMIs
 
 The IPI-based installer creates an encrypted AMI by default. If you wish to have an encrypted AMI for UPI-based
-installs, you will need to create it directly. You can find a list of the appropriate base AMIs 
-[here](../../../data/data/rhcos.json).
+installs, you will need to create it directly.  See [CoreOS bootimages](../overview.md#coreos-bootimages) for more information
+about bootimages, including how to find the AMI identifiers.
 
 You will make an encrypted copy of the AMI according to the [AWS documentation][encrypted-copy].
 

docs/user/metal/customization_ipi.md

@@ -24,13 +24,15 @@
 
 When doing a disconnected installation, the baremetal platform has the
 additional requirement that we have locations to download the RHCOS
-images. The installer downloads these from a location described in
-[/data/data/rhcos.json](/data/data/rhcos.json), but they can be
+images. The installer downloads these from a CoreOS stream metadata
+embedded in the installer code, but they can be
 overridden to point to a local mirror.
 
-The SHA256 parameter in the URLs are required, and should match the
-uncompressed SHA256 from rhcos.json.
+See [CoreOS bootimages](../overview.md#coreos-bootimages) for more information
+about bootimages.
 
+The SHA256 parameter in the URL is required, and should match the
+uncompressed SHA256 from the stream metadata JSON.
 
 * `bootstrapOSImage` (optional string): Override the image used for the
     bootstrap virtual machine.

docs/user/openstack/customization.md

@@ -112,13 +112,18 @@ sshKey: ssh-ed25519 AAAA...
 
 ## Image Overrides
 
-Normally the installer downloads the RHCOS image from a predetermined location described in [data/data/rhcos.json](/data/data/rhcos.json)). But the download URL can be overridden, notably for disconnected installations.
+The OpenShift installer pins the version of RHEL CoreOS and normally handles uploading the image to the target OpenStack instance.
 
-To do so and upload binary data from a custom location the user may set `clusterOSImage` parameter in the install config that points to that location, and then start the installation. In all other respects the process will be consistent with the default.
+If you want to download the image manually, see [CoreOS bootimages](../overview.md#coreos-bootimages) for more information
+about bootimages.  This is useful, for example, to perform a disconnected installation.  To do this,
+download the `qcow2` and host it at a custom location.  Then set the `openstack.clusterOSImage`
+parameter field in the install config to point to that location.   The install process will
+then use that mirrored image.
+In all other respects the process will be consistent with the default.
 
 **NOTE:** For this to work, the parameter value must be a valid http(s) URL.
 
-**NOTE:** The optional `sha256` query parameter can be attached to the URL, which will force the installer to check the image file checksum before uploading it into Glance.
+**NOTE:** The optional `sha256` query parameter can be attached to the URL. This will force the installer to check the uncompressed image file checksum before uploading it into Glance.
 
 Example:
 

docs/user/overview.md

@@ -84,3 +84,33 @@ As the unstable warning suggests, the presence of `manifests` and the names and
 It is occasionally useful to make alterations like this as one-off changes, but don't expect them to work on subsequent installer releases.
 
 [cluster-version]: https://github.com/openshift/cluster-version-operator/blob/master/docs/dev/clusterversion.md
+
+### CoreOS bootimages
+
+The `openshift-install` binary contains pinned versions of RHEL CoreOS "bootimages" (e.g. OpenStack `qcow2`, AWS AMI, bare metal `.iso`).
+Fully automated installs use these by default.
+
+For UPI (User Provisioned Infrastructure) installs, you can use the `openshift-install coreos print-stream-json` command to access information
+about the bootimages in [CoreOS Stream Metadata](https://github.com/coreos/stream-metadata-go) format.
+
+For example, this command will print the `x86_64` AMI for `us-west-1`:
+
+```
+$ openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.images.aws.regions["us-west-1"].image'
+ami-0c548bdf93b74cd59
+```
+
+For on-premise clouds (e.g. OpenStack) with UPI installs, you may need to manually copy
+a bootimage into the infrastructure.  Here's an example command to print the `x86_64` `qcow2` file for `openstack`:
+
+```
+$ openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.openstack.formats["qcow2.gz"]'
+{
+  "disk": {
+    "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-openstack.x86_64.qcow2.gz",
+    "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-openstack.x86_64.qcow2.gz.sig",
+    "sha256": "abc2add9746eb7be82e6919ec13aad8e9eae8cf073d8da6126d7c95ea0dee962",
+    "uncompressed-sha256": "9ed73a4e415ac670535c2188221e5a4a5f3e945bc2e03a65b1ed4fc76e5db6f2"
+  }
+}
+```

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/clarketm/json v1.14.1
 	github.com/containers/image v3.0.2+incompatible
 	github.com/coreos/ignition/v2 v2.9.0
+	github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3
 	github.com/dmacvicar/terraform-provider-libvirt v0.6.4-0.20201216193629-2b60d7626ff8
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/frankban/quicktest v1.7.2 // indirect

go.sum

@@ -302,6 +302,19 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
 github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/prometheus-operator v0.35.0/go.mod h1:XHYZUStZWcwd1yk/1DjZv/fywqKIyAJ6pSwvIr+v9BQ=
+github.com/coreos/prometheus-operator v0.38.0/go.mod h1:xZC7/TgeC0/mBaJk+1H9dbHaiEvLYHgX6Mi1h40UPh8=
+github.com/coreos/stream-metadata-go v0.0.0-20210115160721-ba77d4e64952 h1:t7IgMcyflINfXWPISnHTXwa/F+NxLgWGRGejoyfHUII=
+github.com/coreos/stream-metadata-go v0.0.0-20210115160721-ba77d4e64952/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
+github.com/coreos/stream-metadata-go v0.0.0-20210119202048-fb3ac10789fb h1:a1FamXpn1idyCsvXOAEdlmAWDZi6baC6sP2mJ/iHiHs=
+github.com/coreos/stream-metadata-go v0.0.0-20210119202048-fb3ac10789fb/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
+github.com/coreos/stream-metadata-go v0.0.0-20210120211222-7575c72c6f05 h1:GXgmcV54WyU3LDXRwf3vXtgkmajMHoYfQ/qCuPblr4A=
+github.com/coreos/stream-metadata-go v0.0.0-20210120211222-7575c72c6f05/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
+github.com/coreos/stream-metadata-go v0.0.0-20210121193119-2fbf8747cee7 h1:6LNM7FalzC7155uc5ecuUmD1x70QsOexZYGVHnLc4OE=
+github.com/coreos/stream-metadata-go v0.0.0-20210121193119-2fbf8747cee7/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
+github.com/coreos/stream-metadata-go v0.0.0-20210216154348-7c4d5d7d95b5 h1:rFS+eq6Wghj4SOylbWmWcYWkAho4+35NHIMZBv9EozA=
+github.com/coreos/stream-metadata-go v0.0.0-20210216154348-7c4d5d7d95b5/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
+github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 h1:0JspqV66RwYqYfvi8lCUoL5zUZMh9uN4hx/J5+NRXIE=
+github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
 github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE=
 github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE=
 github.com/coreos/vcontext v0.0.0-20200225161404-ee043618d38d/go.mod h1:z4pMVvaUrxs98RROlIYdAQCKhEicjnTirOaVyDRH5h8=

hack/update-rhcos-bootimage.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# Usage: ./hack/update-rhcos-bootimage.py https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.6/46.82.202008260918-0/x86_64/meta.json amd64
+# This script updates the legacy metadata.  We hope to remove it soon.
+# See docs/dev/pinned-coreos.md for more information.
 import codecs,os,sys,json,argparse
 import urllib.parse
 import urllib.request

pkg/asset/cluster/tfvars.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
+	coreosarch "github.com/coreos/stream-metadata-go/arch"
 	gcpprovider "github.com/openshift/cluster-api-provider-gcp/pkg/apis/gcpprovider/v1beta1"
 	kubevirtprovider "github.com/openshift/cluster-api-provider-kubevirt/pkg/apis/kubevirtprovider/v1alpha1"
 	kubevirtutils "github.com/openshift/cluster-api-provider-kubevirt/pkg/utils"
@@ -338,16 +339,30 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		}
 		preexistingnetwork := installConfig.Config.GCP.Network != ""
 
-		imageRaw, err := rhcospkg.GCPRaw(ctx, installConfig.Config.ControlPlane.Architecture)
+		archName := coreosarch.RpmArch(string(installConfig.Config.ControlPlane.Architecture))
+		st, err := rhcospkg.FetchCoreOSBuild(ctx)
 		if err != nil {
-			return errors.Wrap(err, "failed to find Raw GCP image URL")
+			return err
+		}
+		streamArch, err := st.GetArchitecture(archName)
+		if err != nil {
+			return err
+		}
+
+		img := streamArch.Images.Gcp
+		if img == nil {
+			return fmt.Errorf("%s: No GCP build found", st.FormatPrefix(archName))
 		}
+		// For backwards compatibility, we generate this URL to the image (only applies to RHCOS, not FCOS/OKD)
+		// right now.  It will only be used if nested virt or other licenses are enabled, which we
+		// really should deprecate and remove - xref https://github.com/openshift/installer/pull/4696
+		imageURL := fmt.Sprintf("https://storage.googleapis.com/rhcos/rhcos/%s.tar.gz", img.Name)
 		data, err := gcptfvars.TFVars(
 			gcptfvars.TFVarsSources{
 				Auth:               auth,
 				MasterConfigs:      masterConfigs,
 				WorkerConfigs:      workerConfigs,
-				ImageURI:           imageRaw,
+				ImageURI:           imageURL,
 				ImageLicenses:      installConfig.Config.GCP.Licenses,
 				PublicZoneName:     publicZoneName,
 				PublishStrategy:    installConfig.Config.Publish,

pkg/asset/rhcos/bootstrap_image.go

@@ -3,8 +3,11 @@ package rhcos
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/coreos/stream-metadata-go/arch"
+
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/rhcos"
@@ -37,28 +40,44 @@ func (i *BootstrapImage) Generate(p asset.Parents) error {
 	p.Get(ic)
 	config := ic.Config
 
-	var osimage string
-	var err error
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
+
 	switch config.Platform.Name() {
 	case baremetal.Name:
-		// Check for RHCOS image URL override
-		if boi := config.Platform.BareMetal.BootstrapOSImage; boi != "" {
-			osimage = boi
-			break
+		archName := arch.RpmArch(string(config.ControlPlane.Architecture))
+		st, err := rhcos.FetchCoreOSBuild(ctx)
+		if err != nil {
+			return err
+		}
+		streamArch, err := st.GetArchitecture(archName)
+		if err != nil {
+			return err
 		}
 
+		// Check for CoreOS image URL override
+		if boi := config.Platform.BareMetal.BootstrapOSImage; boi != "" {
+			*i = BootstrapImage(boi)
+			return nil
+		}
 		// Baremetal IPI launches a local VM for the bootstrap node
 		// Hence requires the QEMU image to use the libvirt backend
-		osimage, err = rhcos.QEMU(ctx, config.ControlPlane.Architecture)
+		if a, ok := streamArch.Artifacts["qemu"]; ok {
+			u, err := rhcos.FindArtifactURL(a)
+			if err != nil {
+				return err
+			}
+			*i = BootstrapImage(u)
+			return nil
+		}
+		return fmt.Errorf("%s: No qemu build found", st.FormatPrefix(archName))
 	default:
 		// other platforms use the same image for all nodes
-		osimage, err = osImage(config)
-	}
-	if err != nil {
-		return err
+		u, err := osImage(config)
+		if err != nil {
+			return err
+		}
+		*i = BootstrapImage(u)
+		return nil
 	}
-	*i = BootstrapImage(osimage)
-	return nil
 }