From 565ee45b3510df19a749d43a8739b3bf0289a518 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 6 Mar 2019 14:59:32 -0500 Subject: [PATCH] Create an initial release payload as `release:initial` When we tag images in, create a second image stream `stable-initial` that will not be modified and then let a secondary task create an image stream from that alternate source. If RELEASE_IMAGE_INITIAL is already specified, skip the action and instead tag it to that location. Simplify the structure of the AssembleReleaseStep to handle both initial and latest. Makes it easy to test an upgrade from the previous state to the current state (post change) while also allowing for injecting an initial step. --- pkg/defaults/defaults.go | 11 ++- pkg/steps/release/create_release.go | 105 +++++++++++++++++++++++----- pkg/steps/release/release_images.go | 9 +++ 3 files changed, 106 insertions(+), 19 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index bd13c1e7..f636dffd 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -101,7 +101,7 @@ func FromConfig( params.Add("NAMESPACE", nil, func() (string, error) { return jobSpec.Namespace, nil }) var imageStepLinks []api.StepLink - var releaseStep api.Step + var releaseStep, initialReleaseStep api.Step for _, rawStep := range stepConfigsForBuild(config, jobSpec) { var step api.Step var stepLinks []api.StepLink @@ -141,7 +141,13 @@ func FromConfig( step = release.ReleaseImagesTagStep(*rawStep.ReleaseImagesTagStepConfiguration, srcClient, imageClient, routeGetter, configMapGetter, params, jobSpec) stepLinks = append(stepLinks, step.Creates()...) - releaseStep = release.AssembleReleaseStep(*rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) + releaseStep = release.AssembleReleaseStep(true, *rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) + checkForFullyQualifiedStep(releaseStep, params) + + initialReleaseStep = release.AssembleReleaseStep(false, *rawStep.ReleaseImagesTagStepConfiguration, config.Resources, podClient, imageClient, artifactDir, jobSpec) + checkForFullyQualifiedStep(initialReleaseStep, params) + // initial release is always added + buildSteps = append(buildSteps, initialReleaseStep) } else if rawStep.TestStepConfiguration != nil { step = steps.TestStep(*rawStep.TestStepConfiguration, config.Resources, podClient, artifactDir, jobSpec) @@ -164,7 +170,6 @@ func FromConfig( } if releaseStep != nil { - releaseStep, _ = checkForFullyQualifiedStep(releaseStep, params) buildSteps = append(buildSteps, releaseStep) } else { buildSteps = append(buildSteps, release.StableImagesTagStep(imageClient, jobSpec)) diff --git a/pkg/steps/release/create_release.go b/pkg/steps/release/create_release.go index d2e00e6a..e1ea4313 100644 --- a/pkg/steps/release/create_release.go +++ b/pkg/steps/release/create_release.go @@ -4,6 +4,12 @@ import ( "context" "fmt" "log" + "os" + "strings" + "time" + + coreapi "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" imageapi "github.com/openshift/api/image/v1" imageclientset "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" @@ -20,6 +26,7 @@ import ( // created, then invoking the admin command for building a new release. type assembleReleaseStep struct { config api.ReleaseTagConfiguration + latest bool resources api.ResourceConfiguration imageClient imageclientset.ImageV1Interface podClient steps.PodClient @@ -29,13 +36,54 @@ type assembleReleaseStep struct { } func (s *assembleReleaseStep) Inputs(ctx context.Context, dry bool) (api.InputDefinition, error) { + if val := os.Getenv(s.envVar()); len(val) > 0 { + return api.InputDefinition{val}, nil + } return nil, nil } func (s *assembleReleaseStep) Run(ctx context.Context, dry bool) error { - stable, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get(api.StableImageStream, meta.GetOptions{}) + // if we receive an input, we tag it in instead of generating it + providedImage := os.Getenv(s.envVar()) + if len(providedImage) > 0 { + log.Printf("Setting release image %s to %s", s.tag(), providedImage) + if _, err := s.imageClient.ImageStreamTags(s.jobSpec.Namespace).Update(&imageapi.ImageStreamTag{ + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("release:%s", s.tag()), + }, + Tag: &imageapi.TagReference{ + From: &coreapi.ObjectReference{ + Kind: "DockerImage", + Name: providedImage, + }, + }, + }); err != nil { + return err + } + if err := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { + is, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get("release", meta.GetOptions{}) + if err != nil { + return false, err + } + ref, _ := findStatusTag(is, s.tag()) + return ref != nil, nil + }); err != nil { + return fmt.Errorf("unable to import %s release image: %v", s.tag(), err) + } + return nil + } + + tag := s.tag() + var streamName string + if s.latest { + streamName = api.StableImageStream + } else { + streamName = fmt.Sprintf("%s-initial", api.StableImageStream) + } + + stable, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get(streamName, meta.GetOptions{}) if err != nil { - return fmt.Errorf("could not resolve stable imagestream: %v", err) + return fmt.Errorf("could not resolve imagestream %s: %v", streamName, err) } cvo, ok := resolvePullSpec(stable, "cluster-version-operator", true) if !ok { @@ -43,7 +91,7 @@ func (s *assembleReleaseStep) Run(ctx context.Context, dry bool) error { return nil } if _, ok := resolvePullSpec(stable, "cli", true); !ok { - return fmt.Errorf("no 'cli' image was tagged into the stable stream, that image is required for building a release") + return fmt.Errorf("no 'cli' image was tagged into the %s stream, that image is required for building a release", streamName) } release, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Create(&imageapi.ImageStream{ @@ -61,10 +109,10 @@ func (s *assembleReleaseStep) Run(ctx context.Context, dry bool) error { } } - destination := fmt.Sprintf("%s:%s", release.Status.PublicDockerImageRepository, "latest") - log.Printf("Create a new update payload image %s", destination) + destination := fmt.Sprintf("%s:%s", release.Status.PublicDockerImageRepository, tag) + log.Printf("Create release image %s", destination) podConfig := steps.PodStepConfiguration{ - As: "release-latest", + As: fmt.Sprintf("release-%s", tag), From: api.ImageStreamTagReference{ Name: api.StableImageStream, Tag: "cli", @@ -76,8 +124,8 @@ set -euo pipefail export HOME=/tmp oc registry login oc adm release new --max-per-registry=32 -n %q --from-image-stream %q --to-image-base %q --to-image %q -oc adm release extract --from=%q --to=/tmp/artifacts/release-payload -`, s.jobSpec.Namespace, api.StableImageStream, cvo, destination, destination), +oc adm release extract --from=%q --to=/tmp/artifacts/release-payload-%s +`, s.jobSpec.Namespace, api.StableImageStream, cvo, destination, destination, tag), } // set an explicit default for release-latest resources, but allow customization if necessary @@ -103,16 +151,35 @@ func (s *assembleReleaseStep) Done() (bool, error) { } func (s *assembleReleaseStep) Requires() []api.StepLink { - return []api.StepLink{api.ImagesReadyLink()} + // if our prereq is provided, we don't need any prereqs + if len(os.Getenv(s.envVar())) > 0 { + return nil + } + if s.latest { + return []api.StepLink{api.ImagesReadyLink()} + } + return []api.StepLink{api.ReleaseImagesLink()} } func (s *assembleReleaseStep) Creates() []api.StepLink { - return []api.StepLink{api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference("latest"))} + return []api.StepLink{api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference(s.tag()))} +} + +func (s *assembleReleaseStep) tag() string { + if s.latest { + return "latest" + } + return "initial" +} + +func (s *assembleReleaseStep) envVar() string { + return fmt.Sprintf("RELEASE_IMAGE_%s", strings.ToUpper(s.tag())) } func (s *assembleReleaseStep) Provides() (api.ParameterMap, api.StepLink) { + tag := s.tag() return api.ParameterMap{ - "RELEASE_IMAGE_LATEST": func() (string, error) { + s.envVar(): func() (string, error) { is, err := s.imageClient.ImageStreams(s.jobSpec.Namespace).Get("release", meta.GetOptions{}) if err != nil { return "", fmt.Errorf("could not retrieve output imagestream: %v", err) @@ -125,22 +192,28 @@ func (s *assembleReleaseStep) Provides() (api.ParameterMap, api.StepLink) { } else { return "", fmt.Errorf("image stream %s has no accessible image registry value", "release") } - return fmt.Sprintf("%s:%s", registry, "latest"), nil + return fmt.Sprintf("%s:%s", registry, tag), nil }, - }, api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference("latest")) + }, api.ReleasePayloadImageLink(api.PipelineImageStreamTagReference(tag)) } -func (s *assembleReleaseStep) Name() string { return "[release:latest]" } +func (s *assembleReleaseStep) Name() string { + return fmt.Sprintf("[release:%s]", s.tag()) +} func (s *assembleReleaseStep) Description() string { - return fmt.Sprintf("Create a release image in the release image stream") + if s.latest { + return "Create the release image containing all images built by this job" + } + return "Create initial release image from the images that were in the input tag_specification" } // AssembleReleaseStep builds a new update payload image based on the cluster version operator // and the operators defined in the release configuration. -func AssembleReleaseStep(config api.ReleaseTagConfiguration, resources api.ResourceConfiguration, podClient steps.PodClient, imageClient imageclientset.ImageV1Interface, artifactDir string, jobSpec *api.JobSpec) api.Step { +func AssembleReleaseStep(latest bool, config api.ReleaseTagConfiguration, resources api.ResourceConfiguration, podClient steps.PodClient, imageClient imageclientset.ImageV1Interface, artifactDir string, jobSpec *api.JobSpec) api.Step { return &assembleReleaseStep{ config: config, + latest: latest, resources: resources, podClient: podClient, imageClient: imageClient, diff --git a/pkg/steps/release/release_images.go b/pkg/steps/release/release_images.go index 705671d9..c5300920 100644 --- a/pkg/steps/release/release_images.go +++ b/pkg/steps/release/release_images.go @@ -194,11 +194,20 @@ func (s *releaseImagesTagStep) Run(ctx context.Context, dry bool) error { fmt.Printf("%s\n", istJSON) return nil } + + initialIS := newIS.DeepCopy() + initialIS.Name = fmt.Sprintf("%s-initial", api.StableImageStream) + is, err = s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(newIS) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("could not copy stable imagestreamtag: %v", err) } + is, err = s.dstClient.ImageStreams(s.jobSpec.Namespace).Create(initialIS) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("could not copy stable-initial imagestreamtag: %v", err) + } + for _, tag := range is.Spec.Tags { spec, ok := resolvePullSpec(is, tag.Name, false) if !ok {