diff --git a/cmd/ci-operator/main_test.go b/cmd/ci-operator/main_test.go index 1e42570194a..374e09087bc 100644 --- a/cmd/ci-operator/main_test.go +++ b/cmd/ci-operator/main_test.go @@ -992,7 +992,7 @@ func TestBuildPartialGraph(t *testing.T) { loggingclient.New(fakectrlruntimeclient.NewFakeClient(&imagev1.ImageStreamTag{ObjectMeta: metav1.ObjectMeta{Name: ":"}})), nil, ), - steps.SourceStep(api.SourceStepConfiguration{From: api.PipelineImageStreamTagReferenceRoot, To: api.PipelineImageStreamTagReferenceSource}, api.ResourceConfiguration{}, nil, &api.JobSpec{}, nil, nil), + steps.SourceStep(api.SourceStepConfiguration{From: api.PipelineImageStreamTagReferenceRoot, To: api.PipelineImageStreamTagReferenceSource}, api.ResourceConfiguration{}, nil, nil, &api.JobSpec{}, nil, nil), steps.ProjectDirectoryImageBuildStep( api.ProjectDirectoryImageBuildStepConfiguration{ From: api.PipelineImageStreamTagReferenceSource, @@ -1001,7 +1001,7 @@ func TestBuildPartialGraph(t *testing.T) { }, To: api.PipelineImageStreamTagReference("oc-bin-image"), }, - &api.ReleaseBuildConfiguration{}, api.ResourceConfiguration{}, nil, nil, nil, + &api.ReleaseBuildConfiguration{}, api.ResourceConfiguration{}, nil, nil, nil, nil, ), steps.OutputImageTagStep(api.OutputImageTagStepConfiguration{From: api.PipelineImageStreamTagReference("oc-bin-image")}, nil, nil), steps.ImagesReadyStep(steps.OutputImageTagStep(api.OutputImageTagStepConfiguration{From: api.PipelineImageStreamTagReference("oc-bin-image")}, nil, nil).Creates()), diff --git a/pkg/api/constant.go b/pkg/api/constant.go index 5bdebc3b68e..27d50dca33b 100644 --- a/pkg/api/constant.go +++ b/pkg/api/constant.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "time" "k8s.io/apimachinery/pkg/util/sets" ) @@ -51,6 +52,15 @@ const ( APPCIKubeAPIURL = "https://api.ci.l2s4.p1.openshiftapps.com:6443" + // ReasonPending is the error reason for pods not scheduled in time. + // It is generated when pods are for whatever reason not scheduled before + // `podStartTimeout`. + ReasonPending = "pod_pending" + // PodStartTimeout is the maximum amount of time for pods to begin running. + // It causes builds and tests to fail if their pods cannot be scheduled for + // whatever reason, instead of failing due to the Prow job timeout, which is + // much longer and generates errors that are less descriptive. + PodStartTimeout = 30 * time.Minute // CliEnv if the env we use to expose the path to the cli CliEnv = "CLI_DIR" DefaultLeaseEnv = "LEASED_RESOURCE" diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 4495c7eff2c..f516abb863c 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -244,19 +244,19 @@ func fromConfig( step = steps.InputImageTagStep(&conf, client, jobSpec) inputImages[conf.InputImage] = struct{}{} } else if rawStep.PipelineImageCacheStepConfiguration != nil { - step = steps.PipelineImageCacheStep(*rawStep.PipelineImageCacheStepConfiguration, config.Resources, buildClient, jobSpec, pullSecret) + step = steps.PipelineImageCacheStep(*rawStep.PipelineImageCacheStepConfiguration, config.Resources, buildClient, podClient, jobSpec, pullSecret) } else if rawStep.SourceStepConfiguration != nil { - step = steps.SourceStep(*rawStep.SourceStepConfiguration, config.Resources, buildClient, jobSpec, cloneAuthConfig, pullSecret) + step = steps.SourceStep(*rawStep.SourceStepConfiguration, config.Resources, buildClient, podClient, jobSpec, cloneAuthConfig, pullSecret) } else if rawStep.BundleSourceStepConfiguration != nil { - step = steps.BundleSourceStep(*rawStep.BundleSourceStepConfiguration, config, config.Resources, buildClient, jobSpec, pullSecret) + step = steps.BundleSourceStep(*rawStep.BundleSourceStepConfiguration, config, config.Resources, buildClient, podClient, jobSpec, pullSecret) } else if rawStep.IndexGeneratorStepConfiguration != nil { - step = steps.IndexGeneratorStep(*rawStep.IndexGeneratorStepConfiguration, config, config.Resources, buildClient, jobSpec, pullSecret) + step = steps.IndexGeneratorStep(*rawStep.IndexGeneratorStepConfiguration, config, config.Resources, buildClient, podClient, jobSpec, pullSecret) } else if rawStep.ProjectDirectoryImageBuildStepConfiguration != nil { - step = steps.ProjectDirectoryImageBuildStep(*rawStep.ProjectDirectoryImageBuildStepConfiguration, config, config.Resources, buildClient, jobSpec, pullSecret) + step = steps.ProjectDirectoryImageBuildStep(*rawStep.ProjectDirectoryImageBuildStepConfiguration, config, config.Resources, buildClient, podClient, jobSpec, pullSecret) } else if rawStep.ProjectDirectoryImageBuildInputs != nil { - step = steps.GitSourceStep(*rawStep.ProjectDirectoryImageBuildInputs, config.Resources, buildClient, jobSpec, cloneAuthConfig, pullSecret) + step = steps.GitSourceStep(*rawStep.ProjectDirectoryImageBuildInputs, config.Resources, buildClient, podClient, jobSpec, cloneAuthConfig, pullSecret) } else if rawStep.RPMImageInjectionStepConfiguration != nil { - step = steps.RPMImageInjectionStep(*rawStep.RPMImageInjectionStepConfiguration, config.Resources, buildClient, jobSpec, pullSecret) + step = steps.RPMImageInjectionStep(*rawStep.RPMImageInjectionStepConfiguration, config.Resources, buildClient, podClient, jobSpec, pullSecret) } else if rawStep.RPMServeStepConfiguration != nil { step = steps.RPMServerStep(*rawStep.RPMServeStepConfiguration, client, jobSpec) } else if rawStep.OutputImageTagStepConfiguration != nil { diff --git a/pkg/steps/bundle_source.go b/pkg/steps/bundle_source.go index 9629a1d7029..6c73fe48b02 100644 --- a/pkg/steps/bundle_source.go +++ b/pkg/steps/bundle_source.go @@ -13,6 +13,7 @@ import ( buildapi "github.com/openshift/api/build/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" "github.com/openshift/ci-tools/pkg/steps/utils" ) @@ -22,6 +23,7 @@ type bundleSourceStep struct { releaseBuildConfig *api.ReleaseBuildConfiguration resources api.ResourceConfiguration client BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec pullSecret *coreapi.Secret } @@ -75,7 +77,7 @@ func (s *bundleSourceStep) run(ctx context.Context) error { s.pullSecret, nil, ) - return handleBuild(ctx, s.client, *build) + return handleBuild(ctx, s.client, s.podClient, *build) } func replaceCommand(pullSpec, with string) string { @@ -129,12 +131,21 @@ func (s *bundleSourceStep) Description() string { return fmt.Sprintf("Build image %s from the repository", api.PipelineImageStreamTagReferenceBundleSource) } -func BundleSourceStep(config api.BundleSourceStepConfiguration, releaseBuildConfig *api.ReleaseBuildConfiguration, resources api.ResourceConfiguration, client BuildClient, jobSpec *api.JobSpec, pullSecret *coreapi.Secret) api.Step { +func BundleSourceStep( + config api.BundleSourceStepConfiguration, + releaseBuildConfig *api.ReleaseBuildConfiguration, + resources api.ResourceConfiguration, + client BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + pullSecret *coreapi.Secret, +) api.Step { return &bundleSourceStep{ config: config, releaseBuildConfig: releaseBuildConfig, resources: resources, client: client, + podClient: podClient, jobSpec: jobSpec, pullSecret: pullSecret, } diff --git a/pkg/steps/git_source.go b/pkg/steps/git_source.go index 85a60e7b07d..be7a82297f6 100644 --- a/pkg/steps/git_source.go +++ b/pkg/steps/git_source.go @@ -11,6 +11,7 @@ import ( buildapi "github.com/openshift/api/build/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" ) @@ -18,6 +19,7 @@ type gitSourceStep struct { config api.ProjectDirectoryImageBuildInputs resources api.ResourceConfiguration buildClient BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec cloneAuthConfig *CloneAuthConfig pullSecret *coreapi.Secret @@ -42,7 +44,7 @@ func (s *gitSourceStep) run(ctx context.Context) error { secretName = s.cloneAuthConfig.Secret.Name } - return handleBuild(ctx, s.buildClient, *buildFromSource(s.jobSpec, "", api.PipelineImageStreamTagReferenceRoot, buildapi.BuildSource{ + return handleBuild(ctx, s.buildClient, s.podClient, *buildFromSource(s.jobSpec, "", api.PipelineImageStreamTagReferenceRoot, buildapi.BuildSource{ Type: buildapi.BuildSourceGit, Dockerfile: s.config.DockerfileLiteral, ContextDir: s.config.ContextDir, @@ -99,11 +101,20 @@ func determineRefsWorkdir(refs *prowapi.Refs, extraRefs []prowapi.Refs) *prowapi } // GitSourceStep returns gitSourceStep that holds all the required information to create a build from a git source. -func GitSourceStep(config api.ProjectDirectoryImageBuildInputs, resources api.ResourceConfiguration, buildClient BuildClient, jobSpec *api.JobSpec, cloneAuthConfig *CloneAuthConfig, pullSecret *coreapi.Secret) api.Step { +func GitSourceStep( + config api.ProjectDirectoryImageBuildInputs, + resources api.ResourceConfiguration, + buildClient BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + cloneAuthConfig *CloneAuthConfig, + pullSecret *coreapi.Secret, +) api.Step { return &gitSourceStep{ config: config, resources: resources, buildClient: buildClient, + podClient: podClient, jobSpec: jobSpec, cloneAuthConfig: cloneAuthConfig, pullSecret: pullSecret, diff --git a/pkg/steps/index_generator.go b/pkg/steps/index_generator.go index 3fae17c7ab8..09d76141dad 100644 --- a/pkg/steps/index_generator.go +++ b/pkg/steps/index_generator.go @@ -11,6 +11,7 @@ import ( buildapi "github.com/openshift/api/build/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" "github.com/openshift/ci-tools/pkg/steps/utils" ) @@ -20,6 +21,7 @@ type indexGeneratorStep struct { releaseBuildConfig *api.ReleaseBuildConfiguration resources api.ResourceConfiguration client BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec pullSecret *coreapi.Secret } @@ -79,7 +81,7 @@ func (s *indexGeneratorStep) run(ctx context.Context) error { s.pullSecret, nil, ) - err = handleBuild(ctx, s.client, *build) + err = handleBuild(ctx, s.client, s.podClient, *build) if err != nil && strings.Contains(err.Error(), "error checking provided apis") { return results.ForReason("generating_index").WithError(err).Errorf("failed to generate operator index due to invalid bundle info: %v", err) } @@ -153,12 +155,21 @@ func (s *indexGeneratorStep) Objects() []ctrlruntimeclient.Object { return s.client.Objects() } -func IndexGeneratorStep(config api.IndexGeneratorStepConfiguration, releaseBuildConfig *api.ReleaseBuildConfiguration, resources api.ResourceConfiguration, buildClient BuildClient, jobSpec *api.JobSpec, pullSecret *coreapi.Secret) api.Step { +func IndexGeneratorStep( + config api.IndexGeneratorStepConfiguration, + releaseBuildConfig *api.ReleaseBuildConfiguration, + resources api.ResourceConfiguration, + buildClient BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + pullSecret *coreapi.Secret, +) api.Step { return &indexGeneratorStep{ config: config, releaseBuildConfig: releaseBuildConfig, resources: resources, client: buildClient, + podClient: podClient, jobSpec: jobSpec, pullSecret: pullSecret, } diff --git a/pkg/steps/pipeline_image_cache.go b/pkg/steps/pipeline_image_cache.go index cb4598c8b8b..5b8f399f89f 100644 --- a/pkg/steps/pipeline_image_cache.go +++ b/pkg/steps/pipeline_image_cache.go @@ -11,6 +11,7 @@ import ( buildapi "github.com/openshift/api/build/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" "github.com/openshift/ci-tools/pkg/steps/utils" ) @@ -24,6 +25,7 @@ type pipelineImageCacheStep struct { config api.PipelineImageCacheStepConfiguration resources api.ResourceConfiguration client BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec pullSecret *coreapi.Secret } @@ -44,7 +46,7 @@ func (s *pipelineImageCacheStep) run(ctx context.Context) error { if err != nil { return err } - return handleBuild(ctx, s.client, *buildFromSource( + return handleBuild(ctx, s.client, s.podClient, *buildFromSource( s.jobSpec, s.config.From, s.config.To, buildapi.BuildSource{ Type: buildapi.BuildSourceDockerfile, @@ -85,11 +87,19 @@ func (s *pipelineImageCacheStep) Objects() []ctrlruntimeclient.Object { return s.client.Objects() } -func PipelineImageCacheStep(config api.PipelineImageCacheStepConfiguration, resources api.ResourceConfiguration, client BuildClient, jobSpec *api.JobSpec, pullSecret *coreapi.Secret) api.Step { +func PipelineImageCacheStep( + config api.PipelineImageCacheStepConfiguration, + resources api.ResourceConfiguration, + client BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + pullSecret *coreapi.Secret, +) api.Step { return &pipelineImageCacheStep{ config: config, resources: resources, client: client, + podClient: podClient, jobSpec: jobSpec, pullSecret: pullSecret, } diff --git a/pkg/steps/project_image.go b/pkg/steps/project_image.go index 0e4a0446bfa..74c63d3173b 100644 --- a/pkg/steps/project_image.go +++ b/pkg/steps/project_image.go @@ -14,6 +14,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" "github.com/openshift/ci-tools/pkg/steps/utils" ) @@ -23,6 +24,7 @@ type projectDirectoryImageBuildStep struct { releaseBuildConfig *api.ReleaseBuildConfiguration resources api.ResourceConfiguration client BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec pullSecret *coreapi.Secret } @@ -61,7 +63,7 @@ func (s *projectDirectoryImageBuildStep) run(ctx context.Context) error { s.pullSecret, s.config.BuildArgs, ) - return handleBuild(ctx, s.client, *build) + return handleBuild(ctx, s.client, s.podClient, *build) } type workingDir func(tag string) (string, error) @@ -161,12 +163,21 @@ func (s *projectDirectoryImageBuildStep) Objects() []ctrlruntimeclient.Object { return s.client.Objects() } -func ProjectDirectoryImageBuildStep(config api.ProjectDirectoryImageBuildStepConfiguration, releaseBuildConfig *api.ReleaseBuildConfiguration, resources api.ResourceConfiguration, buildClient BuildClient, jobSpec *api.JobSpec, pullSecret *coreapi.Secret) api.Step { +func ProjectDirectoryImageBuildStep( + config api.ProjectDirectoryImageBuildStepConfiguration, + releaseBuildConfig *api.ReleaseBuildConfiguration, + resources api.ResourceConfiguration, + buildClient BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + pullSecret *coreapi.Secret, +) api.Step { return &projectDirectoryImageBuildStep{ config: config, releaseBuildConfig: releaseBuildConfig, resources: resources, client: buildClient, + podClient: podClient, jobSpec: jobSpec, pullSecret: pullSecret, } diff --git a/pkg/steps/rpm_injection.go b/pkg/steps/rpm_injection.go index bb002311085..8a36da2c62d 100644 --- a/pkg/steps/rpm_injection.go +++ b/pkg/steps/rpm_injection.go @@ -11,6 +11,7 @@ import ( routev1 "github.com/openshift/api/route/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" ) @@ -23,6 +24,7 @@ type rpmImageInjectionStep struct { config api.RPMImageInjectionStepConfiguration resources api.ResourceConfiguration client BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec pullSecret *coreapi.Secret } @@ -48,7 +50,7 @@ func (s *rpmImageInjectionStep) run(ctx context.Context) error { if err != nil { return err } - return handleBuild(ctx, s.client, *buildFromSource( + return handleBuild(ctx, s.client, s.podClient, *buildFromSource( s.jobSpec, s.config.From, s.config.To, buildapi.BuildSource{ Type: buildapi.BuildSourceDockerfile, @@ -84,11 +86,19 @@ func (s *rpmImageInjectionStep) Objects() []ctrlruntimeclient.Object { return s.client.Objects() } -func RPMImageInjectionStep(config api.RPMImageInjectionStepConfiguration, resources api.ResourceConfiguration, buildClient BuildClient, jobSpec *api.JobSpec, pullSecret *coreapi.Secret) api.Step { +func RPMImageInjectionStep( + config api.RPMImageInjectionStepConfiguration, + resources api.ResourceConfiguration, + buildClient BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + pullSecret *coreapi.Secret, +) api.Step { return &rpmImageInjectionStep{ config: config, resources: resources, client: buildClient, + podClient: podClient, jobSpec: jobSpec, pullSecret: pullSecret, } diff --git a/pkg/steps/source.go b/pkg/steps/source.go index 7c32915c0c9..3a4e52296b5 100644 --- a/pkg/steps/source.go +++ b/pkg/steps/source.go @@ -15,7 +15,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/errors" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" prowv1 "k8s.io/test-infra/prow/apis/prowjobs/v1" "k8s.io/test-infra/prow/clonerefs" @@ -26,6 +26,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" "github.com/openshift/ci-tools/pkg/results" "github.com/openshift/ci-tools/pkg/steps/loggingclient" "github.com/openshift/ci-tools/pkg/steps/utils" @@ -134,6 +135,7 @@ type sourceStep struct { config api.SourceStepConfiguration resources api.ResourceConfiguration client BuildClient + podClient kubernetes.PodClient jobSpec *api.JobSpec cloneAuthConfig *CloneAuthConfig pullSecret *corev1.Secret @@ -159,7 +161,7 @@ func (s *sourceStep) run(ctx context.Context) error { if err != nil { return err } - return handleBuild(ctx, s.client, *createBuild(s.config, s.jobSpec, clonerefsRef, s.resources, s.cloneAuthConfig, s.pullSecret, fromDigest)) + return handleBuild(ctx, s.client, s.podClient, *createBuild(s.config, s.jobSpec, clonerefsRef, s.resources, s.cloneAuthConfig, s.pullSecret, fromDigest)) } func createBuild(config api.SourceStepConfiguration, jobSpec *api.JobSpec, clonerefsRef corev1.ObjectReference, resources api.ResourceConfiguration, cloneAuthConfig *CloneAuthConfig, pullSecret *corev1.Secret, fromDigest string) *buildapi.Build { @@ -369,7 +371,7 @@ func isBuildPhaseTerminated(phase buildapi.BuildPhase) bool { return true } -func handleBuild(ctx context.Context, buildClient BuildClient, build buildapi.Build) error { +func handleBuild(ctx context.Context, buildClient BuildClient, podClient kubernetes.PodClient, build buildapi.Build) error { var buildErrs []error attempts := 5 if boErr := wait.ExponentialBackoff(wait.Backoff{Duration: time.Minute, Factor: 1.5, Steps: attempts}, func() (bool, error) { @@ -379,7 +381,7 @@ func handleBuild(ctx context.Context, buildClient BuildClient, build buildapi.Bu return false, fmt.Errorf("could not create build %s: %w", buildAttempt.Name, err) } - buildErr := waitForBuildOrTimeout(ctx, buildClient, buildAttempt.Namespace, buildAttempt.Name) + buildErr := waitForBuildOrTimeout(ctx, buildClient, podClient, buildAttempt.Namespace, buildAttempt.Name) if buildErr == nil { if err := gatherSuccessfulBuildLog(buildClient, buildAttempt.Namespace, buildAttempt.Name); err != nil { // log error but do not fail successful build @@ -419,7 +421,7 @@ func handleBuild(ctx context.Context, buildClient BuildClient, build buildapi.Bu return false, nil }); boErr != nil { if boErr == wait.ErrWaitTimeout { - return fmt.Errorf("build not successful after %d attempts: %w", attempts, errors.NewAggregate(buildErrs)) + return fmt.Errorf("build not successful after %d attempts: %w", attempts, utilerrors.NewAggregate(buildErrs)) } return boErr } @@ -484,7 +486,7 @@ func hintsAtInfraReason(logSnippet string) bool { strings.Contains(logSnippet, "connection reset by peer") } -func waitForBuildOrTimeout(ctx context.Context, buildClient BuildClient, namespace, name string) error { +func waitForBuildOrTimeout(ctx context.Context, buildClient BuildClient, podClient kubernetes.PodClient, namespace, name string) error { isOK := func(b *buildapi.Build) bool { return b.Status.Phase == buildapi.BuildPhaseComplete } @@ -512,6 +514,7 @@ func waitForBuildOrTimeout(ctx context.Context, buildClient BuildClient, namespa } ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() + var ran bool for { select { case <-ctx.Done(): @@ -521,6 +524,12 @@ func waitForBuildOrTimeout(ctx context.Context, buildClient BuildClient, namespa logrus.WithError(err).Warnf("Failed to get build %s.", name) continue } + ran = ran || (build.Status.Phase == buildapi.BuildPhaseRunning) + if !ran && api.PodStartTimeout < time.Since(build.CreationTimestamp.Time) { + err := util.PendingBuildError(ctx, podClient, build) + logrus.Infof(err.Error()) + return err + } if isOK(build) { logrus.Infof("Build %s succeeded after %s", build.Name, buildDuration(build).Truncate(time.Second)) return nil @@ -609,12 +618,20 @@ func (s *sourceStep) Objects() []ctrlruntimeclient.Object { return s.client.Objects() } -func SourceStep(config api.SourceStepConfiguration, resources api.ResourceConfiguration, buildClient BuildClient, - jobSpec *api.JobSpec, cloneAuthConfig *CloneAuthConfig, pullSecret *corev1.Secret) api.Step { +func SourceStep( + config api.SourceStepConfiguration, + resources api.ResourceConfiguration, + buildClient BuildClient, + podClient kubernetes.PodClient, + jobSpec *api.JobSpec, + cloneAuthConfig *CloneAuthConfig, + pullSecret *corev1.Secret, +) api.Step { return &sourceStep{ config: config, resources: resources, client: buildClient, + podClient: podClient, jobSpec: jobSpec, cloneAuthConfig: cloneAuthConfig, pullSecret: pullSecret, diff --git a/pkg/util/builds.go b/pkg/util/builds.go new file mode 100644 index 00000000000..4786b357666 --- /dev/null +++ b/pkg/util/builds.go @@ -0,0 +1,35 @@ +package util + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + + corev1 "k8s.io/api/core/v1" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + buildapi "github.com/openshift/api/build/v1" + + "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/kubernetes" + "github.com/openshift/ci-tools/pkg/results" +) + +// PendingBuildError fetches scheduling errors from the build pod's events +func PendingBuildError(ctx context.Context, client kubernetes.PodClient, build *buildapi.Build) error { + msg := fmt.Sprintf("build didn't start running within %s (phase: %s)", api.PodStartTimeout, build.Status.Phase) + var ret error + var pod corev1.Pod + if podName, ok := build.Annotations[buildapi.BuildPodNameAnnotation]; !ok { + logrus.Debug("build pod annotation missing") + ret = errors.New(msg) + } else if err := client.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: build.Namespace, Name: podName}, &pod); err != nil { + logrus.Warnf("failed to get build pod: %v", err) + ret = errors.New(msg) + } else { + ret = fmt.Errorf("%s: %s\n%s", msg, getReasonsForUnreadyContainers(&pod), getEventsForPod(ctx, &pod, client)) + } + return results.ForReason(api.ReasonPending).ForError(ret) +} diff --git a/pkg/util/pods.go b/pkg/util/pods.go index bced8cdb75e..e00e702e955 100644 --- a/pkg/util/pods.go +++ b/pkg/util/pods.go @@ -20,7 +20,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/openshift/ci-tools/pkg/api" "github.com/openshift/ci-tools/pkg/kubernetes" + "github.com/openshift/ci-tools/pkg/results" ) func CreateOrRestartPod(ctx context.Context, podClient ctrlruntimeclient.Client, pod *corev1.Pod) (*corev1.Pod, error) { @@ -178,7 +180,6 @@ func waitForPodCompletionOrTimeout(ctx context.Context, podClient kubernetes.Pod podCheckTicker := time.NewTicker(10 * time.Second) defer podCheckTicker.Stop() - podStartTimeout := 30 * time.Minute var podSeenRunning bool for { @@ -197,11 +198,11 @@ func waitForPodCompletionOrTimeout(ctx context.Context, podClient kubernetes.Pod if !podSeenRunning { if podHasStarted(pod) { podSeenRunning = true - } else if time.Since(pod.CreationTimestamp.Time) > podStartTimeout { - message := fmt.Sprintf("pod didn't start running within %s: %s\n%s", podStartTimeout, getReasonsForUnreadyContainers(pod), getEventsForPod(ctx, pod, podClient)) + } else if time.Since(pod.CreationTimestamp.Time) > api.PodStartTimeout { + message := fmt.Sprintf("pod didn't start running within %s: %s\n%s", api.PodStartTimeout, getReasonsForUnreadyContainers(pod), getEventsForPod(ctx, pod, podClient)) logrus.Infof(message) notifier.Complete(name) - return pod, errors.New(message) + return pod, results.ForReason(api.ReasonPending).ForError(errors.New(message)) } } podLogNewFailedContainers(podClient, pod, completed, notifier, skipLogs)