From 34af777bc26bee8067ab6a3c3c9849245be2e46a Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 1 Jul 2021 14:28:50 +0200 Subject: [PATCH 1/3] prowgen: generate stricter `branch:` stanzas for presubmits In the past, we considered any `$branch$suffix` a feature branch of "base" CI branch (e.g. PRs to `master-next` would receive presubmits from `master` config). We need to make this convention stricter to not break on `4.10`, because the current convention means that `release-4.10` is a feature branch for `release-4.1`. All generated presubmits now have two items in their `branches:` stanza. One item that matches exactly the branch (`^branch$`), just like postsubmits have, and one item that matches branches with `$base-` prefix. Practically this means that we enforce exactly the `-` separator for a branch to be considered a feature branch. --- ...is_YAML_and_it_is_correctly_processed.yaml | 6 +++-- ...._Existing_job_should_not_be_changed..yaml | 6 +++-- ...resubmit__pre_post_submit_images_jobs.yaml | 6 +++-- pkg/prowgen/prowgen.go | 21 ++++++++++----- ...auses_promote_job_with_unique_targets.yaml | 3 ++- ...submit_for_a_test_in_a_variant_config.yaml | 3 ++- ...itForTest_presubmit_for_standard_test.yaml | 3 ++- ..._presubmit_with_job_release_specified.yaml | 3 ++- ..._with_job_release_specified_and_clone.yaml | 3 ++- .../private-org-duper-master-presubmits.yaml | 3 ++- .../private-duper-master-presubmits.yaml | 9 ++++--- .../repo/subdir-repo-master-presubmits.yaml | 3 ++- .../duper/super-duper-master-presubmits.yaml | 27 ++++++++++++------- ...r-master-removed-promotion-presubmits.yaml | 3 ++- .../super-duper-release-3.11-presubmits.yaml | 9 ++++--- .../openshift-ci-tools-master-presubmits.yaml | 6 +++-- .../openshift-origin-master-presubmits.yaml | 6 +++-- .../org-other-nonstandard-presubmits.yaml | 3 ++- .../org/repo/org-repo-master-presubmits.yaml | 15 +++++++---- .../org-third-nonstandard-presubmits.yaml | 3 ++- 20 files changed, 95 insertions(+), 46 deletions(-) diff --git a/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Input_is_YAML_and_it_is_correctly_processed.yaml b/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Input_is_YAML_and_it_is_correctly_processed.yaml index b4d0eb371c9..2ec2f27f692 100644 --- a/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Input_is_YAML_and_it_is_correctly_processed.yaml +++ b/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Input_is_YAML_and_it_is_correctly_processed.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/images decorate: true decoration_config: @@ -50,7 +51,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/unit decorate: true decoration_config: diff --git a/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Using_a_variant_config__one_test_and_images__one_existing_job._Expect_one_presubmit__pre_post_submit_images_jobs._Existing_job_should_not_be_changed..yaml b/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Using_a_variant_config__one_test_and_images__one_existing_job._Expect_one_presubmit__pre_post_submit_images_jobs._Existing_job_should_not_be_changed..yaml index 35799118146..bda770fdc9a 100644 --- a/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Using_a_variant_config__one_test_and_images__one_existing_job._Expect_one_presubmit__pre_post_submit_images_jobs._Existing_job_should_not_be_changed..yaml +++ b/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Using_a_variant_config__one_test_and_images__one_existing_job._Expect_one_presubmit__pre_post_submit_images_jobs._Existing_job_should_not_be_changed..yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/rhel-images decorate: true decoration_config: @@ -52,7 +53,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/rhel-unit decorate: true decoration_config: diff --git a/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_one_test_and_images__no_previous_jobs._Expect_test_presubmit__pre_post_submit_images_jobs.yaml b/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_one_test_and_images__no_previous_jobs._Expect_test_presubmit__pre_post_submit_images_jobs.yaml index b4d0eb371c9..2ec2f27f692 100644 --- a/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_one_test_and_images__no_previous_jobs._Expect_test_presubmit__pre_post_submit_images_jobs.yaml +++ b/cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_one_test_and_images__no_previous_jobs._Expect_test_presubmit__pre_post_submit_images_jobs.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/images decorate: true decoration_config: @@ -50,7 +51,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/unit decorate: true decoration_config: diff --git a/pkg/prowgen/prowgen.go b/pkg/prowgen/prowgen.go index 0fc80873d4f..44caeb42a50 100644 --- a/pkg/prowgen/prowgen.go +++ b/pkg/prowgen/prowgen.go @@ -445,7 +445,7 @@ func generatePresubmitForTest(name string, info *ProwgenInfo, podSpec *corev1.Po return &prowconfig.Presubmit{ JobBase: base, AlwaysRun: true, - Brancher: prowconfig.Brancher{Branches: []string{info.Branch}}, + Brancher: prowconfig.Brancher{Branches: sets.NewString(exactlyBranch(info.Branch), featureBranch(info.Branch)).List()}, Reporter: prowconfig.Reporter{ Context: fmt.Sprintf("ci/prow/%s", shortName), }, @@ -458,7 +458,7 @@ func generatePostsubmitForTest(name string, info *ProwgenInfo, podSpec *corev1.P base := generateJobBase(name, jc.PostsubmitPrefix, info, podSpec, false, pathAlias, jobRelease, skipCloning) return &prowconfig.Postsubmit{ JobBase: base, - Brancher: prowconfig.Brancher{Branches: []string{makeBranchExplicit(info.Branch)}}, + Brancher: prowconfig.Brancher{Branches: []string{exactlyBranch(info.Branch)}}, } } @@ -618,16 +618,25 @@ func generateJobBase(name, prefix string, info *ProwgenInfo, podSpec *corev1.Pod // not. var simpleBranchRegexp = regexp.MustCompile(`^[\w\-.]+$`) -// makeBranchExplicit updates the provided branch to prevent wildcard matches to the given branch -// if the branch value does not appear to contain an explicit regex pattern. I.e. 'master' -// is turned into '^master$'. -func makeBranchExplicit(branch string) string { +// exactlyBranch returns a regex string that matches exactly the given branch name: I.e. returns +// '^master$' for 'master'. If the given branch name already looks like a regex, return it unchanged. +func exactlyBranch(branch string) string { if !simpleBranchRegexp.MatchString(branch) { return branch } return fmt.Sprintf("^%s$", regexp.QuoteMeta(branch)) } +// featureBranch returns a regex string that matches feature branch prefixes for the given branch name: +// I.e. returns '^master-' for 'master'. If the given branch name already looks like a regex, +// return it unchanged. +func featureBranch(branch string) string { + if !simpleBranchRegexp.MatchString(branch) { + return branch + } + return fmt.Sprintf("^%s-", regexp.QuoteMeta(branch)) +} + // IsGenerated returns true if the job was generated using prowgen func IsGenerated(job prowconfig.JobBase) bool { _, generated := job.Labels[prowJobLabelGenerated] diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_Promotion_configuration_causes_promote_job_with_unique_targets.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_Promotion_configuration_causes_promote_job_with_unique_targets.yaml index cbefdaec32c..1849acdad29 100644 --- a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_Promotion_configuration_causes_promote_job_with_unique_targets.yaml +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_Promotion_configuration_causes_promote_job_with_unique_targets.yaml @@ -58,7 +58,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - branch + - ^branch$ + - ^branch- context: ci/prow/images decorate: true decoration_config: diff --git a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_a_test_in_a_variant_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_a_test_in_a_variant_config.yaml index 49977208189..e86cdf01de6 100644 --- a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_a_test_in_a_variant_config.yaml +++ b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_a_test_in_a_variant_config.yaml @@ -1,7 +1,8 @@ agent: kubernetes always_run: true branches: -- branch +- ^branch$ +- ^branch- context: ci/prow/also-testname decorate: true decoration_config: diff --git a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_standard_test.yaml b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_standard_test.yaml index 2b7b562003c..89628637d8b 100644 --- a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_standard_test.yaml +++ b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_for_standard_test.yaml @@ -1,7 +1,8 @@ agent: kubernetes always_run: true branches: -- branch +- ^branch$ +- ^branch- context: ci/prow/testname decorate: true decoration_config: diff --git a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified.yaml b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified.yaml index 710052bdca3..bcca4ac44e4 100644 --- a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified.yaml +++ b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified.yaml @@ -1,7 +1,8 @@ agent: kubernetes always_run: true branches: -- branch +- ^branch$ +- ^branch- context: ci/prow/testname decorate: true decoration_config: diff --git a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified_and_clone.yaml b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified_and_clone.yaml index 4e56c62be05..e68b4eb518c 100644 --- a/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified_and_clone.yaml +++ b/pkg/prowgen/testdata/zz_fixture_TestGeneratePresubmitForTest_presubmit_with_job_release_specified_and_clone.yaml @@ -1,7 +1,8 @@ agent: kubernetes always_run: true branches: -- branch +- ^branch$ +- ^branch- context: ci/prow/testname decorate: true labels: diff --git a/test/integration/ci-operator-prowgen/output/jobs/private-org/duper/private-org-duper-master-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/private-org/duper/private-org-duper-master-presubmits.yaml index ee53923911a..f1ad2701984 100644 --- a/test/integration/ci-operator-prowgen/output/jobs/private-org/duper/private-org-duper-master-presubmits.yaml +++ b/test/integration/ci-operator-prowgen/output/jobs/private-org/duper/private-org-duper-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/unit decorate: true decoration_config: diff --git a/test/integration/ci-operator-prowgen/output/jobs/private/duper/private-duper-master-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/private/duper/private-duper-master-presubmits.yaml index 30cd88eb8b2..32ae2c87fb4 100644 --- a/test/integration/ci-operator-prowgen/output/jobs/private/duper/private-duper-master-presubmits.yaml +++ b/test/integration/ci-operator-prowgen/output/jobs/private/duper/private-duper-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/e2e decorate: true decoration_config: @@ -84,7 +85,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/images decorate: true decoration_config: @@ -140,7 +142,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/unit decorate: true decoration_config: diff --git a/test/integration/ci-operator-prowgen/output/jobs/subdir/repo/subdir-repo-master-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/subdir/repo/subdir-repo-master-presubmits.yaml index a0f351e468a..892d122375d 100644 --- a/test/integration/ci-operator-prowgen/output/jobs/subdir/repo/subdir-repo-master-presubmits.yaml +++ b/test/integration/ci-operator-prowgen/output/jobs/subdir/repo/subdir-repo-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/test decorate: true decoration_config: diff --git a/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-presubmits.yaml index 4e805cf7b8d..25aaee32409 100644 --- a/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-presubmits.yaml +++ b/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/ci-index decorate: true decoration_config: @@ -50,7 +51,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/ci-index-my-bundle decorate: true decoration_config: @@ -97,7 +99,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/e2e decorate: true decoration_config: @@ -170,7 +173,8 @@ presubmits: - agent: kubernetes always_run: false branches: - - master + - ^master$ + - ^master- context: ci/prow/images decorate: true decoration_config: @@ -232,7 +236,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/lint decorate: true decoration_config: @@ -279,7 +284,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/steps decorate: true decoration_config: @@ -333,7 +339,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/unit decorate: true decoration_config: @@ -380,7 +387,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/variant-images decorate: true decoration_config: @@ -431,7 +439,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/variant-unit decorate: true decoration_config: diff --git a/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-removed-promotion-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-removed-promotion-presubmits.yaml index f71fcde4caf..204f1126a2f 100644 --- a/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-removed-promotion-presubmits.yaml +++ b/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-removed-promotion-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master-removed-promotion + - ^master-removed-promotion$ + - ^master-removed-promotion- context: ci/prow/images decorate: true decoration_config: diff --git a/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml index f58bb7c0cb2..857ac31a2b5 100644 --- a/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml +++ b/test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - release-3.11 + - ^release-3\.11$ + - ^release-3\.11- context: ci/prow/images decorate: true decoration_config: @@ -61,7 +62,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - release-3.11 + - ^release-3\.11$ + - ^release-3\.11- context: ci/prow/lint decorate: true decoration_config: @@ -108,7 +110,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - release-3.11 + - ^release-3\.11$ + - ^release-3\.11- context: ci/prow/unit decorate: true decoration_config: diff --git a/test/integration/repo-init/expected/ci-operator/jobs/openshift/ci-tools/openshift-ci-tools-master-presubmits.yaml b/test/integration/repo-init/expected/ci-operator/jobs/openshift/ci-tools/openshift-ci-tools-master-presubmits.yaml index 488572ae56d..472706f1551 100644 --- a/test/integration/repo-init/expected/ci-operator/jobs/openshift/ci-tools/openshift-ci-tools-master-presubmits.yaml +++ b/test/integration/repo-init/expected/ci-operator/jobs/openshift/ci-tools/openshift-ci-tools-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/images decorate: true @@ -51,7 +52,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/unit decorate: true diff --git a/test/integration/repo-init/expected/ci-operator/jobs/openshift/origin/openshift-origin-master-presubmits.yaml b/test/integration/repo-init/expected/ci-operator/jobs/openshift/origin/openshift-origin-master-presubmits.yaml index 3199925ac64..db8c23c80af 100644 --- a/test/integration/repo-init/expected/ci-operator/jobs/openshift/origin/openshift-origin-master-presubmits.yaml +++ b/test/integration/repo-init/expected/ci-operator/jobs/openshift/origin/openshift-origin-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/images decorate: true @@ -53,7 +54,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: ci/api-build01-ci-devcluster-openshift-com:6443 context: ci/prow/unit decorate: true diff --git a/test/integration/repo-init/expected/ci-operator/jobs/org/other/org-other-nonstandard-presubmits.yaml b/test/integration/repo-init/expected/ci-operator/jobs/org/other/org-other-nonstandard-presubmits.yaml index a67833fda11..0c2422a9446 100644 --- a/test/integration/repo-init/expected/ci-operator/jobs/org/other/org-other-nonstandard-presubmits.yaml +++ b/test/integration/repo-init/expected/ci-operator/jobs/org/other/org-other-nonstandard-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - nonstandard + - ^nonstandard$ + - ^nonstandard- cluster: ci/api-build01-ci-devcluster-openshift-com:6443 context: ci/prow/unit decorate: true diff --git a/test/integration/repo-init/expected/ci-operator/jobs/org/repo/org-repo-master-presubmits.yaml b/test/integration/repo-init/expected/ci-operator/jobs/org/repo/org-repo-master-presubmits.yaml index 39977b43f4a..ea055d742c5 100644 --- a/test/integration/repo-init/expected/ci-operator/jobs/org/repo/org-repo-master-presubmits.yaml +++ b/test/integration/repo-init/expected/ci-operator/jobs/org/repo/org-repo-master-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/cmd decorate: true @@ -51,7 +52,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/e2e decorate: true @@ -117,7 +119,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/e2e-aws decorate: true @@ -183,7 +186,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/race decorate: true @@ -231,7 +235,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- cluster: api.ci context: ci/prow/unit decorate: true diff --git a/test/integration/repo-init/expected/ci-operator/jobs/org/third/org-third-nonstandard-presubmits.yaml b/test/integration/repo-init/expected/ci-operator/jobs/org/third/org-third-nonstandard-presubmits.yaml index 6ebd9f67eab..ed04dcaa206 100644 --- a/test/integration/repo-init/expected/ci-operator/jobs/org/third/org-third-nonstandard-presubmits.yaml +++ b/test/integration/repo-init/expected/ci-operator/jobs/org/third/org-third-nonstandard-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - nonstandard + - ^nonstandard$ + - ^nonstandard- cluster: api.ci context: ci/prow/e2e decorate: true From daf022d36c2886c0976d6278018b5c61df037056 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Fri, 30 Jul 2021 15:42:03 +0200 Subject: [PATCH 2/3] prowgen: regenerate pj-rehearse test data --- ...uper-duper-ciop-cfg-change-presubmits.yaml | 3 ++- .../duper/super-duper-master-presubmits.yaml | 3 ++- .../super-trooper-master-presubmits.yaml | 21 ++++++++++++------- ...uper-duper-ciop-cfg-change-presubmits.yaml | 3 ++- ...uper-duper-cluster-profile-presubmits.yaml | 3 ++- .../duper/super-duper-master-presubmits.yaml | 3 ++- .../super-trooper-master-presubmits.yaml | 18 ++++++++++------ 7 files changed, 36 insertions(+), 18 deletions(-) diff --git a/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml b/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml index 3a0755e259c..fe9a50aa89f 100644 --- a/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml +++ b/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml @@ -36,7 +36,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - ciop-cfg-change + - ^ciop-cfg-change$ + - ^ciop-cfg-change- context: ci/prow/images decorate: true decoration_config: diff --git a/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml b/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml index 24bbd32ceac..1799dfc4d3a 100644 --- a/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml +++ b/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml @@ -86,7 +86,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/images decorate: true decoration_config: diff --git a/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml b/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml index 40e541e4d94..e8507552738 100644 --- a/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml +++ b/test/integration/pj-rehearse/candidate/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml @@ -85,7 +85,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/images decorate: true decoration_config: @@ -161,7 +162,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage decorate: true decoration_config: @@ -208,7 +210,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage2 decorate: true decoration_config: @@ -255,7 +258,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage3 decorate: true decoration_config: @@ -320,7 +324,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage4 decorate: true decoration_config: @@ -367,7 +372,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage5 decorate: true decoration_config: @@ -414,7 +420,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/unit decorate: true decoration_config: diff --git a/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml b/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml index 3a0755e259c..fe9a50aa89f 100644 --- a/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml +++ b/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-ciop-cfg-change-presubmits.yaml @@ -36,7 +36,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - ciop-cfg-change + - ^ciop-cfg-change$ + - ^ciop-cfg-change- context: ci/prow/images decorate: true decoration_config: diff --git a/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-cluster-profile-presubmits.yaml b/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-cluster-profile-presubmits.yaml index a14274bbf88..fb50d3f1c91 100644 --- a/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-cluster-profile-presubmits.yaml +++ b/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-cluster-profile-presubmits.yaml @@ -3,7 +3,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - cluster-profile + - ^cluster-profile$ + - ^cluster-profile- context: ci/prow/images decorate: true decoration_config: diff --git a/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml b/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml index 51ab06971f0..3f024aeb3ff 100644 --- a/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml +++ b/test/integration/pj-rehearse/master/ci-operator/jobs/super/duper/super-duper-master-presubmits.yaml @@ -81,7 +81,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/images decorate: true decoration_config: diff --git a/test/integration/pj-rehearse/master/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml b/test/integration/pj-rehearse/master/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml index 496a8afc090..19d8abb1661 100644 --- a/test/integration/pj-rehearse/master/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml +++ b/test/integration/pj-rehearse/master/ci-operator/jobs/super/trooper/super-trooper-master-presubmits.yaml @@ -85,7 +85,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/images decorate: true decoration_config: @@ -161,7 +162,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage decorate: true decoration_config: @@ -208,7 +210,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage2 decorate: true decoration_config: @@ -255,7 +258,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage3 decorate: true decoration_config: @@ -320,7 +324,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/multistage4 decorate: true decoration_config: @@ -367,7 +372,8 @@ presubmits: - agent: kubernetes always_run: true branches: - - master + - ^master$ + - ^master- context: ci/prow/unit decorate: true decoration_config: From 24f0d9376f83b7dfdd6acd0d8f5a4f02b218bf9c Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Fri, 30 Jul 2021 15:57:04 +0200 Subject: [PATCH 3/3] pj-rehearse: handle presubmits with multiple branches Instead of bailing out on multiple presubmits, iterate over them and try to find a branch name that is a "simple name" after stripping regex decoration. --- pkg/jobconfig/files.go | 5 +++++ pkg/prowgen/prowgen.go | 9 ++------- pkg/rehearse/jobs.go | 23 ++++++++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pkg/jobconfig/files.go b/pkg/jobconfig/files.go index 686c1c42d1e..5ccab32dabb 100644 --- a/pkg/jobconfig/files.go +++ b/pkg/jobconfig/files.go @@ -34,6 +34,11 @@ const ( PeriodicPrefix = "periodic" ) +// SimpleBranchRegexp matches a branch name that does not appear to be a regex (lacks wildcard, +// group, or other modifiers). For instance, `master` is considered simple, `master-.*` would +// not. +var SimpleBranchRegexp = regexp.MustCompile(`^[\w\-.]+$`) + // Info describes the metadata for a Prow job configuration file type Info struct { Org string diff --git a/pkg/prowgen/prowgen.go b/pkg/prowgen/prowgen.go index 44caeb42a50..72de8cf693f 100644 --- a/pkg/prowgen/prowgen.go +++ b/pkg/prowgen/prowgen.go @@ -613,15 +613,10 @@ func generateJobBase(name, prefix string, info *ProwgenInfo, podSpec *corev1.Pod return base } -// simpleBranchRegexp matches a branch name that does not appear to be a regex (lacks wildcard, -// group, or other modifiers). For instance, `master` is considered simple, `master-.*` would -// not. -var simpleBranchRegexp = regexp.MustCompile(`^[\w\-.]+$`) - // exactlyBranch returns a regex string that matches exactly the given branch name: I.e. returns // '^master$' for 'master'. If the given branch name already looks like a regex, return it unchanged. func exactlyBranch(branch string) string { - if !simpleBranchRegexp.MatchString(branch) { + if !jc.SimpleBranchRegexp.MatchString(branch) { return branch } return fmt.Sprintf("^%s$", regexp.QuoteMeta(branch)) @@ -631,7 +626,7 @@ func exactlyBranch(branch string) string { // I.e. returns '^master-' for 'master'. If the given branch name already looks like a regex, // return it unchanged. func featureBranch(branch string) string { - if !simpleBranchRegexp.MatchString(branch) { + if !jc.SimpleBranchRegexp.MatchString(branch) { return branch } return fmt.Sprintf("^%s-", regexp.QuoteMeta(branch)) diff --git a/pkg/rehearse/jobs.go b/pkg/rehearse/jobs.go index 3499ab7c35a..3b1f411963d 100644 --- a/pkg/rehearse/jobs.go +++ b/pkg/rehearse/jobs.go @@ -94,8 +94,17 @@ func NewCMClient(clusterConfig *rest.Config, namespace string, dry bool) (corecl // and we know that `\` is an invalid character in Git branch names, so any // that exist in the name have been placed there by regexp.QuoteMeta() and // can simply be removed as well. +// Iterates over all branches and returns an empty string when no branch +// is a simple branch name after the stripping func BranchFromRegexes(branches []string) string { - return strings.ReplaceAll(strings.TrimPrefix(strings.TrimSuffix(branches[0], "$"), "^"), "\\", "") + for i := range branches { + branch := strings.ReplaceAll(strings.TrimPrefix(strings.TrimSuffix(branches[i], "$"), "^"), "\\", "") + if branch != "" && jobconfig.SimpleBranchRegexp.MatchString(branch) { + return branch + } + } + + return "" } func makeRehearsalPresubmit(source *prowconfig.Presubmit, repo string, prNumber int, refs *pjapi.Refs) (*prowconfig.Presubmit, error) { @@ -196,11 +205,6 @@ func filterPresubmits(changedPresubmits map[string][]prowconfig.Presubmit, logge continue } - if len(job.Branches) != 1 { - jobLogger.Warn("cannot rehearse jobs that run over multiple branches") - continue - } - presubmits.Add(repo, job, config.GetSourceType(job.Labels)) } } @@ -440,10 +444,15 @@ func (jc *JobConfigurer) ConfigurePresubmitRehearsals(presubmits config.Presubmi if len(splitOrgRepo) != 2 { jobLogger.WithError(fmt.Errorf("failed to identify org and repo from string %s", orgrepo)).Warn("Failed to inline ci-operator-config into rehearsal presubmit job") } + branch := BranchFromRegexes(job.Branches) + if branch == "" { + jobLogger.Warn("failed to extract a simple branch name for a presubmit") + continue + } metadata := api.Metadata{ Org: splitOrgRepo[0], Repo: splitOrgRepo[1], - Branch: BranchFromRegexes(job.Branches), + Branch: branch, Variant: VariantFromLabels(job.Labels), } testname := metadata.TestNameFromJobName(job.Name, jobconfig.PresubmitPrefix)