From 2c7cc0c27bbe7eaa16790158e477cf48e3f5d60f Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Thu, 13 Feb 2020 12:55:33 +0100 Subject: [PATCH 1/4] Added periodic job for OCP 4.4 with realtime workers For this a new CLUSTER_VARIANT "rt" is introduced --- ...enshift-release-release-4.4-periodics.yaml | 79 +++++++++++++++++++ .../cluster-launch-installer-e2e.yaml | 20 ++++- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml index ca67981b7a414..689d08be8c884 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml @@ -1549,6 +1549,85 @@ periodics: - name: release-pull-secret secret: secretName: ci-pull-credentials +- agent: kubernetes + decorate: true + interval: 48h + labels: + ci.openshift.io/release-type: informing + job-release: "4.4" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: release-openshift-ocp-installer-e2e-gcp-rt-4.4 + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --input-hash=$(BUILD_ID) + - --input-hash=$(JOB_NAME) + - --lease-server=http://boskos + - --secret-dir=/usr/local/e2e-gcp-rt-cluster-profile + - --secret-dir=/usr/local/pull-secret + - --target=e2e-gcp + - --template=/usr/local/e2e-gcp + command: + - ci-operator + env: + - name: RELEASE_IMAGE_LATEST + value: registry.svc.ci.openshift.org/ocp/release:4.4 + - name: BRANCH + value: "4.4" + - name: CLUSTER_TYPE + value: gcp + - name: CLUSTER_VARIANT + value: rt + - name: CONFIG_SPEC + value: | + tag_specification: + name: "$(BRANCH)" + namespace: ocp + resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi + tests: + - as: e2e-$(CLUSTER_TYPE)-rt + commands: TEST_SUITE=openshift/conformance/parallel run-tests + openshift_installer: + cluster_profile: "$(CLUSTER_TYPE)" + - name: JOB_NAME_SAFE + value: e2e-gcp-rt + - name: TEST_COMMAND + value: TEST_SUITE=openshift/conformance/parallel run-tests + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /usr/local/e2e-gcp-rt-cluster-profile + name: cluster-profile + - mountPath: /usr/local/e2e-gcp + name: job-definition + subPath: cluster-launch-installer-e2e.yaml + - mountPath: /usr/local/pull-secret + name: pull-secret + serviceAccountName: ci-operator + volumes: + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-gcp + - configMap: + name: prow-job-cluster-launch-installer-e2e + name: job-definition + - name: pull-secret + secret: + secretName: ci-pull-credentials - agent: kubernetes decorate: true interval: 48h diff --git a/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml b/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml index b387f4ae86c64..476f37d632129 100644 --- a/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml +++ b/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml @@ -736,9 +736,25 @@ objects: fi # TODO: Replace with a more concise manifest injection approach - if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]]; then + if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]] || has_variant "rt"; then openshift-install --dir=/tmp/artifacts/installer/ create manifests - echo "${CLUSTER_NETWORK_MANIFEST}" > /tmp/artifacts/installer/manifests/cluster-network-03-config.yml + + if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]]; then + echo "${CLUSTER_NETWORK_MANIFEST}" > /tmp/artifacts/installer/manifests/cluster-network-03-config.yml + fi + + if has_variant "rt"; then + cat > /tmp/artifacts/installer/manifests/realtime-worker-machine-config.yml << EOF + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: realtime-worker + spec: + kernelType: realtime + EOF + fi fi TF_LOG=debug openshift-install --dir=/tmp/artifacts/installer create cluster 2>&1 | grep --line-buffered -v password & From 46000a370365ac8694eb8cac328d8fe12c142aea Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Thu, 13 Feb 2020 18:40:03 +0100 Subject: [PATCH 2/4] Added periodic jobs for OCP 4.5 and 4.6, added all to release configs --- ...enshift-release-release-4.5-periodics.yaml | 79 +++++++++++++++++++ ...enshift-release-release-4.6-periodics.yaml | 79 +++++++++++++++++++ .../_releases/release-ocp-4.4.json | 4 + .../_releases/release-ocp-4.5.json | 4 + .../_releases/release-ocp-4.6.json | 4 + 5 files changed, 170 insertions(+) diff --git a/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml index 7f20261801c28..7be75e6a973db 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml @@ -1183,6 +1183,85 @@ periodics: - name: release-pull-secret secret: secretName: ci-pull-credentials +- agent: kubernetes + decorate: true + interval: 48h + labels: + ci.openshift.io/release-type: informing + job-release: "4.5" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: release-openshift-ocp-installer-e2e-gcp-rt-4.5 + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --input-hash=$(BUILD_ID) + - --input-hash=$(JOB_NAME) + - --lease-server=http://boskos + - --secret-dir=/usr/local/e2e-gcp-rt-cluster-profile + - --secret-dir=/usr/local/pull-secret + - --target=e2e-gcp + - --template=/usr/local/e2e-gcp + command: + - ci-operator + env: + - name: RELEASE_IMAGE_LATEST + value: registry.svc.ci.openshift.org/ocp/release:4.5 + - name: BRANCH + value: "4.5" + - name: CLUSTER_TYPE + value: gcp + - name: CLUSTER_VARIANT + value: rt + - name: CONFIG_SPEC + value: | + tag_specification: + name: "$(BRANCH)" + namespace: ocp + resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi + tests: + - as: e2e-$(CLUSTER_TYPE)-rt + commands: TEST_SUITE=openshift/conformance/parallel run-tests + openshift_installer: + cluster_profile: "$(CLUSTER_TYPE)" + - name: JOB_NAME_SAFE + value: e2e-gcp-rt + - name: TEST_COMMAND + value: TEST_SUITE=openshift/conformance/parallel run-tests + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /usr/local/e2e-gcp-rt-cluster-profile + name: cluster-profile + - mountPath: /usr/local/e2e-gcp + name: job-definition + subPath: cluster-launch-installer-e2e.yaml + - mountPath: /usr/local/pull-secret + name: pull-secret + serviceAccountName: ci-operator + volumes: + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-gcp + - configMap: + name: prow-job-cluster-launch-installer-e2e + name: job-definition + - name: pull-secret + secret: + secretName: ci-pull-credentials - agent: kubernetes decorate: true interval: 48h diff --git a/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml index 7095ab74f2964..a27409312383b 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml @@ -1183,6 +1183,85 @@ periodics: - name: release-pull-secret secret: secretName: ci-pull-credentials +- agent: kubernetes + decorate: true + interval: 48h + labels: + ci.openshift.io/release-type: informing + job-release: "4.6" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: release-openshift-ocp-installer-e2e-gcp-rt-4.6 + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --input-hash=$(BUILD_ID) + - --input-hash=$(JOB_NAME) + - --lease-server=http://boskos + - --secret-dir=/usr/local/e2e-gcp-rt-cluster-profile + - --secret-dir=/usr/local/pull-secret + - --target=e2e-gcp + - --template=/usr/local/e2e-gcp + command: + - ci-operator + env: + - name: RELEASE_IMAGE_LATEST + value: registry.svc.ci.openshift.org/ocp/release:4.6 + - name: BRANCH + value: "4.6" + - name: CLUSTER_TYPE + value: gcp + - name: CLUSTER_VARIANT + value: rt + - name: CONFIG_SPEC + value: | + tag_specification: + name: "$(BRANCH)" + namespace: ocp + resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi + tests: + - as: e2e-$(CLUSTER_TYPE)-rt + commands: TEST_SUITE=openshift/conformance/parallel run-tests + openshift_installer: + cluster_profile: "$(CLUSTER_TYPE)" + - name: JOB_NAME_SAFE + value: e2e-gcp-rt + - name: TEST_COMMAND + value: TEST_SUITE=openshift/conformance/parallel run-tests + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /usr/local/e2e-gcp-rt-cluster-profile + name: cluster-profile + - mountPath: /usr/local/e2e-gcp + name: job-definition + subPath: cluster-launch-installer-e2e.yaml + - mountPath: /usr/local/pull-secret + name: pull-secret + serviceAccountName: ci-operator + volumes: + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-gcp + - configMap: + name: prow-job-cluster-launch-installer-e2e + name: job-definition + - name: pull-secret + secret: + secretName: ci-pull-credentials - agent: kubernetes decorate: true interval: 48h diff --git a/core-services/release-controller/_releases/release-ocp-4.4.json b/core-services/release-controller/_releases/release-ocp-4.4.json index d8ac4b6b81b0e..9d067771d0831 100644 --- a/core-services/release-controller/_releases/release-ocp-4.4.json +++ b/core-services/release-controller/_releases/release-ocp-4.4.json @@ -41,6 +41,10 @@ "optional":true, "prowJob":{"name":"release-openshift-ocp-installer-e2e-gcp-serial-4.4"} }, + "gcp-rt":{ + "optional":true, + "prowJob":{"name":"release-openshift-ocp-installer-e2e-gcp-rt-4.4"} + }, "metal":{ "optional":true, "prowJob":{"name":"release-openshift-ocp-installer-e2e-metal-4.4"} diff --git a/core-services/release-controller/_releases/release-ocp-4.5.json b/core-services/release-controller/_releases/release-ocp-4.5.json index 6cf2524ae27ee..734c6aaa40592 100644 --- a/core-services/release-controller/_releases/release-ocp-4.5.json +++ b/core-services/release-controller/_releases/release-ocp-4.5.json @@ -41,6 +41,10 @@ "optional":true, "prowJob":{"name":"release-openshift-ocp-installer-e2e-gcp-serial-4.5"} }, + "gcp-rt":{ + "optional":true, + "prowJob":{"name":"release-openshift-ocp-installer-e2e-gcp-rt-4.5"} + }, "metal":{ "optional":true, "prowJob":{"name":"release-openshift-ocp-installer-e2e-metal-4.5"} diff --git a/core-services/release-controller/_releases/release-ocp-4.6.json b/core-services/release-controller/_releases/release-ocp-4.6.json index f68eb8442157c..eab1cd84c30eb 100644 --- a/core-services/release-controller/_releases/release-ocp-4.6.json +++ b/core-services/release-controller/_releases/release-ocp-4.6.json @@ -41,6 +41,10 @@ "optional":true, "prowJob":{"name":"release-openshift-ocp-installer-e2e-gcp-serial-4.6"} }, + "gcp-rt":{ + "optional":true, + "prowJob":{"name":"release-openshift-ocp-installer-e2e-gcp-rt-4.6"} + }, "metal":{ "optional":true, "prowJob":{"name":"release-openshift-ocp-installer-e2e-metal-4.6"} From d83c7794e11a35b7f99a54296572a1f890347c05 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Mon, 9 Mar 2020 13:28:52 +0100 Subject: [PATCH 3/4] Apply config changes after rebase --- ...enshift-release-release-4.4-periodics.yaml | 40 ++++++++++++++++++- ...enshift-release-release-4.5-periodics.yaml | 40 ++++++++++++++++++- ...enshift-release-release-4.6-periodics.yaml | 40 ++++++++++++++++++- 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml index 689d08be8c884..7a4ca3ef80607 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-release-4.4-periodics.yaml @@ -1562,9 +1562,13 @@ periodics: - args: - --artifact-dir=$(ARTIFACTS) - --give-pr-author-access-to-namespace=true + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - --input-hash=$(BUILD_ID) - --input-hash=$(JOB_NAME) - - --lease-server=http://boskos + - --kubeconfig=/etc/apici/kubeconfig + - --lease-server-password-file=/etc/boskos/password + - --lease-server-username=ci + - --lease-server=https://boskos-ci.svc.ci.openshift.org - --secret-dir=/usr/local/e2e-gcp-rt-cluster-profile - --secret-dir=/usr/local/pull-secret - --target=e2e-gcp @@ -1582,7 +1586,15 @@ periodics: value: rt - name: CONFIG_SPEC value: | + resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi tag_specification: + cluster: https://api.ci.openshift.org name: "$(BRANCH)" namespace: ocp resources: @@ -1608,15 +1620,36 @@ periodics: requests: cpu: 10m volumeMounts: + - mountPath: /etc/apici + name: apici-ci-operator-credentials + readOnly: true + - mountPath: /etc/boskos + name: boskos + readOnly: true - mountPath: /usr/local/e2e-gcp-rt-cluster-profile name: cluster-profile - mountPath: /usr/local/e2e-gcp name: job-definition subPath: cluster-launch-installer-e2e.yaml - - mountPath: /usr/local/pull-secret + - mountPath: /etc/pull-secret name: pull-secret + readOnly: true + - mountPath: /usr/local/pull-secret + name: release-pull-secret serviceAccountName: ci-operator volumes: + - name: apici-ci-operator-credentials + secret: + items: + - key: sa.ci-operator.apici.config + path: kubeconfig + secretName: apici-ci-operator-credentials + - name: boskos + secret: + items: + - key: password + path: password + secretName: boskos-credentials - name: cluster-profile projected: sources: @@ -1626,6 +1659,9 @@ periodics: name: prow-job-cluster-launch-installer-e2e name: job-definition - name: pull-secret + secret: + secretName: regcred + - name: release-pull-secret secret: secretName: ci-pull-credentials - agent: kubernetes diff --git a/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml index 7be75e6a973db..33b6c1b8d6c9b 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-release-4.5-periodics.yaml @@ -1196,9 +1196,13 @@ periodics: - args: - --artifact-dir=$(ARTIFACTS) - --give-pr-author-access-to-namespace=true + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - --input-hash=$(BUILD_ID) - --input-hash=$(JOB_NAME) - - --lease-server=http://boskos + - --kubeconfig=/etc/apici/kubeconfig + - --lease-server-password-file=/etc/boskos/password + - --lease-server-username=ci + - --lease-server=https://boskos-ci.svc.ci.openshift.org - --secret-dir=/usr/local/e2e-gcp-rt-cluster-profile - --secret-dir=/usr/local/pull-secret - --target=e2e-gcp @@ -1216,7 +1220,15 @@ periodics: value: rt - name: CONFIG_SPEC value: | + resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi tag_specification: + cluster: https://api.ci.openshift.org name: "$(BRANCH)" namespace: ocp resources: @@ -1242,15 +1254,36 @@ periodics: requests: cpu: 10m volumeMounts: + - mountPath: /etc/apici + name: apici-ci-operator-credentials + readOnly: true + - mountPath: /etc/boskos + name: boskos + readOnly: true - mountPath: /usr/local/e2e-gcp-rt-cluster-profile name: cluster-profile - mountPath: /usr/local/e2e-gcp name: job-definition subPath: cluster-launch-installer-e2e.yaml - - mountPath: /usr/local/pull-secret + - mountPath: /etc/pull-secret name: pull-secret + readOnly: true + - mountPath: /usr/local/pull-secret + name: release-pull-secret serviceAccountName: ci-operator volumes: + - name: apici-ci-operator-credentials + secret: + items: + - key: sa.ci-operator.apici.config + path: kubeconfig + secretName: apici-ci-operator-credentials + - name: boskos + secret: + items: + - key: password + path: password + secretName: boskos-credentials - name: cluster-profile projected: sources: @@ -1260,6 +1293,9 @@ periodics: name: prow-job-cluster-launch-installer-e2e name: job-definition - name: pull-secret + secret: + secretName: regcred + - name: release-pull-secret secret: secretName: ci-pull-credentials - agent: kubernetes diff --git a/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml index a27409312383b..dc1d09338e1f9 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-release-4.6-periodics.yaml @@ -1196,9 +1196,13 @@ periodics: - args: - --artifact-dir=$(ARTIFACTS) - --give-pr-author-access-to-namespace=true + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - --input-hash=$(BUILD_ID) - --input-hash=$(JOB_NAME) - - --lease-server=http://boskos + - --kubeconfig=/etc/apici/kubeconfig + - --lease-server-password-file=/etc/boskos/password + - --lease-server-username=ci + - --lease-server=https://boskos-ci.svc.ci.openshift.org - --secret-dir=/usr/local/e2e-gcp-rt-cluster-profile - --secret-dir=/usr/local/pull-secret - --target=e2e-gcp @@ -1216,7 +1220,15 @@ periodics: value: rt - name: CONFIG_SPEC value: | + resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi tag_specification: + cluster: https://api.ci.openshift.org name: "$(BRANCH)" namespace: ocp resources: @@ -1242,15 +1254,36 @@ periodics: requests: cpu: 10m volumeMounts: + - mountPath: /etc/apici + name: apici-ci-operator-credentials + readOnly: true + - mountPath: /etc/boskos + name: boskos + readOnly: true - mountPath: /usr/local/e2e-gcp-rt-cluster-profile name: cluster-profile - mountPath: /usr/local/e2e-gcp name: job-definition subPath: cluster-launch-installer-e2e.yaml - - mountPath: /usr/local/pull-secret + - mountPath: /etc/pull-secret name: pull-secret + readOnly: true + - mountPath: /usr/local/pull-secret + name: release-pull-secret serviceAccountName: ci-operator volumes: + - name: apici-ci-operator-credentials + secret: + items: + - key: sa.ci-operator.apici.config + path: kubeconfig + secretName: apici-ci-operator-credentials + - name: boskos + secret: + items: + - key: password + path: password + secretName: boskos-credentials - name: cluster-profile projected: sources: @@ -1260,6 +1293,9 @@ periodics: name: prow-job-cluster-launch-installer-e2e name: job-definition - name: pull-secret + secret: + secretName: regcred + - name: release-pull-secret secret: secretName: ci-pull-credentials - agent: kubernetes From 05e04711b5cc3c7207136e13d7642c29133f80cf Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Mon, 16 Mar 2020 19:07:16 +0100 Subject: [PATCH 4/4] separated CLUSTER_NETWORK_MANIFEST and rt variant flows Signed-off-by: Marc Sluiter --- .../cluster-launch-installer-e2e.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml b/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml index 476f37d632129..dd9f2e0b42d7c 100644 --- a/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml +++ b/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml @@ -736,15 +736,18 @@ objects: fi # TODO: Replace with a more concise manifest injection approach - if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]] || has_variant "rt"; then + if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]]; then openshift-install --dir=/tmp/artifacts/installer/ create manifests + echo "${CLUSTER_NETWORK_MANIFEST}" > /tmp/artifacts/installer/manifests/cluster-network-03-config.yml + fi - if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]]; then - echo "${CLUSTER_NETWORK_MANIFEST}" > /tmp/artifacts/installer/manifests/cluster-network-03-config.yml - fi - - if has_variant "rt"; then - cat > /tmp/artifacts/installer/manifests/realtime-worker-machine-config.yml << EOF + if has_variant "rt"; then + if [[ -n "${CLUSTER_NETWORK_MANIFEST:-}" ]]; then + echo 'error: CLUSTER_NETWORK_MANIFEST is incompatible with the `rt` variant' + exit 1 + fi + openshift-install --dir=/tmp/artifacts/installer/ create manifests + cat > /tmp/artifacts/installer/manifests/realtime-worker-machine-config.yml << EOF apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: @@ -754,7 +757,6 @@ objects: spec: kernelType: realtime EOF - fi fi TF_LOG=debug openshift-install --dir=/tmp/artifacts/installer create cluster 2>&1 | grep --line-buffered -v password &