From 6fe40487249c41d986642d7ee685fa666cfa7297 Mon Sep 17 00:00:00 2001 From: Varsha B Date: Wed, 4 Feb 2026 17:16:52 +0530 Subject: [PATCH 01/10] change arch Signed-off-by: Varsha B --- .../redhat-developer-gitops-operator-master__v4.19.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index d65d7ce4ac74b..48163264d7d1a 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -55,6 +55,7 @@ tests: cluster_profile: gitops-aws env: BASE_DOMAIN: ocp-gitops-qe.com + OCP_ARCH: arm64 test: - as: install cli: latest @@ -88,6 +89,7 @@ tests: cluster_profile: gitops-aws env: BASE_DOMAIN: ocp-gitops-qe.com + OCP_ARCH: arm64 test: - as: install cli: latest @@ -120,6 +122,7 @@ tests: cluster_profile: gitops-aws env: BASE_DOMAIN: ocp-gitops-qe.com + OCP_ARCH: arm64 test: - as: e2e-steps commands: scripts/openshiftci-presubmit-all-tests.sh From f0ce92947f5f8642476012df719eee98dc9e755d Mon Sep 17 00:00:00 2001 From: Varsha B Date: Thu, 5 Feb 2026 18:43:52 +0530 Subject: [PATCH 02/10] add arm64 release Signed-off-by: Varsha B --- ...dhat-developer-gitops-operator-master__v4.19.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index 48163264d7d1a..f4e264a31d013 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -34,6 +34,12 @@ operator: - pullspec: quay.io/redhat-developer/gitops-operator:latest with: pipeline:gitops-operator releases: + arm64-latest: + candidate: + architecture: arm64 + product: ocp + stream: nightly + version: "4.19" initial: integration: name: "4.19" @@ -53,6 +59,8 @@ tests: skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:arm64-latest env: BASE_DOMAIN: ocp-gitops-qe.com OCP_ARCH: arm64 @@ -87,6 +95,8 @@ tests: skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:arm64-latest env: BASE_DOMAIN: ocp-gitops-qe.com OCP_ARCH: arm64 @@ -120,6 +130,8 @@ tests: skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:arm64-latest env: BASE_DOMAIN: ocp-gitops-qe.com OCP_ARCH: arm64 From 4909289227e5b53dedb73550351bfed4c88fc18e Mon Sep 17 00:00:00 2001 From: Varsha B Date: Fri, 6 Feb 2026 11:40:15 +0530 Subject: [PATCH 03/10] compatible gitops image Signed-off-by: Varsha B --- .../redhat-developer-gitops-operator-master__v4.19.yaml | 6 +++++- .../redhat-developer-gitops-operator-master-presubmits.yaml | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index f4e264a31d013..17ee0dcd5fa7a 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -22,12 +22,16 @@ images: - dockerfile_path: openshift-ci/build-root/Dockerfile from: oc-bin-image to: dependencies - - context_dir: . + - additional_architectures: + - arm64 + context_dir: . from: src to: gitops-operator operator: bundles: - as: gitops-operator-bundle + capabilities: + - arm64 context_dir: bundle/ dockerfile_path: bundle.Dockerfile substitutions: diff --git a/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml b/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml index c323fe406fa42..d5f0efeec2a2f 100644 --- a/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml +++ b/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml @@ -441,6 +441,7 @@ presubmits: context: ci/prow/v4.19-ci-index-gitops-operator-bundle decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/variant: v4.19 ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -579,6 +580,7 @@ presubmits: context: ci/prow/v4.19-images decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/variant: v4.19 ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" From f7ae9dafbb56520f7f0bc4769ba7d2ede4e626f7 Mon Sep 17 00:00:00 2001 From: Varsha B Date: Tue, 17 Feb 2026 15:38:48 +0530 Subject: [PATCH 04/10] diagnostic steps Signed-off-by: Varsha B --- ...veloper-gitops-operator-master__v4.19.yaml | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index 17ee0dcd5fa7a..a0a43b56534ae 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -74,7 +74,26 @@ tests: commands: | oc create namespace openshift-gitops-operator oc get ns openshift-gitops-operator -o yaml - operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m + if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then + echo "=== Operator installation failed, gathering diagnostics ===" + echo "=== CSV Status ===" + oc get csv -n openshift-gitops-operator -o yaml || true + echo "=== CSV Describe ===" + oc describe csv -n openshift-gitops-operator || true + echo "=== Operator Pods ===" + oc get pods -n openshift-gitops-operator || true + echo "=== Operator Pod Describe ===" + oc describe pods -n openshift-gitops-operator || true + echo "=== Operator Pod Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=controller-manager --tail=100 || true + echo "=== Events ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true + echo "=== Deployment Status ===" + oc get deployment -n openshift-gitops-operator -o yaml || true + echo "=== Image being used ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[0].image}' || true + exit 1 + fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager dependencies: - env: OO_BUNDLE @@ -110,7 +129,26 @@ tests: commands: | oc create namespace openshift-gitops-operator oc get ns openshift-gitops-operator -o yaml - operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m + if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then + echo "=== Operator installation failed, gathering diagnostics ===" + echo "=== CSV Status ===" + oc get csv -n openshift-gitops-operator -o yaml || true + echo "=== CSV Describe ===" + oc describe csv -n openshift-gitops-operator || true + echo "=== Operator Pods ===" + oc get pods -n openshift-gitops-operator || true + echo "=== Operator Pod Describe ===" + oc describe pods -n openshift-gitops-operator || true + echo "=== Operator Pod Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=controller-manager --tail=100 || true + echo "=== Events ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true + echo "=== Deployment Status ===" + oc get deployment -n openshift-gitops-operator -o yaml || true + echo "=== Image being used ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[0].image}' || true + exit 1 + fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager dependencies: - env: OO_BUNDLE From 075b1a89a9ec864efd578fe492d927a4b31feb2e Mon Sep 17 00:00:00 2001 From: Varsha B Date: Fri, 13 Mar 2026 17:49:23 +0530 Subject: [PATCH 05/10] more diagnostic step Signed-off-by: Varsha B --- ...veloper-gitops-operator-master__v4.19.yaml | 122 ++++++++++++++++-- ...per-gitops-operator-master-presubmits.yaml | 3 + 2 files changed, 111 insertions(+), 14 deletions(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index a0a43b56534ae..eafbb6a3e57ef 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -60,6 +60,8 @@ resources: memory: 6Gi tests: - as: kuttl-sequential + capabilities: + - arm64 skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws @@ -76,22 +78,66 @@ tests: oc get ns openshift-gitops-operator -o yaml if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then echo "=== Operator installation failed, gathering diagnostics ===" + echo "=== Cluster Architecture Info ===" + oc get nodes -o wide || true + echo "=== Node Architecture Details ===" + for node in $(oc get nodes -o jsonpath='{.items[*].metadata.name}'); do + echo "Node: $node" + oc get node "$node" -o jsonpath='{.status.nodeInfo.architecture}' && echo || true + oc get node "$node" -o jsonpath='{.status.nodeInfo.operatingSystem}' && echo || true + oc get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' && echo || true + done echo "=== CSV Status ===" oc get csv -n openshift-gitops-operator -o yaml || true echo "=== CSV Describe ===" oc describe csv -n openshift-gitops-operator || true echo "=== Operator Pods ===" - oc get pods -n openshift-gitops-operator || true - echo "=== Operator Pod Describe ===" + oc get pods -n openshift-gitops-operator -o wide || true + echo "=== Operator Pod Describe (Detailed) ===" oc describe pods -n openshift-gitops-operator || true - echo "=== Operator Pod Logs ===" - oc logs -n openshift-gitops-operator -l control-plane=controller-manager --tail=100 || true - echo "=== Events ===" + echo "=== Manager Container Logs (Current) ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=200 || true + echo "=== Manager Container Logs (Previous - if crashed) ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --previous --tail=200 || true + echo "=== Kube-RBAC-Proxy Container Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c kube-rbac-proxy --tail=100 || true + echo "=== All Container Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator --all-containers=true --tail=100 || true + echo "=== Pod Status Details ===" + oc get pods -n openshift-gitops-operator -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{" Phase: "}{.status.phase}{"\n"}{" Node: "}{.spec.nodeName}{"\n"}{range .status.containerStatuses[*]}{" Container: "}{.name}{" - Ready: "}{.ready}{" - Restarts: "}{.restartCount}{"\n"}{" State: "}{.state}{"\n"}{" Last State: "}{.lastState}{"\n"}{end}{end}' || true + echo "=== Container Exit Codes ===" + oc get pods -n openshift-gitops-operator -o json | jq -r '.items[] | select(.metadata.labels."control-plane"=="controller-manager") | .status.containerStatuses[]? | "\(.name): exitCode=\(.lastState.terminated.exitCode // .state.terminated.exitCode // "N/A"), reason=\(.lastState.terminated.reason // .state.terminated.reason // "N/A")"' || true + echo "=== Events (All) ===" oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true + echo "=== Events (Recent, Sorted) ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -50 || true echo "=== Deployment Status ===" oc get deployment -n openshift-gitops-operator -o yaml || true - echo "=== Image being used ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[0].image}' || true + echo "=== Deployment Conditions ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.status.conditions}' | jq '.' || true + echo "=== Images being used ===" + echo "Manager image:" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].image}' && echo || true + echo "Kube-rbac-proxy image:" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="kube-rbac-proxy")].image}' && echo || true + echo "=== Pod Node Assignment ===" + MANAGER_POD=$(oc get pod -n openshift-gitops-operator -l control-plane=gitops-operator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + if [ -n "$MANAGER_POD" ]; then + echo "Manager pod: $MANAGER_POD" + NODE_NAME=$(oc get pod "$MANAGER_POD" -n openshift-gitops-operator -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "") + if [ -n "$NODE_NAME" ]; then + echo "Scheduled on node: $NODE_NAME" + echo "Node architecture: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.architecture}' 2>/dev/null || echo 'unknown')" + echo "Node OS: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.operatingSystem}' 2>/dev/null || echo 'unknown')" + echo "Node labels:" + oc get node "$NODE_NAME" --show-labels || true + fi + fi + echo "=== Security Context ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.securityContext}' | jq '.' || true + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].securityContext}' | jq '.' || true + echo "=== Resource Limits/Requests ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].resources}' | jq '.' || true exit 1 fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager @@ -115,6 +161,8 @@ tests: timeout: 3h30m0s workflow: ipi-aws - as: kuttl-parallel + capabilities: + - arm64 skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws @@ -131,22 +179,66 @@ tests: oc get ns openshift-gitops-operator -o yaml if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then echo "=== Operator installation failed, gathering diagnostics ===" + echo "=== Cluster Architecture Info ===" + oc get nodes -o wide || true + echo "=== Node Architecture Details ===" + for node in $(oc get nodes -o jsonpath='{.items[*].metadata.name}'); do + echo "Node: $node" + oc get node "$node" -o jsonpath='{.status.nodeInfo.architecture}' && echo || true + oc get node "$node" -o jsonpath='{.status.nodeInfo.operatingSystem}' && echo || true + oc get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' && echo || true + done echo "=== CSV Status ===" oc get csv -n openshift-gitops-operator -o yaml || true echo "=== CSV Describe ===" oc describe csv -n openshift-gitops-operator || true echo "=== Operator Pods ===" - oc get pods -n openshift-gitops-operator || true - echo "=== Operator Pod Describe ===" + oc get pods -n openshift-gitops-operator -o wide || true + echo "=== Operator Pod Describe (Detailed) ===" oc describe pods -n openshift-gitops-operator || true - echo "=== Operator Pod Logs ===" - oc logs -n openshift-gitops-operator -l control-plane=controller-manager --tail=100 || true - echo "=== Events ===" + echo "=== Manager Container Logs (Current) ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=200 || true + echo "=== Manager Container Logs (Previous - if crashed) ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --previous --tail=200 || true + echo "=== Kube-RBAC-Proxy Container Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c kube-rbac-proxy --tail=100 || true + echo "=== All Container Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator --all-containers=true --tail=100 || true + echo "=== Pod Status Details ===" + oc get pods -n openshift-gitops-operator -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{" Phase: "}{.status.phase}{"\n"}{" Node: "}{.spec.nodeName}{"\n"}{range .status.containerStatuses[*]}{" Container: "}{.name}{" - Ready: "}{.ready}{" - Restarts: "}{.restartCount}{"\n"}{" State: "}{.state}{"\n"}{" Last State: "}{.lastState}{"\n"}{end}{end}' || true + echo "=== Container Exit Codes ===" + oc get pods -n openshift-gitops-operator -o json | jq -r '.items[] | select(.metadata.labels."control-plane"=="controller-manager") | .status.containerStatuses[]? | "\(.name): exitCode=\(.lastState.terminated.exitCode // .state.terminated.exitCode // "N/A"), reason=\(.lastState.terminated.reason // .state.terminated.reason // "N/A")"' || true + echo "=== Events (All) ===" oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true + echo "=== Events (Recent, Sorted) ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -50 || true echo "=== Deployment Status ===" oc get deployment -n openshift-gitops-operator -o yaml || true - echo "=== Image being used ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[0].image}' || true + echo "=== Deployment Conditions ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.status.conditions}' | jq '.' || true + echo "=== Images being used ===" + echo "Manager image:" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].image}' && echo || true + echo "Kube-rbac-proxy image:" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="kube-rbac-proxy")].image}' && echo || true + echo "=== Pod Node Assignment ===" + MANAGER_POD=$(oc get pod -n openshift-gitops-operator -l control-plane=gitops-operator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + if [ -n "$MANAGER_POD" ]; then + echo "Manager pod: $MANAGER_POD" + NODE_NAME=$(oc get pod "$MANAGER_POD" -n openshift-gitops-operator -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "") + if [ -n "$NODE_NAME" ]; then + echo "Scheduled on node: $NODE_NAME" + echo "Node architecture: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.architecture}' 2>/dev/null || echo 'unknown')" + echo "Node OS: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.operatingSystem}' 2>/dev/null || echo 'unknown')" + echo "Node labels:" + oc get node "$NODE_NAME" --show-labels || true + fi + fi + echo "=== Security Context ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.securityContext}' | jq '.' || true + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].securityContext}' | jq '.' || true + echo "=== Resource Limits/Requests ===" + oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].resources}' | jq '.' || true exit 1 fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager @@ -169,6 +261,8 @@ tests: memory: 6Gi workflow: ipi-aws - as: e2e + capabilities: + - arm64 skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws diff --git a/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml b/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml index d5f0efeec2a2f..93cf1749dc66c 100644 --- a/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml +++ b/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml @@ -497,6 +497,7 @@ presubmits: context: ci/prow/v4.19-e2e decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: gitops-aws ci-operator.openshift.io/variant: v4.19 @@ -636,6 +637,7 @@ presubmits: context: ci/prow/v4.19-kuttl-parallel decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: gitops-aws ci-operator.openshift.io/variant: v4.19 @@ -719,6 +721,7 @@ presubmits: context: ci/prow/v4.19-kuttl-sequential decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: gitops-aws ci-operator.openshift.io/variant: v4.19 From 294dd8f0c174520afca93233fb40994afc0270be Mon Sep 17 00:00:00 2001 From: Varsha B Date: Mon, 16 Mar 2026 21:56:22 +0530 Subject: [PATCH 06/10] add diagnostic steps Signed-off-by: Varsha B --- ...veloper-gitops-operator-master__v4.19.yaml | 148 +++++------------- 1 file changed, 40 insertions(+), 108 deletions(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index eafbb6a3e57ef..dbd9c26d1191c 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -78,66 +78,16 @@ tests: oc get ns openshift-gitops-operator -o yaml if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then echo "=== Operator installation failed, gathering diagnostics ===" - echo "=== Cluster Architecture Info ===" - oc get nodes -o wide || true - echo "=== Node Architecture Details ===" - for node in $(oc get nodes -o jsonpath='{.items[*].metadata.name}'); do - echo "Node: $node" - oc get node "$node" -o jsonpath='{.status.nodeInfo.architecture}' && echo || true - oc get node "$node" -o jsonpath='{.status.nodeInfo.operatingSystem}' && echo || true - oc get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' && echo || true - done echo "=== CSV Status ===" oc get csv -n openshift-gitops-operator -o yaml || true - echo "=== CSV Describe ===" - oc describe csv -n openshift-gitops-operator || true echo "=== Operator Pods ===" oc get pods -n openshift-gitops-operator -o wide || true - echo "=== Operator Pod Describe (Detailed) ===" + echo "=== Operator Pod Describe ===" oc describe pods -n openshift-gitops-operator || true - echo "=== Manager Container Logs (Current) ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=200 || true - echo "=== Manager Container Logs (Previous - if crashed) ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --previous --tail=200 || true - echo "=== Kube-RBAC-Proxy Container Logs ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c kube-rbac-proxy --tail=100 || true - echo "=== All Container Logs ===" + echo "=== Operator Logs ===" oc logs -n openshift-gitops-operator -l control-plane=gitops-operator --all-containers=true --tail=100 || true - echo "=== Pod Status Details ===" - oc get pods -n openshift-gitops-operator -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{" Phase: "}{.status.phase}{"\n"}{" Node: "}{.spec.nodeName}{"\n"}{range .status.containerStatuses[*]}{" Container: "}{.name}{" - Ready: "}{.ready}{" - Restarts: "}{.restartCount}{"\n"}{" State: "}{.state}{"\n"}{" Last State: "}{.lastState}{"\n"}{end}{end}' || true - echo "=== Container Exit Codes ===" - oc get pods -n openshift-gitops-operator -o json | jq -r '.items[] | select(.metadata.labels."control-plane"=="controller-manager") | .status.containerStatuses[]? | "\(.name): exitCode=\(.lastState.terminated.exitCode // .state.terminated.exitCode // "N/A"), reason=\(.lastState.terminated.reason // .state.terminated.reason // "N/A")"' || true - echo "=== Events (All) ===" - oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true - echo "=== Events (Recent, Sorted) ===" - oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -50 || true - echo "=== Deployment Status ===" - oc get deployment -n openshift-gitops-operator -o yaml || true - echo "=== Deployment Conditions ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.status.conditions}' | jq '.' || true - echo "=== Images being used ===" - echo "Manager image:" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].image}' && echo || true - echo "Kube-rbac-proxy image:" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="kube-rbac-proxy")].image}' && echo || true - echo "=== Pod Node Assignment ===" - MANAGER_POD=$(oc get pod -n openshift-gitops-operator -l control-plane=gitops-operator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") - if [ -n "$MANAGER_POD" ]; then - echo "Manager pod: $MANAGER_POD" - NODE_NAME=$(oc get pod "$MANAGER_POD" -n openshift-gitops-operator -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "") - if [ -n "$NODE_NAME" ]; then - echo "Scheduled on node: $NODE_NAME" - echo "Node architecture: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.architecture}' 2>/dev/null || echo 'unknown')" - echo "Node OS: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.operatingSystem}' 2>/dev/null || echo 'unknown')" - echo "Node labels:" - oc get node "$NODE_NAME" --show-labels || true - fi - fi - echo "=== Security Context ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.securityContext}' | jq '.' || true - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].securityContext}' | jq '.' || true - echo "=== Resource Limits/Requests ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].resources}' | jq '.' || true + echo "=== Events ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -30 || true exit 1 fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager @@ -151,6 +101,22 @@ tests: memory: 6Gi - as: sequential-e2e-steps commands: | + echo "=== Manager pod is up, checking if operator is reconciling default instance ===" + sleep 10s + echo "=== Check openshift-gitops namespace ===" + oc get namespace openshift-gitops || echo "Namespace openshift-gitops does not exist yet" + echo "=== GitOpsService CR (should be created automatically) ===" + oc get gitopsservices -A -o yaml || echo "No GitOpsService CR found" + echo "=== ArgoCD CRs ===" + oc get argocds -A -o yaml || echo "No ArgoCD CRs found" + echo "=== All resources in openshift-gitops namespace ===" + oc get all -n openshift-gitops 2>/dev/null || echo "No resources in openshift-gitops namespace" + echo "=== Operator Manager Logs (checking for reconciliation activity) ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=100 || true + echo "=== Events in openshift-gitops-operator namespace ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true + echo "=== Events in openshift-gitops namespace (if exists) ===" + oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" scripts/openshift-CI-kuttl-tests.sh make e2e-tests-sequential from: dependencies @@ -179,66 +145,16 @@ tests: oc get ns openshift-gitops-operator -o yaml if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then echo "=== Operator installation failed, gathering diagnostics ===" - echo "=== Cluster Architecture Info ===" - oc get nodes -o wide || true - echo "=== Node Architecture Details ===" - for node in $(oc get nodes -o jsonpath='{.items[*].metadata.name}'); do - echo "Node: $node" - oc get node "$node" -o jsonpath='{.status.nodeInfo.architecture}' && echo || true - oc get node "$node" -o jsonpath='{.status.nodeInfo.operatingSystem}' && echo || true - oc get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' && echo || true - done echo "=== CSV Status ===" oc get csv -n openshift-gitops-operator -o yaml || true - echo "=== CSV Describe ===" - oc describe csv -n openshift-gitops-operator || true echo "=== Operator Pods ===" oc get pods -n openshift-gitops-operator -o wide || true - echo "=== Operator Pod Describe (Detailed) ===" + echo "=== Operator Pod Describe ===" oc describe pods -n openshift-gitops-operator || true - echo "=== Manager Container Logs (Current) ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=200 || true - echo "=== Manager Container Logs (Previous - if crashed) ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --previous --tail=200 || true - echo "=== Kube-RBAC-Proxy Container Logs ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c kube-rbac-proxy --tail=100 || true - echo "=== All Container Logs ===" + echo "=== Operator Logs ===" oc logs -n openshift-gitops-operator -l control-plane=gitops-operator --all-containers=true --tail=100 || true - echo "=== Pod Status Details ===" - oc get pods -n openshift-gitops-operator -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{" Phase: "}{.status.phase}{"\n"}{" Node: "}{.spec.nodeName}{"\n"}{range .status.containerStatuses[*]}{" Container: "}{.name}{" - Ready: "}{.ready}{" - Restarts: "}{.restartCount}{"\n"}{" State: "}{.state}{"\n"}{" Last State: "}{.lastState}{"\n"}{end}{end}' || true - echo "=== Container Exit Codes ===" - oc get pods -n openshift-gitops-operator -o json | jq -r '.items[] | select(.metadata.labels."control-plane"=="controller-manager") | .status.containerStatuses[]? | "\(.name): exitCode=\(.lastState.terminated.exitCode // .state.terminated.exitCode // "N/A"), reason=\(.lastState.terminated.reason // .state.terminated.reason // "N/A")"' || true - echo "=== Events (All) ===" - oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true - echo "=== Events (Recent, Sorted) ===" - oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -50 || true - echo "=== Deployment Status ===" - oc get deployment -n openshift-gitops-operator -o yaml || true - echo "=== Deployment Conditions ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.status.conditions}' | jq '.' || true - echo "=== Images being used ===" - echo "Manager image:" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].image}' && echo || true - echo "Kube-rbac-proxy image:" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="kube-rbac-proxy")].image}' && echo || true - echo "=== Pod Node Assignment ===" - MANAGER_POD=$(oc get pod -n openshift-gitops-operator -l control-plane=gitops-operator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") - if [ -n "$MANAGER_POD" ]; then - echo "Manager pod: $MANAGER_POD" - NODE_NAME=$(oc get pod "$MANAGER_POD" -n openshift-gitops-operator -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo "") - if [ -n "$NODE_NAME" ]; then - echo "Scheduled on node: $NODE_NAME" - echo "Node architecture: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.architecture}' 2>/dev/null || echo 'unknown')" - echo "Node OS: $(oc get node "$NODE_NAME" -o jsonpath='{.status.nodeInfo.operatingSystem}' 2>/dev/null || echo 'unknown')" - echo "Node labels:" - oc get node "$NODE_NAME" --show-labels || true - fi - fi - echo "=== Security Context ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.securityContext}' | jq '.' || true - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].securityContext}' | jq '.' || true - echo "=== Resource Limits/Requests ===" - oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].resources}' | jq '.' || true + echo "=== Events ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -30 || true exit 1 fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager @@ -252,6 +168,22 @@ tests: memory: 6Gi - as: parallel-e2e-steps commands: | + echo "=== Manager pod is up, checking if operator is reconciling default instance ===" + sleep 10s + echo "=== Check openshift-gitops namespace ===" + oc get namespace openshift-gitops || echo "Namespace openshift-gitops does not exist yet" + echo "=== GitOpsService CR (should be created automatically) ===" + oc get gitopsservices -A -o yaml || echo "No GitOpsService CR found" + echo "=== ArgoCD CRs ===" + oc get argocds -A -o yaml || echo "No ArgoCD CRs found" + echo "=== All resources in openshift-gitops namespace ===" + oc get all -n openshift-gitops 2>/dev/null || echo "No resources in openshift-gitops namespace" + echo "=== Operator Manager Logs (checking for reconciliation activity) ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=100 || true + echo "=== Events in openshift-gitops-operator namespace ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true + echo "=== Events in openshift-gitops namespace (if exists) ===" + oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" scripts/openshift-CI-kuttl-tests.sh make e2e-tests-parallel from: dependencies From b05dc8eeb16995d4e2323b81b2f95af45f116f6b Mon Sep 17 00:00:00 2001 From: Varsha B Date: Tue, 17 Mar 2026 08:29:52 +0530 Subject: [PATCH 07/10] pod diagnostics Signed-off-by: Varsha B --- ...veloper-gitops-operator-master__v4.19.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index dbd9c26d1191c..b0e4d06588a46 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -117,6 +117,28 @@ tests: oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true echo "=== Events in openshift-gitops namespace (if exists) ===" oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" + echo "=== Detailed Pod Status (Ready Condition Check) ===" + oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true + echo "=== Pod Status with Wide Output ===" + oc get pods -n openshift-gitops -o wide || true + echo "=== Checking All Pods for Issues (Not Ready or CrashLoopBackOff) ===" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") + if [ "$ready_status" != "True" ]; then + echo "--- Pod $pod is NOT Ready (Status: $ready_status) ---" + echo "=== Describe pod: $pod ===" + oc describe pod -n openshift-gitops $pod || true + echo "=== Container Statuses for pod: $pod ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[*].name}{"\n"}{.status.containerStatuses[*].ready}{"\n"}{.status.containerStatuses[*].state}{"\n"}{.status.containerStatuses[*].restartCount}{"\n"}' || true + echo "=== Init Container Statuses for pod: $pod ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{.status.initContainerStatuses[*].name}{"\n"}{.status.initContainerStatuses[*].ready}{"\n"}{.status.initContainerStatuses[*].state}{"\n"}' || true + echo "=== Logs from pod: $pod (all containers) ===" + oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true + echo "=== Previous container logs (if any): $pod ===" + oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=100 2>/dev/null || echo "No previous logs" + echo "" + fi + done scripts/openshift-CI-kuttl-tests.sh make e2e-tests-sequential from: dependencies @@ -184,6 +206,28 @@ tests: oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true echo "=== Events in openshift-gitops namespace (if exists) ===" oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" + echo "=== Detailed Pod Status (Ready Condition Check) ===" + oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true + echo "=== Pod Status with Wide Output ===" + oc get pods -n openshift-gitops -o wide || true + echo "=== Checking All Pods for Issues (Not Ready or CrashLoopBackOff) ===" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") + if [ "$ready_status" != "True" ]; then + echo "--- Pod $pod is NOT Ready (Status: $ready_status) ---" + echo "=== Describe pod: $pod ===" + oc describe pod -n openshift-gitops $pod || true + echo "=== Container Statuses for pod: $pod ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[*].name}{"\n"}{.status.containerStatuses[*].ready}{"\n"}{.status.containerStatuses[*].state}{"\n"}{.status.containerStatuses[*].restartCount}{"\n"}' || true + echo "=== Init Container Statuses for pod: $pod ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{.status.initContainerStatuses[*].name}{"\n"}{.status.initContainerStatuses[*].ready}{"\n"}{.status.initContainerStatuses[*].state}{"\n"}' || true + echo "=== Logs from pod: $pod (all containers) ===" + oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true + echo "=== Previous container logs (if any): $pod ===" + oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=100 2>/dev/null || echo "No previous logs" + echo "" + fi + done scripts/openshift-CI-kuttl-tests.sh make e2e-tests-parallel from: dependencies From d0b947f0b4ead9aacc34aa8ec0a03d20fc7bf35a Mon Sep 17 00:00:00 2001 From: Varsha B Date: Tue, 17 Mar 2026 10:59:33 +0530 Subject: [PATCH 08/10] pod diagnostics Signed-off-by: Varsha B --- ...veloper-gitops-operator-master__v4.19.yaml | 94 ++++++++++++++----- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index b0e4d06588a46..0323cb4f05dcd 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -117,25 +117,48 @@ tests: oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true echo "=== Events in openshift-gitops namespace (if exists) ===" oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" - echo "=== Detailed Pod Status (Ready Condition Check) ===" - oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true echo "=== Pod Status with Wide Output ===" oc get pods -n openshift-gitops -o wide || true - echo "=== Checking All Pods for Issues (Not Ready or CrashLoopBackOff) ===" + echo "=== Detailed Pod Status (Phase and Ready Condition) ===" + oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}Phase={.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true + echo "=== All Pod Conditions (to see why pods aren't Ready) ===" for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + echo "--- Conditions for pod: $pod ---" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" (Reason: "}{.reason}{", Message: "}{.message}{")\n"}{end}' || true + done + echo "=== Logs from ALL Pods (to identify why Running pods aren't Ready) ===" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") + echo "--- Logs from pod: $pod (Phase: $pod_phase, Ready: $ready_status) ---" + oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true + echo "" + done + echo "=== Checking ALL Pods for Issues (CrashLoopBackOff, Not Ready, or Error states) ===" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") - if [ "$ready_status" != "True" ]; then - echo "--- Pod $pod is NOT Ready (Status: $ready_status) ---" + container_state=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].state}' 2>/dev/null || echo "Unknown") + restart_count=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0") + echo "--- Checking pod: $pod (Phase: $pod_phase, Ready: $ready_status, Restarts: $restart_count) ---" + if [ "$pod_phase" = "CrashLoopBackOff" ] || [ "$pod_phase" = "Error" ] || [ "$ready_status" != "True" ] || [ "$restart_count" != "0" ]; then + echo ">>> Pod $pod has issues - gathering detailed diagnostics <<<" + echo "=== Full pod status (JSON) ===" + oc get pod -n openshift-gitops $pod -o json || true echo "=== Describe pod: $pod ===" oc describe pod -n openshift-gitops $pod || true - echo "=== Container Statuses for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[*].name}{"\n"}{.status.containerStatuses[*].ready}{"\n"}{.status.containerStatuses[*].state}{"\n"}{.status.containerStatuses[*].restartCount}{"\n"}' || true + echo "=== Container Statuses with Detailed State for pod: $pod ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{":\n"}{" ready="}{.ready}{"\n"}{" restartCount="}{.restartCount}{"\n"}{" state="}{.state}{"\n"}{" lastState="}{.lastState}{"\n"}{end}' || true + echo "=== Container Waiting States (if any containers are waiting) ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[?(@.state.waiting)]}{.name}{": waiting - reason="}{.state.waiting.reason}{" message="}{.state.waiting.message}{"\n"}{end}' || echo "No containers in waiting state" + echo "=== Readiness Probe Status ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{": started="}{.started}{" ready="}{.ready}{"\n"}{end}' || true echo "=== Init Container Statuses for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{.status.initContainerStatuses[*].name}{"\n"}{.status.initContainerStatuses[*].ready}{"\n"}{.status.initContainerStatuses[*].state}{"\n"}' || true - echo "=== Logs from pod: $pod (all containers) ===" - oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true - echo "=== Previous container logs (if any): $pod ===" - oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=100 2>/dev/null || echo "No previous logs" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.initContainerStatuses[*]}{.name}{": "}{"ready="}{.ready}{" state="}{.state}{"\n"}{end}' || true + echo "=== Current Logs from pod: $pod (all containers) ===" + oc logs -n openshift-gitops $pod --all-containers=true --tail=200 || true + echo "=== Previous container logs (if crashed): $pod ===" + oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=200 2>/dev/null || echo "No previous logs available" echo "" fi done @@ -206,25 +229,48 @@ tests: oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true echo "=== Events in openshift-gitops namespace (if exists) ===" oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" - echo "=== Detailed Pod Status (Ready Condition Check) ===" - oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true echo "=== Pod Status with Wide Output ===" oc get pods -n openshift-gitops -o wide || true - echo "=== Checking All Pods for Issues (Not Ready or CrashLoopBackOff) ===" + echo "=== Detailed Pod Status (Phase and Ready Condition) ===" + oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}Phase={.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true + echo "=== All Pod Conditions (to see why pods aren't Ready) ===" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + echo "--- Conditions for pod: $pod ---" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" (Reason: "}{.reason}{", Message: "}{.message}{")\n"}{end}' || true + done + echo "=== Logs from ALL Pods (to identify why Running pods aren't Ready) ===" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") + echo "--- Logs from pod: $pod (Phase: $pod_phase, Ready: $ready_status) ---" + oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true + echo "" + done + echo "=== Checking ALL Pods for Issues (CrashLoopBackOff, Not Ready, or Error states) ===" for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do + pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") - if [ "$ready_status" != "True" ]; then - echo "--- Pod $pod is NOT Ready (Status: $ready_status) ---" + container_state=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].state}' 2>/dev/null || echo "Unknown") + restart_count=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0") + echo "--- Checking pod: $pod (Phase: $pod_phase, Ready: $ready_status, Restarts: $restart_count) ---" + if [ "$pod_phase" = "CrashLoopBackOff" ] || [ "$pod_phase" = "Error" ] || [ "$ready_status" != "True" ] || [ "$restart_count" != "0" ]; then + echo ">>> Pod $pod has issues - gathering detailed diagnostics <<<" + echo "=== Full pod status (JSON) ===" + oc get pod -n openshift-gitops $pod -o json || true echo "=== Describe pod: $pod ===" oc describe pod -n openshift-gitops $pod || true - echo "=== Container Statuses for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[*].name}{"\n"}{.status.containerStatuses[*].ready}{"\n"}{.status.containerStatuses[*].state}{"\n"}{.status.containerStatuses[*].restartCount}{"\n"}' || true + echo "=== Container Statuses with Detailed State for pod: $pod ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{":\n"}{" ready="}{.ready}{"\n"}{" restartCount="}{.restartCount}{"\n"}{" state="}{.state}{"\n"}{" lastState="}{.lastState}{"\n"}{end}' || true + echo "=== Container Waiting States (if any containers are waiting) ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[?(@.state.waiting)]}{.name}{": waiting - reason="}{.state.waiting.reason}{" message="}{.state.waiting.message}{"\n"}{end}' || echo "No containers in waiting state" + echo "=== Readiness Probe Status ===" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{": started="}{.started}{" ready="}{.ready}{"\n"}{end}' || true echo "=== Init Container Statuses for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{.status.initContainerStatuses[*].name}{"\n"}{.status.initContainerStatuses[*].ready}{"\n"}{.status.initContainerStatuses[*].state}{"\n"}' || true - echo "=== Logs from pod: $pod (all containers) ===" - oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true - echo "=== Previous container logs (if any): $pod ===" - oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=100 2>/dev/null || echo "No previous logs" + oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.initContainerStatuses[*]}{.name}{": "}{"ready="}{.ready}{" state="}{.state}{"\n"}{end}' || true + echo "=== Current Logs from pod: $pod (all containers) ===" + oc logs -n openshift-gitops $pod --all-containers=true --tail=200 || true + echo "=== Previous container logs (if crashed): $pod ===" + oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=200 2>/dev/null || echo "No previous logs available" echo "" fi done From 7c4686a95cdca1ef52520353a0988085a66af9c1 Mon Sep 17 00:00:00 2001 From: Varsha B Date: Wed, 8 Apr 2026 13:43:51 +0530 Subject: [PATCH 09/10] merge conflic resolution Signed-off-by: Varsha B --- .../redhat-developer-gitops-operator-master__v4.19.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index 0323cb4f05dcd..ee00d443836d3 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -23,8 +23,8 @@ images: from: oc-bin-image to: dependencies - additional_architectures: - - arm64 - context_dir: . + - arm64 + context_dir: . from: src to: gitops-operator operator: From 4add2ca3f6cefcafeefb670c12e7d5aa6c06099b Mon Sep 17 00:00:00 2001 From: Varsha B Date: Fri, 17 Apr 2026 16:27:15 +0530 Subject: [PATCH 10/10] changes to 4.14 config Signed-off-by: Varsha B --- ...veloper-gitops-operator-master__v4.14.yaml | 58 ++++++++- ...veloper-gitops-operator-master__v4.19.yaml | 122 ------------------ ...per-gitops-operator-master-presubmits.yaml | 6 + 3 files changed, 61 insertions(+), 125 deletions(-) diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.14.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.14.yaml index 57faf7d17ecc3..aa82d41976b44 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.14.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.14.yaml @@ -22,18 +22,28 @@ images: - dockerfile_path: openshift-ci/build-root/Dockerfile from: oc-bin-image to: dependencies - - context_dir: . + - additional_architectures: + - arm64 + context_dir: . from: src to: gitops-operator operator: bundles: - as: gitops-operator-bundle + capabilities: + - arm64 context_dir: bundle/ dockerfile_path: bundle.Dockerfile substitutions: - pullspec: quay.io/redhat-developer/gitops-operator:latest with: pipeline:gitops-operator releases: + arm64-latest: + candidate: + architecture: arm64 + product: ocp + stream: nightly + version: "4.14" initial: integration: name: "4.14" @@ -50,19 +60,37 @@ resources: memory: 6Gi tests: - as: kuttl-sequential + capabilities: + - arm64 optional: true skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:arm64-latest env: BASE_DOMAIN: ocp-gitops-qe.com + OCP_ARCH: arm64 test: - as: install cli: latest commands: | oc create namespace openshift-gitops-operator oc get ns openshift-gitops-operator -o yaml - operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m + if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then + echo "=== Operator installation failed, gathering diagnostics ===" + echo "=== CSV Status ===" + oc get csv -n openshift-gitops-operator -o yaml || true + echo "=== Operator Pods ===" + oc get pods -n openshift-gitops-operator -o wide || true + echo "=== Operator Pod Describe ===" + oc describe pods -n openshift-gitops-operator || true + echo "=== Operator Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator --all-containers=true --tail=100 || true + echo "=== Events ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -30 || true + exit 1 + fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager dependencies: - env: OO_BUNDLE @@ -84,19 +112,37 @@ tests: timeout: 3h30m0s workflow: ipi-aws - as: kuttl-parallel + capabilities: + - arm64 optional: true skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:arm64-latest env: BASE_DOMAIN: ocp-gitops-qe.com + OCP_ARCH: arm64 test: - as: install cli: latest commands: | oc create namespace openshift-gitops-operator oc get ns openshift-gitops-operator -o yaml - operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m + if ! operator-sdk run bundle --security-context-config restricted -n openshift-gitops-operator "$OO_BUNDLE" --timeout 10m; then + echo "=== Operator installation failed, gathering diagnostics ===" + echo "=== CSV Status ===" + oc get csv -n openshift-gitops-operator -o yaml || true + echo "=== Operator Pods ===" + oc get pods -n openshift-gitops-operator -o wide || true + echo "=== Operator Pod Describe ===" + oc describe pods -n openshift-gitops-operator || true + echo "=== Operator Logs ===" + oc logs -n openshift-gitops-operator -l control-plane=gitops-operator --all-containers=true --tail=100 || true + echo "=== Events ===" + oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -30 || true + exit 1 + fi oc wait --for condition=Available -n openshift-gitops-operator deployment openshift-gitops-operator-controller-manager dependencies: - env: OO_BUNDLE @@ -117,11 +163,17 @@ tests: memory: 6Gi workflow: ipi-aws - as: e2e + capabilities: + - arm64 + optional: true skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ steps: cluster_profile: gitops-aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:arm64-latest env: BASE_DOMAIN: ocp-gitops-qe.com + OCP_ARCH: arm64 test: - as: e2e-steps commands: scripts/openshiftci-presubmit-all-tests.sh diff --git a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml index ee00d443836d3..c6a6edf0c27b4 100644 --- a/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml +++ b/ci-operator/config/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master__v4.19.yaml @@ -101,67 +101,6 @@ tests: memory: 6Gi - as: sequential-e2e-steps commands: | - echo "=== Manager pod is up, checking if operator is reconciling default instance ===" - sleep 10s - echo "=== Check openshift-gitops namespace ===" - oc get namespace openshift-gitops || echo "Namespace openshift-gitops does not exist yet" - echo "=== GitOpsService CR (should be created automatically) ===" - oc get gitopsservices -A -o yaml || echo "No GitOpsService CR found" - echo "=== ArgoCD CRs ===" - oc get argocds -A -o yaml || echo "No ArgoCD CRs found" - echo "=== All resources in openshift-gitops namespace ===" - oc get all -n openshift-gitops 2>/dev/null || echo "No resources in openshift-gitops namespace" - echo "=== Operator Manager Logs (checking for reconciliation activity) ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=100 || true - echo "=== Events in openshift-gitops-operator namespace ===" - oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true - echo "=== Events in openshift-gitops namespace (if exists) ===" - oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" - echo "=== Pod Status with Wide Output ===" - oc get pods -n openshift-gitops -o wide || true - echo "=== Detailed Pod Status (Phase and Ready Condition) ===" - oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}Phase={.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true - echo "=== All Pod Conditions (to see why pods aren't Ready) ===" - for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do - echo "--- Conditions for pod: $pod ---" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" (Reason: "}{.reason}{", Message: "}{.message}{")\n"}{end}' || true - done - echo "=== Logs from ALL Pods (to identify why Running pods aren't Ready) ===" - for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do - pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") - echo "--- Logs from pod: $pod (Phase: $pod_phase, Ready: $ready_status) ---" - oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true - echo "" - done - echo "=== Checking ALL Pods for Issues (CrashLoopBackOff, Not Ready, or Error states) ===" - for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do - pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") - container_state=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].state}' 2>/dev/null || echo "Unknown") - restart_count=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0") - echo "--- Checking pod: $pod (Phase: $pod_phase, Ready: $ready_status, Restarts: $restart_count) ---" - if [ "$pod_phase" = "CrashLoopBackOff" ] || [ "$pod_phase" = "Error" ] || [ "$ready_status" != "True" ] || [ "$restart_count" != "0" ]; then - echo ">>> Pod $pod has issues - gathering detailed diagnostics <<<" - echo "=== Full pod status (JSON) ===" - oc get pod -n openshift-gitops $pod -o json || true - echo "=== Describe pod: $pod ===" - oc describe pod -n openshift-gitops $pod || true - echo "=== Container Statuses with Detailed State for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{":\n"}{" ready="}{.ready}{"\n"}{" restartCount="}{.restartCount}{"\n"}{" state="}{.state}{"\n"}{" lastState="}{.lastState}{"\n"}{end}' || true - echo "=== Container Waiting States (if any containers are waiting) ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[?(@.state.waiting)]}{.name}{": waiting - reason="}{.state.waiting.reason}{" message="}{.state.waiting.message}{"\n"}{end}' || echo "No containers in waiting state" - echo "=== Readiness Probe Status ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{": started="}{.started}{" ready="}{.ready}{"\n"}{end}' || true - echo "=== Init Container Statuses for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.initContainerStatuses[*]}{.name}{": "}{"ready="}{.ready}{" state="}{.state}{"\n"}{end}' || true - echo "=== Current Logs from pod: $pod (all containers) ===" - oc logs -n openshift-gitops $pod --all-containers=true --tail=200 || true - echo "=== Previous container logs (if crashed): $pod ===" - oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=200 2>/dev/null || echo "No previous logs available" - echo "" - fi - done scripts/openshift-CI-kuttl-tests.sh make e2e-tests-sequential from: dependencies @@ -213,67 +152,6 @@ tests: memory: 6Gi - as: parallel-e2e-steps commands: | - echo "=== Manager pod is up, checking if operator is reconciling default instance ===" - sleep 10s - echo "=== Check openshift-gitops namespace ===" - oc get namespace openshift-gitops || echo "Namespace openshift-gitops does not exist yet" - echo "=== GitOpsService CR (should be created automatically) ===" - oc get gitopsservices -A -o yaml || echo "No GitOpsService CR found" - echo "=== ArgoCD CRs ===" - oc get argocds -A -o yaml || echo "No ArgoCD CRs found" - echo "=== All resources in openshift-gitops namespace ===" - oc get all -n openshift-gitops 2>/dev/null || echo "No resources in openshift-gitops namespace" - echo "=== Operator Manager Logs (checking for reconciliation activity) ===" - oc logs -n openshift-gitops-operator -l control-plane=gitops-operator -c manager --tail=100 || true - echo "=== Events in openshift-gitops-operator namespace ===" - oc get events -n openshift-gitops-operator --sort-by='.lastTimestamp' | tail -20 || true - echo "=== Events in openshift-gitops namespace (if exists) ===" - oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || echo "No events in openshift-gitops namespace" - echo "=== Pod Status with Wide Output ===" - oc get pods -n openshift-gitops -o wide || true - echo "=== Detailed Pod Status (Phase and Ready Condition) ===" - oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}Phase={.status.phase}{"\t"}Ready={.status.conditions[?(@.type=="Ready")].status}{"\t"}Reason={.status.conditions[?(@.type=="Ready")].reason}{"\t"}Message={.status.conditions[?(@.type=="Ready")].message}{"\n"}{end}' || true - echo "=== All Pod Conditions (to see why pods aren't Ready) ===" - for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do - echo "--- Conditions for pod: $pod ---" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" (Reason: "}{.reason}{", Message: "}{.message}{")\n"}{end}' || true - done - echo "=== Logs from ALL Pods (to identify why Running pods aren't Ready) ===" - for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do - pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") - echo "--- Logs from pod: $pod (Phase: $pod_phase, Ready: $ready_status) ---" - oc logs -n openshift-gitops $pod --all-containers=true --tail=100 || true - echo "" - done - echo "=== Checking ALL Pods for Issues (CrashLoopBackOff, Not Ready, or Error states) ===" - for pod in $(oc get pods -n openshift-gitops -o jsonpath='{.items[*].metadata.name}'); do - pod_phase=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - ready_status=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") - container_state=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].state}' 2>/dev/null || echo "Unknown") - restart_count=$(oc get pod -n openshift-gitops $pod -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0") - echo "--- Checking pod: $pod (Phase: $pod_phase, Ready: $ready_status, Restarts: $restart_count) ---" - if [ "$pod_phase" = "CrashLoopBackOff" ] || [ "$pod_phase" = "Error" ] || [ "$ready_status" != "True" ] || [ "$restart_count" != "0" ]; then - echo ">>> Pod $pod has issues - gathering detailed diagnostics <<<" - echo "=== Full pod status (JSON) ===" - oc get pod -n openshift-gitops $pod -o json || true - echo "=== Describe pod: $pod ===" - oc describe pod -n openshift-gitops $pod || true - echo "=== Container Statuses with Detailed State for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{":\n"}{" ready="}{.ready}{"\n"}{" restartCount="}{.restartCount}{"\n"}{" state="}{.state}{"\n"}{" lastState="}{.lastState}{"\n"}{end}' || true - echo "=== Container Waiting States (if any containers are waiting) ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[?(@.state.waiting)]}{.name}{": waiting - reason="}{.state.waiting.reason}{" message="}{.state.waiting.message}{"\n"}{end}' || echo "No containers in waiting state" - echo "=== Readiness Probe Status ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.containerStatuses[*]}{.name}{": started="}{.started}{" ready="}{.ready}{"\n"}{end}' || true - echo "=== Init Container Statuses for pod: $pod ===" - oc get pod -n openshift-gitops $pod -o jsonpath='{range .status.initContainerStatuses[*]}{.name}{": "}{"ready="}{.ready}{" state="}{.state}{"\n"}{end}' || true - echo "=== Current Logs from pod: $pod (all containers) ===" - oc logs -n openshift-gitops $pod --all-containers=true --tail=200 || true - echo "=== Previous container logs (if crashed): $pod ===" - oc logs -n openshift-gitops $pod --all-containers=true --previous --tail=200 2>/dev/null || echo "No previous logs available" - echo "" - fi - done scripts/openshift-CI-kuttl-tests.sh make e2e-tests-parallel from: dependencies diff --git a/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml b/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml index 93cf1749dc66c..8e7edfd6c1804 100644 --- a/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml +++ b/ci-operator/jobs/redhat-developer/gitops-operator/redhat-developer-gitops-operator-master-presubmits.yaml @@ -80,6 +80,7 @@ presubmits: context: ci/prow/v4.14-ci-index-gitops-operator-bundle decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/variant: v4.14 ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -135,12 +136,14 @@ presubmits: context: ci/prow/v4.14-e2e decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: gitops-aws ci-operator.openshift.io/variant: v4.14 ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" name: pull-ci-redhat-developer-gitops-operator-master-v4.14-e2e + optional: true rerun_command: /test v4.14-e2e skip_if_only_changed: ^docs/|\.(md|adoc)$|^(README|LICENSE|OWNERS)$ spec: @@ -218,6 +221,7 @@ presubmits: context: ci/prow/v4.14-images decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/variant: v4.14 ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -273,6 +277,7 @@ presubmits: context: ci/prow/v4.14-kuttl-parallel decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: gitops-aws ci-operator.openshift.io/variant: v4.14 @@ -357,6 +362,7 @@ presubmits: context: ci/prow/v4.14-kuttl-sequential decorate: true labels: + capability/arm64: arm64 ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: gitops-aws ci-operator.openshift.io/variant: v4.14