From bca4aab4e4f5c44e8f9110592fd801fe15a387e9 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Mon, 6 Oct 2025 07:50:14 -0400 Subject: [PATCH 01/10] Add ExternalDNS deployment for OpenShift clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change enables automatic deployment of ExternalDNS via ArgoCD when running e2e tests on OpenShift clusters (including OpenShift CI). Key changes: - Add ArgoCD Application manifest to deploy ExternalDNS from acscs-manifests - Create helper script to fetch cluster's Infrastructure name for unique naming - Update bootstrap.sh to deploy ExternalDNS on OpenShift clusters The ExternalDNS instance uses the Infrastructure CR's infrastructureName to generate a globally unique name, preventing DNS record conflicts across different clusters/CI jobs. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- README.md | 1 + .../external-dns/00-application.yaml | 36 +++++++++++++++++++ dev/env/scripts/bootstrap.sh | 9 +++++ dev/env/scripts/get-infrastructure-name.sh | 22 ++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 dev/env/manifests/external-dns/00-application.yaml create mode 100755 dev/env/scripts/get-infrastructure-name.sh diff --git a/README.md b/README.md index 22f9218047..fbe0cdf5fb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ ACS fleet-manager repository for the ACS managed service. + ## Quickstart ### Overview diff --git a/dev/env/manifests/external-dns/00-application.yaml b/dev/env/manifests/external-dns/00-application.yaml new file mode 100644 index 0000000000..388a895d03 --- /dev/null +++ b/dev/env/manifests/external-dns/00-application.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: rhacs-external-dns + namespace: "$ARGOCD_NAMESPACE" +spec: + destination: + namespace: external-dns-operator + server: https://kubernetes.default.svc + project: default + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + managedNamespaceMetadata: + labels: + argocd.argoproj.io/managed-by: "$ARGOCD_NAMESPACE" + app.kubernetes.io/managed-by: "$ARGOCD_NAMESPACE" + retry: + limit: -1 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + source: + chart: external-dns + repoURL: https://github.com/stackrox/acscs-manifests + targetRevision: HEAD + path: external-dns + helm: + valuesObject: + name: "$INFRASTRUCTURE_NAME" + zones: [Z07321763RS4X0PD14EUT] # dev.acs.rhcloud.com Route53 zone + region: us-east-1 diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh index b09f377f25..fd956ef257 100755 --- a/dev/env/scripts/bootstrap.sh +++ b/dev/env/scripts/bootstrap.sh @@ -87,6 +87,15 @@ fi apply "${MANIFESTS_DIR}/addons" +if is_openshift_cluster "$CLUSTER_TYPE"; then + log "Installing ExternalDNS for OpenShift" + wait_for_crd "applications.argoproj.io" + source "${GITROOT}/dev/env/scripts/get-infrastructure-name.sh" + apply "${MANIFESTS_DIR}/external-dns" +else + log "Skipping installation of ExternalDNS (only installed on openshift)" +fi + if [[ "$CLUSTER_TYPE" == "kind" ]]; then log "Ensuring operator images exist from dev GitOps config" ensure_operator_image_exists.sh diff --git a/dev/env/scripts/get-infrastructure-name.sh b/dev/env/scripts/get-infrastructure-name.sh new file mode 100755 index 0000000000..e8571f0e19 --- /dev/null +++ b/dev/env/scripts/get-infrastructure-name.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This script retrieves the Infrastructure CR's infrastructureName from the cluster +# and exports it as INFRASTRUCTURE_NAME for use in manifest templating. + +set -euo pipefail + +GITROOT="$(git rev-parse --show-toplevel)" +export GITROOT +# shellcheck source=/dev/null +source "${GITROOT}/scripts/lib/log.sh" + +KUBECTL_BIN=${KUBECTL:-kubectl} + +INFRASTRUCTURE_NAME=$($KUBECTL_BIN get infrastructures.config.openshift.io cluster -o jsonpath='{.status.infrastructureName}') + +if [[ -z "$INFRASTRUCTURE_NAME" ]]; then + die "Error: Could not retrieve infrastructure name from cluster" +fi + +export INFRASTRUCTURE_NAME +log "Infrastructure name: $INFRASTRUCTURE_NAME" From 19c5a32c45db06ae1f632a24a4594e4b7521e4c2 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Wed, 8 Oct 2025 10:44:22 -0400 Subject: [PATCH 02/10] install addon crd before applying fleetshard --- dev/env/scripts/bootstrap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh index fd956ef257..21bdacd4f3 100755 --- a/dev/env/scripts/bootstrap.sh +++ b/dev/env/scripts/bootstrap.sh @@ -85,6 +85,8 @@ if ! is_openshift_cluster "$CLUSTER_TYPE"; then apply "${MANIFESTS_DIR}/monitoring" fi +apply "${MANIFESTS_DIR}/addons/00-addon-crd.yaml" +wait_for_crd "addons.addons.managed.openshift.io" apply "${MANIFESTS_DIR}/addons" if is_openshift_cluster "$CLUSTER_TYPE"; then From 579da97f46f383f1a7643cf6bb3deb3273652ffd Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Thu, 9 Oct 2025 09:20:25 -0400 Subject: [PATCH 03/10] commit these changes --- dev/env/scripts/bootstrap.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh index 21bdacd4f3..3a3396f331 100755 --- a/dev/env/scripts/bootstrap.sh +++ b/dev/env/scripts/bootstrap.sh @@ -22,11 +22,29 @@ fi log "** Preparing ACSCS Environment **" print_env -if ! kc_output=$($KUBECTL api-versions 2>&1); then - die "Error: Sanity check for contacting Kubernetes cluster failed: +# Retry for up to 30 minutes to contact the Kubernetes cluster +MAX_RETRIES=180 # 30 minutes with 10 second intervals +RETRY_COUNT=0 +RETRY_DELAY=10 + +log "Attempting to contact Kubernetes cluster..." +while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if kc_output=$($KUBECTL api-versions 2>&1); then + log "Successfully contacted Kubernetes cluster" + break + fi + + RETRY_COUNT=$((RETRY_COUNT + 1)) + ELAPSED=$((RETRY_COUNT * RETRY_DELAY)) + log "Failed to contact cluster (attempt $RETRY_COUNT/$MAX_RETRIES, elapsed: ${ELAPSED}s). Retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY +done + +if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then + die "Error: Sanity check for contacting Kubernetes cluster failed after $((MAX_RETRIES * RETRY_DELAY)) seconds: Command tried: '$KUBECTL api-versions' -Output: +Last output: ${kc_output:-(no output)}" fi From 885b61b54fbb95d64f629112c5ad2dd2c997741b Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Thu, 9 Oct 2025 13:04:46 -0400 Subject: [PATCH 04/10] fix chart --- dev/env/manifests/external-dns/00-application.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/env/manifests/external-dns/00-application.yaml b/dev/env/manifests/external-dns/00-application.yaml index 388a895d03..4385473480 100644 --- a/dev/env/manifests/external-dns/00-application.yaml +++ b/dev/env/manifests/external-dns/00-application.yaml @@ -25,7 +25,6 @@ spec: factor: 2 maxDuration: 3m source: - chart: external-dns repoURL: https://github.com/stackrox/acscs-manifests targetRevision: HEAD path: external-dns From 43f6f71a0b5999e2232e76be31504e4a2bde30d5 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Thu, 9 Oct 2025 16:07:56 -0400 Subject: [PATCH 05/10] fix permissions --- .../external-dns-operator/00-application.yaml | 30 +++++++++++++++++++ .../external-dns/00-application.yaml | 1 + .../openshift-gitops/04-clusterrole.yaml | 4 +++ dev/env/scripts/bootstrap.sh | 1 + 4 files changed, 36 insertions(+) create mode 100644 dev/env/manifests/external-dns-operator/00-application.yaml diff --git a/dev/env/manifests/external-dns-operator/00-application.yaml b/dev/env/manifests/external-dns-operator/00-application.yaml new file mode 100644 index 0000000000..e273bab9b5 --- /dev/null +++ b/dev/env/manifests/external-dns-operator/00-application.yaml @@ -0,0 +1,30 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: rhacs-external-dns-operator + namespace: "$ARGOCD_NAMESPACE" +spec: + destination: + namespace: external-dns-operator + server: https://kubernetes.default.svc + project: default + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + managedNamespaceMetadata: + labels: + argocd.argoproj.io/managed-by: "$ARGOCD_NAMESPACE" + app.kubernetes.io/managed-by: "$ARGOCD_NAMESPACE" + retry: + limit: -1 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + source: + repoURL: https://github.com/stackrox/acscs-manifests + targetRevision: HEAD + path: external-dns-operator \ No newline at end of file diff --git a/dev/env/manifests/external-dns/00-application.yaml b/dev/env/manifests/external-dns/00-application.yaml index 4385473480..87fac982a6 100644 --- a/dev/env/manifests/external-dns/00-application.yaml +++ b/dev/env/manifests/external-dns/00-application.yaml @@ -33,3 +33,4 @@ spec: name: "$INFRASTRUCTURE_NAME" zones: [Z07321763RS4X0PD14EUT] # dev.acs.rhcloud.com Route53 zone region: us-east-1 + routerName: default diff --git a/dev/env/manifests/openshift-gitops/04-clusterrole.yaml b/dev/env/manifests/openshift-gitops/04-clusterrole.yaml index e28d5cc73f..066ee1a7e9 100644 --- a/dev/env/manifests/openshift-gitops/04-clusterrole.yaml +++ b/dev/env/manifests/openshift-gitops/04-clusterrole.yaml @@ -17,6 +17,10 @@ rules: - apiGroups: [ "admissionregistration.k8s.io" ] resources: [ "validatingwebhookconfigurations" ] verbs: [ "*" ] + # Allow managing external dnses + - apiGroups: [ "externaldns.olm.openshift.io" ] + resources: [ "externaldnses" ] + verbs: [ "*" ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh index 3a3396f331..2b1ac80086 100755 --- a/dev/env/scripts/bootstrap.sh +++ b/dev/env/scripts/bootstrap.sh @@ -111,6 +111,7 @@ if is_openshift_cluster "$CLUSTER_TYPE"; then log "Installing ExternalDNS for OpenShift" wait_for_crd "applications.argoproj.io" source "${GITROOT}/dev/env/scripts/get-infrastructure-name.sh" + apply "${MANIFESTS_DIR}/external-dns-operator" apply "${MANIFESTS_DIR}/external-dns" else log "Skipping installation of ExternalDNS (only installed on openshift)" From 647b95da5b1c09c5c218d6c1356dee4f4f70871b Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Thu, 9 Oct 2025 16:29:02 -0400 Subject: [PATCH 06/10] fix permissions --- dev/env/manifests/external-dns-operator/00-application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/env/manifests/external-dns-operator/00-application.yaml b/dev/env/manifests/external-dns-operator/00-application.yaml index e273bab9b5..635a377520 100644 --- a/dev/env/manifests/external-dns-operator/00-application.yaml +++ b/dev/env/manifests/external-dns-operator/00-application.yaml @@ -27,4 +27,4 @@ spec: source: repoURL: https://github.com/stackrox/acscs-manifests targetRevision: HEAD - path: external-dns-operator \ No newline at end of file + path: external-dns-operator From fa900192cb8819b1e7739cda6be3cd8b2d5ddd74 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Thu, 9 Oct 2025 17:07:12 -0400 Subject: [PATCH 07/10] fix permissions --- dev/env/manifests/external-dns/00-application.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/env/manifests/external-dns/00-application.yaml b/dev/env/manifests/external-dns/00-application.yaml index 87fac982a6..536e76a85a 100644 --- a/dev/env/manifests/external-dns/00-application.yaml +++ b/dev/env/manifests/external-dns/00-application.yaml @@ -34,3 +34,4 @@ spec: zones: [Z07321763RS4X0PD14EUT] # dev.acs.rhcloud.com Route53 zone region: us-east-1 routerName: default + roleArn: arn:aws:iam::047735621815:role/ExternalDnsRole-acs-dev-dp-01 From e477ca279efd1ed97d5a604e54bcb6371cc0155a Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Fri, 10 Oct 2025 15:23:54 -0400 Subject: [PATCH 08/10] fix permissions --- .../external-dns/00-application.yaml | 37 ------------------- .../manifests/external-dns/00-namespace.yaml | 4 ++ .../external-dns/01-external-dns.yaml | 23 ++++++++++++ dev/env/manifests/external-dns/02-secret.yaml | 10 +++++ dev/env/scripts/bootstrap.sh | 8 +++- 5 files changed, 43 insertions(+), 39 deletions(-) delete mode 100644 dev/env/manifests/external-dns/00-application.yaml create mode 100644 dev/env/manifests/external-dns/00-namespace.yaml create mode 100644 dev/env/manifests/external-dns/01-external-dns.yaml create mode 100644 dev/env/manifests/external-dns/02-secret.yaml diff --git a/dev/env/manifests/external-dns/00-application.yaml b/dev/env/manifests/external-dns/00-application.yaml deleted file mode 100644 index 536e76a85a..0000000000 --- a/dev/env/manifests/external-dns/00-application.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: rhacs-external-dns - namespace: "$ARGOCD_NAMESPACE" -spec: - destination: - namespace: external-dns-operator - server: https://kubernetes.default.svc - project: default - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - managedNamespaceMetadata: - labels: - argocd.argoproj.io/managed-by: "$ARGOCD_NAMESPACE" - app.kubernetes.io/managed-by: "$ARGOCD_NAMESPACE" - retry: - limit: -1 - backoff: - duration: 5s - factor: 2 - maxDuration: 3m - source: - repoURL: https://github.com/stackrox/acscs-manifests - targetRevision: HEAD - path: external-dns - helm: - valuesObject: - name: "$INFRASTRUCTURE_NAME" - zones: [Z07321763RS4X0PD14EUT] # dev.acs.rhcloud.com Route53 zone - region: us-east-1 - routerName: default - roleArn: arn:aws:iam::047735621815:role/ExternalDnsRole-acs-dev-dp-01 diff --git a/dev/env/manifests/external-dns/00-namespace.yaml b/dev/env/manifests/external-dns/00-namespace.yaml new file mode 100644 index 0000000000..b8730d5083 --- /dev/null +++ b/dev/env/manifests/external-dns/00-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-dns-operator diff --git a/dev/env/manifests/external-dns/01-external-dns.yaml b/dev/env/manifests/external-dns/01-external-dns.yaml new file mode 100644 index 0000000000..05283a358c --- /dev/null +++ b/dev/env/manifests/external-dns/01-external-dns.yaml @@ -0,0 +1,23 @@ +apiVersion: externaldns.olm.openshift.io/v1beta1 +kind: ExternalDNS +metadata: + name: "${EXTERNAL_DNS_NAME}" +spec: + domains: + - filterType: Include + matchType: Pattern + pattern: ".*\\.rhcloud.com" + provider: + type: AWS + aws: + credentials: + name: "${EXTERNAL_DNS_NAME}-aws-credentials" + source: + type: OpenShiftRoute + labelFilter: + matchLabels: + external-dns.rhacs.redhat.com/enabled: "true" + openshiftRouteOptions: + routerName: default + zones: + - "${ROUTE53_ZONE_ID}" \ No newline at end of file diff --git a/dev/env/manifests/external-dns/02-secret.yaml b/dev/env/manifests/external-dns/02-secret.yaml new file mode 100644 index 0000000000..3ea6db5d09 --- /dev/null +++ b/dev/env/manifests/external-dns/02-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${EXTERNAL_DNS_NAME}-aws-credentials" + namespace: external-dns-operator +stringData: + credentials: |- + [default] + aws_access_key_id = ${AWS_ACCESS_KEY_ID} + aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh index 2b1ac80086..ef9c6b4d43 100755 --- a/dev/env/scripts/bootstrap.sh +++ b/dev/env/scripts/bootstrap.sh @@ -110,9 +110,13 @@ apply "${MANIFESTS_DIR}/addons" if is_openshift_cluster "$CLUSTER_TYPE"; then log "Installing ExternalDNS for OpenShift" wait_for_crd "applications.argoproj.io" - source "${GITROOT}/dev/env/scripts/get-infrastructure-name.sh" + apply "${MANIFESTS_DIR}/external-dns-operator" - apply "${MANIFESTS_DIR}/external-dns" + wait_for_crd externaldnses.externaldns.olm.openshift.io + + source "${GITROOT}/dev/env/scripts/get-infrastructure-name.sh" + export EXTERNAL_DNS_NAME=${INFRASTRUCTURE_NAME} + chamber exec e2e-external-dns -- apply "${MANIFESTS_DIR}/external-dns" else log "Skipping installation of ExternalDNS (only installed on openshift)" fi From 71046aaa2ec8061e9d8d743f2f2b75c79726dfd0 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Fri, 10 Oct 2025 16:04:38 -0400 Subject: [PATCH 09/10] fix permissions --- dev/env/manifests/external-dns/01-external-dns.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/env/manifests/external-dns/01-external-dns.yaml b/dev/env/manifests/external-dns/01-external-dns.yaml index 05283a358c..272f8b82bf 100644 --- a/dev/env/manifests/external-dns/01-external-dns.yaml +++ b/dev/env/manifests/external-dns/01-external-dns.yaml @@ -6,7 +6,7 @@ spec: domains: - filterType: Include matchType: Pattern - pattern: ".*\\.rhcloud.com" + pattern: ".*\\.dev\.rhcloud.com" provider: type: AWS aws: @@ -20,4 +20,4 @@ spec: openshiftRouteOptions: routerName: default zones: - - "${ROUTE53_ZONE_ID}" \ No newline at end of file + - "${ROUTE53_ZONE_ID}" From c04a4f5d3315b4a77fae5086f9363a64be60edcc Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Fri, 10 Oct 2025 16:27:51 -0400 Subject: [PATCH 10/10] fix permissions --- dev/env/manifests/external-dns/01-external-dns.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/env/manifests/external-dns/01-external-dns.yaml b/dev/env/manifests/external-dns/01-external-dns.yaml index 272f8b82bf..ff42e66258 100644 --- a/dev/env/manifests/external-dns/01-external-dns.yaml +++ b/dev/env/manifests/external-dns/01-external-dns.yaml @@ -6,7 +6,7 @@ spec: domains: - filterType: Include matchType: Pattern - pattern: ".*\\.dev\.rhcloud.com" + pattern: ".*\\.dev\\.rhcloud.com" provider: type: AWS aws: