From 927482536d7177576fda4acb88c2777bf8285427 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 6 Feb 2026 16:28:55 -0500 Subject: [PATCH 01/21] feat: BYO container registry support Restructure registry configuration to support three deployment states: - Fresh install: No registry configured (both disabled by default) - Built-in Quay: quay.enabled=true uses hub/infra/quay/ vault path - External/BYO: externalRegistry.enabled=true uses hub/infra/registry/ path Changes: - Add externalRegistry.enabled flag to supply-chain and qtodo charts - Separate vault paths for built-in Quay vs external registry - Templates conditionally select vault path based on enabled flags - Update supply-chain.md with BYO registry setup instructions - Add helm template method and oc monitoring commands to supply-chain.md - Follow VP best practice: external registry secrets in local ~/values-secret.yaml To enable supply-chain: 1. Uncomment openshift-pipelines namespace and subscription 2. Uncomment supply-chain vault role (JWT auth) 3. Configure registry (BYO or built-in Quay) in application overrides - For BYO registry: - Set externalRegistry.enabled=true and configure registry settings - Add registry credentials to ~/values-secret.yaml - For built-in Quay: - Enable openshift-storage namespace - Enable ODF, NooBaa MCG - Enable Quay operator subscription, quay-registry application 4. RHTAS (signing): Enable rhtas-operator subscription and trusted-artifact-signer namespace 5. RHTPA (SBOM): Enable rhtpa-operator subscription, ODF, NooBaa, and trusted-profile-analyzer Signed-off-by: Min Zhang --- .../templates/registry-external-secret.yaml | 11 +- charts/qtodo/values.yaml | 23 ++- .../templates/pipeline-qtodo.yaml | 11 +- .../templates/pipelinerun-qtodo.yaml | 21 +++ .../rbac/registry-image-namespace.yaml | 28 ++++ ...egistry-pass.yaml => qtodo-quay-pass.yaml} | 11 +- .../secrets/qtodo-registry-auth.yaml | 34 ++++- charts/supply-chain/values.yaml | 47 ++++-- docs/supply-chain.md | 136 ++++++++++++++++++ values-hub.yaml | 81 ++++++++--- values-secret.yaml.template | 45 +++--- 11 files changed, 383 insertions(+), 65 deletions(-) create mode 100644 charts/supply-chain/templates/pipelinerun-qtodo.yaml create mode 100644 charts/supply-chain/templates/rbac/registry-image-namespace.yaml rename charts/supply-chain/templates/secrets/{qtodo-registry-pass.yaml => qtodo-quay-pass.yaml} (55%) diff --git a/charts/qtodo/templates/registry-external-secret.yaml b/charts/qtodo/templates/registry-external-secret.yaml index 8646909d..6b6979c9 100644 --- a/charts/qtodo/templates/registry-external-secret.yaml +++ b/charts/qtodo/templates/registry-external-secret.yaml @@ -18,7 +18,7 @@ spec: .dockerconfigjson: | { "auths": { - "{{ .Values.app.images.main.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain) }}": { + "{{ required "app.images.main.registry.domain is required when registry.auth is enabled" .Values.app.images.main.registry.domain }}": { "auth": "{{ `{{ printf "%s:%s" "` }}{{ .Values.app.images.main.registry.user }}{{ `" .password | b64enc }}` }}" } } @@ -26,6 +26,11 @@ spec: data: - secretKey: password remoteRef: - key: {{ .Values.app.images.main.registry.vaultPath }} - property: {{ .Values.app.images.main.registry.passwordVaultKey }} + {{- if .Values.app.images.main.registry.builtinQuay.enabled }} + key: {{ .Values.app.images.main.registry.builtinQuay.vaultPath }} + property: {{ .Values.app.images.main.registry.builtinQuay.passwordVaultKey }} + {{- else if .Values.app.images.main.registry.externalRegistry.enabled }} + key: {{ .Values.app.images.main.registry.externalRegistry.vaultPath }} + property: {{ .Values.app.images.main.registry.externalRegistry.passwordVaultKey }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index d5fd70c8..52b13d33 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -15,13 +15,26 @@ app: # Modified to Always to force a pull so we can test changes to the container image without requiring manual deletion of images or restarts of argo pullPolicy: Always registry: + # auth: controls whether to create registry auth secret + # Set to true when using private registry (built-in Quay or external) auth: false secretName: qtodo-registry-auth - user: quay-user - # domain: quay-registry-quay-quay-enterprise.apps.example.com - # Registry credentials - stored in quay path - vaultPath: secret/data/hub/infra/quay/quay-users - passwordVaultKey: quay-user-password + user: registry-user + # domain: registry.example.com # REQUIRED when auth is enabled + + # Built-in Quay registry (optional) + # When enabled, uses auto-generated credentials from Vault + builtinQuay: + enabled: false + vaultPath: secret/data/hub/infra/quay/quay-users + passwordVaultKey: quay-user-password + + # External/BYO registry (optional) + # When enabled, uses user-provided credentials from Vault + externalRegistry: + enabled: false + vaultPath: secret/data/hub/infra/registry/registry-user + passwordVaultKey: registry-password spiffeHelper: name: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9 version: v0.10.0 diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index 13ae2c8c..39fe8494 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -1,3 +1,12 @@ +{{- /* Determine registry domain: auto-construct for built-in Quay, require for external */ -}} +{{- $registryDomain := "" -}} +{{- if .Values.registry.domain -}} + {{- $registryDomain = .Values.registry.domain -}} +{{- else if .Values.quay.enabled -}} + {{- $registryDomain = printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain -}} +{{- else -}} + {{- fail "registry.domain is required for external registry" -}} +{{- end -}} --- apiVersion: tekton.dev/v1beta1 kind: Pipeline @@ -25,7 +34,7 @@ spec: - name: image-target type: string description: qtodo image push destination (e.g. quay.io/ztvp/qtodo:latest) - default: {{ .Values.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain) }}/{{ .Values.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} + default: {{ $registryDomain }}/{{ .Values.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} - name: image-tls-verify type: string description: Whether to verify TLS when pushing to the OCI registry diff --git a/charts/supply-chain/templates/pipelinerun-qtodo.yaml b/charts/supply-chain/templates/pipelinerun-qtodo.yaml new file mode 100644 index 00000000..820c8da4 --- /dev/null +++ b/charts/supply-chain/templates/pipelinerun-qtodo.yaml @@ -0,0 +1,21 @@ +{{- if .Values.pipelinerun.enabled }} +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: qtodo-supply-chain- + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + pipelineRef: + name: qtodo-supply-chain + workspaces: + - name: qtodo-source + persistentVolumeClaim: + claimName: qtodo-workspace-source + - name: registry-auth-config + secret: + secretName: {{ .Values.registry.authSecretName }} +{{- end }} diff --git a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml new file mode 100644 index 00000000..35f3ab76 --- /dev/null +++ b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml @@ -0,0 +1,28 @@ +{{- if and (index .Values.registry "embeddedOCP") (index .Values.registry.embeddedOCP "ensureImageNamespaceRBAC") }} +# When using the embedded OCP image registry, the pipeline pushes to a namespace +# that matches registry.org (e.g. ztvp). This ensures that namespace exists and +# the pipeline SA has system:image-builder so the push succeeds (transparent to the user). +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.registry.org }} + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pipeline-image-builder + namespace: {{ .Values.registry.org }} + annotations: + argocd.argoproj.io/sync-wave: "0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:image-builder +subjects: + - kind: ServiceAccount + name: pipeline + namespace: {{ .Values.global.namespace }} +{{- end }} diff --git a/charts/supply-chain/templates/secrets/qtodo-registry-pass.yaml b/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml similarity index 55% rename from charts/supply-chain/templates/secrets/qtodo-registry-pass.yaml rename to charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml index 65406f8d..d66f6507 100644 --- a/charts/supply-chain/templates/secrets/qtodo-registry-pass.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml @@ -1,3 +1,10 @@ +{{/* + Quay User Provisioner Secret + Purpose: Provides password for the Quay user provisioner job to create/update users in built-in Quay + Used by: quay-user-job.yaml (CronJob that provisions Quay users) + Only created when: quay.enabled=true (built-in Quay registry) + Not used for: BYO/external registry (use qtodo-registry-auth.yaml instead) +*/}} {{- if eq .Values.quay.enabled true }} --- apiVersion: "external-secrets.io/v1beta1" @@ -19,6 +26,6 @@ spec: data: - secretKey: password remoteRef: - key: {{ .Values.registry.vaultPath }} - property: {{ .Values.registry.passwordVaultKey }} + key: {{ .Values.quay.vaultPath }} + property: {{ .Values.quay.passwordVaultKey }} {{- end }} \ No newline at end of file diff --git a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml index 416e8020..eb889c9f 100644 --- a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml @@ -1,3 +1,25 @@ +{{/* + Pipeline Registry Auth Secret + Purpose: Provides dockerconfigjson for pipeline to push/pull images + Used by: Tekton pipeline tasks (build-image, sign-image, verify-image) + Created when: quay.enabled=true OR externalRegistry.enabled=true + Vault path: Automatically selects based on which registry is enabled + - Built-in Quay: quay.vaultPath (auto-generated credentials) + - BYO Registry: externalRegistry.vaultPath (user-provided credentials) + Registry domain: + - Built-in Quay: auto-constructed as quay-registry-quay-quay-enterprise. + - BYO Registry: must be explicitly set via registry.domain +*/}} +{{- if or .Values.quay.enabled .Values.externalRegistry.enabled }} +{{- /* Determine registry domain: auto-construct for built-in Quay, require for external */ -}} +{{- $registryDomain := "" -}} +{{- if .Values.registry.domain -}} + {{- $registryDomain = .Values.registry.domain -}} +{{- else if .Values.quay.enabled -}} + {{- $registryDomain = printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain -}} +{{- else -}} + {{- fail "registry.domain is required for external registry" -}} +{{- end -}} --- apiVersion: "external-secrets.io/v1beta1" kind: ExternalSecret @@ -17,7 +39,7 @@ spec: .dockerconfigjson: | { "auths": { - "{{ .Values.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain) }}": { + "{{ $registryDomain }}": { "auth": "{{ `{{ printf "%s:%s" "` }}{{ .Values.registry.user }}{{ `" .password | b64enc }}` }}" } } @@ -25,5 +47,11 @@ spec: data: - secretKey: password remoteRef: - key: {{ .Values.registry.vaultPath }} - property: {{ .Values.registry.passwordVaultKey }} \ No newline at end of file + {{- if .Values.quay.enabled }} + key: {{ .Values.quay.vaultPath }} + property: {{ .Values.quay.passwordVaultKey }} + {{- else if .Values.externalRegistry.enabled }} + key: {{ .Values.externalRegistry.vaultPath }} + property: {{ .Values.externalRegistry.passwordVaultKey }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index 4a54d048..aed322b4 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -26,26 +26,55 @@ qtodo: buildCmd: "./mvnw -s settings.xml package -DskipTests -Dquarkus.package.jar.type=uber-jar" containerfile: "./Containerfile" -# quay registry configuration -# used to create a new user in quay. Generic registry configuration is below. +# =========================================================================== +# BUILT-IN QUAY REGISTRY (optional) +# When enabled, deploys internal Quay registry with auto-generated credentials +# =========================================================================== quay: enabled: true email: "quay-user@example.com" + # Vault path for auto-generated Quay credentials + vaultPath: "secret/data/hub/infra/quay/quay-users" + passwordVaultKey: "quay-user-password" + # User provisioner job settings job: image: registry.access.redhat.com/ubi9/ubi:9.7-1764794285 schedule: "*/5 * * * *" -# container registry configuration +# =========================================================================== +# EXTERNAL/BYO REGISTRY (optional) +# User-provided credentials for external registry (quay.io, ghcr.io, etc.) +# Enable this when using an external registry instead of built-in Quay +# =========================================================================== +externalRegistry: + enabled: false + # Vault path for user-provided credentials + vaultPath: "secret/data/hub/infra/registry/registry-user" + passwordVaultKey: "registry-password" + +# =========================================================================== +# COMMON REGISTRY SETTINGS (shared by both built-in Quay and external registry) +# =========================================================================== registry: - # Commented to generate it dynamically - # domain: "quay-registry-quay-quay-enterprise.hub.example.com" + # For built-in Quay: domain is auto-constructed from hubClusterDomain + # For external registry: REQUIRED - set explicitly (e.g., quay.io, ghcr.io) + # domain: "registry.example.com" org: "ztvp" repo: "qtodo" tlsVerify: "true" - user: "quay-user" - passwordVaultKey: "quay-user-password" - # Infrastructure secrets - stored in quay path - vaultPath: "secret/data/hub/infra/quay/quay-users" + user: "registry-user" + # Secret name for registry auth (dockerconfigjson) + authSecretName: "qtodo-registry-auth" + # Embedded OCP registry only: create image namespace (registry.org) and grant + # pipeline SA system:image-builder so the pipeline can push. Set to true only when + # using the in-cluster OpenShift image registry; leave false for quay.io or other external registries. + embeddedOCP: + ensureImageNamespaceRBAC: false + +# pipeline run configuration +pipelinerun: + # Set to true to automatically trigger a pipeline run on ArgoCD sync + enabled: false # spire configuration spire: diff --git a/docs/supply-chain.md b/docs/supply-chain.md index d2459fad..55cb4d9e 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -20,6 +20,99 @@ In our demo, we will use a number of additional ZTVP components. These component * [Multicloud Object Gateway](https://docs.redhat.com/en/documentation/red_hat_openshift_container_storage/4.8/html/managing_hybrid_and_multicloud_resources/index) is a data service for OpenShift that provides an S3-compatible object storage. In our case, this component is necessary to provide a storage system to Quay. * [Red Hat OpenShift Pipelines](https://docs.redhat.com/en/documentation/red_hat_openshift_pipelines/1.20) is a cloud-native CI/CD solution built on the Tekton framework. We will use this product to automate our secure supply chain process, but you could use your own CI/CD solution if one exists. +## Bring Your Own (BYO) Container Registry + +By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use your own container registry (e.g., quay.io, Docker Hub, GitHub Container Registry, or a private registry) instead. + +### Configuration Steps + +1. **Disable built-in Quay registry** (optional - if not using Quay): Comment out the Quay-related applications in `values-hub.yaml`: `quay-enterprise` namespace, `quay-operator` subscription, and `quay-registry` application. + +2. **Configure registry credentials in Vault**: Per VP rule, add your registry credentials to `~/values-secrets.yaml` (or `~/values-secret.yaml` / `~/values-secret-layered-zero-trust.yaml` per VP lookup order): + + ```bash + # Copy template to local file if not already done + cp values-secret.yaml.template ~/values-secrets.yaml + ``` + + Add the registry-user secret (same format for **BYO external registry** and **embedded OCP registry**): + + ```yaml + - name: registry-user + vaultPrefixes: + - hub/infra/registry + fields: + - name: registry-password + value: "REPLACE_WITH_REGISTRY_TOKEN" + onMissingValue: error + ``` + + Replace `REPLACE_WITH_REGISTRY_TOKEN` with: + * **Embedded OCP registry:** output of `oc whoami -t` (after `oc login`). + * **External registry (BYO):** your registry token or password (e.g. quay.io, ghcr.io). + + > **Note**: Never commit `~/values-secrets.yaml` (or your local values-secret file) to git. This file contains sensitive credentials and should remain local. + +3. **Set registry configuration in values-hub.yaml**: For the supply-chain application, add these overrides: + + ```yaml + overrides: + # Disable built-in Quay + - name: quay.enabled + value: "false" + # Enable external registry + - name: externalRegistry.enabled + value: "true" + # External registry settings + - name: registry.domain + value: "your-registry.example.com" + - name: registry.user + value: "your-username" + - name: registry.org + value: "your-org" + ``` + +4. **Configure qtodo for custom registry** (if pulling from custom registry): + + ```yaml + overrides: + - name: app.images.main.registry.auth + value: true + - name: app.images.main.registry.domain + value: "your-registry.example.com" + - name: app.images.main.registry.user + value: "your-username" + ``` + +### Required Configuration + +| Parameter | Description | Example | +| --------- | ----------- | ------- | +| `registry.domain` | Registry hostname (required for BYO only) | `quay.io`, `ghcr.io`, `registry.example.com` | +| `registry.org` | Organization/namespace | `my-org` | +| `registry.repo` | Repository name | `qtodo` | +| `registry.user` | Registry username | `my-robot-account` | +| `quay.enabled` | Set to `false` for BYO registry | `false` | + +> **Note**: For built-in Quay registry, `registry.domain` is automatically constructed as `quay-registry-quay-quay-enterprise.` and does not need to be specified. For BYO/external registries, `registry.domain` is **required**. + +### Vault Paths + +Registry credentials are stored at different paths based on registry type: + +| Registry Type | Vault Path | Password Key | +| --------------- | ---------------------------------------------- | -------------------- | +| Built-in Quay | `secret/data/hub/infra/quay/quay-users` | `quay-user-password` | +| BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | + +The chart automatically selects the correct vault path based on the enabled flags: + +* `quay.enabled=true`: Uses built-in Quay vault path +* `externalRegistry.enabled=true`: Uses external registry vault path +* Both disabled (default): No registry auth secret created (fresh install state) + +The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. + ## Automatic approach To automate the application building and certifying process, we will use _Red Hat OpenShift Pipelines_. @@ -78,12 +171,55 @@ Using the previously created definition, start a new execution of the pipeline u oc create -f qtodo-pipeline.yaml ``` +#### Using Helm Template + +You can also trigger a pipeline run using the Helm template included in the chart. + +**For Built-in Quay Registry:** + +```shell +helm template supply-chain charts/supply-chain \ + --set pipelinerun.enabled=true \ + --set quay.enabled=true \ + --set global.namespace=layered-zero-trust-hub \ + --set global.hubClusterDomain=apps.example.com \ + --show-only templates/pipelinerun-qtodo.yaml | oc create -f - +``` + +> **Note**: For built-in Quay, `registry.domain` is auto-constructed from `global.hubClusterDomain`. + +**For BYO/External Registry:** + +```shell +helm template supply-chain charts/supply-chain \ + --set pipelinerun.enabled=true \ + --set externalRegistry.enabled=true \ + --set global.namespace=layered-zero-trust-hub \ + --set registry.domain=quay.io \ + --show-only templates/pipelinerun-qtodo.yaml | oc create -f - +``` + +This renders the PipelineRun template with the correct PVC and secret workspace bindings, then creates it in the cluster. + You can review the current pipeline logs using the [Tekton CLI](https://tekton.dev/docs/cli/). ```shell tkn pipeline logs -n layered-zero-trust-hub -L -f ``` +Or use `oc` commands to monitor progress: + +```shell +# List pipeline runs +oc get pipelinerun -n layered-zero-trust-hub + +# Check task status for a specific run +oc get taskruns -n layered-zero-trust-hub -l tekton.dev/pipelineRun= + +# View logs for a specific task +oc logs -n layered-zero-trust-hub -l tekton.dev/pipelineRun=,tekton.dev/pipelineTask= +``` + ### Pipeline tasks The pipeline we have prepared has the following steps: diff --git a/values-hub.yaml b/values-hub.yaml index b8da45d9..c5b506d4 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -286,6 +286,9 @@ clusterGroup: path "secret/data/hub/infra/quay/*" { capabilities = ["read"] } + path "secret/data/hub/infra/registry/*" { + capabilities = ["read"] + } path "secret/data/hub/infra/rhtpa/rhtpa-oidc-cli" { capabilities = ["read"] } @@ -434,8 +437,7 @@ clusterGroup: project: hub path: charts/qtodo ignoreDifferences: - - group: "" - kind: ServiceAccount + - kind: ServiceAccount jqPathExpressions: - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) overrides: @@ -463,26 +465,61 @@ clusterGroup: # value: quay-user-password # Secure Supply Chain - Uncomment to enable # supply-chain: - # name: supply-chain - # project: hub - # path: charts/supply-chain - # ignoreDifferences: - # - group: "" - # kind: ServiceAccount - # jqPathExpressions: - # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) - # overrides: - # # Don't forget to uncomment the RHTAS and RHTPA components in this same file - # - name: rhtas.enabled - # value: true - # - name: rhtpa.enabled - # value: true - # - name: registry.tlsVerify - # value: "false" - # - name: registry.user - # value: quay-admin - # - name: registry.passwordVaultKey - # value: quay-admin-password + # name: supply-chain + # project: hub + # path: charts/supply-chain + # ignoreDifferences: + # - kind: ServiceAccount + # jqPathExpressions: + # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) + # overrides: + # # ============================================================ + # # OPTION 1: Built-in Quay Registry + # # Requires: quay-enterprise namespace, quay-operator, quay-registry app + # # Note: registry.domain is auto-constructed from hubClusterDomain + # # ============================================================ + # # - name: quay.enabled + # # value: "true" + # # - name: externalRegistry.enabled + # # value: "false" + # # - name: registry.tlsVerify + # # value: "false" + # # - name: registry.user + # # value: quay-user + # # ============================================================ + # # OPTION 2: BYO/External Registry + # # Requires: registry credentials in ~/values-secret.yaml + # # Note: registry.domain is REQUIRED for external registry + # # ============================================================ + # # - name: quay.enabled + # # value: "false" + # # - name: externalRegistry.enabled + # # value: "true" + # # - name: registry.domain + # # value: quay.io + # # - name: registry.org + # # value: your-org + # # - name: registry.user + # # value: your-username + # # ============================================================ + # # OPTION 3: Embedded OCP Registry (comment out Option 1, 2; uncomment below) + # # ============================================================ + # # - name: registry.domain + # # value: default-route-openshift-image-registry.apps. + # # - name: registry.org + # # value: ztvp + # # - name: registry.user + # # value: admin + # # Embedded OCP registry only: create image namespace and grant pipeline push (transparent) + # # - name: registry.embeddedOCP.ensureImageNamespaceRBAC + # # value: "true" + # # ============================================================ + # # Enable RHTAS signing + # # - name: rhtas.enabled + # # value: "true" + # # Enable RHTPA SBOM upload + # # - name: rhtpa.enabled + # # value: "true" argoCD: resourceHealthChecks: - check: | diff --git a/values-secret.yaml.template b/values-secret.yaml.template index e1f48ac9..16cbc205 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -16,7 +16,8 @@ version: "2.0" # Infrastructure Secrets (hub/infra/*): # hub/infra/keycloak/ - Keycloak infrastructure secrets # hub/infra/rhtpa/ - RHTPA infrastructure secrets -# hub/infra/quay/ - Quay registry credentials +# hub/infra/quay/ - Built-in Quay registry credentials (auto-generated) +# hub/infra/registry/ - BYO container registry credentials (user-provided) # hub/infra/users/ - User credentials managed by IdP # # Framework Secrets: @@ -150,34 +151,38 @@ secrets: vaultPolicy: alphaNumericPolicy # =========================================================================== - # QUAY INFRASTRUCTURE SECRETS (hub/infra/quay/) - # Registry credentials for Quay - # Policy: hub-infra-quay-secret (read access to hub/infra/quay/*) + # BUILT-IN QUAY REGISTRY SECRETS (hub/infra/quay/) + # Auto-generated credentials for built-in Quay registry + # Used by: Quay user provisioner job, supply-chain pipeline (when quay.enabled=true) + # Policy: hub-supply-chain-jwt-secret (read access to hub/infra/quay/*) # =========================================================================== - name: quay-users vaultPrefixes: - hub/infra/quay fields: - - name: quay-admin-password - onMissingValue: generate - vaultPolicy: validatedPatternDefaultPolicy - name: quay-user-password onMissingValue: generate vaultPolicy: validatedPatternDefaultPolicy - # External Registry Credentials (e.g., Quay.io, Docker Hub, GHCR) - # Reserved for future use with container signing workflows - # Uncomment and provide your credentials when needed - #- name: external-registry - # vaultPrefixes: - # - hub/infra - # fields: - # - name: username - # value: "your-registry-username" # Replace with your username - # onMissingValue: error - # - name: password - # value: "your-registry-token" # Replace with your token/password - # onMissingValue: error + # =========================================================================== + # BYO / EMBEDDED OCP REGISTRY SECRETS (hub/infra/registry/) + # User-provided credentials for external or embedded OCP registry. + # Used by: supply-chain pipeline (push), qtodo (pull) when externalRegistry.enabled=true + # Policy: hub-supply-chain-jwt-secret (read access to hub/infra/registry/*) + # + # VP rule: add this (with your token) to ~/values-secrets.yaml (or + # ~/values-secret.yaml / ~/values-secret-layered-zero-trust.yaml per VP lookup). + # Replace REPLACE_WITH_REGISTRY_TOKEN in your local file: + # - Embedded OCP registry: use output of oc whoami -t + # - External registry (BYO): use your registry token/password + # =========================================================================== + - name: registry-user + vaultPrefixes: + - hub/infra/registry + fields: + - name: registry-password + value: "REPLACE_WITH_REGISTRY_TOKEN" + onMissingValue: error # =========================================================================== # HUB-SPECIFIC SECRETS (hub/) From b1203c18a3986b9b4b6f353739bc777a18157a0a Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Wed, 18 Feb 2026 10:08:11 -0500 Subject: [PATCH 02/21] feat: unified registry configuration with multi-registry support Refactor supply-chain and qtodo charts to use a single, option-agnostic registry configuration instead of separate per-registry blocks. Registry options (configure one in values-hub.yaml): - Option 1: Built-in Quay Registry - Option 2: BYO/External Registry (quay.io, ghcr.io, etc.) - Option 3: Embedded OCP Image Registry Key changes: Supply-chain chart: * Unified registry.* parameters (domain, org, user, vaultPath, passwordVaultKey) * Use tpl function to resolve template expressions in registry.domain values passed as --set parameters from the validated patterns framework * Embedded OCP registry automation (registry.embeddedOCP.ensureImageNamespaceRBAC): - Auto-create image namespace matching registry.org - Grant pipeline SA system:image-builder via RoleBinding - Enable default route on OCP image registry via Kubernetes API (curl-based Job using ServiceAccount token, no oc CLI dependency) * ArgoCD hook annotations on the route-enabler Job (Sync + HookSucceeded) * Rename qtodo-registry-pass to qtodo-quay-pass for clarity Qtodo chart: * Unified app.images.main.registry.* parameters * Use tpl function in registry-external-secret.yaml for domain resolution ztvp-certificates chart: * Node-level image pull trust for kubelet (imagePullTrust.*) * Create ConfigMap with ingress CA per registry hostname in openshift-config * Patch image.config.openshift.io/cluster additionalTrustedCA * RBAC for patching image.config.openshift.io resources Documentation: * Comprehensive supply-chain.md with configuration steps for all three registry options, vault paths, and example overrides * Updated values-secret.yaml.template with registry credential examples Signed-off-by: Min Zhang --- .gitignore | 1 + .../templates/registry-external-secret.yaml | 13 +-- charts/qtodo/values.yaml | 20 +--- .../templates/pipeline-qtodo.yaml | 11 +- .../templates/quay/quay-user-job.yaml | 2 +- .../rbac/registry-image-namespace.yaml | 91 +++++++++++++++ .../templates/secrets/qtodo-quay-pass.yaml | 8 +- .../secrets/qtodo-registry-auth.yaml | 38 ++----- charts/supply-chain/values.yaml | 58 ++++++---- .../files/extract-certificates.sh.tpl | 74 +++++++++++- charts/ztvp-certificates/templates/rbac.yaml | 33 ++++++ charts/ztvp-certificates/values.yaml | 19 ++++ docs/supply-chain.md | 106 ++++++++++++++---- values-hub.yaml | 98 +++++++++++++--- 14 files changed, 439 insertions(+), 133 deletions(-) diff --git a/.gitignore b/.gitignore index 0d29468e..9e951fa7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,6 @@ super-linter-output github_conf # Editor and IDE specific files +.cursor/ .cursorrules .vscode/ diff --git a/charts/qtodo/templates/registry-external-secret.yaml b/charts/qtodo/templates/registry-external-secret.yaml index 6b6979c9..0fe1e1ad 100644 --- a/charts/qtodo/templates/registry-external-secret.yaml +++ b/charts/qtodo/templates/registry-external-secret.yaml @@ -18,7 +18,7 @@ spec: .dockerconfigjson: | { "auths": { - "{{ required "app.images.main.registry.domain is required when registry.auth is enabled" .Values.app.images.main.registry.domain }}": { + "{{ tpl (required "app.images.main.registry.domain is required when registry.auth is enabled" .Values.app.images.main.registry.domain) $ }}": { "auth": "{{ `{{ printf "%s:%s" "` }}{{ .Values.app.images.main.registry.user }}{{ `" .password | b64enc }}` }}" } } @@ -26,11 +26,6 @@ spec: data: - secretKey: password remoteRef: - {{- if .Values.app.images.main.registry.builtinQuay.enabled }} - key: {{ .Values.app.images.main.registry.builtinQuay.vaultPath }} - property: {{ .Values.app.images.main.registry.builtinQuay.passwordVaultKey }} - {{- else if .Values.app.images.main.registry.externalRegistry.enabled }} - key: {{ .Values.app.images.main.registry.externalRegistry.vaultPath }} - property: {{ .Values.app.images.main.registry.externalRegistry.passwordVaultKey }} - {{- end }} -{{- end }} \ No newline at end of file + key: {{ required "app.images.main.registry.vaultPath is required when registry.auth is enabled" .Values.app.images.main.registry.vaultPath }} + property: {{ required "app.images.main.registry.passwordVaultKey is required when registry.auth is enabled" .Values.app.images.main.registry.passwordVaultKey }} +{{- end }} diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index 52b13d33..0270e896 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -15,26 +15,14 @@ app: # Modified to Always to force a pull so we can test changes to the container image without requiring manual deletion of images or restarts of argo pullPolicy: Always registry: - # auth: controls whether to create registry auth secret - # Set to true when using private registry (built-in Quay or external) + # Set to true to create registry auth secret for image pulls auth: false secretName: qtodo-registry-auth user: registry-user # domain: registry.example.com # REQUIRED when auth is enabled - - # Built-in Quay registry (optional) - # When enabled, uses auto-generated credentials from Vault - builtinQuay: - enabled: false - vaultPath: secret/data/hub/infra/quay/quay-users - passwordVaultKey: quay-user-password - - # External/BYO registry (optional) - # When enabled, uses user-provided credentials from Vault - externalRegistry: - enabled: false - vaultPath: secret/data/hub/infra/registry/registry-user - passwordVaultKey: registry-password + # Vault path and key for registry password (set for your scenario) + vaultPath: "" + passwordVaultKey: "" spiffeHelper: name: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9 version: v0.10.0 diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index 39fe8494..add87310 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -1,12 +1,3 @@ -{{- /* Determine registry domain: auto-construct for built-in Quay, require for external */ -}} -{{- $registryDomain := "" -}} -{{- if .Values.registry.domain -}} - {{- $registryDomain = .Values.registry.domain -}} -{{- else if .Values.quay.enabled -}} - {{- $registryDomain = printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain -}} -{{- else -}} - {{- fail "registry.domain is required for external registry" -}} -{{- end -}} --- apiVersion: tekton.dev/v1beta1 kind: Pipeline @@ -34,7 +25,7 @@ spec: - name: image-target type: string description: qtodo image push destination (e.g. quay.io/ztvp/qtodo:latest) - default: {{ $registryDomain }}/{{ .Values.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} + default: {{ tpl (required "registry.domain is required" .Values.registry.domain) $ }}/{{ .Values.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} - name: image-tls-verify type: string description: Whether to verify TLS when pushing to the OCI registry diff --git a/charts/supply-chain/templates/quay/quay-user-job.yaml b/charts/supply-chain/templates/quay/quay-user-job.yaml index 417afcc7..4fa0f7c4 100644 --- a/charts/supply-chain/templates/quay/quay-user-job.yaml +++ b/charts/supply-chain/templates/quay/quay-user-job.yaml @@ -19,7 +19,7 @@ spec: command: ["python3", "/app/create_user.py"] env: - name: QUAY_HOST - value: {{ .Values.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain) }} + value: {{ tpl (.Values.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain)) $ }} - name: QUAY_ADMIN_USER value: {{ .Values.registry.user }} - name: QUAY_ADMIN_PASSWORD diff --git a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml index 35f3ab76..ebd5bf45 100644 --- a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml +++ b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml @@ -25,4 +25,95 @@ subjects: - kind: ServiceAccount name: pipeline namespace: {{ .Values.global.namespace }} +--- +# Enable the default route on the embedded OCP image registry so that +# the pipeline can push and external clients can pull images via the route. +# Uses a Job because the imageregistry config is a cluster-singleton managed +# by the image-registry operator; declarative ownership would conflict. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: registry-route-enabler + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Values.global.namespace }}-registry-route-enabler + annotations: + argocd.argoproj.io/sync-wave: "0" +rules: +- apiGroups: ["imageregistry.operator.openshift.io"] + resources: ["configs"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Values.global.namespace }}-registry-route-enabler + annotations: + argocd.argoproj.io/sync-wave: "0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Values.global.namespace }}-registry-route-enabler +subjects: +- kind: ServiceAccount + name: registry-route-enabler + namespace: {{ .Values.global.namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: enable-registry-default-route + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "1" + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded +spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: registry-route-enabler + restartPolicy: Never + containers: + - name: enable-route + image: registry.access.redhat.com/ubi9/ubi:9.7-1764794285 + command: + - /bin/sh + - -ce + - | + APISERVER="https://kubernetes.default.svc" + TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + CACERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + RESOURCE_URL="${APISERVER}/apis/imageregistry.operator.openshift.io/v1/configs/cluster" + AUTH_HEADER="Authorization: Bearer ${TOKEN}" + + echo "Checking current defaultRoute status..." + BODY=$(curl -sS --cacert "${CACERT}" -H "${AUTH_HEADER}" "${RESOURCE_URL}") + rc=$?; if [ $rc -ne 0 ]; then echo "ERROR: GET failed (curl rc=${rc})"; exit 1; fi + + # Parse defaultRoute from JSON without jq/grep dependency + case "${BODY}" in + *'"defaultRoute":true'*) echo "Default route already enabled, nothing to do."; exit 0 ;; + esac + + echo "Enabling default route on embedded OCP image registry..." + RESP=$(curl -sS -w "\n%{http_code}" --cacert "${CACERT}" \ + -H "${AUTH_HEADER}" \ + -H "Content-Type: application/merge-patch+json" \ + -X PATCH -d '{"spec":{"defaultRoute":true}}' \ + "${RESOURCE_URL}") + HTTP_CODE=$(echo "${RESP}" | tail -1) + + if [ "${HTTP_CODE}" -ge 200 ] 2>/dev/null && [ "${HTTP_CODE}" -lt 300 ] 2>/dev/null; then + echo "Default route enabled successfully (HTTP ${HTTP_CODE})." + else + echo "ERROR: PATCH failed (HTTP ${HTTP_CODE})." + echo "${RESP}" | head -5 + exit 1 + fi {{- end }} diff --git a/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml b/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml index d66f6507..2f42cedd 100644 --- a/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml @@ -3,7 +3,7 @@ Purpose: Provides password for the Quay user provisioner job to create/update users in built-in Quay Used by: quay-user-job.yaml (CronJob that provisions Quay users) Only created when: quay.enabled=true (built-in Quay registry) - Not used for: BYO/external registry (use qtodo-registry-auth.yaml instead) + Uses unified registry.vaultPath and registry.passwordVaultKey */}} {{- if eq .Values.quay.enabled true }} --- @@ -26,6 +26,6 @@ spec: data: - secretKey: password remoteRef: - key: {{ .Values.quay.vaultPath }} - property: {{ .Values.quay.passwordVaultKey }} -{{- end }} \ No newline at end of file + key: {{ .Values.registry.vaultPath }} + property: {{ .Values.registry.passwordVaultKey }} +{{- end }} diff --git a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml index eb889c9f..ea3e1e2b 100644 --- a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml @@ -2,29 +2,16 @@ Pipeline Registry Auth Secret Purpose: Provides dockerconfigjson for pipeline to push/pull images Used by: Tekton pipeline tasks (build-image, sign-image, verify-image) - Created when: quay.enabled=true OR externalRegistry.enabled=true - Vault path: Automatically selects based on which registry is enabled - - Built-in Quay: quay.vaultPath (auto-generated credentials) - - BYO Registry: externalRegistry.vaultPath (user-provided credentials) - Registry domain: - - Built-in Quay: auto-constructed as quay-registry-quay-quay-enterprise. - - BYO Registry: must be explicitly set via registry.domain + Created when: registry.enabled=true + Registry-agnostic: works for built-in Quay, BYO (quay.io, ghcr.io), or embedded OCP. + Set registry.domain, registry.vaultPath, and registry.passwordVaultKey for your scenario. */}} -{{- if or .Values.quay.enabled .Values.externalRegistry.enabled }} -{{- /* Determine registry domain: auto-construct for built-in Quay, require for external */ -}} -{{- $registryDomain := "" -}} -{{- if .Values.registry.domain -}} - {{- $registryDomain = .Values.registry.domain -}} -{{- else if .Values.quay.enabled -}} - {{- $registryDomain = printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain -}} -{{- else -}} - {{- fail "registry.domain is required for external registry" -}} -{{- end -}} +{{- if .Values.registry.enabled }} --- apiVersion: "external-secrets.io/v1beta1" kind: ExternalSecret metadata: - name: qtodo-registry-auth + name: {{ .Values.registry.authSecretName }} namespace: {{ .Release.Namespace | default .Values.global.namespace }} spec: refreshInterval: 15s @@ -32,14 +19,14 @@ spec: name: {{ .Values.global.secretStore.name }} kind: {{ .Values.global.secretStore.kind }} target: - name: qtodo-registry-auth + name: {{ .Values.registry.authSecretName }} template: type: kubernetes.io/dockerconfigjson data: .dockerconfigjson: | { "auths": { - "{{ $registryDomain }}": { + "{{ tpl (required "registry.domain is required when registry.enabled=true" .Values.registry.domain) $ }}": { "auth": "{{ `{{ printf "%s:%s" "` }}{{ .Values.registry.user }}{{ `" .password | b64enc }}` }}" } } @@ -47,11 +34,6 @@ spec: data: - secretKey: password remoteRef: - {{- if .Values.quay.enabled }} - key: {{ .Values.quay.vaultPath }} - property: {{ .Values.quay.passwordVaultKey }} - {{- else if .Values.externalRegistry.enabled }} - key: {{ .Values.externalRegistry.vaultPath }} - property: {{ .Values.externalRegistry.passwordVaultKey }} - {{- end }} -{{- end }} \ No newline at end of file + key: {{ required "registry.vaultPath is required when registry.enabled=true" .Values.registry.vaultPath }} + property: {{ required "registry.passwordVaultKey is required when registry.enabled=true" .Values.registry.passwordVaultKey }} +{{- end }} diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index aed322b4..0b6be272 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -27,47 +27,59 @@ qtodo: containerfile: "./Containerfile" # =========================================================================== -# BUILT-IN QUAY REGISTRY (optional) -# When enabled, deploys internal Quay registry with auto-generated credentials +# QUAY USER PROVISIONER (only for built-in Quay registry) +# When enabled, runs a CronJob that provisions users in the built-in Quay instance. +# This is Quay-specific and not needed for BYO or embedded OCP registries. # =========================================================================== quay: - enabled: true + enabled: false email: "quay-user@example.com" - # Vault path for auto-generated Quay credentials - vaultPath: "secret/data/hub/infra/quay/quay-users" - passwordVaultKey: "quay-user-password" - # User provisioner job settings job: image: registry.access.redhat.com/ubi9/ubi:9.7-1764794285 schedule: "*/5 * * * *" # =========================================================================== -# EXTERNAL/BYO REGISTRY (optional) -# User-provided credentials for external registry (quay.io, ghcr.io, etc.) -# Enable this when using an external registry instead of built-in Quay -# =========================================================================== -externalRegistry: - enabled: false - # Vault path for user-provided credentials - vaultPath: "secret/data/hub/infra/registry/registry-user" - passwordVaultKey: "registry-password" - -# =========================================================================== -# COMMON REGISTRY SETTINGS (shared by both built-in Quay and external registry) +# REGISTRY CONFIGURATION (option-agnostic) +# Works for all registry types: built-in Quay, BYO (quay.io, ghcr.io, etc.), +# or embedded OCP image registry. Set the values for your scenario. +# +# Scenario-specific values (set in values-hub.yaml overrides): +# Built-in Quay: +# domain: quay-registry-quay-quay-enterprise.apps. +# vaultPath: secret/data/hub/infra/quay/quay-users +# passwordVaultKey: quay-user-password +# BYO (quay.io, ghcr.io, etc.): +# domain: quay.io (or your registry hostname) +# vaultPath: secret/data/hub/infra/registry/registry-user +# passwordVaultKey: registry-password +# Embedded OCP: +# domain: default-route-openshift-image-registry.apps. +# vaultPath: secret/data/hub/infra/registry/registry-user +# passwordVaultKey: registry-password +# embeddedOCP.ensureImageNamespaceRBAC: true # =========================================================================== registry: - # For built-in Quay: domain is auto-constructed from hubClusterDomain - # For external registry: REQUIRED - set explicitly (e.g., quay.io, ghcr.io) - # domain: "registry.example.com" + # Set to true to create the registry auth secret (dockerconfigjson) + enabled: false + # Registry hostname (REQUIRED when enabled) + domain: "" + # Organization/namespace within the registry org: "ztvp" + # Repository name repo: "qtodo" + # Whether to verify TLS when pushing to the registry tlsVerify: "true" + # Registry username user: "registry-user" + # Vault path to the secret containing the registry password + vaultPath: "" + # Key within the Vault secret that holds the password + passwordVaultKey: "" # Secret name for registry auth (dockerconfigjson) authSecretName: "qtodo-registry-auth" # Embedded OCP registry only: create image namespace (registry.org) and grant # pipeline SA system:image-builder so the pipeline can push. Set to true only when - # using the in-cluster OpenShift image registry; leave false for quay.io or other external registries. + # using the in-cluster OpenShift image registry; leave false for other registries. embeddedOCP: ensureImageNamespaceRBAC: false diff --git a/charts/ztvp-certificates/files/extract-certificates.sh.tpl b/charts/ztvp-certificates/files/extract-certificates.sh.tpl index 5db24275..6c97e6f8 100644 --- a/charts/ztvp-certificates/files/extract-certificates.sh.tpl +++ b/charts/ztvp-certificates/files/extract-certificates.sh.tpl @@ -336,7 +336,79 @@ else fi # =================================================================== -# PHASE 9: Automatic Rollout (if enabled) +# PHASE 9: Configure Node-Level Image Pull Trust (if enabled) +# Creates a ConfigMap with registry-hostname keys containing the ingress CA, +# then patches image.config.openshift.io/cluster to reference it. +# This allows kubelet to pull images from registries behind the cluster ingress +# (e.g. built-in Quay) without "x509: certificate signed by unknown authority". +# =================================================================== + +{{- if .Values.imagePullTrust.enabled }} +{{- if .Values.imagePullTrust.registries }} +log "Configuring node-level image pull trust" + +if [[ "$INGRESS_CA_FOUND" != "true" ]]; then + error "imagePullTrust is enabled but no ingress CA was extracted. Cannot configure image pull trust." + error "Ensure autoDetect is true or provide a custom ingress CA source." + exit 1 +fi + +# Build the ConfigMap data with registry hostnames as keys +# Each key is a registry hostname, value is the ingress CA PEM +REGISTRY_CM_DATA="" +{{- range .Values.imagePullTrust.registries }} +log "Adding registry trust: {{ tpl . $ }}" +{{- end }} + +log "Creating ConfigMap: {{ .Values.global.namespace }}/{{ .Values.imagePullTrust.configMapName }}" + +# Combine all ingress CA files into one PEM for registry trust +COMBINED_INGRESS_CA="${TEMP_DIR}/combined-ingress-ca.pem" +> "${COMBINED_INGRESS_CA}" +for f in "${TEMP_DIR}"/ingress-ca-*.crt; do + [[ -f "$f" ]] || continue + cat "$f" >> "${COMBINED_INGRESS_CA}" + echo "" >> "${COMBINED_INGRESS_CA}" +done + +# Create the ConfigMap with registry hostnames as keys +cat <<'CMEOF' > "${TEMP_DIR}/registry-cas-cm.yaml" +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.imagePullTrust.configMapName }} + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: ztvp-certificates + app.kubernetes.io/component: image-pull-trust + app.kubernetes.io/managed-by: ztvp-certificate-manager +data: {} +CMEOF + +oc apply -f "${TEMP_DIR}/registry-cas-cm.yaml" + +# Patch each registry hostname as a key with the ingress CA PEM +{{- range .Values.imagePullTrust.registries }} +log "Patching ConfigMap key: {{ tpl . $ }}" +oc create configmap {{ $.Values.imagePullTrust.configMapName }} \ + -n {{ $.Values.global.namespace }} \ + --from-file="{{ tpl . $ }}=${COMBINED_INGRESS_CA}" \ + --dry-run=client -o yaml | oc apply -f - +{{- end }} + +# Patch image.config.openshift.io/cluster to reference the ConfigMap +log "Patching image.config.openshift.io/cluster additionalTrustedCA" +oc patch image.config.openshift.io/cluster --type merge \ + -p "{\"spec\":{\"additionalTrustedCA\":{\"name\":\"{{ .Values.imagePullTrust.configMapName }}\"}}}" + +log "Node-level image pull trust configured successfully" +log "Note: MCO will roll this out to nodes (may take a few minutes)" + +{{- end }} +{{- end }} + +# =================================================================== +# PHASE 10: Automatic Rollout (if enabled) # =================================================================== {{- if .Values.rollout.enabled }} diff --git a/charts/ztvp-certificates/templates/rbac.yaml b/charts/ztvp-certificates/templates/rbac.yaml index 94949dc7..9d6b9ccd 100644 --- a/charts/ztvp-certificates/templates/rbac.yaml +++ b/charts/ztvp-certificates/templates/rbac.yaml @@ -80,6 +80,39 @@ subjects: - kind: ServiceAccount name: {{ include "ztvp-certificates.serviceAccountName" . }} namespace: {{ .Values.global.namespace }} +{{- if .Values.imagePullTrust.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ztvp-certificates.fullname" . }}-image-config + annotations: + argocd.argoproj.io/sync-wave: "-9" + labels: + {{- include "ztvp-certificates.labels" . | nindent 4 }} +rules: +# Patch image.config.openshift.io/cluster to set additionalTrustedCA +- apiGroups: ["config.openshift.io"] + resources: ["images"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ztvp-certificates.fullname" . }}-image-config + annotations: + argocd.argoproj.io/sync-wave: "-9" + labels: + {{- include "ztvp-certificates.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ztvp-certificates.fullname" . }}-image-config +subjects: +- kind: ServiceAccount + name: {{ include "ztvp-certificates.serviceAccountName" . }} + namespace: {{ .Values.global.namespace }} +{{- end }} {{- if .Values.rollout.enabled }} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/ztvp-certificates/values.yaml b/charts/ztvp-certificates/values.yaml index 354ee4c7..53d374f7 100644 --- a/charts/ztvp-certificates/values.yaml +++ b/charts/ztvp-certificates/values.yaml @@ -177,6 +177,25 @@ distribution: # Requires: ManagedClusterSetBinding in the namespace method: "acm-policy" +# Node-level image pull trust for kubelet +# Configures image.config.openshift.io/cluster additionalTrustedCA so that +# kubelet can pull images from registries behind the cluster's ingress (e.g. +# built-in Quay). Without this, kubelet image pulls fail with +# "x509: certificate signed by unknown authority" when the ingress uses a +# self-signed or cluster-internal CA. +imagePullTrust: + # Set to true to create the registry-CA ConfigMap and patch image.config + enabled: false + # ConfigMap name created in openshift-config for image.config additionalTrustedCA + configMapName: ztvp-registry-cas + # Registry hostnames that need the ingress CA for image pulls. + # Each becomes a key in the ConfigMap with the ingress CA as the value. + # Use {{ .Values.global.hubClusterDomain }} in values-hub.yaml overrides. + registries: [] + # Example (built-in Quay): + # registries: + # - quay-registry-quay-quay-enterprise.apps.example.com + # Debugging options debug: # Enable verbose logging in extraction job diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 55cb4d9e..5e92aa73 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -57,19 +57,18 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use ```yaml overrides: - # Disable built-in Quay - - name: quay.enabled - value: "false" - # Enable external registry - - name: externalRegistry.enabled + - name: registry.enabled value: "true" - # External registry settings - name: registry.domain value: "your-registry.example.com" - name: registry.user value: "your-username" - name: registry.org value: "your-org" + - name: registry.vaultPath + value: "secret/data/hub/infra/registry/registry-user" + - name: registry.passwordVaultKey + value: "registry-password" ``` 4. **Configure qtodo for custom registry** (if pulling from custom registry): @@ -88,31 +87,88 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use | Parameter | Description | Example | | --------- | ----------- | ------- | -| `registry.domain` | Registry hostname (required for BYO only) | `quay.io`, `ghcr.io`, `registry.example.com` | +| `registry.enabled` | Enable registry auth secret creation | `true` | +| `registry.domain` | Registry hostname (REQUIRED) | `quay.io`, `ghcr.io`, `registry.example.com` | | `registry.org` | Organization/namespace | `my-org` | | `registry.repo` | Repository name | `qtodo` | | `registry.user` | Registry username | `my-robot-account` | -| `quay.enabled` | Set to `false` for BYO registry | `false` | +| `registry.vaultPath` | Vault path for registry password | `secret/data/hub/infra/registry/registry-user` | +| `registry.passwordVaultKey` | Key within the Vault secret | `registry-password` | -> **Note**: For built-in Quay registry, `registry.domain` is automatically constructed as `quay-registry-quay-quay-enterprise.` and does not need to be specified. For BYO/external registries, `registry.domain` is **required**. +> **Note**: All registry types (built-in Quay, BYO, embedded OCP) use the same parameters. Set `registry.domain`, `registry.vaultPath`, and `registry.passwordVaultKey` to the appropriate values for your scenario. See the Vault Paths table below for scenario-specific values. ### Vault Paths Registry credentials are stored at different paths based on registry type: -| Registry Type | Vault Path | Password Key | -| --------------- | ---------------------------------------------- | -------------------- | -| Built-in Quay | `secret/data/hub/infra/quay/quay-users` | `quay-user-password` | -| BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | +| Registry Type | Vault Path | Password Key | +| ------------------ | ---------------------------------------------- | -------------------- | +| Built-in Quay | `secret/data/hub/infra/quay/quay-users` | `quay-user-password` | +| BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | +| Embedded OCP | `secret/data/hub/infra/registry/registry-user` | `registry-password` | -The chart automatically selects the correct vault path based on the enabled flags: - -* `quay.enabled=true`: Uses built-in Quay vault path -* `externalRegistry.enabled=true`: Uses external registry vault path -* Both disabled (default): No registry auth secret created (fresh install state) +Set `registry.vaultPath` and `registry.passwordVaultKey` in your `values-hub.yaml` overrides to match your scenario. When `registry.enabled=false` (default), no registry auth secret is created (fresh install state). The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. +### Embedded OCP Registry + +To use the in-cluster OpenShift image registry instead of an external registry: + +1. **Enable `registry.embeddedOCP.ensureImageNamespaceRBAC`** in the supply-chain overrides. The chart will automatically: + * Create the image namespace matching `registry.org` (e.g. `ztvp`) + * Grant the pipeline ServiceAccount `system:image-builder` in that namespace + * Enable the default route on the image registry (via a one-time Job) + +2. **Set the registry domain** to `default-route-openshift-image-registry.apps.`. + +3. **Set the registry user** to `admin` (or a user with push permissions). + +4. **Store the token in Vault**: Use `oc whoami -t` output as the `registry-password` value in `~/values-secrets.yaml`. + +Example supply-chain overrides: + +```yaml +overrides: + - name: registry.enabled + value: "true" + - name: registry.domain + value: default-route-openshift-image-registry.apps. + - name: registry.org + value: ztvp + - name: registry.user + value: admin + - name: registry.vaultPath + value: "secret/data/hub/infra/registry/registry-user" + - name: registry.passwordVaultKey + value: "registry-password" + - name: registry.embeddedOCP.ensureImageNamespaceRBAC + value: "true" +``` + +### Node-Level Image Pull Trust + +When using a registry behind the cluster ingress (Option 1: Built-in Quay or Option 3: Embedded OCP Registry), kubelet cannot pull images by default because the ingress certificate is self-signed and not trusted at the node level. + +The `ztvp-certificates` application handles this by patching `image.config.openshift.io/cluster` with the ingress CA certificate for the configured registry hostnames. Enable it by uncommenting the `imagePullTrust` overrides in `values-hub.yaml`: + +```yaml +# ztvp-certificates overrides +- name: imagePullTrust.enabled + value: "true" +- name: imagePullTrust.registries[0] + value: +``` + +Set `` to match your registry option: + +| Option | Registry Hostname | +| ------ | ----------------- | +| Option 1: Built-in Quay | `quay-registry-quay-quay-enterprise.apps.` | +| Option 3: Embedded OCP | `default-route-openshift-image-registry.apps.` | + +> **Note**: Option 2 (BYO/External Registry) does not require `imagePullTrust` because external registries like quay.io and ghcr.io use publicly trusted certificates. + ## Automatic approach To automate the application building and certifying process, we will use _Red Hat OpenShift Pipelines_. @@ -180,22 +236,24 @@ You can also trigger a pipeline run using the Helm template included in the char ```shell helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ - --set quay.enabled=true \ + --set registry.enabled=true \ + --set registry.domain=quay-registry-quay-quay-enterprise.apps.example.com \ + --set registry.vaultPath=secret/data/hub/infra/quay/quay-users \ + --set registry.passwordVaultKey=quay-user-password \ --set global.namespace=layered-zero-trust-hub \ - --set global.hubClusterDomain=apps.example.com \ --show-only templates/pipelinerun-qtodo.yaml | oc create -f - ``` -> **Note**: For built-in Quay, `registry.domain` is auto-constructed from `global.hubClusterDomain`. - **For BYO/External Registry:** ```shell helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ - --set externalRegistry.enabled=true \ - --set global.namespace=layered-zero-trust-hub \ + --set registry.enabled=true \ --set registry.domain=quay.io \ + --set registry.vaultPath=secret/data/hub/infra/registry/registry-user \ + --set registry.passwordVaultKey=registry-password \ + --set global.namespace=layered-zero-trust-hub \ --show-only templates/pipelinerun-qtodo.yaml | oc create -f - ``` diff --git a/values-hub.yaml b/values-hub.yaml index c5b506d4..048fbadc 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -209,6 +209,15 @@ clusterGroup: - name: rollout.strategy value: labeled + # Node-level image pull trust for kubelet + # Required when pulling images from registries behind the cluster ingress + # (e.g. built-in Quay, embedded OCP registry). Patches image.config.openshift.io/cluster. + # Uncomment and set the registry hostname when enabling a registry option. + # - name: imagePullTrust.enabled + # value: "true" + # - name: imagePullTrust.registries[0] + # value: + # Note: additionalCertificates (complex nested array) temporarily disabled # Need to find proper way to pass complex structures in Validated Patterns acm: @@ -451,18 +460,61 @@ clusterGroup: value: qtodo - name: app.vault.secretPath value: secret/data/apps/qtodo/qtodo-db - # For Secure Supply Chain, we changed the qtodo image to use the one built in the secure supply chain + # ============================================================ + # Secure Supply Chain: pull pipeline-built image from registry + # Uncomment the option matching your supply-chain registry choice. + # ============================================================ + # OPTION 1: Built-in Quay Registry # - name: app.images.main.name # value: quay-registry-quay-quay-enterprise.apps.{{ $.Values.global.clusterDomain }}/ztvp/qtodo # - name: app.images.main.version # value: latest - # Uncomment to enable registry authentication # - name: app.images.main.registry.auth - # value: true + # value: "true" + # - name: app.images.main.registry.domain + # value: quay-registry-quay-quay-enterprise.apps.{{ $.Values.global.clusterDomain }} # - name: app.images.main.registry.user # value: quay-user + # - name: app.images.main.registry.vaultPath + # value: "secret/data/hub/infra/quay/quay-users" # - name: app.images.main.registry.passwordVaultKey - # value: quay-user-password + # value: "quay-user-password" + # ============================================================ + # OPTION 2: BYO/External Registry + # - name: app.images.main.name + # value: quay.io/minzhang/qtodo + # - name: app.images.main.version + # value: latest + # - name: app.images.main.registry.auth + # value: "true" + # - name: app.images.main.registry.domain + # value: quay.io + # - name: app.images.main.registry.user + # value: minzhang + # - name: app.images.main.registry.vaultPath + # value: "secret/data/hub/infra/registry/registry-user" + # - name: app.images.main.registry.passwordVaultKey + # value: "registry-password" + # ============================================================ + # OPTION 3: Embedded OCP Registry + # - name: app.images.main.name + # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }}/ztvp/qtodo + # - name: app.images.main.version + # value: latest + # - name: app.images.main.registry.auth + # value: "true" + # - name: app.images.main.registry.domain + # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }} + # - name: app.images.main.registry.user + # value: admin + # - name: app.images.main.registry.vaultPath + # value: "secret/data/hub/infra/registry/registry-user" + # - name: app.images.main.registry.passwordVaultKey + # value: "registry-password" + # ============================================================ + # DEFAULT: No pipeline image (comment out all options above) + # qtodo uses the upstream image from the chart's default values.yaml + # ============================================================ # Secure Supply Chain - Uncomment to enable # supply-chain: # name: supply-chain @@ -474,26 +526,29 @@ clusterGroup: # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) # overrides: # # ============================================================ - # # OPTION 1: Built-in Quay Registry + # # OPTION 1: Built-in Quay Registry (comment out Option 2, uncomment below) # # Requires: quay-enterprise namespace, quay-operator, quay-registry app - # # Note: registry.domain is auto-constructed from hubClusterDomain # # ============================================================ # # - name: quay.enabled # # value: "true" - # # - name: externalRegistry.enabled - # # value: "false" + # # - name: registry.enabled + # # value: "true" + # # - name: registry.domain + # # value: quay-registry-quay-quay-enterprise.apps.{{ $.Values.global.clusterDomain }} # # - name: registry.tlsVerify # # value: "false" # # - name: registry.user # # value: quay-user + # # - name: registry.vaultPath + # # value: "secret/data/hub/infra/quay/quay-users" + # # - name: registry.passwordVaultKey + # # value: "quay-user-password" # # ============================================================ - # # OPTION 2: BYO/External Registry - # # Requires: registry credentials in ~/values-secret.yaml - # # Note: registry.domain is REQUIRED for external registry + # # OPTION 2: BYO/External Registry (comment out Option 3, uncomment below) + # # Store token in Vault (hub/infra/registry/registry-user, field registry-password) + # # and in ~/values-secrets.yaml. # # ============================================================ - # # - name: quay.enabled - # # value: "false" - # # - name: externalRegistry.enabled + # # - name: registry.enabled # # value: "true" # # - name: registry.domain # # value: quay.io @@ -501,16 +556,25 @@ clusterGroup: # # value: your-org # # - name: registry.user # # value: your-username + # # - name: registry.vaultPath + # # value: "secret/data/hub/infra/registry/registry-user" + # # - name: registry.passwordVaultKey + # # value: "registry-password" # # ============================================================ - # # OPTION 3: Embedded OCP Registry (comment out Option 1, 2; uncomment below) + # # OPTION 3: Embedded OCP Registry # # ============================================================ + # # - name: registry.enabled + # # value: "true" # # - name: registry.domain - # # value: default-route-openshift-image-registry.apps. + # # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }} # # - name: registry.org # # value: ztvp # # - name: registry.user # # value: admin - # # Embedded OCP registry only: create image namespace and grant pipeline push (transparent) + # # - name: registry.vaultPath + # # value: "secret/data/hub/infra/registry/registry-user" + # # - name: registry.passwordVaultKey + # # value: "registry-password" # # - name: registry.embeddedOCP.ensureImageNamespaceRBAC # # value: "true" # # ============================================================ From 2c00ce413afd4691682bd80b648068731df66d5d Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Thu, 2 Apr 2026 15:32:04 -0400 Subject: [PATCH 03/21] feat: add script to generate registry option test variants Add scripts/gen-byo-container-registry-variants.py that reads the base values-hub.yaml (all supply-chain components commented out) and produces up to 3 variants with the chosen registry option enabled: Option 1: Built-in Quay Registry Option 2: BYO / External Registry Option 3: Embedded OCP Image Registry Each variant also enables the common supply-chain stack (OpenShift Pipelines, ODF, NooBaa, RHTAS, RHTPA, and their namespaces, subscriptions, vault roles). Signed-off-by: Min Zhang --- .gitignore | 1 - .../gen-byo-container-registry-variants.py | 534 ++++++++++++++++++ 2 files changed, 534 insertions(+), 1 deletion(-) create mode 100755 scripts/gen-byo-container-registry-variants.py diff --git a/.gitignore b/.gitignore index 598cd782..4bae3c6a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ super-linter-output github_conf # Editor and IDE specific files -.cursor/ .cursorrules .cursor/ .vscode/ diff --git a/scripts/gen-byo-container-registry-variants.py b/scripts/gen-byo-container-registry-variants.py new file mode 100755 index 00000000..ee12e3fe --- /dev/null +++ b/scripts/gen-byo-container-registry-variants.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +"""Generate values-hub.yaml variants for BYO container registry options. + +Reads the default values-hub.yaml (all supply-chain components commented out) +and produces up to 3 variants with the chosen registry option enabled: + + Option 1: Built-in Quay Registry + Option 2: BYO / External Registry (e.g. quay.io, ghcr.io) + Option 3: Embedded OCP Image Registry + +Each variant also enables the common supply-chain stack (OpenShift Pipelines, +ODF, NooBaa, RHTAS, RHTPA, and their namespaces/subscriptions/vault roles). + +Usage: + # Generate all 3 variants under /tmp + python3 scripts/gen-byo-container-registry-options.py + + # Generate a single variant + python3 scripts/gen-byo-container-registry-options.py --option 2 + + # Custom base file and output directory + python3 scripts/gen-byo-container-registry-options.py \\ + --base my-values-hub.yaml --outdir /tmp/variants +""" + +import argparse +import os +import re +import sys + + +def uncomment_line(line): + """Remove one layer of comment: ' # foo' -> ' foo'.""" + return re.sub(r"^(\s*)# ?", r"\1", line, count=1) + + +def uncomment_lines_matching(lines, patterns): + """Uncomment individual lines matching any of the given patterns.""" + result = [] + for line in lines: + matched = False + for pat in patterns: + if re.search(pat, line): + result.append(uncomment_line(line)) + matched = True + break + if not matched: + result.append(line) + return result + + +def _uncomment_multiline_block(lines, trigger_re, body_re): + """Uncomment a contiguous block: first line matches *trigger_re*, + subsequent lines match *body_re*. Both the trigger and body + lines are uncommented.""" + new = [] + i = 0 + while i < len(lines): + if re.search(trigger_re, lines[i]): + while i < len(lines) and re.search(body_re, lines[i]): + new.append(uncomment_line(lines[i])) + i += 1 + continue + new.append(lines[i]) + i += 1 + return new + + +def _uncomment_until_sentinel(lines, trigger_re, sentinel_re, prev_re=None): + """Uncomment from trigger line until a sentinel (exclusive).""" + new = [] + i = 0 + while i < len(lines): + prev_ok = prev_re is None or (i > 0 and re.search(prev_re, lines[i - 1])) + if re.search(trigger_re, lines[i]) and prev_ok: + while i < len(lines): + if re.match(r"^\s*$", lines[i]): + break + if re.match(r"^\s{4}\w", lines[i]): + break + if re.search(sentinel_re, lines[i]): + break + new.append(uncomment_line(lines[i])) + i += 1 + continue + new.append(lines[i]) + i += 1 + return new + + +# --------------------------------------------------------------------------- +# Common supply-chain components (shared by all 3 options) +# --------------------------------------------------------------------------- +def apply_common_supply_chain(lines): + """Uncomment all components common to every supply-chain option.""" + + # Namespace: openshift-pipelines + lines = uncomment_lines_matching(lines, [r"^\s*# - openshift-pipelines\s*$"]) + + # Namespace: openshift-storage + lines = _uncomment_multiline_block( + lines, + r"# - openshift-storage:", + r"#\s+(- openshift-storage:|operatorGroup:|targetNamespace:" + r"|annotations:|labels:" + r"|openshift\.io/cluster-monitoring" + r"|argocd\.argoproj\.io/sync-wave.*26)", + ) + + # Namespace: trusted-artifact-signer + lines = _uncomment_multiline_block( + lines, + r"# - trusted-artifact-signer:", + r"#\s+(- trusted-artifact-signer:" + r"|annotations:|labels:" + r"|argocd\.argoproj\.io/sync-wave.*32.*Auto-created" + r"|openshift\.io/cluster-monitoring)", + ) + + # Namespace: rhtpa-operator + lines = _uncomment_multiline_block( + lines, + r"# - rhtpa-operator:", + r"#\s+(- rhtpa-operator:|operatorGroup:" + r"|targetNamespace: rhtpa" + r"|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*26.*Create before operator)", + ) + + # Namespace: trusted-profile-analyzer + lines = _uncomment_multiline_block( + lines, + r"# - trusted-profile-analyzer:", + r"#\s+(- trusted-profile-analyzer:" + r"|annotations:|labels:" + r"|argocd\.argoproj\.io/sync-wave.*32.*Create before RHTPA" + r"|openshift\.io/cluster-monitoring)", + ) + + # Subscription: openshift-pipelines + new = [] + i = 0 + while i < len(lines): + prev = lines[i - 1] if i > 0 else "" + if re.search(r"# openshift-pipelines:", lines[i]) and re.search( + r"Uncomment to enable OpenShift Pipelines", prev + ): + while i < len(lines) and re.search( + r"#\s*(openshift-pipelines:" + r"|name: openshift-pipelines" + r"|namespace: openshift-operators)", + lines[i], + ): + new.append(uncomment_line(lines[i])) + i += 1 + continue + new.append(lines[i]) + i += 1 + lines = new + + # Subscription: odf + lines = _uncomment_multiline_block( + lines, + r"# odf:", + r"#\s*(odf:|name: odf-operator|namespace: openshift-storage" + r"|channel: stable-4" + r"|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*27.*Install after OperatorGroup)", + ) + + # Subscription: rhtas-operator + lines = _uncomment_multiline_block( + lines, + r"# rhtas-operator:", + r"#\s*(rhtas-operator:|name: rhtas-operator" + r"|namespace: openshift-operators|channel: stable\s*$" + r"|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*29" + r"|catalogSource: redhat-operators)", + ) + + # Subscription: rhtpa-operator + new = [] + i = 0 + while i < len(lines): + prev2 = lines[i - 2] if i > 1 else "" + if re.search(r"# rhtpa-operator:", lines[i]) and re.search(r"Channel:", prev2): + while i < len(lines) and re.search( + r"#\s*(rhtpa-operator:|name: rhtpa-operator" + r"|namespace: rhtpa-operator" + r"|channel: stable-v1\.1" + r"|catalogSource: redhat-operators" + r"|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*27" + r".*Install after OperatorGroup.*before applications)", + lines[i], + ): + new.append(uncomment_line(lines[i])) + i += 1 + continue + new.append(lines[i]) + i += 1 + lines = new + + # Vault JWT roles: rhtpa and supply-chain + lines = uncomment_lines_matching( + lines, + [ + r"#\s+- name: rhtpa\s*$", + r"#\s+audience: rhtpa", + r"#\s+subject: spiffe://.*ns/trusted-profile-analyzer", + r"#\s+policies:\s*$", + r"#\s+- hub-infra-rhtpa-jwt-secret", + r"#\s+- name: supply-chain\s*$", + r"#\s+audience: supply-chain", + r"#\s+subject: spiffe://.*ns/pipeline", + r"#\s+- hub-supply-chain-jwt-secret", + ], + ) + + # Application: noobaa-mcg + lines = _uncomment_multiline_block( + lines, + r"# noobaa-mcg:", + r"#\s*(noobaa-mcg:|name: noobaa-mcg|namespace: openshift-storage" + r"|project: hub|path: charts/noobaa-mcg|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*36)", + ) + + # Application: trusted-artifact-signer + lines = _uncomment_until_sentinel( + lines, + r"# trusted-artifact-signer:", + r"# RHTPA \(Red Hat", + prev_re=r"Depends on:", + ) + + # Application: trusted-profile-analyzer + lines = _uncomment_until_sentinel( + lines, + r"# trusted-profile-analyzer:", + r"PLACEHOLDER_NEVER_MATCH", + prev_re=r"Depends on:", + ) + + return lines + + +# --------------------------------------------------------------------------- +# Per-option enablers +# --------------------------------------------------------------------------- +def enable_quay_namespace_and_sub(lines): + """Enable quay-enterprise namespace, quay-operator sub, quay-registry app.""" + + lines = _uncomment_multiline_block( + lines, + r"# - quay-enterprise:", + r"#\s+(- quay-enterprise:" + r"|annotations:|labels:" + r"|argocd\.argoproj\.io/sync-wave.*32.*Create before" + r"|openshift\.io/cluster-monitoring)", + ) + + lines = _uncomment_multiline_block( + lines, + r"# quay-operator:", + r"#\s*(quay-operator:|name: quay-operator" + r"|namespace: openshift-operators|channel: stable-3" + r"|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*28)", + ) + + lines = _uncomment_multiline_block( + lines, + r"# quay-registry:", + r"#\s*(quay-registry:|name: quay-registry" + r"|namespace: quay-enterprise|project: hub" + r"|chart: quay|chartVersion: 0\.1|annotations:" + r"|argocd\.argoproj\.io/sync-wave.*41)", + ) + + return lines + + +def enable_image_pull_trust(lines, hostname): + """Enable imagePullTrust in ztvp-certificates overrides.""" + result = [] + for line in lines: + if re.search(r"# - name: imagePullTrust\.enabled", line): + result.append(uncomment_line(line)) + elif ( + re.search(r'#\s+value: "true"\s*$', line) + and result + and "imagePullTrust.enabled" in result[-1] + ): + result.append(uncomment_line(line)) + elif re.search(r"# - name: imagePullTrust\.registries\[0\]", line): + result.append(uncomment_line(line)) + elif re.search(r"#\s+value: ", line): + result.append( + line.replace("# ", "").replace("", hostname) + ) + else: + result.append(line) + return result + + +def enable_qtodo_option(lines, option_num): + """Uncomment the specified qtodo registry option (1, 2, or 3).""" + option_header = f"OPTION {option_num}:" + result = [] + i = 0 + in_qtodo_options = False + + while i < len(lines): + line = lines[i] + + if re.search(r"# Secure Supply Chain: pull pipeline-built image", line): + in_qtodo_options = True + result.append(line) + i += 1 + continue + + if in_qtodo_options and re.search(r"# DEFAULT: No pipeline image", line): + in_qtodo_options = False + result.append(line) + i += 1 + continue + + if in_qtodo_options and re.search(f"# {re.escape(option_header)}", line): + result.append(line) + i += 1 + while i < len(lines): + line = lines[i] + if re.search(r"# ====", line): + result.append(line) + i += 1 + break + result.append(uncomment_line(line)) + i += 1 + continue + + result.append(line) + i += 1 + return result + + +def enable_supply_chain_option(lines, option_num): + """Enable the supply-chain app with the specified registry option. + + Two-pass approach: + Pass 1 - Remove the outer structural comment layer from every line + in the supply-chain block. Single-commented structural lines + become bare YAML; double-commented option lines become + single-commented. + Pass 2 - Within the now-single-commented option lines, uncomment the + selected option's overrides and the common RHTAS/RHTPA flags. + """ + option_header = f"OPTION {option_num}:" + option_any_re = re.compile(r"OPTION\s+\d+:") + + # --- Pass 1: strip outer comment from all supply-chain lines ---------- + pass1 = [] + in_block = False + block_start = -1 + block_end = -1 + + for idx, line in enumerate(lines): + if re.search(r"# Secure Supply Chain - Uncomment to enable", line): + in_block = True + block_start = idx + 1 + pass1.append(line) + continue + if in_block and re.match(r"^\s{4}#\s*$", line): + in_block = False + block_end = idx + pass1.append(line) + continue + if in_block: + pass1.append(uncomment_line(line)) + else: + pass1.append(line) + + if block_start < 0: + return pass1 + + # --- Pass 2: selectively uncomment option overrides ------------------- + final = [] + in_target_option = False + past_header = False + + for idx, line in enumerate(pass1): + if not (block_start <= idx < block_end): + final.append(line) + continue + + stripped = line.lstrip() + if not stripped.startswith("#"): + final.append(line) + continue + + # Detect OPTION headers + if re.search(option_header, line): + in_target_option = True + past_header = False + final.append(line) + continue + if option_any_re.search(line) and not re.search(option_header, line): + in_target_option = False + past_header = False + final.append(line) + continue + + is_separator = bool(re.search(r"# ====", line)) + + if is_separator: + if in_target_option and not past_header: + past_header = True + elif in_target_option and past_header: + in_target_option = False + final.append(line) + continue + + # Selected option overrides + if in_target_option and past_header: + final.append(uncomment_line(line)) + continue + + # Common overrides after all option blocks (RHTAS / RHTPA) + if not in_target_option and ( + re.search(r"# - name: rhtas\.enabled", line) + or re.search(r"# - name: rhtpa\.enabled", line) + ): + final.append(uncomment_line(line)) + continue + + if not in_target_option and re.search(r"#\s+value:", line): + if final and ("rhtas.enabled" in final[-1] or "rhtpa.enabled" in final[-1]): + final.append(uncomment_line(line)) + continue + + final.append(line) + + return final + + +# --------------------------------------------------------------------------- +# Top-level generator +# --------------------------------------------------------------------------- +OPTION_LABELS = { + 1: "built-in-quay-registry", + 2: "byo-external-registry", + 3: "embedded-ocp-registry", +} + + +def generate_variant(base_path, option_num, output_path): + with open(base_path) as fh: + lines = fh.readlines() + + lines = apply_common_supply_chain(lines) + + if option_num == 1: + lines = enable_quay_namespace_and_sub(lines) + lines = enable_image_pull_trust( + lines, + "quay-registry-quay-quay-enterprise.apps." + "{{ $.Values.global.clusterDomain }}", + ) + + if option_num == 3: + lines = enable_image_pull_trust( + lines, + "default-route-openshift-image-registry.apps." + "{{ $.Values.global.clusterDomain }}", + ) + + lines = enable_qtodo_option(lines, option_num) + lines = enable_supply_chain_option(lines, option_num) + + with open(output_path, "w") as fh: + fh.writelines(lines) + + label = OPTION_LABELS.get(option_num, f"option-{option_num}") + print(f" Option {option_num} ({label}) -> {output_path}") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--base", + default=None, + help="Base values-hub.yaml to read (default: /values-hub.yaml)", + ) + parser.add_argument( + "--outdir", + default=None, + help="Output directory (default: /tmp)", + ) + parser.add_argument( + "--option", + type=int, + choices=[1, 2, 3], + default=None, + help="Generate only this option (default: all 3)", + ) + args = parser.parse_args() + + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + base = args.base or os.path.join(repo_root, "values-hub.yaml") + outdir = args.outdir or "/tmp" + + if not os.path.isfile(base): + print(f"ERROR: base file not found: {base}", file=sys.stderr) + sys.exit(1) + + os.makedirs(outdir, exist_ok=True) + + options = [args.option] if args.option else [1, 2, 3] + print(f"Base: {base}") + print(f"Output directory: {outdir}") + for opt in options: + label = OPTION_LABELS[opt] + out = os.path.join(outdir, f"values-hub-{label}.yaml") + generate_variant(base, opt, out) + + print("Done.") + + +if __name__ == "__main__": + main() From af5d8547a6c5e0a69c0ffcdc6949c446e1271b34 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Mon, 23 Mar 2026 15:19:43 -0400 Subject: [PATCH 04/21] fix(acs-central): handle CA trust race on fresh cluster deployment On a fresh bare-metal cluster the proxy trustedCA injection may not have propagated to Central's mounted CA bundle by the time the create-auth-provider Job runs. Central caches its TLS trust pool at startup, so all Job retries fail with "x509: certificate signed by unknown authority" when Central tries to validate the Keycloak OIDC discovery endpoint. - Add retry loop in create-auth-provider Job that detects the specific TLS CA error, restarts Central to reload the CA bundle, then retries (up to 3 times) - Add apps/deployments get+patch to the service account Role so the Job can run "oc rollout restart" - Refactor script: extract wait_for_central() and escape_sed() helpers Signed-off-by: Min Zhang --- .../templates/jobs/create-auth-provider.yaml | 168 ++++++++++-------- .../templates/rbac/cluster-init-role.yaml | 9 +- 2 files changed, 103 insertions(+), 74 deletions(-) diff --git a/charts/acs-central/templates/jobs/create-auth-provider.yaml b/charts/acs-central/templates/jobs/create-auth-provider.yaml index 46af4787..ae5dc4fb 100644 --- a/charts/acs-central/templates/jobs/create-auth-provider.yaml +++ b/charts/acs-central/templates/jobs/create-auth-provider.yaml @@ -45,51 +45,50 @@ spec: - | #!/usr/bin/env bash - echo "🔄 Configuring Keycloak OIDC authentication provider..." + echo "Configuring Keycloak OIDC authentication provider..." + + wait_for_central() { + local max_retries=30 + local retry_count=0 + echo "Waiting for ACS Central API to be available..." + until curl -sk -u "admin:$PASSWORD" https://central/v1/ping > /dev/null 2>&1; do + retry_count=$((retry_count + 1)) + if [ $retry_count -ge $max_retries ]; then + echo "ERROR: Timeout waiting for ACS Central API" + return 1 + fi + echo " Retry $retry_count/$max_retries..." + sleep 10 + done + echo "ACS Central API is ready" + } - # Wait for ACS Central to be ready - echo "⏳ Waiting for ACS Central API to be available..." - max_retries=30 - retry_count=0 - until curl -sk -u "admin:$PASSWORD" https://central/v1/ping > /dev/null 2>&1; do - retry_count=$((retry_count + 1)) - if [ $retry_count -ge $max_retries ]; then - echo "❌ Timeout waiting for ACS Central API" - exit 1 - fi - echo " Retry $retry_count/$max_retries..." - sleep 10 - done - echo "✅ ACS Central API is ready" + wait_for_central || exit 1 - # Wait for Keycloak OIDC discovery endpoint to be available - echo "⏳ Waiting for Keycloak OIDC discovery endpoint..." + echo "Waiting for Keycloak OIDC discovery endpoint..." max_retries=30 retry_count=0 until curl -sk "$KEYCLOAK_ISSUER/.well-known/openid-configuration" > /dev/null 2>&1; do retry_count=$((retry_count + 1)) if [ $retry_count -ge $max_retries ]; then - echo "❌ Timeout waiting for Keycloak OIDC discovery endpoint" - echo " Tried: $KEYCLOAK_ISSUER/.well-known/openid-configuration" + echo "ERROR: Timeout waiting for Keycloak OIDC discovery endpoint" + echo " Tried: $KEYCLOAK_ISSUER/.well-known/openid-configuration" exit 1 fi echo " Retry $retry_count/$max_retries..." sleep 10 done - echo "✅ Keycloak OIDC discovery endpoint is ready" + echo "Keycloak OIDC discovery endpoint is ready" - # Check if auth provider already exists AUTH_PROVIDERS=$(curl -sk -u "admin:$PASSWORD" https://central/v1/authProviders) if echo "$AUTH_PROVIDERS" | grep -q "OIDC"; then - echo "✅ OIDC provider already configured" + echo "OIDC provider already configured -- nothing to do" exit 0 fi - # Get ACS Central hostname (without https://) ACS_CENTRAL_HOSTNAME="$(oc get route central -n stackrox -o jsonpath='{.spec.host}')" echo "ACS Central hostname: $ACS_CENTRAL_HOSTNAME" - # Create OIDC provider JSON cat > /tmp/oidc-config.json << 'OIDCEOF' { "name": "OIDC", @@ -111,69 +110,92 @@ spec: } OIDCEOF - # Remove leading spaces from JSON sed -i 's/^ //g' /tmp/oidc-config.json - # Replace placeholders using printf and sed to safely handle special characters - # Escape special sed characters in variables: & \ / and newlines - ACS_CENTRAL_HOSTNAME_ESC=$(printf '%s\n' "$ACS_CENTRAL_HOSTNAME" | sed 's:[&\\/]:\\&:g') - KEYCLOAK_ISSUER_ESC=$(printf '%s\n' "$KEYCLOAK_ISSUER" | sed 's:[&\\/]:\\&:g') - KEYCLOAK_CLIENT_ID_ESC=$(printf '%s\n' "$KEYCLOAK_CLIENT_ID" | sed 's:[&\\/]:\\&:g') - KEYCLOAK_CLIENT_SECRET_ESC=$(printf '%s\n' "$KEYCLOAK_CLIENT_SECRET" | sed 's:[&\\/]:\\&:g') - CLAIM_NAME_ESC=$(printf '%s\n' "$CLAIM_NAME" | sed 's:[&\\/]:\\&:g') - CLAIM_EMAIL_ESC=$(printf '%s\n' "$CLAIM_EMAIL" | sed 's:[&\\/]:\\&:g') - CLAIM_GROUPS_ESC=$(printf '%s\n' "$CLAIM_GROUPS" | sed 's:[&\\/]:\\&:g') - CLAIM_ROLES_ESC=$(printf '%s\n' "$CLAIM_ROLES" | sed 's:[&\\/]:\\&:g') - - sed -i "s|UI_ENDPOINT_PLACEHOLDER|$ACS_CENTRAL_HOSTNAME_ESC|g" /tmp/oidc-config.json - sed -i "s|ISSUER_PLACEHOLDER|$KEYCLOAK_ISSUER_ESC|g" /tmp/oidc-config.json - sed -i "s|CLIENT_ID_PLACEHOLDER|$KEYCLOAK_CLIENT_ID_ESC|g" /tmp/oidc-config.json - sed -i "s|CLIENT_SECRET_PLACEHOLDER|$KEYCLOAK_CLIENT_SECRET_ESC|g" /tmp/oidc-config.json - sed -i "s|CLAIM_NAME_PLACEHOLDER|$CLAIM_NAME_ESC|g" /tmp/oidc-config.json - sed -i "s|CLAIM_EMAIL_PLACEHOLDER|$CLAIM_EMAIL_ESC|g" /tmp/oidc-config.json - sed -i "s|CLAIM_GROUPS_PLACEHOLDER|$CLAIM_GROUPS_ESC|g" /tmp/oidc-config.json - sed -i "s|CLAIM_ROLES_PLACEHOLDER|$CLAIM_ROLES_ESC|g" /tmp/oidc-config.json - - # Debug: Show the JSON payload - echo "📝 OIDC Configuration JSON:" + escape_sed() { printf '%s\n' "$1" | sed 's:[&\\/]:\\&:g'; } + + sed -i "s|UI_ENDPOINT_PLACEHOLDER|$(escape_sed "$ACS_CENTRAL_HOSTNAME")|g" /tmp/oidc-config.json + sed -i "s|ISSUER_PLACEHOLDER|$(escape_sed "$KEYCLOAK_ISSUER")|g" /tmp/oidc-config.json + sed -i "s|CLIENT_ID_PLACEHOLDER|$(escape_sed "$KEYCLOAK_CLIENT_ID")|g" /tmp/oidc-config.json + sed -i "s|CLIENT_SECRET_PLACEHOLDER|$(escape_sed "$KEYCLOAK_CLIENT_SECRET")|g" /tmp/oidc-config.json + sed -i "s|CLAIM_NAME_PLACEHOLDER|$(escape_sed "$CLAIM_NAME")|g" /tmp/oidc-config.json + sed -i "s|CLAIM_EMAIL_PLACEHOLDER|$(escape_sed "$CLAIM_EMAIL")|g" /tmp/oidc-config.json + sed -i "s|CLAIM_GROUPS_PLACEHOLDER|$(escape_sed "$CLAIM_GROUPS")|g" /tmp/oidc-config.json + sed -i "s|CLAIM_ROLES_PLACEHOLDER|$(escape_sed "$CLAIM_ROLES")|g" /tmp/oidc-config.json + + echo "OIDC Configuration JSON:" cat /tmp/oidc-config.json echo "" - # Verify Keycloak discovery endpoint one more time before creating provider - echo "🔍 Verifying Keycloak discovery endpoint..." - curl -sk "$KEYCLOAK_ISSUER/.well-known/openid-configuration" | head -20 - echo "" + # --- Create OIDC auth provider with TLS retry logic --- + # On fresh clusters the proxy trustedCA injection may not have + # propagated to Central's mounted CA bundle yet. If Central + # rejects the Keycloak issuer with "certificate signed by unknown + # authority", restart the Central deployment so it reloads the + # (now-correct) CA bundle, then retry. + create_oidc_provider() { + local http_code + http_code=$(curl -X POST -u "admin:$PASSWORD" -k https://central/v1/authProviders \ + -H "Content-Type: application/json" \ + --data @/tmp/oidc-config.json \ + -w "%{http_code}" \ + -o /tmp/output.json) + + if [ "$http_code" = "200" ]; then + return 0 + fi - echo "📤 Creating auth provider in ACS..." - HTTP_CODE=$(curl -X POST -u "admin:$PASSWORD" -k https://central/v1/authProviders \ - -H "Content-Type: application/json" \ - --data @/tmp/oidc-config.json \ - -w "%{http_code}" \ - -o /tmp/output.json) + local body + body=$(cat /tmp/output.json) + echo "Auth provider creation returned HTTP $http_code: $body" - echo "📥 Response HTTP Code: $HTTP_CODE" - echo "📥 Response Body:" - cat /tmp/output.json - echo "" + if echo "$body" | grep -q "certificate signed by unknown authority"; then + return 2 + fi + return 1 + } + + MAX_TLS_RETRIES=3 + tls_attempt=0 + + echo "Creating auth provider in ACS..." + while true; do + create_oidc_provider + rc=$? - if [ "$HTTP_CODE" != "200" ]; then - echo "❌ Failed to create auth provider (HTTP $HTTP_CODE)" + if [ $rc -eq 0 ]; then + break + fi + + if [ $rc -eq 2 ]; then + tls_attempt=$((tls_attempt + 1)) + if [ $tls_attempt -gt $MAX_TLS_RETRIES ]; then + echo "ERROR: Central still does not trust the ingress CA after $MAX_TLS_RETRIES restart(s)" + exit 1 + fi + echo "Central does not trust the ingress CA yet (attempt $tls_attempt/$MAX_TLS_RETRIES)" + echo "Restarting Central to reload CA bundle..." + oc rollout restart deployment/central -n stackrox + oc rollout status deployment/central -n stackrox --timeout=180s + wait_for_central || exit 1 + continue + fi + + echo "ERROR: Failed to create auth provider" exit 1 - fi + done - # Extract provider ID AUTH_PROVIDER_ID=$(sed 's/,/\n/g' /tmp/output.json | grep -w id | awk -F\" '{ print $4 }') if [ -z "$AUTH_PROVIDER_ID" ]; then - echo "❌ Failed to extract auth provider ID" + echo "ERROR: Failed to extract auth provider ID" cat /tmp/output.json exit 1 fi - echo "✅ Auth provider created with ID: $AUTH_PROVIDER_ID" + echo "Auth provider created with ID: $AUTH_PROVIDER_ID" - # Create admin role mapping for acs-admin role - echo "📝 Creating role mapping: acs-admin → Admin" + echo "Creating role mapping: acs-admin -> Admin" JSON_PAYLOAD="{\"roleName\":\"Admin\",\"props\":{\"authProviderId\":\"$AUTH_PROVIDER_ID\",\"key\":\"roles\",\"value\":\"acs-admin\"}}" HTTP_CODE=$(curl -X POST -u "admin:$PASSWORD" -k https://central/v1/groups \ @@ -182,17 +204,17 @@ spec: -w "%{http_code}" \ -o /tmp/role-mapping.json) - echo "📥 Role mapping response (HTTP $HTTP_CODE):" + echo "Role mapping response (HTTP $HTTP_CODE):" cat /tmp/role-mapping.json echo "" if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ]; then - echo "✅ Admin role mapping created for acs-admin" + echo "Admin role mapping created for acs-admin" else - echo "⚠️ Warning: Failed to create admin role mapping (HTTP $HTTP_CODE, may already exist)" + echo "WARNING: Failed to create admin role mapping (HTTP $HTTP_CODE, may already exist)" fi - echo "🎉 Keycloak OIDC configuration complete" + echo "Keycloak OIDC configuration complete" name: create-auth-provider dnsPolicy: ClusterFirst restartPolicy: Never diff --git a/charts/acs-central/templates/rbac/cluster-init-role.yaml b/charts/acs-central/templates/rbac/cluster-init-role.yaml index 5355d0de..cb1b8697 100644 --- a/charts/acs-central/templates/rbac/cluster-init-role.yaml +++ b/charts/acs-central/templates/rbac/cluster-init-role.yaml @@ -33,4 +33,11 @@ rules: - routes verbs: - get - - list \ No newline at end of file + - list + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - patch \ No newline at end of file From 0d93e08f4bb0c85a9d1cd607894f4dae0d005600 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 3 Apr 2026 21:34:04 -0400 Subject: [PATCH 05/21] refactor: centralize registry config in global.registry and derive qtodo image in chart Move shared registry credentials (domain, org, user, vaultPath, passwordVaultKey) into a single global.registry block in values-hub.yaml. Supply-chain and qtodo charts fall back to global.registry.* when local registry values are empty. Derive the qtodo container image from global.registry.domain/org when registry is enabled, avoiding Validated Patterns --set overrides (Helm templates are not available there). - Add global.registry defaults to supply-chain and qtodo chart values - Update templates to use | default .Values.global.registry.* - Simplify values-hub.yaml application overrides for option-specific flags - Rewrite gen-byo-container-registry-variants.py for the structure - Update docs/supply-chain.md for global.registry architecture Signed-off-by: Min Zhang --- charts/qtodo/templates/_helpers.tpl | 13 +- charts/qtodo/templates/app-deployment.yaml | 2 +- .../qtodo/templates/app-serviceaccount.yaml | 2 +- .../templates/registry-external-secret.yaml | 15 +- charts/qtodo/values.yaml | 11 +- .../templates/pipeline-qtodo.yaml | 2 +- .../templates/quay/quay-user-job.yaml | 4 +- .../rbac/registry-image-namespace.yaml | 4 +- .../templates/secrets/qtodo-quay-pass.yaml | 4 +- .../secrets/qtodo-registry-auth.yaml | 15 +- charts/supply-chain/values.yaml | 15 +- docs/supply-chain.md | 126 ++++---- .../gen-byo-container-registry-variants.py | 288 +++++++++--------- values-hub.yaml | 146 +++------ 14 files changed, 306 insertions(+), 341 deletions(-) diff --git a/charts/qtodo/templates/_helpers.tpl b/charts/qtodo/templates/_helpers.tpl index 3666c0a0..1b90bdcf 100644 --- a/charts/qtodo/templates/_helpers.tpl +++ b/charts/qtodo/templates/_helpers.tpl @@ -1,10 +1,17 @@ {{/* -Create the image path for the passed in image field +Create the image path for the passed in image field. +For the main app image (isMain=true), when global.registry is enabled the +name is derived from global.registry.domain/org so no values-hub override +is needed (VP overrides don't support template expressions). */}} {{- define "qtodo.image" -}} +{{- $name := tpl .value.name .context -}} +{{- if and (.isMain) .context.Values.global.registry.enabled .context.Values.global.registry.domain .context.Values.global.registry.org -}} +{{- $name = printf "%s/%s/qtodo" (tpl .context.Values.global.registry.domain .context) .context.Values.global.registry.org -}} +{{- end -}} {{- if eq (substr 0 7 (tpl .value.version .context)) "sha256:" -}} -{{- printf "%s@%s" (tpl .value.name .context) (tpl .value.version .context) -}} +{{- printf "%s@%s" $name (tpl .value.version .context) -}} {{- else -}} -{{- printf "%s:%s" (tpl .value.name .context) (tpl .value.version .context) -}} +{{- printf "%s:%s" $name (tpl .value.version .context) -}} {{- end -}} {{- end -}} diff --git a/charts/qtodo/templates/app-deployment.yaml b/charts/qtodo/templates/app-deployment.yaml index 41dc0355..f93e2e26 100644 --- a/charts/qtodo/templates/app-deployment.yaml +++ b/charts/qtodo/templates/app-deployment.yaml @@ -193,7 +193,7 @@ spec: readOnly: true {{- end }} - name: qtodo - image: {{ template "qtodo.image" (dict "value" .Values.app.images.main "context" $) }} + image: {{ template "qtodo.image" (dict "value" .Values.app.images.main "isMain" true "context" $) }} imagePullPolicy: {{ .Values.app.images.main.pullPolicy }} ports: - containerPort: 8080 diff --git a/charts/qtodo/templates/app-serviceaccount.yaml b/charts/qtodo/templates/app-serviceaccount.yaml index 984df984..71341fec 100644 --- a/charts/qtodo/templates/app-serviceaccount.yaml +++ b/charts/qtodo/templates/app-serviceaccount.yaml @@ -5,7 +5,7 @@ metadata: app: qtodo name: qtodo namespace: qtodo -{{- if .Values.app.images.main.registry.auth }} +{{- if or .Values.app.images.main.registry.auth .Values.global.registry.enabled }} imagePullSecrets: - name: {{ .Values.app.images.main.registry.secretName }} {{- end }} \ No newline at end of file diff --git a/charts/qtodo/templates/registry-external-secret.yaml b/charts/qtodo/templates/registry-external-secret.yaml index 0fe1e1ad..5c7edc39 100644 --- a/charts/qtodo/templates/registry-external-secret.yaml +++ b/charts/qtodo/templates/registry-external-secret.yaml @@ -1,4 +1,9 @@ -{{- if .Values.app.images.main.registry.auth }} +{{- $regAuth := or .Values.app.images.main.registry.auth .Values.global.registry.enabled }} +{{- $regDomain := .Values.app.images.main.registry.domain | default .Values.global.registry.domain }} +{{- $regUser := .Values.app.images.main.registry.user | default .Values.global.registry.user }} +{{- $regVaultPath := .Values.app.images.main.registry.vaultPath | default .Values.global.registry.vaultPath }} +{{- $regPasswordKey := .Values.app.images.main.registry.passwordVaultKey | default .Values.global.registry.passwordVaultKey }} +{{- if $regAuth }} --- apiVersion: "external-secrets.io/v1beta1" kind: ExternalSecret @@ -18,14 +23,14 @@ spec: .dockerconfigjson: | { "auths": { - "{{ tpl (required "app.images.main.registry.domain is required when registry.auth is enabled" .Values.app.images.main.registry.domain) $ }}": { - "auth": "{{ `{{ printf "%s:%s" "` }}{{ .Values.app.images.main.registry.user }}{{ `" .password | b64enc }}` }}" + "{{ tpl (required "registry domain is required (set app.images.main.registry.domain or global.registry.domain)" $regDomain) $ }}": { + "auth": "{{ `{{ printf "%s:%s" "` }}{{ $regUser }}{{ `" .password | b64enc }}` }}" } } } data: - secretKey: password remoteRef: - key: {{ required "app.images.main.registry.vaultPath is required when registry.auth is enabled" .Values.app.images.main.registry.vaultPath }} - property: {{ required "app.images.main.registry.passwordVaultKey is required when registry.auth is enabled" .Values.app.images.main.registry.passwordVaultKey }} + key: {{ required "registry vaultPath is required (set app.images.main.registry.vaultPath or global.registry.vaultPath)" $regVaultPath }} + property: {{ required "registry passwordVaultKey is required (set app.images.main.registry.passwordVaultKey or global.registry.passwordVaultKey)" $regPasswordKey }} {{- end }} diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index 60732643..7f792058 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -4,6 +4,13 @@ global: secretStore: name: "vault-backend" kind: "ClusterSecretStore" + registry: + enabled: false + domain: "" + org: "" + user: "" + vaultPath: "" + passwordVaultKey: "" # QTodo application configuration app: @@ -16,9 +23,11 @@ app: pullPolicy: Always registry: # Set to true to create registry auth secret for image pulls + # (also enabled when global.registry.enabled=true) auth: false secretName: qtodo-registry-auth - user: registry-user + # Falls back to global.registry.user + user: "" # domain: registry.example.com # REQUIRED when auth is enabled # Vault path and key for registry password (set for your scenario) vaultPath: "" diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index add87310..0de9d1de 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -25,7 +25,7 @@ spec: - name: image-target type: string description: qtodo image push destination (e.g. quay.io/ztvp/qtodo:latest) - default: {{ tpl (required "registry.domain is required" .Values.registry.domain) $ }}/{{ .Values.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} + default: {{ tpl (required "registry.domain (or global.registry.domain) is required" (.Values.registry.domain | default .Values.global.registry.domain)) $ }}/{{ .Values.registry.org | default .Values.global.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} - name: image-tls-verify type: string description: Whether to verify TLS when pushing to the OCI registry diff --git a/charts/supply-chain/templates/quay/quay-user-job.yaml b/charts/supply-chain/templates/quay/quay-user-job.yaml index 4fa0f7c4..5d97bd8f 100644 --- a/charts/supply-chain/templates/quay/quay-user-job.yaml +++ b/charts/supply-chain/templates/quay/quay-user-job.yaml @@ -19,9 +19,9 @@ spec: command: ["python3", "/app/create_user.py"] env: - name: QUAY_HOST - value: {{ tpl (.Values.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain)) $ }} + value: {{ tpl (.Values.registry.domain | default .Values.global.registry.domain | default (printf "quay-registry-quay-quay-enterprise.%s" .Values.global.hubClusterDomain)) $ }} - name: QUAY_ADMIN_USER - value: {{ .Values.registry.user }} + value: {{ .Values.registry.user | default .Values.global.registry.user }} - name: QUAY_ADMIN_PASSWORD valueFrom: secretKeyRef: diff --git a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml index ebd5bf45..8ccd79a3 100644 --- a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml +++ b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml @@ -6,7 +6,7 @@ apiVersion: v1 kind: Namespace metadata: - name: {{ .Values.registry.org }} + name: {{ .Values.registry.org | default .Values.global.registry.org }} annotations: argocd.argoproj.io/sync-wave: "0" --- @@ -14,7 +14,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: pipeline-image-builder - namespace: {{ .Values.registry.org }} + namespace: {{ .Values.registry.org | default .Values.global.registry.org }} annotations: argocd.argoproj.io/sync-wave: "0" roleRef: diff --git a/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml b/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml index 2f42cedd..22a89ee3 100644 --- a/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-quay-pass.yaml @@ -26,6 +26,6 @@ spec: data: - secretKey: password remoteRef: - key: {{ .Values.registry.vaultPath }} - property: {{ .Values.registry.passwordVaultKey }} + key: {{ .Values.registry.vaultPath | default .Values.global.registry.vaultPath }} + property: {{ .Values.registry.passwordVaultKey | default .Values.global.registry.passwordVaultKey }} {{- end }} diff --git a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml index ea3e1e2b..ac293d4e 100644 --- a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml @@ -6,7 +6,12 @@ Registry-agnostic: works for built-in Quay, BYO (quay.io, ghcr.io), or embedded OCP. Set registry.domain, registry.vaultPath, and registry.passwordVaultKey for your scenario. */}} -{{- if .Values.registry.enabled }} +{{- $regEnabled := or .Values.registry.enabled .Values.global.registry.enabled }} +{{- $regDomain := .Values.registry.domain | default .Values.global.registry.domain }} +{{- $regUser := .Values.registry.user | default .Values.global.registry.user }} +{{- $regVaultPath := .Values.registry.vaultPath | default .Values.global.registry.vaultPath }} +{{- $regPasswordKey := .Values.registry.passwordVaultKey | default .Values.global.registry.passwordVaultKey }} +{{- if $regEnabled }} --- apiVersion: "external-secrets.io/v1beta1" kind: ExternalSecret @@ -26,14 +31,14 @@ spec: .dockerconfigjson: | { "auths": { - "{{ tpl (required "registry.domain is required when registry.enabled=true" .Values.registry.domain) $ }}": { - "auth": "{{ `{{ printf "%s:%s" "` }}{{ .Values.registry.user }}{{ `" .password | b64enc }}` }}" + "{{ tpl (required "registry.domain (or global.registry.domain) is required" $regDomain) $ }}": { + "auth": "{{ `{{ printf "%s:%s" "` }}{{ $regUser }}{{ `" .password | b64enc }}` }}" } } } data: - secretKey: password remoteRef: - key: {{ required "registry.vaultPath is required when registry.enabled=true" .Values.registry.vaultPath }} - property: {{ required "registry.passwordVaultKey is required when registry.enabled=true" .Values.registry.passwordVaultKey }} + key: {{ required "registry.vaultPath (or global.registry.vaultPath) is required" $regVaultPath }} + property: {{ required "registry.passwordVaultKey (or global.registry.passwordVaultKey) is required" $regPasswordKey }} {{- end }} diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index 0b6be272..f7f80caa 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -5,6 +5,13 @@ global: secretStore: name: "vault-backend" kind: "ClusterSecretStore" + registry: + enabled: false + domain: "" + org: "" + user: "" + vaultPath: "" + passwordVaultKey: "" rhtas: enabled: false @@ -63,14 +70,14 @@ registry: enabled: false # Registry hostname (REQUIRED when enabled) domain: "" - # Organization/namespace within the registry - org: "ztvp" + # Organization/namespace within the registry (falls back to global.registry.org) + org: "" # Repository name repo: "qtodo" # Whether to verify TLS when pushing to the registry tlsVerify: "true" - # Registry username - user: "registry-user" + # Registry username (falls back to global.registry.user) + user: "" # Vault path to the secret containing the registry password vaultPath: "" # Key within the Vault secret that holds the password diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 5e92aa73..83e64f08 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -53,49 +53,44 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use > **Note**: Never commit `~/values-secrets.yaml` (or your local values-secret file) to git. This file contains sensitive credentials and should remain local. -3. **Set registry configuration in values-hub.yaml**: For the supply-chain application, add these overrides: +3. **Set the global registry configuration in values-hub.yaml**: Uncomment the matching `global.registry` block at the top of `values-hub.yaml`. All registry credentials are defined once here; both the `supply-chain` and `qtodo` charts inherit them automatically. ```yaml - overrides: - - name: registry.enabled - value: "true" - - name: registry.domain - value: "your-registry.example.com" - - name: registry.user - value: "your-username" - - name: registry.org - value: "your-org" - - name: registry.vaultPath - value: "secret/data/hub/infra/registry/registry-user" - - name: registry.passwordVaultKey - value: "registry-password" + # Example: BYO/External Registry (Option 2) + global: + registry: + enabled: true + domain: quay.io + org: your-org + user: your-username + vaultPath: "secret/data/hub/infra/registry/registry-user" + passwordVaultKey: "registry-password" ``` -4. **Configure qtodo for custom registry** (if pulling from custom registry): + See the **Registry Options** section below for the full set of option blocks. - ```yaml - overrides: - - name: app.images.main.registry.auth - value: true - - name: app.images.main.registry.domain - value: "your-registry.example.com" - - name: app.images.main.registry.user - value: "your-username" - ``` +4. **Enable supply-chain-specific overrides** (if needed): The `supply-chain` application may need additional overrides depending on the registry type. These are set in the `supply-chain` overrides section of `values-hub.yaml`: + + * **Built-in Quay**: Enable `quay.enabled` (Quay user provisioner CronJob) and `registry.tlsVerify: "false"` (self-signed certs). + * **Embedded OCP**: Enable `registry.embeddedOCP.ensureImageNamespaceRBAC` (creates image namespace and push RBAC). + * **BYO/External**: No extra overrides needed. + + > **Note**: The qtodo chart automatically derives its image name from `global.registry.domain` and `global.registry.org` when `global.registry.enabled=true`. No per-app image override is needed. ### Required Configuration +These parameters are set in the `global.registry` block at the top of `values-hub.yaml`: + | Parameter | Description | Example | | --------- | ----------- | ------- | -| `registry.enabled` | Enable registry auth secret creation | `true` | -| `registry.domain` | Registry hostname (REQUIRED) | `quay.io`, `ghcr.io`, `registry.example.com` | -| `registry.org` | Organization/namespace | `my-org` | -| `registry.repo` | Repository name | `qtodo` | -| `registry.user` | Registry username | `my-robot-account` | -| `registry.vaultPath` | Vault path for registry password | `secret/data/hub/infra/registry/registry-user` | -| `registry.passwordVaultKey` | Key within the Vault secret | `registry-password` | +| `global.registry.enabled` | Enable registry auth secret creation | `true` | +| `global.registry.domain` | Registry hostname (REQUIRED) | `quay.io`, `ghcr.io`, `registry.example.com` | +| `global.registry.org` | Organization/namespace | `my-org` | +| `global.registry.user` | Registry username | `my-robot-account` | +| `global.registry.vaultPath` | Vault path for registry password | `secret/data/hub/infra/registry/registry-user` | +| `global.registry.passwordVaultKey` | Key within the Vault secret | `registry-password` | -> **Note**: All registry types (built-in Quay, BYO, embedded OCP) use the same parameters. Set `registry.domain`, `registry.vaultPath`, and `registry.passwordVaultKey` to the appropriate values for your scenario. See the Vault Paths table below for scenario-specific values. +> **Note**: All registry types (built-in Quay, BYO, embedded OCP) use the same `global.registry` parameters. Both the `supply-chain` and `qtodo` charts fall back to these values when their local registry values are empty. See the Vault Paths table below for scenario-specific values. ### Vault Paths @@ -107,7 +102,7 @@ Registry credentials are stored at different paths based on registry type: | BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | | Embedded OCP | `secret/data/hub/infra/registry/registry-user` | `registry-password` | -Set `registry.vaultPath` and `registry.passwordVaultKey` in your `values-hub.yaml` overrides to match your scenario. When `registry.enabled=false` (default), no registry auth secret is created (fresh install state). +Set `global.registry.vaultPath` and `global.registry.passwordVaultKey` in the `global.registry` block to match your scenario. When `global.registry.enabled` is false or unset (default), no registry auth secret is created (fresh install state). The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. @@ -115,36 +110,31 @@ The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths To use the in-cluster OpenShift image registry instead of an external registry: -1. **Enable `registry.embeddedOCP.ensureImageNamespaceRBAC`** in the supply-chain overrides. The chart will automatically: - * Create the image namespace matching `registry.org` (e.g. `ztvp`) - * Grant the pipeline ServiceAccount `system:image-builder` in that namespace - * Enable the default route on the image registry (via a one-time Job) - -2. **Set the registry domain** to `default-route-openshift-image-registry.apps.`. +1. **Uncomment the Option 3 `global.registry` block** in `values-hub.yaml`: -3. **Set the registry user** to `admin` (or a user with push permissions). + ```yaml + global: + registry: + enabled: true + domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} + org: ztvp + user: admin + vaultPath: "secret/data/hub/infra/registry/registry-user" + passwordVaultKey: "registry-password" + ``` -4. **Store the token in Vault**: Use `oc whoami -t` output as the `registry-password` value in `~/values-secrets.yaml`. +2. **Enable `registry.embeddedOCP.ensureImageNamespaceRBAC`** in the supply-chain overrides. The chart will automatically: + * Create the image namespace matching `global.registry.org` (e.g. `ztvp`) + * Grant the pipeline ServiceAccount `system:image-builder` in that namespace + * Enable the default route on the image registry (via a one-time Job) -Example supply-chain overrides: + ```yaml + # In the supply-chain application overrides: + - name: registry.embeddedOCP.ensureImageNamespaceRBAC + value: "true" + ``` -```yaml -overrides: - - name: registry.enabled - value: "true" - - name: registry.domain - value: default-route-openshift-image-registry.apps. - - name: registry.org - value: ztvp - - name: registry.user - value: admin - - name: registry.vaultPath - value: "secret/data/hub/infra/registry/registry-user" - - name: registry.passwordVaultKey - value: "registry-password" - - name: registry.embeddedOCP.ensureImageNamespaceRBAC - value: "true" -``` +3. **Store the token in Vault**: Use `oc whoami -t` output as the `registry-password` value in `~/values-secrets.yaml`. ### Node-Level Image Pull Trust @@ -236,10 +226,11 @@ You can also trigger a pipeline run using the Helm template included in the char ```shell helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ - --set registry.enabled=true \ - --set registry.domain=quay-registry-quay-quay-enterprise.apps.example.com \ - --set registry.vaultPath=secret/data/hub/infra/quay/quay-users \ - --set registry.passwordVaultKey=quay-user-password \ + --set global.registry.enabled=true \ + --set global.registry.domain=quay-registry-quay-quay-enterprise.apps.example.com \ + --set global.registry.org=ztvp \ + --set global.registry.vaultPath=secret/data/hub/infra/quay/quay-users \ + --set global.registry.passwordVaultKey=quay-user-password \ --set global.namespace=layered-zero-trust-hub \ --show-only templates/pipelinerun-qtodo.yaml | oc create -f - ``` @@ -249,10 +240,11 @@ helm template supply-chain charts/supply-chain \ ```shell helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ - --set registry.enabled=true \ - --set registry.domain=quay.io \ - --set registry.vaultPath=secret/data/hub/infra/registry/registry-user \ - --set registry.passwordVaultKey=registry-password \ + --set global.registry.enabled=true \ + --set global.registry.domain=quay.io \ + --set global.registry.org=your-org \ + --set global.registry.vaultPath=secret/data/hub/infra/registry/registry-user \ + --set global.registry.passwordVaultKey=registry-password \ --set global.namespace=layered-zero-trust-hub \ --show-only templates/pipelinerun-qtodo.yaml | oc create -f - ``` diff --git a/scripts/gen-byo-container-registry-variants.py b/scripts/gen-byo-container-registry-variants.py index ee12e3fe..44cca9b8 100755 --- a/scripts/gen-byo-container-registry-variants.py +++ b/scripts/gen-byo-container-registry-variants.py @@ -11,15 +11,19 @@ Each variant also enables the common supply-chain stack (OpenShift Pipelines, ODF, NooBaa, RHTAS, RHTPA, and their namespaces/subscriptions/vault roles). +Registry credentials are centralized in a single `global.registry` block at +the top of values-hub.yaml. Both the supply-chain and qtodo charts fall back +to `global.registry.*` when their local registry values are empty. + Usage: # Generate all 3 variants under /tmp - python3 scripts/gen-byo-container-registry-options.py + python3 scripts/gen-byo-container-registry-variants.py # Generate a single variant - python3 scripts/gen-byo-container-registry-options.py --option 2 + python3 scripts/gen-byo-container-registry-variants.py --option 2 # Custom base file and output directory - python3 scripts/gen-byo-container-registry-options.py \\ + python3 scripts/gen-byo-container-registry-variants.py \\ --base my-values-hub.yaml --outdir /tmp/variants """ @@ -88,6 +92,140 @@ def _uncomment_until_sentinel(lines, trigger_re, sentinel_re, prev_re=None): return new +# --------------------------------------------------------------------------- +# Global registry block +# --------------------------------------------------------------------------- +def enable_global_registry(lines, option_num): + """Uncomment the global.registry block for the selected option. + + The base file contains three commented blocks: + # OPTION 1: Built-in Quay Registry + # global: + # registry: + # ... + # OPTION 2: ... + # global: + # registry: + # ... + # OPTION 3: ... + # global: + # registry: + # ... + + This function uncomments only the block matching option_num. + """ + target_header = f"# OPTION {option_num}:" + result = [] + i = 0 + while i < len(lines): + line = lines[i] + + if re.search(re.escape(target_header), line): + result.append(line) + i += 1 + while i < len(lines): + if re.match(r"^# OPTION \d+:", lines[i]): + break + if re.match(r"^$", lines[i]): + break + if re.match(r"^[^#]", lines[i]): + break + result.append(uncomment_line(lines[i])) + i += 1 + continue + + result.append(line) + i += 1 + return result + + +# --------------------------------------------------------------------------- +# Supply-chain app enabler +# --------------------------------------------------------------------------- +def enable_supply_chain_app(lines, option_num): + """Enable the supply-chain app and its option-specific overrides. + + Pass 1: strip one comment layer from all supply-chain block lines. + Pass 2: selectively uncomment option-specific and common overrides. + """ + # --- Pass 1: strip outer comment from all supply-chain lines ---------- + pass1 = [] + in_block = False + block_start = -1 + block_end = -1 + + for idx, line in enumerate(lines): + if re.search(r"# Secure Supply Chain - Uncomment to enable", line): + in_block = True + block_start = idx + 1 + pass1.append(line) + continue + if in_block and re.match(r"^\s{4}#\s*$", line): + in_block = False + block_end = idx + pass1.append(line) + continue + if in_block: + pass1.append(uncomment_line(line)) + else: + pass1.append(line) + + if block_start < 0: + return pass1 + + # --- Pass 2: selectively uncomment option overrides ------------------- + final = [] + for idx, line in enumerate(pass1): + if not (block_start <= idx < block_end): + final.append(line) + continue + + stripped = line.lstrip() + if not stripped.startswith("#"): + final.append(line) + continue + + # Always uncomment RHTAS and RHTPA flags + if re.search(r"# - name: rhtas\.enabled", line) or re.search( + r"# - name: rhtpa\.enabled", line + ): + final.append(uncomment_line(line)) + continue + if re.search(r"#\s+value:", line) and final: + prev = final[-1] + if "rhtas.enabled" in prev or "rhtpa.enabled" in prev: + final.append(uncomment_line(line)) + continue + + # Option 1 (Built-in Quay): uncomment quay.enabled and tlsVerify + if option_num == 1: + if re.search(r"# - name: quay\.enabled", line) or re.search( + r"# - name: registry\.tlsVerify", line + ): + final.append(uncomment_line(line)) + continue + if re.search(r"#\s+value:", line) and final: + prev = final[-1] + if "quay.enabled" in prev or "registry.tlsVerify" in prev: + final.append(uncomment_line(line)) + continue + + # Option 3 (Embedded OCP): uncomment ensureImageNamespaceRBAC + if option_num == 3: + if re.search(r"# - name: registry\.embeddedOCP", line): + final.append(uncomment_line(line)) + continue + if re.search(r"#\s+value:", line) and final: + prev = final[-1] + if "embeddedOCP" in prev: + final.append(uncomment_line(line)) + continue + + final.append(line) + + return final + + # --------------------------------------------------------------------------- # Common supply-chain components (shared by all 3 options) # --------------------------------------------------------------------------- @@ -305,145 +443,6 @@ def enable_image_pull_trust(lines, hostname): return result -def enable_qtodo_option(lines, option_num): - """Uncomment the specified qtodo registry option (1, 2, or 3).""" - option_header = f"OPTION {option_num}:" - result = [] - i = 0 - in_qtodo_options = False - - while i < len(lines): - line = lines[i] - - if re.search(r"# Secure Supply Chain: pull pipeline-built image", line): - in_qtodo_options = True - result.append(line) - i += 1 - continue - - if in_qtodo_options and re.search(r"# DEFAULT: No pipeline image", line): - in_qtodo_options = False - result.append(line) - i += 1 - continue - - if in_qtodo_options and re.search(f"# {re.escape(option_header)}", line): - result.append(line) - i += 1 - while i < len(lines): - line = lines[i] - if re.search(r"# ====", line): - result.append(line) - i += 1 - break - result.append(uncomment_line(line)) - i += 1 - continue - - result.append(line) - i += 1 - return result - - -def enable_supply_chain_option(lines, option_num): - """Enable the supply-chain app with the specified registry option. - - Two-pass approach: - Pass 1 - Remove the outer structural comment layer from every line - in the supply-chain block. Single-commented structural lines - become bare YAML; double-commented option lines become - single-commented. - Pass 2 - Within the now-single-commented option lines, uncomment the - selected option's overrides and the common RHTAS/RHTPA flags. - """ - option_header = f"OPTION {option_num}:" - option_any_re = re.compile(r"OPTION\s+\d+:") - - # --- Pass 1: strip outer comment from all supply-chain lines ---------- - pass1 = [] - in_block = False - block_start = -1 - block_end = -1 - - for idx, line in enumerate(lines): - if re.search(r"# Secure Supply Chain - Uncomment to enable", line): - in_block = True - block_start = idx + 1 - pass1.append(line) - continue - if in_block and re.match(r"^\s{4}#\s*$", line): - in_block = False - block_end = idx - pass1.append(line) - continue - if in_block: - pass1.append(uncomment_line(line)) - else: - pass1.append(line) - - if block_start < 0: - return pass1 - - # --- Pass 2: selectively uncomment option overrides ------------------- - final = [] - in_target_option = False - past_header = False - - for idx, line in enumerate(pass1): - if not (block_start <= idx < block_end): - final.append(line) - continue - - stripped = line.lstrip() - if not stripped.startswith("#"): - final.append(line) - continue - - # Detect OPTION headers - if re.search(option_header, line): - in_target_option = True - past_header = False - final.append(line) - continue - if option_any_re.search(line) and not re.search(option_header, line): - in_target_option = False - past_header = False - final.append(line) - continue - - is_separator = bool(re.search(r"# ====", line)) - - if is_separator: - if in_target_option and not past_header: - past_header = True - elif in_target_option and past_header: - in_target_option = False - final.append(line) - continue - - # Selected option overrides - if in_target_option and past_header: - final.append(uncomment_line(line)) - continue - - # Common overrides after all option blocks (RHTAS / RHTPA) - if not in_target_option and ( - re.search(r"# - name: rhtas\.enabled", line) - or re.search(r"# - name: rhtpa\.enabled", line) - ): - final.append(uncomment_line(line)) - continue - - if not in_target_option and re.search(r"#\s+value:", line): - if final and ("rhtas.enabled" in final[-1] or "rhtpa.enabled" in final[-1]): - final.append(uncomment_line(line)) - continue - - final.append(line) - - return final - - # --------------------------------------------------------------------------- # Top-level generator # --------------------------------------------------------------------------- @@ -459,6 +458,8 @@ def generate_variant(base_path, option_num, output_path): lines = fh.readlines() lines = apply_common_supply_chain(lines) + lines = enable_global_registry(lines, option_num) + lines = enable_supply_chain_app(lines, option_num) if option_num == 1: lines = enable_quay_namespace_and_sub(lines) @@ -475,9 +476,6 @@ def generate_variant(base_path, option_num, output_path): "{{ $.Values.global.clusterDomain }}", ) - lines = enable_qtodo_option(lines, option_num) - lines = enable_supply_chain_option(lines, option_num) - with open(output_path, "w") as fh: fh.writelines(lines) diff --git a/values-hub.yaml b/values-hub.yaml index 40b4eaeb..f6df2b60 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -11,6 +11,42 @@ spire: route.openshift.io/termination: reencrypt route.openshift.io/destination-ca-certificate-secret: spire-bundle +# =========================================================================== +# Global registry configuration (shared by qtodo and supply-chain) +# Define once here; both charts fall back to global.registry.* when their +# local registry values are empty. Per-app overrides still take precedence. +# +# The gen-byo-container-registry-variants.py script uncomments the matching +# OPTION block for each variant. +# =========================================================================== +# OPTION 1: Built-in Quay Registry +# global: +# registry: +# enabled: true +# domain: quay-registry-quay-quay-enterprise.apps.{{ .Values.global.clusterDomain }} +# org: ztvp +# user: quay-user +# vaultPath: "secret/data/hub/infra/quay/quay-users" +# passwordVaultKey: "quay-user-password" +# OPTION 2: BYO/External Registry (quay.io, ghcr.io, etc.) +# global: +# registry: +# enabled: true +# domain: quay.io +# org: your-org +# user: your-username +# vaultPath: "secret/data/hub/infra/registry/registry-user" +# passwordVaultKey: "registry-password" +# OPTION 3: Embedded OCP Registry +# global: +# registry: +# enabled: true +# domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} +# org: ztvp +# user: admin +# vaultPath: "secret/data/hub/infra/registry/registry-user" +# passwordVaultKey: "registry-password" + clusterGroup: name: hub isHubCluster: true @@ -503,61 +539,9 @@ clusterGroup: value: qtodo - name: app.vault.secretPath value: secret/data/apps/qtodo/qtodo-db - # ============================================================ - # Secure Supply Chain: pull pipeline-built image from registry - # Uncomment the option matching your supply-chain registry choice. - # ============================================================ - # OPTION 1: Built-in Quay Registry - # - name: app.images.main.name - # value: quay-registry-quay-quay-enterprise.apps.{{ $.Values.global.clusterDomain }}/ztvp/qtodo - # - name: app.images.main.version - # value: latest - # - name: app.images.main.registry.auth - # value: "true" - # - name: app.images.main.registry.domain - # value: quay-registry-quay-quay-enterprise.apps.{{ $.Values.global.clusterDomain }} - # - name: app.images.main.registry.user - # value: quay-user - # - name: app.images.main.registry.vaultPath - # value: "secret/data/hub/infra/quay/quay-users" - # - name: app.images.main.registry.passwordVaultKey - # value: "quay-user-password" - # ============================================================ - # OPTION 2: BYO/External Registry - # - name: app.images.main.name - # value: quay.io/minzhang/qtodo - # - name: app.images.main.version - # value: latest - # - name: app.images.main.registry.auth - # value: "true" - # - name: app.images.main.registry.domain - # value: quay.io - # - name: app.images.main.registry.user - # value: minzhang - # - name: app.images.main.registry.vaultPath - # value: "secret/data/hub/infra/registry/registry-user" - # - name: app.images.main.registry.passwordVaultKey - # value: "registry-password" - # ============================================================ - # OPTION 3: Embedded OCP Registry - # - name: app.images.main.name - # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }}/ztvp/qtodo - # - name: app.images.main.version - # value: latest - # - name: app.images.main.registry.auth - # value: "true" - # - name: app.images.main.registry.domain - # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }} - # - name: app.images.main.registry.user - # value: admin - # - name: app.images.main.registry.vaultPath - # value: "secret/data/hub/infra/registry/registry-user" - # - name: app.images.main.registry.passwordVaultKey - # value: "registry-password" - # ============================================================ - # DEFAULT: No pipeline image (comment out all options above) - # qtodo uses the upstream image from the chart's default values.yaml - # ============================================================ + # Secure Supply Chain: when global.registry.enabled=true the chart + # automatically derives the image name from global.registry.domain/org. + # No override needed here. # Secure Supply Chain - Uncomment to enable # supply-chain: # name: supply-chain @@ -570,59 +554,17 @@ clusterGroup: # jqPathExpressions: # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) # overrides: - # # ============================================================ - # # OPTION 1: Built-in Quay Registry (comment out Option 2, uncomment below) - # # Requires: quay-enterprise namespace, quay-operator, quay-registry app - # # ============================================================ + # # Registry credentials are inherited from global.registry. + # # Only set app-specific overrides below. + # # Built-in Quay: uncomment to enable the Quay user provisioner CronJob # # - name: quay.enabled # # value: "true" - # # - name: registry.enabled - # # value: "true" - # # - name: registry.domain - # # value: quay-registry-quay-quay-enterprise.apps.{{ $.Values.global.clusterDomain }} + # # Built-in Quay: self-signed certs require TLS verify off # # - name: registry.tlsVerify # # value: "false" - # # - name: registry.user - # # value: quay-user - # # - name: registry.vaultPath - # # value: "secret/data/hub/infra/quay/quay-users" - # # - name: registry.passwordVaultKey - # # value: "quay-user-password" - # # ============================================================ - # # OPTION 2: BYO/External Registry (comment out Option 3, uncomment below) - # # Store token in Vault (hub/infra/registry/registry-user, field registry-password) - # # and in ~/values-secrets.yaml. - # # ============================================================ - # # - name: registry.enabled - # # value: "true" - # # - name: registry.domain - # # value: quay.io - # # - name: registry.org - # # value: your-org - # # - name: registry.user - # # value: your-username - # # - name: registry.vaultPath - # # value: "secret/data/hub/infra/registry/registry-user" - # # - name: registry.passwordVaultKey - # # value: "registry-password" - # # ============================================================ - # # OPTION 3: Embedded OCP Registry - # # ============================================================ - # # - name: registry.enabled - # # value: "true" - # # - name: registry.domain - # # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }} - # # - name: registry.org - # # value: ztvp - # # - name: registry.user - # # value: admin - # # - name: registry.vaultPath - # # value: "secret/data/hub/infra/registry/registry-user" - # # - name: registry.passwordVaultKey - # # value: "registry-password" + # # Embedded OCP: uncomment to create image namespace and grant push RBAC # # - name: registry.embeddedOCP.ensureImageNamespaceRBAC # # value: "true" - # # ============================================================ # # Enable RHTAS signing # # - name: rhtas.enabled # # value: "true" From da24280106542cabc63f43bae4ad1d6ec548bcc2 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Tue, 7 Apr 2026 10:14:24 -0400 Subject: [PATCH 06/21] fix: align vault-utils JWT placeholders and ACS init RBAC Sync common/scripts/vault-utils.sh (Helm global.pattern substitution for Ansible) and charts/acs-central cluster-init Role (list/watch deployments) from embedded-ocp-registry for parity across registry option branches. Signed-off-by: Min Zhang --- .../acs-central/templates/rbac/cluster-init-role.yaml | 2 ++ common/scripts/vault-utils.sh | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/acs-central/templates/rbac/cluster-init-role.yaml b/charts/acs-central/templates/rbac/cluster-init-role.yaml index cb1b8697..c260f86c 100644 --- a/charts/acs-central/templates/rbac/cluster-init-role.yaml +++ b/charts/acs-central/templates/rbac/cluster-init-role.yaml @@ -40,4 +40,6 @@ rules: - deployments verbs: - get + - list + - watch - patch \ No newline at end of file diff --git a/common/scripts/vault-utils.sh b/common/scripts/vault-utils.sh index 42ef0c31..1b50bb34 100755 --- a/common/scripts/vault-utils.sh +++ b/common/scripts/vault-utils.sh @@ -33,8 +33,15 @@ trap "rm -f ${EXTRA_VARS_FILE}" EXIT if [ "$(yq ".clusterGroup.applications.vault.jwt.enabled // \"false\"" "${MAIN_CLUSTERGROUP_FILE}")" == "true" ]; then OCP_DOMAIN="$(oc get dns cluster -o jsonpath='{.spec.baseDomain}')" - OIDC_DISCOVERY_URL="$(yq ".clusterGroup.applications.vault.jwt.oidcDiscoveryUrl" "${MAIN_CLUSTERGROUP_FILE}" | sed "s/{{ \$.Values.global.clusterDomain }}/${OCP_DOMAIN}/g")" - JWT_ROLES="$(yq -o json ".clusterGroup.applications.vault.jwt.roles" "${MAIN_CLUSTERGROUP_FILE}" | sed "s/{{ \$.Values.global.clusterDomain }}/${OCP_DOMAIN}/g")" + GLOBAL_PATTERN="$(yq -r '.global.pattern // ""' "${PATTERNPATH}/values-global.yaml")" + GLOBAL_PATTERN="${GLOBAL_PATTERN:-${PATTERN_NAME}}" + # Replace Helm-style placeholders so Ansible/Jinja2 never sees "{{ $.Values... }}" (invalid Jinja2). + _subst_vault_yaml() { + sed -e "s/{{ \$.Values.global.clusterDomain }}/${OCP_DOMAIN}/g" \ + -e "s/{{ \$.Values.global.pattern }}/${GLOBAL_PATTERN}/g" + } + OIDC_DISCOVERY_URL="$(yq ".clusterGroup.applications.vault.jwt.oidcDiscoveryUrl" "${MAIN_CLUSTERGROUP_FILE}" | _subst_vault_yaml)" + JWT_ROLES="$(yq -o json ".clusterGroup.applications.vault.jwt.roles" "${MAIN_CLUSTERGROUP_FILE}" | _subst_vault_yaml)" # Extract JWT policies (policies ending in -jwt-secret) JWT_POLICIES="$(yq -o json ".clusterGroup.applications.vault.policies" "${MAIN_CLUSTERGROUP_FILE}" | jq '[.[] | select(.name | test("-jwt-secret$"))]')" From 78229ad0bca2c3d351b797d14bd7e23c2a8cc06c Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Tue, 7 Apr 2026 10:57:15 -0400 Subject: [PATCH 07/21] feat(supply-chain): embedded OCP registry token refresh and Vault JWT Add CronJob and sync-hook seed Job for pipeline SA token refresh to Vault (SPIFFE JWT). Extend supply-chain values, docs/supply-chain.md, and values-hub for embedded OCP (merged with fresh-install baseline). Signed-off-by: Min Zhang --- .../files/refresh_registry_token.sh | 87 +++++++ .../rbac/registry-token-refresher.yaml | 224 ++++++++++++++++++ charts/supply-chain/values.yaml | 5 + docs/supply-chain.md | 36 ++- values-hub.yaml | 71 +++--- 5 files changed, 377 insertions(+), 46 deletions(-) create mode 100755 charts/supply-chain/files/refresh_registry_token.sh create mode 100644 charts/supply-chain/templates/rbac/registry-token-refresher.yaml diff --git a/charts/supply-chain/files/refresh_registry_token.sh b/charts/supply-chain/files/refresh_registry_token.sh new file mode 100755 index 00000000..9a8c2cf5 --- /dev/null +++ b/charts/supply-chain/files/refresh_registry_token.sh @@ -0,0 +1,87 @@ +#!/bin/sh +set -eu + +VAULT_URL="${VAULT_URL:?VAULT_URL is required}" +VAULT_ROLE="${VAULT_ROLE:-supply-chain}" +VAULT_SECRET_PATH="${VAULT_SECRET_PATH:?VAULT_SECRET_PATH is required}" +VAULT_SECRET_KEY="${VAULT_SECRET_KEY:?VAULT_SECRET_KEY is required}" +SA_NAME="${SA_NAME:-pipeline}" +SA_NAMESPACE="${SA_NAMESPACE:?SA_NAMESPACE is required}" +TOKEN_DURATION="${TOKEN_DURATION:-172800}" +JWT_TOKEN_FILE="${JWT_TOKEN_FILE:-/svids/jwt.token}" +CA_CERT="${CA_CERT:-/run/secrets/kubernetes.io/serviceaccount/service-ca.crt}" + +APISERVER="https://kubernetes.default.svc" +SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +CACERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + +log() { echo "[$(date '+%H:%M:%S')] $*"; } + +vault_curl() { + if [ -f "${CA_CERT}" ]; then + curl -sS --cacert "${CA_CERT}" "$@" + else + curl -sSk "$@" + fi +} + +log "Starting OCP registry token refresh" + +# 1. Read SPIFFE JWT for Vault authentication +if [ ! -f "${JWT_TOKEN_FILE}" ]; then + log "ERROR: JWT token file not found at ${JWT_TOKEN_FILE}" + exit 1 +fi +JWT="$(cat "${JWT_TOKEN_FILE}")" +log "Read SPIFFE JWT from ${JWT_TOKEN_FILE}" + +# 2. Authenticate to Vault using SPIFFE JWT +log "Authenticating to Vault at ${VAULT_URL} with role ${VAULT_ROLE}..." +AUTH_RESP=$(vault_curl -X POST "${VAULT_URL}/v1/auth/jwt/login" \ + -H "Content-Type: application/json" \ + -d "{\"role\":\"${VAULT_ROLE}\",\"jwt\":\"${JWT}\"}") + +VAULT_TOKEN=$(echo "${AUTH_RESP}" | python3 -c "import sys,json; print(json.load(sys.stdin)['auth']['client_token'])" 2>/dev/null) || { + log "ERROR: Vault authentication failed" + log "${AUTH_RESP}" + exit 1 +} +log "Vault authentication successful" + +# 3. Create a fresh SA token via the Kubernetes TokenRequest API +log "Creating ${SA_NAME} SA token (duration: ${TOKEN_DURATION}s)..." +TOKEN_RESP=$(curl -sS --cacert "${CACERT}" \ + -X POST "${APISERVER}/api/v1/namespaces/${SA_NAMESPACE}/serviceaccounts/${SA_NAME}/token" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"apiVersion\":\"authentication.k8s.io/v1\",\"kind\":\"TokenRequest\",\"spec\":{\"expirationSeconds\":${TOKEN_DURATION}}}") + +NEW_TOKEN=$(echo "${TOKEN_RESP}" | python3 -c "import sys,json; print(json.load(sys.stdin)['status']['token'])" 2>/dev/null) || { + log "ERROR: TokenRequest API failed" + log "${TOKEN_RESP}" + exit 1 +} +log "SA token created successfully" + +# 4. Write the new token to Vault +log "Writing token to Vault at ${VAULT_SECRET_PATH}..." +WRITE_RESP=$(vault_curl -X POST "${VAULT_URL}/v1/${VAULT_SECRET_PATH}" \ + -H "X-Vault-Token: ${VAULT_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"data\":{\"${VAULT_SECRET_KEY}\":\"${NEW_TOKEN}\"}}") + +# Check for errors in the response +echo "${WRITE_RESP}" | python3 -c " +import sys, json +resp = json.load(sys.stdin) +if 'errors' in resp and resp['errors']: + print('ERROR: ' + str(resp['errors']), file=sys.stderr) + sys.exit(1) +" || { + log "ERROR: Failed to write token to Vault" + log "${WRITE_RESP}" + exit 1 +} + +log "Token successfully written to Vault at ${VAULT_SECRET_PATH}" +log "Registry token refresh complete" diff --git a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml new file mode 100644 index 00000000..4fe14082 --- /dev/null +++ b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml @@ -0,0 +1,224 @@ +{{- if and (index .Values.registry "embeddedOCP") (index .Values.registry.embeddedOCP "tokenRefresher") (index .Values.registry.embeddedOCP.tokenRefresher "enabled") }} +{{- $regVaultPath := .Values.registry.vaultPath | default .Values.global.registry.vaultPath }} +{{- $regPasswordKey := .Values.registry.passwordVaultKey | default .Values.global.registry.passwordVaultKey }} +# ============================================================ +# OCP Registry Token Refresher +# Periodically creates a fresh pipeline SA token and writes it +# to Vault so the ExternalSecret-based dockerconfigjson stays +# valid for the embedded OCP image registry. +# ============================================================ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-token-refresher-script + namespace: {{ .Values.global.namespace }} +data: + refresh_registry_token.sh: | +{{- .Files.Get "files/refresh_registry_token.sh" | nindent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-token-refresher-spiffe-config + namespace: {{ .Values.global.namespace }} +data: + helper.conf: |- + agent_address = "{{ .Values.spire.endpointSocketPath }}" + cmd = "" + cmd_args = "" + cert_dir = "/svids" + renew_signal = "" + svid_file_name = "svid.pem" + svid_key_file_name = "svid_key.pem" + svid_bundle_file_name = "svid_bundle.pem" + jwt_svids = [{jwt_audience="supply-chain", jwt_svid_file_name="jwt.token"}] + jwt_bundle_file_name = "jwt_bundle.json" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: registry-token-refresher + namespace: {{ .Values.global.namespace }} +rules: +- apiGroups: [""] + resources: ["serviceaccounts/token"] + resourceNames: ["pipeline"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: registry-token-refresher + namespace: {{ .Values.global.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: registry-token-refresher +subjects: +- kind: ServiceAccount + name: pipeline + namespace: {{ .Values.global.namespace }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: registry-token-refresher + namespace: {{ .Values.global.namespace }} +spec: + schedule: {{ .Values.registry.embeddedOCP.tokenRefresher.schedule | quote }} + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: pipeline + restartPolicy: Never + initContainers: + - name: fetch-spiffe-jwt + image: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9:v0.10.0 + imagePullPolicy: IfNotPresent + args: + - '-config' + - /etc/helper.conf + - '-daemon-mode=false' + volumeMounts: + - name: spiffe-helper-config + readOnly: true + mountPath: /etc/helper.conf + subPath: helper.conf + - name: spiffe-workload-api + readOnly: true + mountPath: /spiffe-workload-api + - name: svids + mountPath: /svids + containers: + - name: refresh-token + image: {{ .Values.registry.embeddedOCP.tokenRefresher.image }} + command: ["sh", "/app/refresh_registry_token.sh"] + env: + - name: VAULT_URL + value: https://vault.vault.svc.cluster.local:8200 + - name: VAULT_ROLE + value: supply-chain + - name: VAULT_SECRET_PATH + value: {{ required "registry vaultPath is required" $regVaultPath }} + - name: VAULT_SECRET_KEY + value: {{ required "registry passwordVaultKey is required" $regPasswordKey }} + - name: SA_NAME + value: pipeline + - name: SA_NAMESPACE + value: {{ .Values.global.namespace }} + - name: TOKEN_DURATION + value: {{ .Values.registry.embeddedOCP.tokenRefresher.tokenDuration | quote }} + - name: JWT_TOKEN_FILE + value: /svids/jwt.token + - name: CA_CERT + value: /run/secrets/kubernetes.io/serviceaccount/service-ca.crt + volumeMounts: + - name: svids + readOnly: true + mountPath: /svids + - name: script-volume + readOnly: true + mountPath: /app + volumes: + - name: spiffe-workload-api + csi: + driver: csi.spiffe.io + readOnly: true + - name: spiffe-helper-config + configMap: + name: registry-token-refresher-spiffe-config + defaultMode: 420 + - name: svids + emptyDir: {} + - name: script-volume + configMap: + name: registry-token-refresher-script + defaultMode: 0755 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: registry-token-refresher-seed + namespace: {{ .Values.global.namespace }} + annotations: + # Run after wave 0 (ConfigMaps, RBAC, ztvp ns) and wave 1 (enable-registry-default-route hook). + argocd.argoproj.io/sync-wave: "10" + argocd.argoproj.io/hook: Sync + # HookSucceeded: remove completed Job so Argo does not sit on a stale object; BeforeHookCreation: clean retry/sync. + argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation +spec: + backoffLimit: 3 + # Avoid an indefinite sync if Vault/SPIFFE never become ready (Argo waits for hook completion). + activeDeadlineSeconds: 900 + template: + spec: + serviceAccountName: pipeline + restartPolicy: Never + initContainers: + - name: fetch-spiffe-jwt + image: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9:v0.10.0 + imagePullPolicy: IfNotPresent + args: + - '-config' + - /etc/helper.conf + - '-daemon-mode=false' + volumeMounts: + - name: spiffe-helper-config + readOnly: true + mountPath: /etc/helper.conf + subPath: helper.conf + - name: spiffe-workload-api + readOnly: true + mountPath: /spiffe-workload-api + - name: svids + mountPath: /svids + containers: + - name: refresh-token + image: {{ .Values.registry.embeddedOCP.tokenRefresher.image }} + command: ["sh", "/app/refresh_registry_token.sh"] + env: + - name: VAULT_URL + value: https://vault.vault.svc.cluster.local:8200 + - name: VAULT_ROLE + value: supply-chain + - name: VAULT_SECRET_PATH + value: {{ required "registry vaultPath is required" $regVaultPath }} + - name: VAULT_SECRET_KEY + value: {{ required "registry passwordVaultKey is required" $regPasswordKey }} + - name: SA_NAME + value: pipeline + - name: SA_NAMESPACE + value: {{ .Values.global.namespace }} + - name: TOKEN_DURATION + value: {{ .Values.registry.embeddedOCP.tokenRefresher.tokenDuration | quote }} + - name: JWT_TOKEN_FILE + value: /svids/jwt.token + - name: CA_CERT + value: /run/secrets/kubernetes.io/serviceaccount/service-ca.crt + volumeMounts: + - name: svids + readOnly: true + mountPath: /svids + - name: script-volume + readOnly: true + mountPath: /app + volumes: + - name: spiffe-workload-api + csi: + driver: csi.spiffe.io + readOnly: true + - name: spiffe-helper-config + configMap: + name: registry-token-refresher-spiffe-config + defaultMode: 420 + - name: svids + emptyDir: {} + - name: script-volume + configMap: + name: registry-token-refresher-script + defaultMode: 0755 +{{- end }} diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index f7f80caa..ed6ce816 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -89,6 +89,11 @@ registry: # using the in-cluster OpenShift image registry; leave false for other registries. embeddedOCP: ensureImageNamespaceRBAC: false + tokenRefresher: + enabled: false + image: registry.access.redhat.com/ubi9/ubi:9.7-1764794285 + schedule: "0 */6 * * *" + tokenDuration: "172800" # pipeline run configuration pipelinerun: diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 83e64f08..6e63bc8c 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -48,7 +48,7 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use ``` Replace `REPLACE_WITH_REGISTRY_TOKEN` with: - * **Embedded OCP registry:** output of `oc whoami -t` (after `oc login`). + * **Embedded OCP registry:** Not required when the automatic token refresher is enabled (see [Embedded OCP Registry](#embedded-ocp-registry)). If the token refresher is disabled, use the output of `oc whoami -t` (after `oc login`). * **External registry (BYO):** your registry token or password (e.g. quay.io, ghcr.io). > **Note**: Never commit `~/values-secrets.yaml` (or your local values-secret file) to git. This file contains sensitive credentials and should remain local. @@ -67,12 +67,12 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use passwordVaultKey: "registry-password" ``` - See the **Registry Options** section below for the full set of option blocks. + See the **Registry Options** section at the top of `values-hub.yaml` for the full set of option blocks (built-in Quay, BYO, embedded OCP). 4. **Enable supply-chain-specific overrides** (if needed): The `supply-chain` application may need additional overrides depending on the registry type. These are set in the `supply-chain` overrides section of `values-hub.yaml`: * **Built-in Quay**: Enable `quay.enabled` (Quay user provisioner CronJob) and `registry.tlsVerify: "false"` (self-signed certs). - * **Embedded OCP**: Enable `registry.embeddedOCP.ensureImageNamespaceRBAC` (creates image namespace and push RBAC). + * **Embedded OCP**: Enable `registry.embeddedOCP.ensureImageNamespaceRBAC` (creates image namespace and push RBAC) and optionally `registry.embeddedOCP.tokenRefresher.enabled` (see [Embedded OCP Registry](#embedded-ocp-registry)). * **BYO/External**: No extra overrides needed. > **Note**: The qtodo chart automatically derives its image name from `global.registry.domain` and `global.registry.org` when `global.registry.enabled=true`. No per-app image override is needed. @@ -104,13 +104,13 @@ Registry credentials are stored at different paths based on registry type: Set `global.registry.vaultPath` and `global.registry.passwordVaultKey` in the `global.registry` block to match your scenario. When `global.registry.enabled` is false or unset (default), no registry auth secret is created (fresh install state). -The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. +The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. For the embedded OCP registry, the policy also grants `create` and `update` capabilities on the registry path so the automatic token refresher can write fresh tokens back to Vault. ### Embedded OCP Registry To use the in-cluster OpenShift image registry instead of an external registry: -1. **Uncomment the Option 3 `global.registry` block** in `values-hub.yaml`: +1. **Uncomment the Option 3 `global.registry` block** in `values-hub.yaml` so `global.registry` points at the embedded registry (domain, org, vault paths). Use `user: _token` when using automatic token refresh (bearer tokens; the username is not significant to the registry). ```yaml global: @@ -118,7 +118,7 @@ To use the in-cluster OpenShift image registry instead of an external registry: enabled: true domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} org: ztvp - user: admin + user: _token vaultPath: "secret/data/hub/infra/registry/registry-user" passwordVaultKey: "registry-password" ``` @@ -128,13 +128,25 @@ To use the in-cluster OpenShift image registry instead of an external registry: * Grant the pipeline ServiceAccount `system:image-builder` in that namespace * Enable the default route on the image registry (via a one-time Job) - ```yaml - # In the supply-chain application overrides: - - name: registry.embeddedOCP.ensureImageNamespaceRBAC - value: "true" - ``` +3. **Confirm the registry domain** is `default-route-openshift-image-registry.apps.` (set in `global.registry.domain` above). + +4. **Enable automatic token refresh** (recommended): Set `registry.embeddedOCP.tokenRefresher.enabled` to `true`. This deploys: + * A **CronJob** (`registry-token-refresher`) that runs every 6 hours. It uses a SPIFFE JWT to authenticate to Vault, creates a fresh `pipeline` ServiceAccount token via the Kubernetes TokenRequest API, and writes it to Vault. + * A one-shot **Sync hook Job** (`registry-token-refresher-seed`) that seeds the initial token on first deploy so the pipeline is ready immediately. + + When the token refresher is enabled, you do **not** need to manually store a token in `~/values-secrets.yaml` for the embedded OCP registry. The refresher handles credential lifecycle automatically. -3. **Store the token in Vault**: Use `oc whoami -t` output as the `registry-password` value in `~/values-secrets.yaml`. + If you prefer manual token management instead, disable the token refresher and store the output of `oc whoami -t` as the `registry-password` value in `~/values-secrets.yaml`. + +Example `supply-chain` application overrides for embedded OCP (registry host, org, and Vault paths are normally taken from the `global.registry` block): + +```yaml +overrides: + - name: registry.embeddedOCP.ensureImageNamespaceRBAC + value: "true" + - name: registry.embeddedOCP.tokenRefresher.enabled + value: "true" +``` ### Node-Level Image Pull Trust diff --git a/values-hub.yaml b/values-hub.yaml index f6df2b60..000e846b 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -43,7 +43,7 @@ spire: # enabled: true # domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} # org: ztvp -# user: admin +# user: _token # vaultPath: "secret/data/hub/infra/registry/registry-user" # passwordVaultKey: "registry-password" @@ -263,11 +263,11 @@ clusterGroup: # Node-level image pull trust for kubelet # Required when pulling images from registries behind the cluster ingress # (e.g. built-in Quay, embedded OCP registry). Patches image.config.openshift.io/cluster. - # Uncomment and set the registry hostname when enabling a registry option. + # Uncomment and set the registry hostname when enabling Option 1 (Quay) or Option 3 (embedded OCP). # - name: imagePullTrust.enabled # value: "true" # - name: imagePullTrust.registries[0] - # value: + # value: default-route-openshift-image-registry.apps.{{ $.Values.global.clusterDomain }} # Note: additionalCertificates (complex nested array) temporarily disabled # Need to find proper way to pass complex structures in Validated Patterns @@ -354,7 +354,7 @@ clusterGroup: capabilities = ["read"] } path "secret/data/hub/infra/registry/*" { - capabilities = ["read"] + capabilities = ["read", "create", "update"] } path "secret/data/hub/infra/rhtpa/rhtpa-oidc-cli" { capabilities = ["read"] @@ -376,10 +376,10 @@ clusterGroup: # subject: spiffe://apps.{{ $.Values.global.clusterDomain }}/ns/trusted-profile-analyzer/sa/rhtpa # policies: # - hub-infra-rhtpa-jwt-secret - # Supply chain vault role (for Tekton pipelines) + # Supply chain vault role (for Tekton pipelines; enable with supply-chain app / Option 3 or BYO registry) # - name: supply-chain # audience: supply-chain - # subject: spiffe://apps.{{ $.Values.global.clusterDomain }}/ns/pipeline/sa/pipeline + # subject: spiffe://apps.{{ $.Values.global.clusterDomain }}/ns/{{ $.Values.global.pattern }}-hub/sa/pipeline # policies: # - hub-supply-chain-jwt-secret # Shared Object Storage Backend @@ -542,35 +542,38 @@ clusterGroup: # Secure Supply Chain: when global.registry.enabled=true the chart # automatically derives the image name from global.registry.domain/org. # No override needed here. - # Secure Supply Chain - Uncomment to enable + # Secure Supply Chain - Uncomment to enable (required for Option 1, 2, or 3 registry flows in docs) # supply-chain: - # name: supply-chain - # project: hub - # path: charts/supply-chain - # annotations: - # argocd.argoproj.io/sync-wave: "48" - # ignoreDifferences: - # - kind: ServiceAccount - # jqPathExpressions: - # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) - # overrides: - # # Registry credentials are inherited from global.registry. - # # Only set app-specific overrides below. - # # Built-in Quay: uncomment to enable the Quay user provisioner CronJob - # # - name: quay.enabled - # # value: "true" - # # Built-in Quay: self-signed certs require TLS verify off - # # - name: registry.tlsVerify - # # value: "false" - # # Embedded OCP: uncomment to create image namespace and grant push RBAC - # # - name: registry.embeddedOCP.ensureImageNamespaceRBAC - # # value: "true" - # # Enable RHTAS signing - # # - name: rhtas.enabled - # # value: "true" - # # Enable RHTPA SBOM upload - # # - name: rhtpa.enabled - # # value: "true" + # name: supply-chain + # project: hub + # path: charts/supply-chain + # annotations: + # argocd.argoproj.io/sync-wave: "48" + # ignoreDifferences: + # - kind: ServiceAccount + # jqPathExpressions: + # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) + # overrides: + # # Registry credentials are inherited from global.registry. + # # Only set app-specific overrides below. + # # Built-in Quay: uncomment to enable the Quay user provisioner CronJob + # # - name: quay.enabled + # # value: "true" + # # Built-in Quay: self-signed certs require TLS verify off + # # - name: registry.tlsVerify + # # value: "false" + # # Embedded OCP (Option 3): create image namespace and grant push RBAC + # # - name: registry.embeddedOCP.ensureImageNamespaceRBAC + # # value: "true" + # # Embedded OCP (Option 3): periodically refresh pipeline SA token in Vault + # # - name: registry.embeddedOCP.tokenRefresher.enabled + # # value: "true" + # # Enable RHTAS signing + # # - name: rhtas.enabled + # # value: "true" + # # Enable RHTPA SBOM upload + # # - name: rhtpa.enabled + # # value: "true" # # ACS Central Services acs-central: From 3444e02ac0b766406b57d54a219b282ccef3a40e Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Tue, 7 Apr 2026 13:17:40 -0400 Subject: [PATCH 08/21] fix(gen-variants): fix subject regex and imagePullTrust matching Two bugs in gen-byo-container-registry-variants.py: 1. The supply-chain JWT role subject regex used ns/pipeline which no longer matches after the namespace was changed to {{ $.Values.global.pattern }}-hub. Changed to sa/pipeline which matches both old and new formats. 2. enable_image_pull_trust looked for the stale placeholder. Changed to match by position (value line after the imagePullTrust.registries line) so it works regardless of the default value in the base file. Signed-off-by: Min Zhang --- scripts/gen-byo-container-registry-variants.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/gen-byo-container-registry-variants.py b/scripts/gen-byo-container-registry-variants.py index 44cca9b8..e81b54cb 100755 --- a/scripts/gen-byo-container-registry-variants.py +++ b/scripts/gen-byo-container-registry-variants.py @@ -351,7 +351,7 @@ def apply_common_supply_chain(lines): r"#\s+- hub-infra-rhtpa-jwt-secret", r"#\s+- name: supply-chain\s*$", r"#\s+audience: supply-chain", - r"#\s+subject: spiffe://.*ns/pipeline", + r"#\s+subject: spiffe://.*sa/pipeline", r"#\s+- hub-supply-chain-jwt-secret", ], ) @@ -434,10 +434,12 @@ def enable_image_pull_trust(lines, hostname): result.append(uncomment_line(line)) elif re.search(r"# - name: imagePullTrust\.registries\[0\]", line): result.append(uncomment_line(line)) - elif re.search(r"#\s+value: ", line): - result.append( - line.replace("# ", "").replace("", hostname) - ) + elif ( + re.search(r"#\s+value:", line) + and result + and "imagePullTrust.registries" in result[-1] + ): + result.append(re.sub(r"#\s+value:.*", f" value: {hostname}", line)) else: result.append(line) return result From 808914ae5971f2dedbfac9ceb428a62a1eadb42b Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Tue, 7 Apr 2026 17:57:15 -0400 Subject: [PATCH 09/21] docs: clarify registry secret is only needed for BYO registry - Comment out registry-user in values-secret.yaml.template (was active by default but unnecessary for minimal deployments) - Update supply-chain.md step 2 to clarify that only Option 2 (BYO registry) needs the manual registry-user secret - Option 1 (Quay) uses auto-generated quay-users secret - Option 3 (embedded OCP) token refresher writes to Vault automatically Signed-off-by: Min Zhang --- docs/supply-chain.md | 182 ++++++++++++++++++------------------ values-secret.yaml.template | 30 +++--- 2 files changed, 104 insertions(+), 108 deletions(-) diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 6e63bc8c..ced9fdaf 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -28,67 +28,64 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use 1. **Disable built-in Quay registry** (optional - if not using Quay): Comment out the Quay-related applications in `values-hub.yaml`: `quay-enterprise` namespace, `quay-operator` subscription, and `quay-registry` application. -2. **Configure registry credentials in Vault**: Per VP rule, add your registry credentials to `~/values-secrets.yaml` (or `~/values-secret.yaml` / `~/values-secret-layered-zero-trust.yaml` per VP lookup order): +2. **Configure registry credentials in Vault** (**BYO registry only**): Per VP rule, add your registry credentials to `~/values-secrets.yaml` (or `~/values-secret.yaml` / `~/values-secret-layered-zero-trust.yaml` per VP lookup order): - ```bash - # Copy template to local file if not already done - cp values-secret.yaml.template ~/values-secrets.yaml - ``` + ```bash + # Copy template to local file if not already done + cp values-secret.yaml.template ~/values-secrets.yaml + ``` - Add the registry-user secret (same format for **BYO external registry** and **embedded OCP registry**): + Uncomment the `registry-user` secret and replace the placeholder with your registry token or password: - ```yaml - - name: registry-user - vaultPrefixes: - - hub/infra/registry - fields: - - name: registry-password - value: "REPLACE_WITH_REGISTRY_TOKEN" - onMissingValue: error - ``` + ```yaml + - name: registry-user + vaultPrefixes: + - hub/infra/registry + fields: + - name: registry-password + value: "REPLACE_WITH_REGISTRY_TOKEN" + onMissingValue: error + ``` - Replace `REPLACE_WITH_REGISTRY_TOKEN` with: - * **Embedded OCP registry:** Not required when the automatic token refresher is enabled (see [Embedded OCP Registry](#embedded-ocp-registry)). If the token refresher is disabled, use the output of `oc whoami -t` (after `oc login`). - * **External registry (BYO):** your registry token or password (e.g. quay.io, ghcr.io). - - > **Note**: Never commit `~/values-secrets.yaml` (or your local values-secret file) to git. This file contains sensitive credentials and should remain local. + > **Note**: This secret is only required for **BYO/external registries** (Option 2). **Built-in Quay** (Option 1) uses the auto-generated `quay-users` secret. **Embedded OCP registry** (Option 3) does not need a manual secret when the automatic token refresher is enabled (see [Embedded OCP Registry](#embedded-ocp-registry)) -- the refresher creates and rotates the Vault credential automatically. + > + > **Note**: Never commit `~/values-secrets.yaml` (or your local values-secret file) to git. This file contains sensitive credentials and should remain local. 3. **Set the global registry configuration in values-hub.yaml**: Uncomment the matching `global.registry` block at the top of `values-hub.yaml`. All registry credentials are defined once here; both the `supply-chain` and `qtodo` charts inherit them automatically. - ```yaml - # Example: BYO/External Registry (Option 2) - global: - registry: - enabled: true - domain: quay.io - org: your-org - user: your-username - vaultPath: "secret/data/hub/infra/registry/registry-user" - passwordVaultKey: "registry-password" - ``` + ```yaml + # Example: BYO/External Registry (Option 2) + global: + registry: + enabled: true + domain: quay.io + org: your-org + user: your-username + vaultPath: "secret/data/hub/infra/registry/registry-user" + passwordVaultKey: "registry-password" + ``` - See the **Registry Options** section at the top of `values-hub.yaml` for the full set of option blocks (built-in Quay, BYO, embedded OCP). + See the **Registry Options** section at the top of `values-hub.yaml` for the full set of option blocks (built-in Quay, BYO, embedded OCP). 4. **Enable supply-chain-specific overrides** (if needed): The `supply-chain` application may need additional overrides depending on the registry type. These are set in the `supply-chain` overrides section of `values-hub.yaml`: + * **Built-in Quay**: Enable `quay.enabled` (Quay user provisioner CronJob) and `registry.tlsVerify: "false"` (self-signed certs). + * **Embedded OCP**: Enable `registry.embeddedOCP.ensureImageNamespaceRBAC` (creates image namespace and push RBAC) and optionally `registry.embeddedOCP.tokenRefresher.enabled` (see [Embedded OCP Registry](#embedded-ocp-registry)). + * **BYO/External**: No extra overrides needed. - * **Built-in Quay**: Enable `quay.enabled` (Quay user provisioner CronJob) and `registry.tlsVerify: "false"` (self-signed certs). - * **Embedded OCP**: Enable `registry.embeddedOCP.ensureImageNamespaceRBAC` (creates image namespace and push RBAC) and optionally `registry.embeddedOCP.tokenRefresher.enabled` (see [Embedded OCP Registry](#embedded-ocp-registry)). - * **BYO/External**: No extra overrides needed. - - > **Note**: The qtodo chart automatically derives its image name from `global.registry.domain` and `global.registry.org` when `global.registry.enabled=true`. No per-app image override is needed. + > **Note**: The qtodo chart automatically derives its image name from `global.registry.domain` and `global.registry.org` when `global.registry.enabled=true`. No per-app image override is needed. ### Required Configuration These parameters are set in the `global.registry` block at the top of `values-hub.yaml`: -| Parameter | Description | Example | -| --------- | ----------- | ------- | -| `global.registry.enabled` | Enable registry auth secret creation | `true` | -| `global.registry.domain` | Registry hostname (REQUIRED) | `quay.io`, `ghcr.io`, `registry.example.com` | -| `global.registry.org` | Organization/namespace | `my-org` | -| `global.registry.user` | Registry username | `my-robot-account` | -| `global.registry.vaultPath` | Vault path for registry password | `secret/data/hub/infra/registry/registry-user` | -| `global.registry.passwordVaultKey` | Key within the Vault secret | `registry-password` | +| Parameter | Description | Example | +| ---------------------------------- | ------------------------------------ | ---------------------------------------------- | +| `global.registry.enabled` | Enable registry auth secret creation | `true` | +| `global.registry.domain` | Registry hostname (REQUIRED) | `quay.io`, `ghcr.io`, `registry.example.com` | +| `global.registry.org` | Organization/namespace | `my-org` | +| `global.registry.user` | Registry username | `my-robot-account` | +| `global.registry.vaultPath` | Vault path for registry password | `secret/data/hub/infra/registry/registry-user` | +| `global.registry.passwordVaultKey` | Key within the Vault secret | `registry-password` | > **Note**: All registry types (built-in Quay, BYO, embedded OCP) use the same `global.registry` parameters. Both the `supply-chain` and `qtodo` charts fall back to these values when their local registry values are empty. See the Vault Paths table below for scenario-specific values. @@ -96,11 +93,11 @@ These parameters are set in the `global.registry` block at the top of `values-hu Registry credentials are stored at different paths based on registry type: -| Registry Type | Vault Path | Password Key | -| ------------------ | ---------------------------------------------- | -------------------- | -| Built-in Quay | `secret/data/hub/infra/quay/quay-users` | `quay-user-password` | -| BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | -| Embedded OCP | `secret/data/hub/infra/registry/registry-user` | `registry-password` | +| Registry Type | Vault Path | Password Key | +| ------------- | ---------------------------------------------- | -------------------- | +| Built-in Quay | `secret/data/hub/infra/quay/quay-users` | `quay-user-password` | +| BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | +| Embedded OCP | `secret/data/hub/infra/registry/registry-user` | `registry-password` | Set `global.registry.vaultPath` and `global.registry.passwordVaultKey` in the `global.registry` block to match your scenario. When `global.registry.enabled` is false or unset (default), no registry auth secret is created (fresh install state). @@ -112,16 +109,16 @@ To use the in-cluster OpenShift image registry instead of an external registry: 1. **Uncomment the Option 3 `global.registry` block** in `values-hub.yaml` so `global.registry` points at the embedded registry (domain, org, vault paths). Use `user: _token` when using automatic token refresh (bearer tokens; the username is not significant to the registry). - ```yaml - global: - registry: - enabled: true - domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} - org: ztvp - user: _token - vaultPath: "secret/data/hub/infra/registry/registry-user" - passwordVaultKey: "registry-password" - ``` + ```yaml + global: + registry: + enabled: true + domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} + org: ztvp + user: _token + vaultPath: "secret/data/hub/infra/registry/registry-user" + passwordVaultKey: "registry-password" + ``` 2. **Enable `registry.embeddedOCP.ensureImageNamespaceRBAC`** in the supply-chain overrides. The chart will automatically: * Create the image namespace matching `global.registry.org` (e.g. `ztvp`) @@ -164,10 +161,10 @@ The `ztvp-certificates` application handles this by patching `image.config.opens Set `` to match your registry option: -| Option | Registry Hostname | -| ------ | ----------------- | -| Option 1: Built-in Quay | `quay-registry-quay-quay-enterprise.apps.` | -| Option 3: Embedded OCP | `default-route-openshift-image-registry.apps.` | +| Option | Registry Hostname | +| ----------------------- | ------------------------------------------------------------- | +| Option 1: Built-in Quay | `quay-registry-quay-quay-enterprise.apps.` | +| Option 3: Embedded OCP | `default-route-openshift-image-registry.apps.` | > **Note**: Option 2 (BYO/External Registry) does not require `imagePullTrust` because external registries like quay.io and ghcr.io use publicly trusted certificates. @@ -186,12 +183,11 @@ ZTVP will create a `Pipeline` in our cluster called **qtodo-supply-chain** that 3. Locate the **qtodo-supply-chain** pipeline. It's within the **layered-zero-trust-hub** project. 4. In the kebab menu (three vertical dots) from the right-hand, select **Start**. - Review the configurable parameters. Most parameters should be correct with their default values if we are in single-cluster mode. But, double-check their values just in case. - - At the bottom we have the **workspaces**. These must be configured manually. + Review the configurable parameters. Most parameters should be correct with their default values if we are in single-cluster mode. But, double-check their values just in case. - * For **qtodo-source**, select `PersistentVolumeClaim` and the PVC name is `qtodo-workspace-source`. - * For **registry-auth-config**, select `Secret` and the name of the secret is `qtodo-registry-auth`. + At the bottom we have the **workspaces**. These must be configured manually. + * For **qtodo-source**, select `PersistentVolumeClaim` and the PVC name is `qtodo-workspace-source`. + * For **registry-auth-config**, select `Secret` and the name of the secret is `qtodo-registry-auth`. 5. Press **Start** to finish and run the pipeline. @@ -213,12 +209,12 @@ spec: timeouts: pipeline: 1h0m0s workspaces: - - name: qtodo-source - persistentVolumeClaim: - claimName: qtodo-workspace-source - - name: registry-auth-config - secret: - secretName: qtodo-registry-auth + - name: qtodo-source + persistentVolumeClaim: + claimName: qtodo-workspace-source + - name: registry-auth-config + secret: + secretName: qtodo-registry-auth ``` As was described previously, verify the values associated with the PVC storage and registry configuration. @@ -294,7 +290,7 @@ The pipeline we have prepared has the following steps: * **qtodo-sign-image**. Signs the container image. * **qtodo-generate-sbom**. Generates an SBOM from the image. * **qtodo-sbom-attestation**. Creates a (signed) attestation, and attaches it to the image. -* **qtodo-upload-sbom**. Uploads the generated SBOM file to RHTPA. +* **qtodo-upload-sbom**. Uploads the generated SBOM file to RHTPA. * **qtodo-verify-image**. Verifies the attestation and the signature attached to the image. ### Inspecting the results @@ -389,18 +385,18 @@ The credentials to access the Quay web interface can be obtained as follows: * Quay URL - ```shell - echo "https://$(oc get route -n quay-enterprise \ - -l quay-component=quay-app-route \ - -o jsonpath='{.items[0].spec.host}')" - ``` + ```shell + echo "https://$(oc get route -n quay-enterprise \ + -l quay-component=quay-app-route \ + -o jsonpath='{.items[0].spec.host}')" + ``` * Quay username: The same one you specified in `values-hub.yaml` or **quay-user**. * Quay password: - ```shell - oc get secret -n layered-zero-trust-hub qtodo-quay-password -o json | jq '.data["password"] | @base64d' - ``` + ```shell + oc get secret -n layered-zero-trust-hub qtodo-quay-password -o json | jq '.data["password"] | @base64d' + ``` Now that we have the credentials, we can check the content in Quay. @@ -429,19 +425,19 @@ The RHTPA web UI uses OIDC for user authentication. If you are using the **Keycl * RHTPA URL - ```shell - echo "https://$(oc get route -n trusted-profile-analyzer \ - -l app.kubernetes.io/name=server \ - -o jsonpath='{.items[0].spec.host}')" - ``` + ```shell + echo "https://$(oc get route -n trusted-profile-analyzer \ + -l app.kubernetes.io/name=server \ + -o jsonpath='{.items[0].spec.host}')" + ``` * RHTPA user: **rhtpa-user** * RHTPA user password - ```shell - oc get secret keycloak-users -n keycloak-system -o json \ - | jq '.data["rhtpa-user-password"] | @base64d' - ``` + ```shell + oc get secret keycloak-users -n keycloak-system -o json \ + | jq '.data["rhtpa-user-password"] | @base64d' + ``` To review our SBOM within the RHTPA web UI: diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 131d554e..1b626142 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -189,24 +189,24 @@ secrets: vaultPolicy: validatedPatternDefaultPolicy # =========================================================================== - # BYO / EMBEDDED OCP REGISTRY SECRETS (hub/infra/registry/) - # User-provided credentials for external or embedded OCP registry. - # Used by: supply-chain pipeline (push), qtodo (pull) when externalRegistry.enabled=true + # BYO REGISTRY SECRETS (hub/infra/registry/) + # Only needed for Option 2 (BYO/external registry, e.g. quay.io, ghcr.io). + # NOT needed for Option 1 (built-in Quay uses quay-users secret) or + # Option 3 (embedded OCP registry with token refresher writes to Vault + # automatically -- see docs/supply-chain.md). + # Used by: supply-chain pipeline (push), qtodo (pull) when registry enabled # Policy: hub-supply-chain-jwt-secret (read access to hub/infra/registry/*) # - # VP rule: add this (with your token) to ~/values-secrets.yaml (or - # ~/values-secret.yaml / ~/values-secret-layered-zero-trust.yaml per VP lookup). - # Replace REPLACE_WITH_REGISTRY_TOKEN in your local file: - # - Embedded OCP registry: use output of oc whoami -t - # - External registry (BYO): use your registry token/password + # Uncomment and replace REPLACE_WITH_REGISTRY_TOKEN with your registry + # token/password in your local ~/values-secret-layered-zero-trust.yaml. # =========================================================================== - - name: registry-user - vaultPrefixes: - - hub/infra/registry - fields: - - name: registry-password - value: "REPLACE_WITH_REGISTRY_TOKEN" - onMissingValue: error + #- name: registry-user + # vaultPrefixes: + # - hub/infra/registry + # fields: + # - name: registry-password + # value: "REPLACE_WITH_REGISTRY_TOKEN" + # onMissingValue: error # =========================================================================== # COCO (CONFIDENTIAL CONTAINERS) SECRETS From e3d7f26bddab344ce9838f6cbde5635877e5f856 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Tue, 14 Apr 2026 08:31:38 -0400 Subject: [PATCH 10/21] refactor: address PR #99 review feedback - Rename org -> repository throughout (global.registry and supply-chain) - Rename embeddedOCP -> embeddedOpenShift in supply-chain templates and docs - Scope registry image rewrite via useRegistry flag in qtodo.image helper - Guard imagePullSecrets on vaultPath being set (not just registry.enabled) - Add Vault auth retry loop to refresh_registry_token.sh for seed Job timing - Extract image namespace from first path component of repository (splitList) - Update docs/supply-chain.md with new parameter names and examples Signed-off-by: Min Zhang --- charts/qtodo/templates/_helpers.tpl | 11 +++-- charts/qtodo/templates/app-deployment.yaml | 2 +- .../qtodo/templates/app-serviceaccount.yaml | 2 +- .../templates/registry-external-secret.yaml | 2 +- charts/qtodo/values.yaml | 2 +- .../files/refresh_registry_token.sh | 35 ++++++++++---- .../templates/pipeline-qtodo.yaml | 2 +- .../rbac/registry-image-namespace.yaml | 29 ++++++------ .../rbac/registry-token-refresher.yaml | 32 ++++++------- .../secrets/qtodo-registry-auth.yaml | 2 +- charts/supply-chain/values.yaml | 32 ++++++++----- docs/supply-chain.md | 46 +++++++++---------- values-secret.yaml.template | 2 +- 13 files changed, 115 insertions(+), 84 deletions(-) diff --git a/charts/qtodo/templates/_helpers.tpl b/charts/qtodo/templates/_helpers.tpl index 1b90bdcf..866e5d3e 100644 --- a/charts/qtodo/templates/_helpers.tpl +++ b/charts/qtodo/templates/_helpers.tpl @@ -1,13 +1,14 @@ {{/* Create the image path for the passed in image field. -For the main app image (isMain=true), when global.registry is enabled the -name is derived from global.registry.domain/org so no values-hub override -is needed (VP overrides don't support template expressions). +When global.registry is enabled with domain and repository, the image +reference is derived from global.registry.domain/repository (e.g. +quay.io/ztvp/qtodo) so no VP --set override is needed. */}} {{- define "qtodo.image" -}} {{- $name := tpl .value.name .context -}} -{{- if and (.isMain) .context.Values.global.registry.enabled .context.Values.global.registry.domain .context.Values.global.registry.org -}} -{{- $name = printf "%s/%s/qtodo" (tpl .context.Values.global.registry.domain .context) .context.Values.global.registry.org -}} +{{- $useRegistry := default false .useRegistry -}} +{{- if and $useRegistry .context.Values.global.registry.enabled .context.Values.global.registry.domain .context.Values.global.registry.repository -}} +{{- $name = printf "%s/%s" (tpl .context.Values.global.registry.domain .context) .context.Values.global.registry.repository -}} {{- end -}} {{- if eq (substr 0 7 (tpl .value.version .context)) "sha256:" -}} {{- printf "%s@%s" $name (tpl .value.version .context) -}} diff --git a/charts/qtodo/templates/app-deployment.yaml b/charts/qtodo/templates/app-deployment.yaml index f93e2e26..8586221e 100644 --- a/charts/qtodo/templates/app-deployment.yaml +++ b/charts/qtodo/templates/app-deployment.yaml @@ -193,7 +193,7 @@ spec: readOnly: true {{- end }} - name: qtodo - image: {{ template "qtodo.image" (dict "value" .Values.app.images.main "isMain" true "context" $) }} + image: {{ template "qtodo.image" (dict "value" .Values.app.images.main "context" $ "useRegistry" true) }} imagePullPolicy: {{ .Values.app.images.main.pullPolicy }} ports: - containerPort: 8080 diff --git a/charts/qtodo/templates/app-serviceaccount.yaml b/charts/qtodo/templates/app-serviceaccount.yaml index 71341fec..cac8253d 100644 --- a/charts/qtodo/templates/app-serviceaccount.yaml +++ b/charts/qtodo/templates/app-serviceaccount.yaml @@ -5,7 +5,7 @@ metadata: app: qtodo name: qtodo namespace: qtodo -{{- if or .Values.app.images.main.registry.auth .Values.global.registry.enabled }} +{{- if or .Values.app.images.main.registry.auth (and .Values.global.registry.enabled .Values.global.registry.vaultPath) }} imagePullSecrets: - name: {{ .Values.app.images.main.registry.secretName }} {{- end }} \ No newline at end of file diff --git a/charts/qtodo/templates/registry-external-secret.yaml b/charts/qtodo/templates/registry-external-secret.yaml index 5c7edc39..ab47e768 100644 --- a/charts/qtodo/templates/registry-external-secret.yaml +++ b/charts/qtodo/templates/registry-external-secret.yaml @@ -1,4 +1,4 @@ -{{- $regAuth := or .Values.app.images.main.registry.auth .Values.global.registry.enabled }} +{{- $regAuth := or .Values.app.images.main.registry.auth (and .Values.global.registry.enabled .Values.global.registry.vaultPath) }} {{- $regDomain := .Values.app.images.main.registry.domain | default .Values.global.registry.domain }} {{- $regUser := .Values.app.images.main.registry.user | default .Values.global.registry.user }} {{- $regVaultPath := .Values.app.images.main.registry.vaultPath | default .Values.global.registry.vaultPath }} diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index 7f792058..abadf788 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -7,7 +7,7 @@ global: registry: enabled: false domain: "" - org: "" + repository: "" user: "" vaultPath: "" passwordVaultKey: "" diff --git a/charts/supply-chain/files/refresh_registry_token.sh b/charts/supply-chain/files/refresh_registry_token.sh index 9a8c2cf5..a53831f4 100755 --- a/charts/supply-chain/files/refresh_registry_token.sh +++ b/charts/supply-chain/files/refresh_registry_token.sh @@ -25,7 +25,7 @@ vault_curl() { fi } -log "Starting OCP registry token refresh" +log "Starting OpenShift registry token refresh" # 1. Read SPIFFE JWT for Vault authentication if [ ! -f "${JWT_TOKEN_FILE}" ]; then @@ -35,17 +35,36 @@ fi JWT="$(cat "${JWT_TOKEN_FILE}")" log "Read SPIFFE JWT from ${JWT_TOKEN_FILE}" -# 2. Authenticate to Vault using SPIFFE JWT +# 2. Authenticate to Vault using SPIFFE JWT (with retry for seed Job timing) +VAULT_MAX_RETRIES="${VAULT_MAX_RETRIES:-20}" +VAULT_RETRY_INTERVAL="${VAULT_RETRY_INTERVAL:-15}" +VAULT_TOKEN="" + log "Authenticating to Vault at ${VAULT_URL} with role ${VAULT_ROLE}..." -AUTH_RESP=$(vault_curl -X POST "${VAULT_URL}/v1/auth/jwt/login" \ - -H "Content-Type: application/json" \ - -d "{\"role\":\"${VAULT_ROLE}\",\"jwt\":\"${JWT}\"}") +attempt=0 +while [ "${attempt}" -lt "${VAULT_MAX_RETRIES}" ]; do + AUTH_RESP=$(vault_curl -X POST "${VAULT_URL}/v1/auth/jwt/login" \ + -H "Content-Type: application/json" \ + -d "{\"role\":\"${VAULT_ROLE}\",\"jwt\":\"${JWT}\"}" 2>&1) || true + + VAULT_TOKEN=$(echo "${AUTH_RESP}" | python3 -c "import sys,json; print(json.load(sys.stdin)['auth']['client_token'])" 2>/dev/null) || true + + if [ -n "${VAULT_TOKEN}" ]; then + break + fi -VAULT_TOKEN=$(echo "${AUTH_RESP}" | python3 -c "import sys,json; print(json.load(sys.stdin)['auth']['client_token'])" 2>/dev/null) || { - log "ERROR: Vault authentication failed" + attempt=$((attempt + 1)) + if [ "${attempt}" -lt "${VAULT_MAX_RETRIES}" ]; then + log "Vault not ready (attempt ${attempt}/${VAULT_MAX_RETRIES}). Retrying in ${VAULT_RETRY_INTERVAL}s..." + sleep "${VAULT_RETRY_INTERVAL}" + fi +done + +if [ -z "${VAULT_TOKEN}" ]; then + log "ERROR: Vault authentication failed after ${VAULT_MAX_RETRIES} attempts" log "${AUTH_RESP}" exit 1 -} +fi log "Vault authentication successful" # 3. Create a fresh SA token via the Kubernetes TokenRequest API diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index 0de9d1de..8a62af6a 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -25,7 +25,7 @@ spec: - name: image-target type: string description: qtodo image push destination (e.g. quay.io/ztvp/qtodo:latest) - default: {{ tpl (required "registry.domain (or global.registry.domain) is required" (.Values.registry.domain | default .Values.global.registry.domain)) $ }}/{{ .Values.registry.org | default .Values.global.registry.org }}/{{ .Values.registry.repo }}:{{ .Values.qtodo.tag }} + default: {{ tpl (required "registry.domain (or global.registry.domain) is required" (.Values.registry.domain | default .Values.global.registry.domain)) $ }}/{{ .Values.registry.repository | default .Values.global.registry.repository }}:{{ .Values.qtodo.tag }} - name: image-tls-verify type: string description: Whether to verify TLS when pushing to the OCI registry diff --git a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml index 8ccd79a3..6602279d 100644 --- a/charts/supply-chain/templates/rbac/registry-image-namespace.yaml +++ b/charts/supply-chain/templates/rbac/registry-image-namespace.yaml @@ -1,12 +1,15 @@ -{{- if and (index .Values.registry "embeddedOCP") (index .Values.registry.embeddedOCP "ensureImageNamespaceRBAC") }} -# When using the embedded OCP image registry, the pipeline pushes to a namespace -# that matches registry.org (e.g. ztvp). This ensures that namespace exists and -# the pipeline SA has system:image-builder so the push succeeds (transparent to the user). +{{- if and (index .Values.registry "embeddedOpenShift") (index .Values.registry.embeddedOpenShift "ensureImageNamespaceRBAC") }} +{{- $repository := .Values.registry.repository | default .Values.global.registry.repository }} +{{- $imageNamespace := index (splitList "/" $repository) 0 }} +# When using the embedded OpenShift image registry, the pipeline pushes to a +# namespace that matches the first path component of registry.repository +# (e.g. "ztvp" from "ztvp/qtodo"). This ensures that namespace exists and the +# pipeline SA has system:image-builder so the push succeeds. --- apiVersion: v1 kind: Namespace metadata: - name: {{ .Values.registry.org | default .Values.global.registry.org }} + name: {{ $imageNamespace }} annotations: argocd.argoproj.io/sync-wave: "0" --- @@ -14,7 +17,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: pipeline-image-builder - namespace: {{ .Values.registry.org | default .Values.global.registry.org }} + namespace: {{ $imageNamespace }} annotations: argocd.argoproj.io/sync-wave: "0" roleRef: @@ -23,17 +26,17 @@ roleRef: name: system:image-builder subjects: - kind: ServiceAccount - name: pipeline + name: {{ .Values.pipelineServiceAccount }} namespace: {{ .Values.global.namespace }} --- -# Enable the default route on the embedded OCP image registry so that +# Enable the default route on the embedded OpenShift image registry so that # the pipeline can push and external clients can pull images via the route. # Uses a Job because the imageregistry config is a cluster-singleton managed # by the image-registry operator; declarative ownership would conflict. apiVersion: v1 kind: ServiceAccount metadata: - name: registry-route-enabler + name: {{ .Values.registry.embeddedOpenShift.routeEnablerServiceAccount }} namespace: {{ .Values.global.namespace }} annotations: argocd.argoproj.io/sync-wave: "0" @@ -61,7 +64,7 @@ roleRef: name: {{ .Values.global.namespace }}-registry-route-enabler subjects: - kind: ServiceAccount - name: registry-route-enabler + name: {{ .Values.registry.embeddedOpenShift.routeEnablerServiceAccount }} namespace: {{ .Values.global.namespace }} --- apiVersion: batch/v1 @@ -77,11 +80,11 @@ spec: backoffLimit: 3 template: spec: - serviceAccountName: registry-route-enabler + serviceAccountName: {{ .Values.registry.embeddedOpenShift.routeEnablerServiceAccount }} restartPolicy: Never containers: - name: enable-route - image: registry.access.redhat.com/ubi9/ubi:9.7-1764794285 + image: {{ .Values.images.ubi }} command: - /bin/sh - -ce @@ -101,7 +104,7 @@ spec: *'"defaultRoute":true'*) echo "Default route already enabled, nothing to do."; exit 0 ;; esac - echo "Enabling default route on embedded OCP image registry..." + echo "Enabling default route on embedded OpenShift image registry..." RESP=$(curl -sS -w "\n%{http_code}" --cacert "${CACERT}" \ -H "${AUTH_HEADER}" \ -H "Content-Type: application/merge-patch+json" \ diff --git a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml index 4fe14082..932a00bd 100644 --- a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml +++ b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml @@ -1,11 +1,11 @@ -{{- if and (index .Values.registry "embeddedOCP") (index .Values.registry.embeddedOCP "tokenRefresher") (index .Values.registry.embeddedOCP.tokenRefresher "enabled") }} +{{- if and (index .Values.registry "embeddedOpenShift") (index .Values.registry.embeddedOpenShift "tokenRefresher") (index .Values.registry.embeddedOpenShift.tokenRefresher "enabled") }} {{- $regVaultPath := .Values.registry.vaultPath | default .Values.global.registry.vaultPath }} {{- $regPasswordKey := .Values.registry.passwordVaultKey | default .Values.global.registry.passwordVaultKey }} # ============================================================ -# OCP Registry Token Refresher +# OpenShift Registry Token Refresher # Periodically creates a fresh pipeline SA token and writes it # to Vault so the ExternalSecret-based dockerconfigjson stays -# valid for the embedded OCP image registry. +# valid for the embedded OpenShift image registry. # ============================================================ --- apiVersion: v1 @@ -43,7 +43,7 @@ metadata: rules: - apiGroups: [""] resources: ["serviceaccounts/token"] - resourceNames: ["pipeline"] + resourceNames: [{{ .Values.pipelineServiceAccount | quote }}] verbs: ["create"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -57,7 +57,7 @@ roleRef: name: registry-token-refresher subjects: - kind: ServiceAccount - name: pipeline + name: {{ .Values.pipelineServiceAccount }} namespace: {{ .Values.global.namespace }} --- apiVersion: batch/v1 @@ -66,18 +66,18 @@ metadata: name: registry-token-refresher namespace: {{ .Values.global.namespace }} spec: - schedule: {{ .Values.registry.embeddedOCP.tokenRefresher.schedule | quote }} + schedule: {{ .Values.registry.embeddedOpenShift.tokenRefresher.schedule | quote }} concurrencyPolicy: Forbid jobTemplate: spec: backoffLimit: 3 template: spec: - serviceAccountName: pipeline + serviceAccountName: {{ .Values.pipelineServiceAccount }} restartPolicy: Never initContainers: - name: fetch-spiffe-jwt - image: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9:v0.10.0 + image: {{ .Values.images.spiffeHelper }} imagePullPolicy: IfNotPresent args: - '-config' @@ -95,7 +95,7 @@ spec: mountPath: /svids containers: - name: refresh-token - image: {{ .Values.registry.embeddedOCP.tokenRefresher.image }} + image: {{ .Values.images.ubi }} command: ["sh", "/app/refresh_registry_token.sh"] env: - name: VAULT_URL @@ -107,11 +107,11 @@ spec: - name: VAULT_SECRET_KEY value: {{ required "registry passwordVaultKey is required" $regPasswordKey }} - name: SA_NAME - value: pipeline + value: {{ .Values.pipelineServiceAccount }} - name: SA_NAMESPACE value: {{ .Values.global.namespace }} - name: TOKEN_DURATION - value: {{ .Values.registry.embeddedOCP.tokenRefresher.tokenDuration | quote }} + value: {{ .Values.registry.embeddedOpenShift.tokenRefresher.tokenDuration | quote }} - name: JWT_TOKEN_FILE value: /svids/jwt.token - name: CA_CERT @@ -156,11 +156,11 @@ spec: activeDeadlineSeconds: 900 template: spec: - serviceAccountName: pipeline + serviceAccountName: {{ .Values.pipelineServiceAccount }} restartPolicy: Never initContainers: - name: fetch-spiffe-jwt - image: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9:v0.10.0 + image: {{ .Values.images.spiffeHelper }} imagePullPolicy: IfNotPresent args: - '-config' @@ -178,7 +178,7 @@ spec: mountPath: /svids containers: - name: refresh-token - image: {{ .Values.registry.embeddedOCP.tokenRefresher.image }} + image: {{ .Values.images.ubi }} command: ["sh", "/app/refresh_registry_token.sh"] env: - name: VAULT_URL @@ -190,11 +190,11 @@ spec: - name: VAULT_SECRET_KEY value: {{ required "registry passwordVaultKey is required" $regPasswordKey }} - name: SA_NAME - value: pipeline + value: {{ .Values.pipelineServiceAccount }} - name: SA_NAMESPACE value: {{ .Values.global.namespace }} - name: TOKEN_DURATION - value: {{ .Values.registry.embeddedOCP.tokenRefresher.tokenDuration | quote }} + value: {{ .Values.registry.embeddedOpenShift.tokenRefresher.tokenDuration | quote }} - name: JWT_TOKEN_FILE value: /svids/jwt.token - name: CA_CERT diff --git a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml index ac293d4e..726ba0ce 100644 --- a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml @@ -3,7 +3,7 @@ Purpose: Provides dockerconfigjson for pipeline to push/pull images Used by: Tekton pipeline tasks (build-image, sign-image, verify-image) Created when: registry.enabled=true - Registry-agnostic: works for built-in Quay, BYO (quay.io, ghcr.io), or embedded OCP. + Registry-agnostic: works for built-in Quay, BYO (quay.io, ghcr.io), or embedded OpenShift. Set registry.domain, registry.vaultPath, and registry.passwordVaultKey for your scenario. */}} {{- $regEnabled := or .Values.registry.enabled .Values.global.registry.enabled }} diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index ed6ce816..fb8b3170 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -8,7 +8,7 @@ global: registry: enabled: false domain: "" - org: "" + repository: "" user: "" vaultPath: "" passwordVaultKey: "" @@ -36,7 +36,7 @@ qtodo: # =========================================================================== # QUAY USER PROVISIONER (only for built-in Quay registry) # When enabled, runs a CronJob that provisions users in the built-in Quay instance. -# This is Quay-specific and not needed for BYO or embedded OCP registries. +# This is Quay-specific and not needed for BYO or embedded OpenShift registries. # =========================================================================== quay: enabled: false @@ -48,7 +48,7 @@ quay: # =========================================================================== # REGISTRY CONFIGURATION (option-agnostic) # Works for all registry types: built-in Quay, BYO (quay.io, ghcr.io, etc.), -# or embedded OCP image registry. Set the values for your scenario. +# or embedded OpenShift image registry. Set the values for your scenario. # # Scenario-specific values (set in values-hub.yaml overrides): # Built-in Quay: @@ -59,21 +59,20 @@ quay: # domain: quay.io (or your registry hostname) # vaultPath: secret/data/hub/infra/registry/registry-user # passwordVaultKey: registry-password -# Embedded OCP: +# Embedded OpenShift: # domain: default-route-openshift-image-registry.apps. # vaultPath: secret/data/hub/infra/registry/registry-user # passwordVaultKey: registry-password -# embeddedOCP.ensureImageNamespaceRBAC: true +# embeddedOpenShift.ensureImageNamespaceRBAC: true # =========================================================================== registry: # Set to true to create the registry auth secret (dockerconfigjson) enabled: false # Registry hostname (REQUIRED when enabled) domain: "" - # Organization/namespace within the registry (falls back to global.registry.org) - org: "" - # Repository name - repo: "qtodo" + # Repository path within the registry, e.g. "ztvp/qtodo" + # (falls back to global.registry.repository) + repository: "" # Whether to verify TLS when pushing to the registry tlsVerify: "true" # Registry username (falls back to global.registry.user) @@ -84,17 +83,26 @@ registry: passwordVaultKey: "" # Secret name for registry auth (dockerconfigjson) authSecretName: "qtodo-registry-auth" - # Embedded OCP registry only: create image namespace (registry.org) and grant + # Embedded OpenShift registry only: create image namespace and grant # pipeline SA system:image-builder so the pipeline can push. Set to true only when # using the in-cluster OpenShift image registry; leave false for other registries. - embeddedOCP: + embeddedOpenShift: ensureImageNamespaceRBAC: false + # Service account name for the route-enabler Job + routeEnablerServiceAccount: "registry-route-enabler" tokenRefresher: enabled: false - image: registry.access.redhat.com/ubi9/ubi:9.7-1764794285 schedule: "0 */6 * * *" tokenDuration: "172800" +# Container images used by Jobs and CronJobs (externalized for overridability) +images: + ubi: "registry.access.redhat.com/ubi9/ubi:9.7-1764794285" + spiffeHelper: "registry.redhat.io/zero-trust-workload-identity-manager/spiffe-helper-rhel9:v0.10.0" + +# Service account name used by pipeline Jobs (CronJob, seed Job) +pipelineServiceAccount: "pipeline" + # pipeline run configuration pipelinerun: # Set to true to automatically trigger a pipeline run on ArgoCD sync diff --git a/docs/supply-chain.md b/docs/supply-chain.md index ced9fdaf..7f1e584a 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -47,7 +47,7 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use onMissingValue: error ``` - > **Note**: This secret is only required for **BYO/external registries** (Option 2). **Built-in Quay** (Option 1) uses the auto-generated `quay-users` secret. **Embedded OCP registry** (Option 3) does not need a manual secret when the automatic token refresher is enabled (see [Embedded OCP Registry](#embedded-ocp-registry)) -- the refresher creates and rotates the Vault credential automatically. + > **Note**: This secret is only required for **BYO/external registries** (Option 2). **Built-in Quay** (Option 1) uses the auto-generated `quay-users` secret. **Embedded OpenShift registry** (Option 3) does not need a manual secret when the automatic token refresher is enabled (see [Embedded OpenShift Registry](#embedded-openshift-registry)) -- the refresher creates and rotates the Vault credential automatically. > > **Note**: Never commit `~/values-secrets.yaml` (or your local values-secret file) to git. This file contains sensitive credentials and should remain local. @@ -59,20 +59,20 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use registry: enabled: true domain: quay.io - org: your-org + repository: your-org/qtodo user: your-username vaultPath: "secret/data/hub/infra/registry/registry-user" passwordVaultKey: "registry-password" ``` - See the **Registry Options** section at the top of `values-hub.yaml` for the full set of option blocks (built-in Quay, BYO, embedded OCP). + See the **Registry Options** section at the top of `values-hub.yaml` for the full set of option blocks (built-in Quay, BYO, embedded OpenShift). 4. **Enable supply-chain-specific overrides** (if needed): The `supply-chain` application may need additional overrides depending on the registry type. These are set in the `supply-chain` overrides section of `values-hub.yaml`: * **Built-in Quay**: Enable `quay.enabled` (Quay user provisioner CronJob) and `registry.tlsVerify: "false"` (self-signed certs). - * **Embedded OCP**: Enable `registry.embeddedOCP.ensureImageNamespaceRBAC` (creates image namespace and push RBAC) and optionally `registry.embeddedOCP.tokenRefresher.enabled` (see [Embedded OCP Registry](#embedded-ocp-registry)). + * **Embedded OpenShift**: Enable `registry.embeddedOpenShift.ensureImageNamespaceRBAC` (creates image namespace and push RBAC) and optionally `registry.embeddedOpenShift.tokenRefresher.enabled` (see [Embedded OpenShift Registry](#embedded-openshift-registry)). * **BYO/External**: No extra overrides needed. - > **Note**: The qtodo chart automatically derives its image name from `global.registry.domain` and `global.registry.org` when `global.registry.enabled=true`. No per-app image override is needed. + > **Note**: The qtodo chart automatically derives its image from `global.registry.domain` and `global.registry.repository` when `global.registry.enabled=true`. No per-app image override is needed. ### Required Configuration @@ -82,12 +82,12 @@ These parameters are set in the `global.registry` block at the top of `values-hu | ---------------------------------- | ------------------------------------ | ---------------------------------------------- | | `global.registry.enabled` | Enable registry auth secret creation | `true` | | `global.registry.domain` | Registry hostname (REQUIRED) | `quay.io`, `ghcr.io`, `registry.example.com` | -| `global.registry.org` | Organization/namespace | `my-org` | +| `global.registry.repository` | Repository path (org/image) | `ztvp/qtodo`, `my-org/my-app` | | `global.registry.user` | Registry username | `my-robot-account` | | `global.registry.vaultPath` | Vault path for registry password | `secret/data/hub/infra/registry/registry-user` | | `global.registry.passwordVaultKey` | Key within the Vault secret | `registry-password` | -> **Note**: All registry types (built-in Quay, BYO, embedded OCP) use the same `global.registry` parameters. Both the `supply-chain` and `qtodo` charts fall back to these values when their local registry values are empty. See the Vault Paths table below for scenario-specific values. +> **Note**: All registry types (built-in Quay, BYO, embedded OpenShift) use the same `global.registry` parameters. Both the `supply-chain` and `qtodo` charts fall back to these values when their local registry values are empty. See the Vault Paths table below for scenario-specific values. ### Vault Paths @@ -97,57 +97,57 @@ Registry credentials are stored at different paths based on registry type: | ------------- | ---------------------------------------------- | -------------------- | | Built-in Quay | `secret/data/hub/infra/quay/quay-users` | `quay-user-password` | | BYO Registry | `secret/data/hub/infra/registry/registry-user` | `registry-password` | -| Embedded OCP | `secret/data/hub/infra/registry/registry-user` | `registry-password` | +| Embedded OpenShift | `secret/data/hub/infra/registry/registry-user` | `registry-password` | Set `global.registry.vaultPath` and `global.registry.passwordVaultKey` in the `global.registry` block to match your scenario. When `global.registry.enabled` is false or unset (default), no registry auth secret is created (fresh install state). -The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. For the embedded OCP registry, the policy also grants `create` and `update` capabilities on the registry path so the automatic token refresher can write fresh tokens back to Vault. +The Vault policy `hub-supply-chain-jwt-secret` grants read access to both paths for the pipeline service account. For the embedded OpenShift registry, the policy also grants `create` and `update` capabilities on the registry path so the automatic token refresher can write fresh tokens back to Vault. -### Embedded OCP Registry +### Embedded OpenShift Registry To use the in-cluster OpenShift image registry instead of an external registry: -1. **Uncomment the Option 3 `global.registry` block** in `values-hub.yaml` so `global.registry` points at the embedded registry (domain, org, vault paths). Use `user: _token` when using automatic token refresh (bearer tokens; the username is not significant to the registry). +1. **Uncomment the Option 3 `global.registry` block** in `values-hub.yaml` so `global.registry` points at the embedded registry (domain, repository, vault paths). Use `user: _token` when using automatic token refresh (bearer tokens; the username is not significant to the registry). ```yaml global: registry: enabled: true domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} - org: ztvp + repository: ztvp/qtodo user: _token vaultPath: "secret/data/hub/infra/registry/registry-user" passwordVaultKey: "registry-password" ``` -2. **Enable `registry.embeddedOCP.ensureImageNamespaceRBAC`** in the supply-chain overrides. The chart will automatically: - * Create the image namespace matching `global.registry.org` (e.g. `ztvp`) +2. **Enable `registry.embeddedOpenShift.ensureImageNamespaceRBAC`** in the supply-chain overrides. The chart will automatically: + * Create the image namespace from the first component of `global.registry.repository` (e.g. `ztvp` from `ztvp/qtodo`) * Grant the pipeline ServiceAccount `system:image-builder` in that namespace * Enable the default route on the image registry (via a one-time Job) 3. **Confirm the registry domain** is `default-route-openshift-image-registry.apps.` (set in `global.registry.domain` above). -4. **Enable automatic token refresh** (recommended): Set `registry.embeddedOCP.tokenRefresher.enabled` to `true`. This deploys: +4. **Enable automatic token refresh** (recommended): Set `registry.embeddedOpenShift.tokenRefresher.enabled` to `true`. This deploys: * A **CronJob** (`registry-token-refresher`) that runs every 6 hours. It uses a SPIFFE JWT to authenticate to Vault, creates a fresh `pipeline` ServiceAccount token via the Kubernetes TokenRequest API, and writes it to Vault. * A one-shot **Sync hook Job** (`registry-token-refresher-seed`) that seeds the initial token on first deploy so the pipeline is ready immediately. - When the token refresher is enabled, you do **not** need to manually store a token in `~/values-secrets.yaml` for the embedded OCP registry. The refresher handles credential lifecycle automatically. + When the token refresher is enabled, you do **not** need to manually store a token in `~/values-secrets.yaml` for the embedded OpenShift registry. The refresher handles credential lifecycle automatically. If you prefer manual token management instead, disable the token refresher and store the output of `oc whoami -t` as the `registry-password` value in `~/values-secrets.yaml`. -Example `supply-chain` application overrides for embedded OCP (registry host, org, and Vault paths are normally taken from the `global.registry` block): +Example `supply-chain` application overrides for embedded OpenShift (registry host, repository, and Vault paths are normally taken from the `global.registry` block): ```yaml overrides: - - name: registry.embeddedOCP.ensureImageNamespaceRBAC + - name: registry.embeddedOpenShift.ensureImageNamespaceRBAC value: "true" - - name: registry.embeddedOCP.tokenRefresher.enabled + - name: registry.embeddedOpenShift.tokenRefresher.enabled value: "true" ``` ### Node-Level Image Pull Trust -When using a registry behind the cluster ingress (Option 1: Built-in Quay or Option 3: Embedded OCP Registry), kubelet cannot pull images by default because the ingress certificate is self-signed and not trusted at the node level. +When using a registry behind the cluster ingress (Option 1: Built-in Quay or Option 3: Embedded OpenShift Registry), kubelet cannot pull images by default because the ingress certificate is self-signed and not trusted at the node level. The `ztvp-certificates` application handles this by patching `image.config.openshift.io/cluster` with the ingress CA certificate for the configured registry hostnames. Enable it by uncommenting the `imagePullTrust` overrides in `values-hub.yaml`: @@ -164,7 +164,7 @@ Set `` to match your registry option: | Option | Registry Hostname | | ----------------------- | ------------------------------------------------------------- | | Option 1: Built-in Quay | `quay-registry-quay-quay-enterprise.apps.` | -| Option 3: Embedded OCP | `default-route-openshift-image-registry.apps.` | +| Option 3: Embedded OpenShift | `default-route-openshift-image-registry.apps.` | > **Note**: Option 2 (BYO/External Registry) does not require `imagePullTrust` because external registries like quay.io and ghcr.io use publicly trusted certificates. @@ -236,7 +236,7 @@ helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ --set global.registry.enabled=true \ --set global.registry.domain=quay-registry-quay-quay-enterprise.apps.example.com \ - --set global.registry.org=ztvp \ + --set global.registry.repository=ztvp/qtodo \ --set global.registry.vaultPath=secret/data/hub/infra/quay/quay-users \ --set global.registry.passwordVaultKey=quay-user-password \ --set global.namespace=layered-zero-trust-hub \ @@ -250,7 +250,7 @@ helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ --set global.registry.enabled=true \ --set global.registry.domain=quay.io \ - --set global.registry.org=your-org \ + --set global.registry.repository=your-org/qtodo \ --set global.registry.vaultPath=secret/data/hub/infra/registry/registry-user \ --set global.registry.passwordVaultKey=registry-password \ --set global.namespace=layered-zero-trust-hub \ diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 1b626142..945279fd 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -192,7 +192,7 @@ secrets: # BYO REGISTRY SECRETS (hub/infra/registry/) # Only needed for Option 2 (BYO/external registry, e.g. quay.io, ghcr.io). # NOT needed for Option 1 (built-in Quay uses quay-users secret) or - # Option 3 (embedded OCP registry with token refresher writes to Vault + # Option 3 (embedded OpenShift registry with token refresher writes to Vault # automatically -- see docs/supply-chain.md). # Used by: supply-chain pipeline (push), qtodo (pull) when registry enabled # Policy: hub-supply-chain-jwt-secret (read access to hub/infra/registry/*) From cc9b1425963a50ef23778906afc95685c449c915 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Thu, 16 Apr 2026 07:55:34 -0400 Subject: [PATCH 11/21] fix: address PR review feedback (naming, sync-wave, docs) - Rename OCP_DOMAIN to OPENSHIFT_DOMAIN in vault-utils.sh - Use repository: ztvp/qtodo for all 3 registry options in values-hub.yaml - Add sync-wave "15" to qtodo-registry-auth ExternalSecret so it runs after the registry-token-refresher-seed Job at wave 10, preventing a deadlock where the ExternalSecret blocks Argo from reaching the seed - Update SYNC-WAVE-INVENTORY.md with full supply-chain chart internals Signed-off-by: Min Zhang --- .../rbac/registry-token-refresher.yaml | 3 ++- .../templates/secrets/qtodo-registry-auth.yaml | 2 ++ common/scripts/vault-utils.sh | 4 ++-- docs/SYNC-WAVE-INVENTORY.md | 17 ++++++++++++++--- values-hub.yaml | 2 +- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml index 932a00bd..40fd7e58 100644 --- a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml +++ b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml @@ -145,7 +145,8 @@ metadata: name: registry-token-refresher-seed namespace: {{ .Values.global.namespace }} annotations: - # Run after wave 0 (ConfigMaps, RBAC, ztvp ns) and wave 1 (enable-registry-default-route hook). + # Run after wave 0 (ConfigMaps, RBAC) and wave 1 (enable-registry-default-route hook), + # but before the qtodo-registry-auth ExternalSecret at wave 15. argocd.argoproj.io/sync-wave: "10" argocd.argoproj.io/hook: Sync # HookSucceeded: remove completed Job so Argo does not sit on a stale object; BeforeHookCreation: clean retry/sync. diff --git a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml index 726ba0ce..6773fb99 100644 --- a/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-registry-auth.yaml @@ -18,6 +18,8 @@ kind: ExternalSecret metadata: name: {{ .Values.registry.authSecretName }} namespace: {{ .Release.Namespace | default .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "15" spec: refreshInterval: 15s secretStoreRef: diff --git a/common/scripts/vault-utils.sh b/common/scripts/vault-utils.sh index 1b50bb34..a72f1343 100755 --- a/common/scripts/vault-utils.sh +++ b/common/scripts/vault-utils.sh @@ -32,12 +32,12 @@ EXTRA_VARS_FILE=$(mktemp) trap "rm -f ${EXTRA_VARS_FILE}" EXIT if [ "$(yq ".clusterGroup.applications.vault.jwt.enabled // \"false\"" "${MAIN_CLUSTERGROUP_FILE}")" == "true" ]; then - OCP_DOMAIN="$(oc get dns cluster -o jsonpath='{.spec.baseDomain}')" + OPENSHIFT_DOMAIN="$(oc get dns cluster -o jsonpath='{.spec.baseDomain}')" GLOBAL_PATTERN="$(yq -r '.global.pattern // ""' "${PATTERNPATH}/values-global.yaml")" GLOBAL_PATTERN="${GLOBAL_PATTERN:-${PATTERN_NAME}}" # Replace Helm-style placeholders so Ansible/Jinja2 never sees "{{ $.Values... }}" (invalid Jinja2). _subst_vault_yaml() { - sed -e "s/{{ \$.Values.global.clusterDomain }}/${OCP_DOMAIN}/g" \ + sed -e "s/{{ \$.Values.global.clusterDomain }}/${OPENSHIFT_DOMAIN}/g" \ -e "s/{{ \$.Values.global.pattern }}/${GLOBAL_PATTERN}/g" } OIDC_DISCOVERY_URL="$(yq ".clusterGroup.applications.vault.jwt.oidcDiscoveryUrl" "${MAIN_CLUSTERGROUP_FILE}" | _subst_vault_yaml)" diff --git a/docs/SYNC-WAVE-INVENTORY.md b/docs/SYNC-WAVE-INVENTORY.md index d0708d5e..91c53779 100644 --- a/docs/SYNC-WAVE-INVENTORY.md +++ b/docs/SYNC-WAVE-INVENTORY.md @@ -69,6 +69,10 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 46 | └ acs-secured-cluster | chart | secured-cluster-cr | | 46 | └ rhtas-operator | chart | securesign | | 48 | supply-chain | **App** | | +| 48+0 | └ supply-chain | chart | registry-image-namespace (Namespace, RBAC), pipeline-sa, tasks, secrets (quay-pass, rhtpa-pass), quay-user, rhtas/rhtpa-config | +| 48+1 | └ supply-chain | chart (hook) | enable-registry-default-route (Sync hook Job) | +| 48+10 | └ supply-chain | chart (hook) | registry-token-refresher-seed (Sync hook Job — writes SA token to Vault) | +| 48+15 | └ supply-chain | chart | qtodo-registry-auth ExternalSecret (reads token from Vault) | | 49 | └ rhtpa-operator | chart | spiffe-helper-config | | 51 | acs-policies | **App** | After ACS Central + Secured Cluster | | 51 | └ rhtpa-operator | chart | trusted-profile-analyzer (supporting objects) | @@ -235,9 +239,16 @@ Charts marked **(external)** have been externalized to standalone repositories m ### supply-chain (`charts/supply-chain/templates/`) — App wave: 48 -| Resource | Old | Current | -| --- | ---: | ---: | -| workspaces.yaml | 20 | 51 | +Resources without an explicit sync-wave default to wave 0. These include: pipeline-sa, pipeline-qtodo, tasks/*, secrets/qtodo-quay-pass, secrets/qtodo-rhtpa-pass, rhtas-config, rhtpa-config, quay/quay-user-cm, quay/quay-user-job. + +| Resource | Old | Current | Notes | +| --- | ---: | ---: | --- | +| registry-image-namespace.yaml (Namespace, RoleBinding, SA, ClusterRole, ClusterRoleBinding) | — | 0 | Foundation RBAC for embedded OpenShift registry | +| enable-registry-default-route (Sync hook Job) | — | 1 | Exposes the OpenShift image registry route | +| registry-token-refresher-seed (Sync hook Job) | — | 10 | Writes initial SA token to Vault | +| qtodo-registry-auth ExternalSecret | — | 15 | Reads registry token from Vault; must run after seed Job | +| workspaces.yaml | 20 | 51 | Pipeline PVCs | +| pipelinerun-qtodo.yaml (PostSync hook) | — | — | Triggers pipeline run after sync completes | ### docs/DEVELOPMENT.md (example snippet, not deployed) diff --git a/values-hub.yaml b/values-hub.yaml index 000e846b..7c91b696 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -33,7 +33,7 @@ spire: # registry: # enabled: true # domain: quay.io -# org: your-org +# repository: ztvp/qtodo # user: your-username # vaultPath: "secret/data/hub/infra/registry/registry-user" # passwordVaultKey: "registry-password" From 876f94528cdd83dbcc0a8090b365d860f888a3c3 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Thu, 16 Apr 2026 08:16:32 -0400 Subject: [PATCH 12/21] fix: rename OCP to OpenShift in gen-byo-container-registry-variants.py Signed-off-by: Min Zhang --- scripts/gen-byo-container-registry-variants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/gen-byo-container-registry-variants.py b/scripts/gen-byo-container-registry-variants.py index e81b54cb..0448e0a9 100755 --- a/scripts/gen-byo-container-registry-variants.py +++ b/scripts/gen-byo-container-registry-variants.py @@ -6,7 +6,7 @@ Option 1: Built-in Quay Registry Option 2: BYO / External Registry (e.g. quay.io, ghcr.io) - Option 3: Embedded OCP Image Registry + Option 3: Embedded OpenShift Image Registry Each variant also enables the common supply-chain stack (OpenShift Pipelines, ODF, NooBaa, RHTAS, RHTPA, and their namespaces/subscriptions/vault roles). @@ -210,14 +210,14 @@ def enable_supply_chain_app(lines, option_num): final.append(uncomment_line(line)) continue - # Option 3 (Embedded OCP): uncomment ensureImageNamespaceRBAC + # Option 3 (Embedded OpenShift): uncomment ensureImageNamespaceRBAC if option_num == 3: - if re.search(r"# - name: registry\.embeddedOCP", line): + if re.search(r"# - name: registry\.embeddedOpenShift", line): final.append(uncomment_line(line)) continue if re.search(r"#\s+value:", line) and final: prev = final[-1] - if "embeddedOCP" in prev: + if "embeddedOpenShift" in prev: final.append(uncomment_line(line)) continue @@ -451,7 +451,7 @@ def enable_image_pull_trust(lines, hostname): OPTION_LABELS = { 1: "built-in-quay-registry", 2: "byo-external-registry", - 3: "embedded-ocp-registry", + 3: "embedded-openshift-registry", } From a8a56603300260bfb29223b2cc92cae7092e5bb2 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Thu, 16 Apr 2026 08:29:46 -0400 Subject: [PATCH 13/21] fix: apply remaining OCP to OpenShift renames in values-hub.yaml - Rename org -> repository: ztvp/qtodo for Options 1 and 3 - Rename Embedded OCP -> Embedded OpenShift throughout - Rename embeddedOCP -> embeddedOpenShift in supply-chain overrides - Update domain/org -> domain/repository in comments Signed-off-by: Min Zhang --- values-hub.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/values-hub.yaml b/values-hub.yaml index 7c91b696..f5736a49 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -24,7 +24,7 @@ spire: # registry: # enabled: true # domain: quay-registry-quay-quay-enterprise.apps.{{ .Values.global.clusterDomain }} -# org: ztvp +# repository: ztvp/qtodo # user: quay-user # vaultPath: "secret/data/hub/infra/quay/quay-users" # passwordVaultKey: "quay-user-password" @@ -37,12 +37,12 @@ spire: # user: your-username # vaultPath: "secret/data/hub/infra/registry/registry-user" # passwordVaultKey: "registry-password" -# OPTION 3: Embedded OCP Registry +# OPTION 3: Embedded OpenShift Registry # global: # registry: # enabled: true # domain: default-route-openshift-image-registry.apps.{{ .Values.global.clusterDomain }} -# org: ztvp +# repository: ztvp/qtodo # user: _token # vaultPath: "secret/data/hub/infra/registry/registry-user" # passwordVaultKey: "registry-password" @@ -262,8 +262,8 @@ clusterGroup: # Node-level image pull trust for kubelet # Required when pulling images from registries behind the cluster ingress - # (e.g. built-in Quay, embedded OCP registry). Patches image.config.openshift.io/cluster. - # Uncomment and set the registry hostname when enabling Option 1 (Quay) or Option 3 (embedded OCP). + # (e.g. built-in Quay, embedded OpenShift registry). Patches image.config.openshift.io/cluster. + # Uncomment and set the registry hostname when enabling Option 1 (Quay) or Option 3 (embedded OpenShift). # - name: imagePullTrust.enabled # value: "true" # - name: imagePullTrust.registries[0] @@ -540,7 +540,7 @@ clusterGroup: - name: app.vault.secretPath value: secret/data/apps/qtodo/qtodo-db # Secure Supply Chain: when global.registry.enabled=true the chart - # automatically derives the image name from global.registry.domain/org. + # automatically derives the image from global.registry.domain/repository. # No override needed here. # Secure Supply Chain - Uncomment to enable (required for Option 1, 2, or 3 registry flows in docs) # supply-chain: @@ -562,11 +562,11 @@ clusterGroup: # # Built-in Quay: self-signed certs require TLS verify off # # - name: registry.tlsVerify # # value: "false" - # # Embedded OCP (Option 3): create image namespace and grant push RBAC - # # - name: registry.embeddedOCP.ensureImageNamespaceRBAC + # # Embedded OpenShift (Option 3): create image namespace and grant push RBAC + # # - name: registry.embeddedOpenShift.ensureImageNamespaceRBAC # # value: "true" - # # Embedded OCP (Option 3): periodically refresh pipeline SA token in Vault - # # - name: registry.embeddedOCP.tokenRefresher.enabled + # # Embedded OpenShift (Option 3): periodically refresh pipeline SA token in Vault + # # - name: registry.embeddedOpenShift.tokenRefresher.enabled # # value: "true" # # Enable RHTAS signing # # - name: rhtas.enabled From 9c68d623e7d52f0237df723a1e79d8eb707692ce Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Thu, 16 Apr 2026 23:07:47 -0400 Subject: [PATCH 14/21] feat: seed image job, pipeline auto-trigger, PVC health check, and oc mirror fix Bootstrapping: registry seed image Job (qtodo chart) - New registry-seed-job.yaml: mirrors upstream qtodo image into the configured registry (embedded OpenShift, built-in Quay, or BYO) so the deployment can pull before the supply-chain pipeline runs. - Adds seedImage config block to qtodo/values.yaml (disabled by default). - Handles all three registry types: SA token auth for embedded OpenShift, dockerconfigjson secret for Quay/BYO, with registry-reachability retry. - Uses -a flag for oc image mirror auth (not skopeo --dest-creds flags). - Sync hook with HookSucceeded delete policy; runs at wave 0+5 (before the qtodo deployment at wave 51). Pipeline auto-trigger: PostSync Job wrapper (supply-chain chart) - pipelinerun-qtodo.yaml now creates a PostSync Job that imperatively runs `oc create` to launch the PipelineRun. This works around the upstream clustergroup chart's resourceExclusions for tekton.dev PipelineRun/TaskRun, which silently blocks direct PipelineRun hooks. - Dedicated pipelinerun-launcher ServiceAccount + Role + RoleBinding with minimal RBAC (create pipelineruns, get secret/PVC). Post-pipeline qtodo refresh (supply-chain chart) - New restart-qtodo Tekton Task + RBAC (pipeline-qtodo-restarter Role/RoleBinding in qtodo namespace). - Added `finally` section to pipeline-qtodo.yaml that runs restart-qtodo after successful image verification, forcing the qtodo deployment to pull the newly built and signed image. ArgoCD PVC health check (values-hub.yaml) - Added custom resourceHealthCheck for PersistentVolumeClaim: treats Pending as Healthy. WaitForFirstConsumer PVCs stay Pending until a pod mounts them, which causes ArgoCD to report the app as Progressing indefinitely and prevents PostSync hooks from firing. This is a common pattern for gp3-csi (AWS), lvms-vg1 (bare metal), etc. - Preserves existing KeycloakRealmImport health check. Generator script updates - gen-byo-container-registry-variants.py: uncomments pipelinerun.enabled and app.seedImage.enabled flags when generating supply-chain variants. Other - registry-external-secret.yaml: added sync-wave 36 annotation. - values-hub.yaml: clean base with new flags commented out; RHTAS pinned to stable-v1.3. - SYNC-WAVE-INVENTORY.md: updated qtodo and supply-chain sections. Signed-off-by: Min Zhang --- .../templates/registry-external-secret.yaml | 2 + charts/qtodo/templates/registry-seed-job.yaml | 186 ++++++++++++++++++ charts/qtodo/values.yaml | 8 + .../templates/pipeline-qtodo.yaml | 12 +- .../templates/pipelinerun-qtodo.yaml | 93 +++++++-- .../rbac/pipeline-qtodo-restarter.yaml | 34 ++++ .../templates/tasks/restart-qtodo.yaml | 20 ++ docs/SYNC-WAVE-INVENTORY.md | 14 +- .../gen-byo-container-registry-variants.py | 31 ++- values-coco-dev.yaml | 2 +- values-hub.yaml | 79 +++++--- 11 files changed, 433 insertions(+), 48 deletions(-) create mode 100644 charts/qtodo/templates/registry-seed-job.yaml create mode 100644 charts/supply-chain/templates/rbac/pipeline-qtodo-restarter.yaml create mode 100644 charts/supply-chain/templates/tasks/restart-qtodo.yaml diff --git a/charts/qtodo/templates/registry-external-secret.yaml b/charts/qtodo/templates/registry-external-secret.yaml index ab47e768..5ce857c0 100644 --- a/charts/qtodo/templates/registry-external-secret.yaml +++ b/charts/qtodo/templates/registry-external-secret.yaml @@ -10,6 +10,8 @@ kind: ExternalSecret metadata: name: {{ .Values.app.images.main.registry.secretName }} namespace: {{ .Release.Namespace | default .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "36" spec: refreshInterval: 15s secretStoreRef: diff --git a/charts/qtodo/templates/registry-seed-job.yaml b/charts/qtodo/templates/registry-seed-job.yaml new file mode 100644 index 00000000..22bbde6a --- /dev/null +++ b/charts/qtodo/templates/registry-seed-job.yaml @@ -0,0 +1,186 @@ +{{- if .Values.app.seedImage.enabled }} +{{- $regDomain := .Values.global.registry.domain }} +{{- $regRepository := .Values.global.registry.repository }} +{{- $isEmbedded := and (hasKey .Values.global.registry "user") (eq (default "" .Values.global.registry.user) "_token") }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: registry-seed + namespace: {{ .Release.Namespace | default "qtodo" }} + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace | default "qtodo" }}-registry-seed + annotations: + argocd.argoproj.io/sync-wave: "0" +rules: +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "create"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["get", "create"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + verbs: ["bind"] + resourceNames: ["system:image-builder"] +- apiGroups: ["imageregistry.operator.openshift.io"] + resources: ["configs"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace | default "qtodo" }}-registry-seed + annotations: + argocd.argoproj.io/sync-wave: "0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace | default "qtodo" }}-registry-seed +subjects: +- kind: ServiceAccount + name: registry-seed + namespace: {{ .Release.Namespace | default "qtodo" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: registry-seed-image + namespace: {{ .Release.Namespace | default "qtodo" }} + annotations: + argocd.argoproj.io/sync-wave: "5" + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded +spec: + backoffLimit: 3 + activeDeadlineSeconds: 1200 + template: + spec: + serviceAccountName: registry-seed + restartPolicy: Never +{{- if not $isEmbedded }} + volumes: + - name: registry-auth + secret: + secretName: {{ .Values.app.images.main.registry.secretName }} + optional: true +{{- end }} + containers: + - name: seed + image: {{ .Values.app.seedImage.image }} + env: + - name: SOURCE_IMAGE + value: {{ .Values.app.seedImage.source | quote }} + - name: TAG + value: {{ .Values.app.seedImage.tag | quote }} + - name: REGISTRY_DOMAIN + value: {{ tpl (required "global.registry.domain is required" $regDomain) $ | quote }} + - name: REGISTRY_REPOSITORY + value: {{ $regRepository | quote }} + - name: IS_EMBEDDED + value: {{ $isEmbedded | quote }} +{{- if not $isEmbedded }} + volumeMounts: + - name: registry-auth + mountPath: /var/run/secrets/registry + readOnly: true +{{- end }} + command: + - /bin/sh + - -ce + - | + APISERVER="https://kubernetes.default.svc" + SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + CACERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + TARGET_IMAGE="${REGISTRY_DOMAIN}/${REGISTRY_REPOSITORY}:${TAG}" + + if [ "${IS_EMBEDDED}" = "true" ]; then + PUSH_REGISTRY="image-registry.openshift-image-registry.svc:5000" + PUSH_IMAGE="${PUSH_REGISTRY}/${REGISTRY_REPOSITORY}:${TAG}" + + IMAGE_NS=$(echo "${REGISTRY_REPOSITORY}" | cut -d/ -f1) + echo "Ensuring namespace '${IMAGE_NS}' exists..." + NS_BODY="{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"name\":\"${IMAGE_NS}\"}}" + HTTP=$(curl -sS -o /dev/null -w "%{http_code}" --cacert "${CACERT}" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + -H "Content-Type: application/json" \ + -X POST -d "${NS_BODY}" \ + "${APISERVER}/api/v1/namespaces" 2>/dev/null) + case "${HTTP}" in + 20[0-9]|409) echo "Namespace ready (HTTP ${HTTP})." ;; + *) echo "WARNING: namespace create returned HTTP ${HTTP}, continuing..." ;; + esac + + echo "Granting system:image-builder to registry-seed SA..." + RB_BODY="{\"apiVersion\":\"rbac.authorization.k8s.io/v1\",\"kind\":\"RoleBinding\",\"metadata\":{\"name\":\"seed-image-builder\",\"namespace\":\"${IMAGE_NS}\"},\"roleRef\":{\"apiGroup\":\"rbac.authorization.k8s.io\",\"kind\":\"ClusterRole\",\"name\":\"system:image-builder\"},\"subjects\":[{\"kind\":\"ServiceAccount\",\"name\":\"registry-seed\",\"namespace\":\"{{ .Release.Namespace | default "qtodo" }}\"}]}" + HTTP=$(curl -sS -o /dev/null -w "%{http_code}" --cacert "${CACERT}" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + -H "Content-Type: application/json" \ + -X POST -d "${RB_BODY}" \ + "${APISERVER}/apis/rbac.authorization.k8s.io/v1/namespaces/${IMAGE_NS}/rolebindings" 2>/dev/null) + case "${HTTP}" in + 20[0-9]|409) echo "RoleBinding ready (HTTP ${HTTP})." ;; + *) echo "WARNING: RoleBinding create returned HTTP ${HTTP}, continuing..." ;; + esac + + mkdir -p /tmp/auth + AUTH_TOKEN=$(echo -n "ignored:${SA_TOKEN}" | base64 -w0) + printf '{"auths":{"%s":{"auth":"%s"}}}' "${PUSH_REGISTRY}" "${AUTH_TOKEN}" > /tmp/auth/config.json + AUTH_ARGS="-a /tmp/auth/config.json" + else + PUSH_REGISTRY="${REGISTRY_DOMAIN}" + PUSH_IMAGE="${TARGET_IMAGE}" + + if [ -f /var/run/secrets/registry/.dockerconfigjson ]; then + AUTH_ARGS="-a /var/run/secrets/registry/.dockerconfigjson" + else + echo "WARNING: No registry auth secret found, attempting without credentials..." + AUTH_ARGS="" + fi + fi + + echo "Waiting for registry at ${PUSH_REGISTRY} to become reachable..." + MAX_WAIT=900 + WAITED=0 + while [ ${WAITED} -lt ${MAX_WAIT} ]; do + if [ "${IS_EMBEDDED}" = "true" ]; then + HTTP=$(curl -sk -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + "https://${PUSH_REGISTRY}/v2/" 2>/dev/null) + else + HTTP=$(curl -sk -o /dev/null -w "%{http_code}" "https://${PUSH_REGISTRY}/v2/" 2>/dev/null) + fi + if [ "${HTTP}" = "200" ] || [ "${HTTP}" = "401" ] || [ "${HTTP}" = "301" ]; then + echo "Registry is reachable (HTTP ${HTTP})." + break + fi + echo "Registry not ready (HTTP ${HTTP}), retrying in 15s... (${WAITED}/${MAX_WAIT}s)" + sleep 15 + WAITED=$((WAITED + 15)) + done + if [ ${WAITED} -ge ${MAX_WAIT} ]; then + echo "ERROR: Registry not reachable after ${MAX_WAIT}s, giving up." + exit 1 + fi + + echo "Checking if image already exists at ${PUSH_IMAGE}..." + if oc image info "${PUSH_IMAGE}" --insecure=true 2>/dev/null; then + echo "Image already exists, skipping seed." + exit 0 + fi + + echo "Mirroring ${SOURCE_IMAGE} -> ${PUSH_IMAGE}..." + oc image mirror "${SOURCE_IMAGE}" "${PUSH_IMAGE}" \ + --insecure=true --skip-missing=true \ + ${AUTH_ARGS} \ + --filter-by-os="linux/amd64" \ + --keep-manifest-list=false + + echo "Seed image push complete." +{{- end }} diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index abadf788..cd70b59b 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -83,6 +83,14 @@ app: # QTodo OIDC secret path (app-level isolation) vaultPath: "secret/data/apps/qtodo/qtodo-oidc-client" + # Seed image Job: mirrors the upstream qtodo image into the configured + # registry so the deployment can pull before the supply-chain pipeline runs. + seedImage: + enabled: false + source: "quay.io/validatedpatterns/qtodo:latest" + tag: "latest" + image: "registry.redhat.io/openshift4/ose-tools-rhel9:latest" + # Truststore configuration for Java CA certificates (PKCS12 format) truststore: enabled: true diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index 8a62af6a..5c4b6385 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -308,4 +308,14 @@ spec: - name: oidc-identity value: $(params.oidc-identity) - name: oidc-issuer - value: $(params.oidc-issuer) \ No newline at end of file + value: $(params.oidc-issuer) + + finally: + - name: restart-qtodo + when: + - input: $(tasks.qtodo-verify-image.status) + operator: in + values: ["Succeeded"] + taskRef: + name: restart-qtodo + kind: Task \ No newline at end of file diff --git a/charts/supply-chain/templates/pipelinerun-qtodo.yaml b/charts/supply-chain/templates/pipelinerun-qtodo.yaml index 820c8da4..be64f3b5 100644 --- a/charts/supply-chain/templates/pipelinerun-qtodo.yaml +++ b/charts/supply-chain/templates/pipelinerun-qtodo.yaml @@ -1,21 +1,90 @@ {{- if .Values.pipelinerun.enabled }} --- -apiVersion: tekton.dev/v1beta1 -kind: PipelineRun +apiVersion: v1 +kind: ServiceAccount metadata: - generateName: qtodo-supply-chain- + name: pipelinerun-launcher + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pipelinerun-launcher + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +rules: +- apiGroups: ["tekton.dev"] + resources: ["pipelineruns"] + verbs: ["create"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + resourceNames: [{{ .Values.registry.authSecretName | quote }}] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + resourceNames: ["qtodo-workspace-source"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pipelinerun-launcher + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pipelinerun-launcher +subjects: +- kind: ServiceAccount + name: pipelinerun-launcher + namespace: {{ .Values.global.namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: launch-qtodo-pipeline namespace: {{ .Values.global.namespace }} annotations: argocd.argoproj.io/hook: PostSync argocd.argoproj.io/hook-delete-policy: BeforeHookCreation spec: - pipelineRef: - name: qtodo-supply-chain - workspaces: - - name: qtodo-source - persistentVolumeClaim: - claimName: qtodo-workspace-source - - name: registry-auth-config - secret: - secretName: {{ .Values.registry.authSecretName }} + backoffLimit: 2 + activeDeadlineSeconds: 120 + template: + spec: + serviceAccountName: pipelinerun-launcher + restartPolicy: Never + containers: + - name: launcher + image: registry.redhat.io/openshift4/ose-tools-rhel9:latest + command: + - /bin/sh + - -ce + - | + cat <<'MANIFEST' | oc create -f - + apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: qtodo-supply-chain- + namespace: {{ .Values.global.namespace }} + spec: + pipelineRef: + name: qtodo-supply-chain + workspaces: + - name: qtodo-source + persistentVolumeClaim: + claimName: qtodo-workspace-source + - name: registry-auth-config + secret: + secretName: {{ .Values.registry.authSecretName }} + MANIFEST + echo "PipelineRun created successfully." {{- end }} diff --git a/charts/supply-chain/templates/rbac/pipeline-qtodo-restarter.yaml b/charts/supply-chain/templates/rbac/pipeline-qtodo-restarter.yaml new file mode 100644 index 00000000..5eafeefd --- /dev/null +++ b/charts/supply-chain/templates/rbac/pipeline-qtodo-restarter.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pipeline-qtodo-restarter + namespace: qtodo +rules: +- apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["qtodo"] + verbs: ["get", "patch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["list"] +- apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["list", "get"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pipeline-qtodo-restarter + namespace: qtodo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pipeline-qtodo-restarter +subjects: +- kind: ServiceAccount + name: pipeline + namespace: {{ .Values.global.namespace }} diff --git a/charts/supply-chain/templates/tasks/restart-qtodo.yaml b/charts/supply-chain/templates/tasks/restart-qtodo.yaml new file mode 100644 index 00000000..dfc83e71 --- /dev/null +++ b/charts/supply-chain/templates/tasks/restart-qtodo.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: restart-qtodo + namespace: {{ .Values.global.namespace }} +spec: + description: Restart the qtodo deployment to pick up the latest built image. + steps: + - name: restart + image: registry.redhat.io/openshift4/ose-tools-rhel9:latest + resources: {} + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "Restarting qtodo deployment to pull the latest image..." + oc rollout restart deployment/qtodo -n qtodo + echo "Waiting for rollout to complete..." + oc rollout status deployment/qtodo -n qtodo --timeout=120s + echo "qtodo deployment restarted successfully." diff --git a/docs/SYNC-WAVE-INVENTORY.md b/docs/SYNC-WAVE-INVENTORY.md index 91c53779..fa7a57c1 100644 --- a/docs/SYNC-WAVE-INVENTORY.md +++ b/docs/SYNC-WAVE-INVENTORY.md @@ -46,7 +46,9 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 36 | └ keycloak | chart | keycloak.yaml (Keycloak CR) | | 36 | └ quay-registry | chart | object-bucket-claim | | 36 | └ acs-central | chart | admin-password-secret, central-htpasswd-external-secret, keycloak-client-secret-external-secret | -| 36 | └ qtodo | chart | truststore-secret-external-secret | +| 36 | └ qtodo | chart | truststore-secret-external-secret, registry-external-secret | +| 38+0 | └ qtodo | chart | registry-seed SA, ClusterRole, ClusterRoleBinding | +| 38+5 | └ qtodo | chart (hook) | registry-seed-image (Sync hook Job -- mirrors upstream image to configured registry) | | 37 | └ quay-registry | chart | quay-s3-setup-serviceaccount (5 resources) | | 37 | └ acs-central | chart | create-htpasswd-field (Job) | | 38 | qtodo | **App** | | @@ -69,7 +71,7 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 46 | └ acs-secured-cluster | chart | secured-cluster-cr | | 46 | └ rhtas-operator | chart | securesign | | 48 | supply-chain | **App** | | -| 48+0 | └ supply-chain | chart | registry-image-namespace (Namespace, RBAC), pipeline-sa, tasks, secrets (quay-pass, rhtpa-pass), quay-user, rhtas/rhtpa-config | +| 48+0 | └ supply-chain | chart | registry-image-namespace (Namespace, RBAC), pipeline-sa, tasks (incl. restart-qtodo), secrets (quay-pass, rhtpa-pass), quay-user, rhtas/rhtpa-config, pipeline-qtodo-restarter (Role+RoleBinding in qtodo ns) | | 48+1 | └ supply-chain | chart (hook) | enable-registry-default-route (Sync hook Job) | | 48+10 | └ supply-chain | chart (hook) | registry-token-refresher-seed (Sync hook Job — writes SA token to Vault) | | 48+15 | └ supply-chain | chart | qtodo-registry-auth ExternalSecret (reads token from Vault) | @@ -230,7 +232,10 @@ Charts marked **(external)** have been externalized to standalone repositories m | Resource | Old | Current | | --- | ---: | ---: | +| registry-seed-job.yaml (SA, ClusterRole, ClusterRoleBinding) | --- | 0 | +| registry-seed-job.yaml (Sync hook Job) | --- | 5 | | truststore-secret-external-secret.yaml | 5 | 36 | +| registry-external-secret.yaml | --- | 36 | | postgresql-statefulset.yaml | 10 | 41 | | postgresql-service.yaml | 10 | 41 | | qtodo-truststore-config.yaml | 10 | 41 | @@ -239,7 +244,7 @@ Charts marked **(external)** have been externalized to standalone repositories m ### supply-chain (`charts/supply-chain/templates/`) — App wave: 48 -Resources without an explicit sync-wave default to wave 0. These include: pipeline-sa, pipeline-qtodo, tasks/*, secrets/qtodo-quay-pass, secrets/qtodo-rhtpa-pass, rhtas-config, rhtpa-config, quay/quay-user-cm, quay/quay-user-job. +Resources without an explicit sync-wave default to wave 0. These include: pipeline-sa, pipeline-qtodo, tasks/* (incl. restart-qtodo), secrets/qtodo-quay-pass, secrets/qtodo-rhtpa-pass, rhtas-config, rhtpa-config, quay/quay-user-cm, quay/quay-user-job, pipeline-qtodo-restarter (Role+RoleBinding). | Resource | Old | Current | Notes | | --- | ---: | ---: | --- | @@ -248,7 +253,8 @@ Resources without an explicit sync-wave default to wave 0. These include: pipeli | registry-token-refresher-seed (Sync hook Job) | — | 10 | Writes initial SA token to Vault | | qtodo-registry-auth ExternalSecret | — | 15 | Reads registry token from Vault; must run after seed Job | | workspaces.yaml | 20 | 51 | Pipeline PVCs | -| pipelinerun-qtodo.yaml (PostSync hook) | — | — | Triggers pipeline run after sync completes | +| pipelinerun-qtodo.yaml (PostSync hook Job + RBAC) | — | — | Job wraps `oc create` of PipelineRun (PipelineRun is excluded from ArgoCD tracking) | +| pipeline-qtodo.yaml `finally` section | — | — | restart-qtodo Task runs after successful verify-image | ### docs/DEVELOPMENT.md (example snippet, not deployed) diff --git a/scripts/gen-byo-container-registry-variants.py b/scripts/gen-byo-container-registry-variants.py index 0448e0a9..1d6cc374 100755 --- a/scripts/gen-byo-container-registry-variants.py +++ b/scripts/gen-byo-container-registry-variants.py @@ -185,15 +185,21 @@ def enable_supply_chain_app(lines, option_num): final.append(line) continue - # Always uncomment RHTAS and RHTPA flags - if re.search(r"# - name: rhtas\.enabled", line) or re.search( - r"# - name: rhtpa\.enabled", line + # Always uncomment RHTAS, RHTPA, and pipelinerun flags + if ( + re.search(r"# - name: rhtas\.enabled", line) + or re.search(r"# - name: rhtpa\.enabled", line) + or re.search(r"# - name: pipelinerun\.enabled", line) ): final.append(uncomment_line(line)) continue if re.search(r"#\s+value:", line) and final: prev = final[-1] - if "rhtas.enabled" in prev or "rhtpa.enabled" in prev: + if ( + "rhtas.enabled" in prev + or "rhtpa.enabled" in prev + or "pipelinerun.enabled" in prev + ): final.append(uncomment_line(line)) continue @@ -311,7 +317,7 @@ def apply_common_supply_chain(lines): lines, r"# rhtas-operator:", r"#\s*(rhtas-operator:|name: rhtas-operator" - r"|namespace: openshift-operators|channel: stable\s*$" + r"|namespace: openshift-operators|channel: stable-v1\.3" r"|annotations:" r"|argocd\.argoproj\.io/sync-wave.*29" r"|catalogSource: redhat-operators)", @@ -381,6 +387,21 @@ def apply_common_supply_chain(lines): prev_re=r"Depends on:", ) + # qtodo override: enable seed image job + new = [] + for idx, line in enumerate(lines): + if re.search(r"# - name: app\.seedImage\.enabled", line): + new.append(uncomment_line(line)) + elif ( + re.search(r'#\s+value: "true"', line) + and new + and "app.seedImage.enabled" in new[-1] + ): + new.append(uncomment_line(line)) + else: + new.append(line) + lines = new + return lines diff --git a/values-coco-dev.yaml b/values-coco-dev.yaml index 70c956f0..614f7083 100644 --- a/values-coco-dev.yaml +++ b/values-coco-dev.yaml @@ -164,7 +164,7 @@ clusterGroup: # rhtas-operator: # name: rhtas-operator # namespace: openshift-operators - # channel: stable + # channel: stable-v1.3 # annotations: # argocd.argoproj.io/sync-wave: "29" # Install after Quay operator, before applications # catalogSource: redhat-operators diff --git a/values-hub.yaml b/values-hub.yaml index f5736a49..c021c512 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -169,7 +169,7 @@ clusterGroup: # rhtas-operator: # name: rhtas-operator # namespace: openshift-operators - # channel: stable + # channel: stable-v1.3 # annotations: # argocd.argoproj.io/sync-wave: "29" # Install after Quay operator, before applications # catalogSource: redhat-operators @@ -542,6 +542,10 @@ clusterGroup: # Secure Supply Chain: when global.registry.enabled=true the chart # automatically derives the image from global.registry.domain/repository. # No override needed here. + # Seed image: mirror upstream qtodo image into the configured registry + # before the supply-chain pipeline runs (avoids ImagePullBackOff on first install) + # - name: app.seedImage.enabled + # value: "true" # Secure Supply Chain - Uncomment to enable (required for Option 1, 2, or 3 registry flows in docs) # supply-chain: # name: supply-chain @@ -574,6 +578,9 @@ clusterGroup: # # Enable RHTPA SBOM upload # # - name: rhtpa.enabled # # value: "true" + # # Auto-trigger pipeline run on ArgoCD sync + # # - name: pipelinerun.enabled + # # value: "true" # # ACS Central Services acs-central: @@ -633,34 +640,56 @@ clusterGroup: argocd.argoproj.io/sync-wave: "51" argoCD: resourceHealthChecks: - - check: | - local hs = {} + - check: | + local hs = {} + if obj.status ~= nil and obj.status.phase ~= nil then + if obj.status.phase == "Bound" then + hs.status = "Healthy" + hs.message = "PVC is bound" + elseif obj.status.phase == "Pending" then + hs.status = "Healthy" + hs.message = "Waiting for first consumer" + elseif obj.status.phase == "Lost" then + hs.status = "Degraded" + hs.message = "PVC is lost" + else + hs.status = "Progressing" + hs.message = obj.status.phase + end + else hs.status = "Progressing" - hs.message = "Waiting for status update." - if obj.status ~= nil then - if obj.status.conditions ~= nil then - for i, condition in ipairs(obj.status.conditions) do - if condition.type == "Done" and condition.status == "True" then - hs.status = "Healthy" - hs.message = condition.message - return hs - end - if condition.type == "Started" and condition.status == "True" then - hs.status = "Progressing" - hs.message = "Realm import is running" - return hs - end - if condition.type == "HasErrors" and condition.status == "True" then - hs.status = "Degraded" - hs.message = condition.message - return hs - end + hs.message = "Waiting for PVC status" + end + return hs + kind: PersistentVolumeClaim + - check: | + local hs = {} + hs.status = "Progressing" + hs.message = "Waiting for status update." + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Done" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end + if condition.type == "Started" and condition.status == "True" then + hs.status = "Progressing" + hs.message = "Realm import is running" + return hs + end + if condition.type == "HasErrors" and condition.status == "True" then + hs.status = "Degraded" + hs.message = condition.message + return hs end end end - return hs - group: k8s.keycloak.org - kind: KeycloakRealmImport + end + return hs + group: k8s.keycloak.org + kind: KeycloakRealmImport resourceExclusions: | - apiGroups: - internal.open-cluster-management.io From 580d8806d907c773e472a97f14fa0f519ceb6117 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 02:14:58 -0400 Subject: [PATCH 15/21] fix: make registry-seed-image job best-effort to prevent ArgoCD retry loops A Sync hook failure triggers ArgoCD retry loops for the entire sync operation (up to 20 retries with timeouts), blocking all higher sync waves and creating an unrecoverable deadlock. Wrap the seed logic in a function guarded by an unconditional exit 0 so the hook always succeeds regardless of registry availability: - No auth secret (built-in Quay first install): skip gracefully - Registry unreachable: warn and exit 0 - Mirror failure: warn and exit 0 Also tune Job parameters: - backoffLimit: 0 (K8s retries unnecessary since job always exits 0) - activeDeadlineSeconds: 600 (hard safety net) - Registry poll MAX_WAIT: 480s (enough for embedded registry startup) - hook-delete-policy: BeforeHookCreation,HookSucceeded Signed-off-by: Min Zhang --- charts/qtodo/templates/registry-seed-job.yaml | 176 ++++++++++-------- 1 file changed, 96 insertions(+), 80 deletions(-) diff --git a/charts/qtodo/templates/registry-seed-job.yaml b/charts/qtodo/templates/registry-seed-job.yaml index 22bbde6a..68f4aa48 100644 --- a/charts/qtodo/templates/registry-seed-job.yaml +++ b/charts/qtodo/templates/registry-seed-job.yaml @@ -55,10 +55,10 @@ metadata: annotations: argocd.argoproj.io/sync-wave: "5" argocd.argoproj.io/hook: Sync - argocd.argoproj.io/hook-delete-policy: HookSucceeded + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded spec: - backoffLimit: 3 - activeDeadlineSeconds: 1200 + backoffLimit: 0 + activeDeadlineSeconds: 600 template: spec: serviceAccountName: registry-seed @@ -92,95 +92,111 @@ spec: {{- end }} command: - /bin/sh - - -ce + - -c - | - APISERVER="https://kubernetes.default.svc" - SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" - CACERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + # Best-effort seed: this job must NEVER fail because a Sync hook + # failure triggers ArgoCD retry loops that block the entire app sync. + # The supply-chain pipeline is the authoritative image builder; + # this seed is only an optimisation for first-install UX. + seed_image() { + set -euo pipefail + APISERVER="https://kubernetes.default.svc" + SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + CACERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - TARGET_IMAGE="${REGISTRY_DOMAIN}/${REGISTRY_REPOSITORY}:${TAG}" + TARGET_IMAGE="${REGISTRY_DOMAIN}/${REGISTRY_REPOSITORY}:${TAG}" - if [ "${IS_EMBEDDED}" = "true" ]; then - PUSH_REGISTRY="image-registry.openshift-image-registry.svc:5000" - PUSH_IMAGE="${PUSH_REGISTRY}/${REGISTRY_REPOSITORY}:${TAG}" - - IMAGE_NS=$(echo "${REGISTRY_REPOSITORY}" | cut -d/ -f1) - echo "Ensuring namespace '${IMAGE_NS}' exists..." - NS_BODY="{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"name\":\"${IMAGE_NS}\"}}" - HTTP=$(curl -sS -o /dev/null -w "%{http_code}" --cacert "${CACERT}" \ - -H "Authorization: Bearer ${SA_TOKEN}" \ - -H "Content-Type: application/json" \ - -X POST -d "${NS_BODY}" \ - "${APISERVER}/api/v1/namespaces" 2>/dev/null) - case "${HTTP}" in - 20[0-9]|409) echo "Namespace ready (HTTP ${HTTP})." ;; - *) echo "WARNING: namespace create returned HTTP ${HTTP}, continuing..." ;; - esac + if [ "${IS_EMBEDDED}" = "true" ]; then + PUSH_REGISTRY="image-registry.openshift-image-registry.svc:5000" + PUSH_IMAGE="${PUSH_REGISTRY}/${REGISTRY_REPOSITORY}:${TAG}" - echo "Granting system:image-builder to registry-seed SA..." - RB_BODY="{\"apiVersion\":\"rbac.authorization.k8s.io/v1\",\"kind\":\"RoleBinding\",\"metadata\":{\"name\":\"seed-image-builder\",\"namespace\":\"${IMAGE_NS}\"},\"roleRef\":{\"apiGroup\":\"rbac.authorization.k8s.io\",\"kind\":\"ClusterRole\",\"name\":\"system:image-builder\"},\"subjects\":[{\"kind\":\"ServiceAccount\",\"name\":\"registry-seed\",\"namespace\":\"{{ .Release.Namespace | default "qtodo" }}\"}]}" - HTTP=$(curl -sS -o /dev/null -w "%{http_code}" --cacert "${CACERT}" \ - -H "Authorization: Bearer ${SA_TOKEN}" \ - -H "Content-Type: application/json" \ - -X POST -d "${RB_BODY}" \ - "${APISERVER}/apis/rbac.authorization.k8s.io/v1/namespaces/${IMAGE_NS}/rolebindings" 2>/dev/null) - case "${HTTP}" in - 20[0-9]|409) echo "RoleBinding ready (HTTP ${HTTP})." ;; - *) echo "WARNING: RoleBinding create returned HTTP ${HTTP}, continuing..." ;; - esac + IMAGE_NS=$(echo "${REGISTRY_REPOSITORY}" | cut -d/ -f1) + echo "Ensuring namespace '${IMAGE_NS}' exists..." + NS_BODY="{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"name\":\"${IMAGE_NS}\"}}" + HTTP=$(curl -sS -o /dev/null -w "%{http_code}" --cacert "${CACERT}" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + -H "Content-Type: application/json" \ + -X POST -d "${NS_BODY}" \ + "${APISERVER}/api/v1/namespaces" 2>/dev/null) + case "${HTTP}" in + 20[0-9]|409) echo "Namespace ready (HTTP ${HTTP})." ;; + *) echo "WARNING: namespace create returned HTTP ${HTTP}, continuing..." ;; + esac - mkdir -p /tmp/auth - AUTH_TOKEN=$(echo -n "ignored:${SA_TOKEN}" | base64 -w0) - printf '{"auths":{"%s":{"auth":"%s"}}}' "${PUSH_REGISTRY}" "${AUTH_TOKEN}" > /tmp/auth/config.json - AUTH_ARGS="-a /tmp/auth/config.json" - else - PUSH_REGISTRY="${REGISTRY_DOMAIN}" - PUSH_IMAGE="${TARGET_IMAGE}" + echo "Granting system:image-builder to registry-seed SA..." + RB_BODY="{\"apiVersion\":\"rbac.authorization.k8s.io/v1\",\"kind\":\"RoleBinding\",\"metadata\":{\"name\":\"seed-image-builder\",\"namespace\":\"${IMAGE_NS}\"},\"roleRef\":{\"apiGroup\":\"rbac.authorization.k8s.io\",\"kind\":\"ClusterRole\",\"name\":\"system:image-builder\"},\"subjects\":[{\"kind\":\"ServiceAccount\",\"name\":\"registry-seed\",\"namespace\":\"{{ .Release.Namespace | default "qtodo" }}\"}]}" + HTTP=$(curl -sS -o /dev/null -w "%{http_code}" --cacert "${CACERT}" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + -H "Content-Type: application/json" \ + -X POST -d "${RB_BODY}" \ + "${APISERVER}/apis/rbac.authorization.k8s.io/v1/namespaces/${IMAGE_NS}/rolebindings" 2>/dev/null) + case "${HTTP}" in + 20[0-9]|409) echo "RoleBinding ready (HTTP ${HTTP})." ;; + *) echo "WARNING: RoleBinding create returned HTTP ${HTTP}, continuing..." ;; + esac - if [ -f /var/run/secrets/registry/.dockerconfigjson ]; then - AUTH_ARGS="-a /var/run/secrets/registry/.dockerconfigjson" + mkdir -p /tmp/auth + AUTH_TOKEN=$(echo -n "ignored:${SA_TOKEN}" | base64 -w0) + printf '{"auths":{"%s":{"auth":"%s"}}}' "${PUSH_REGISTRY}" "${AUTH_TOKEN}" > /tmp/auth/config.json + AUTH_ARGS="-a /tmp/auth/config.json" else - echo "WARNING: No registry auth secret found, attempting without credentials..." - AUTH_ARGS="" + PUSH_REGISTRY="${REGISTRY_DOMAIN}" + PUSH_IMAGE="${TARGET_IMAGE}" + + if [ -f /var/run/secrets/registry/.dockerconfigjson ]; then + AUTH_ARGS="-a /var/run/secrets/registry/.dockerconfigjson" + else + echo "INFO: Registry auth secret not yet available (expected on first install)." + echo "Skipping seed -- the supply-chain pipeline will push the image later." + return 0 + fi fi - fi - echo "Waiting for registry at ${PUSH_REGISTRY} to become reachable..." - MAX_WAIT=900 - WAITED=0 - while [ ${WAITED} -lt ${MAX_WAIT} ]; do - if [ "${IS_EMBEDDED}" = "true" ]; then - HTTP=$(curl -sk -o /dev/null -w "%{http_code}" \ - -H "Authorization: Bearer ${SA_TOKEN}" \ - "https://${PUSH_REGISTRY}/v2/" 2>/dev/null) - else - HTTP=$(curl -sk -o /dev/null -w "%{http_code}" "https://${PUSH_REGISTRY}/v2/" 2>/dev/null) + echo "Waiting for registry at ${PUSH_REGISTRY} to become reachable..." + MAX_WAIT=480 + WAITED=0 + while [ ${WAITED} -lt ${MAX_WAIT} ]; do + if [ "${IS_EMBEDDED}" = "true" ]; then + HTTP=$(curl -sk -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer ${SA_TOKEN}" \ + "https://${PUSH_REGISTRY}/v2/" 2>/dev/null) + else + HTTP=$(curl -sk -o /dev/null -w "%{http_code}" "https://${PUSH_REGISTRY}/v2/" 2>/dev/null) + fi + if [ "${HTTP}" = "200" ] || [ "${HTTP}" = "401" ] || [ "${HTTP}" = "301" ]; then + echo "Registry is reachable (HTTP ${HTTP})." + break + fi + echo "Registry not ready (HTTP ${HTTP}), retrying in 15s... (${WAITED}/${MAX_WAIT}s)" + sleep 15 + WAITED=$((WAITED + 15)) + done + if [ ${WAITED} -ge ${MAX_WAIT} ]; then + echo "WARNING: Registry not reachable after ${MAX_WAIT}s." + return 1 fi - if [ "${HTTP}" = "200" ] || [ "${HTTP}" = "401" ] || [ "${HTTP}" = "301" ]; then - echo "Registry is reachable (HTTP ${HTTP})." - break + + echo "Checking if image already exists at ${PUSH_IMAGE}..." + if oc image info "${PUSH_IMAGE}" --insecure=true 2>/dev/null; then + echo "Image already exists, skipping seed." + return 0 fi - echo "Registry not ready (HTTP ${HTTP}), retrying in 15s... (${WAITED}/${MAX_WAIT}s)" - sleep 15 - WAITED=$((WAITED + 15)) - done - if [ ${WAITED} -ge ${MAX_WAIT} ]; then - echo "ERROR: Registry not reachable after ${MAX_WAIT}s, giving up." - exit 1 - fi - echo "Checking if image already exists at ${PUSH_IMAGE}..." - if oc image info "${PUSH_IMAGE}" --insecure=true 2>/dev/null; then - echo "Image already exists, skipping seed." - exit 0 - fi + echo "Mirroring ${SOURCE_IMAGE} -> ${PUSH_IMAGE}..." + oc image mirror "${SOURCE_IMAGE}" "${PUSH_IMAGE}" \ + --insecure=true --skip-missing=true \ + ${AUTH_ARGS} \ + --filter-by-os="linux/amd64" \ + --keep-manifest-list=false - echo "Mirroring ${SOURCE_IMAGE} -> ${PUSH_IMAGE}..." - oc image mirror "${SOURCE_IMAGE}" "${PUSH_IMAGE}" \ - --insecure=true --skip-missing=true \ - ${AUTH_ARGS} \ - --filter-by-os="linux/amd64" \ - --keep-manifest-list=false + echo "Seed image push complete." + } - echo "Seed image push complete." + if seed_image; then + echo "SUCCESS: Seed image job finished." + else + echo "WARNING: Seed image job could not push the image (best-effort)." + echo "The supply-chain pipeline will build and push the image later." + fi + exit 0 {{- end }} From facfacd06572c6804e21e012811a844a0ceafe41 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 07:55:19 -0400 Subject: [PATCH 16/21] fix: add registry readiness check to pipeline launcher job On fresh install the PostSync hook can fire before the built-in Quay registry is fully ready, causing the pipeline to fail when pushing images. Add a readiness poll loop that waits up to 480s for the registry /v2/ endpoint to respond before creating the PipelineRun. Also increase activeDeadlineSeconds from 120 to 600 to accommodate the wait. Works for all registry types (built-in Quay, embedded OpenShift, BYO) since it uses global.registry.domain. Signed-off-by: Min Zhang --- .../templates/pipelinerun-qtodo.yaml | 24 +++++- values-hub.yaml | 86 +++++++++---------- 2 files changed, 66 insertions(+), 44 deletions(-) diff --git a/charts/supply-chain/templates/pipelinerun-qtodo.yaml b/charts/supply-chain/templates/pipelinerun-qtodo.yaml index be64f3b5..36b52d4b 100644 --- a/charts/supply-chain/templates/pipelinerun-qtodo.yaml +++ b/charts/supply-chain/templates/pipelinerun-qtodo.yaml @@ -57,7 +57,7 @@ metadata: argocd.argoproj.io/hook-delete-policy: BeforeHookCreation spec: backoffLimit: 2 - activeDeadlineSeconds: 120 + activeDeadlineSeconds: 600 template: spec: serviceAccountName: pipelinerun-launcher @@ -65,10 +65,32 @@ spec: containers: - name: launcher image: registry.redhat.io/openshift4/ose-tools-rhel9:latest + env: + - name: REGISTRY_DOMAIN + value: {{ tpl (default "" .Values.global.registry.domain) $ | quote }} command: - /bin/sh - -ce - | + if [ -n "${REGISTRY_DOMAIN}" ]; then + echo "Waiting for registry ${REGISTRY_DOMAIN} to become ready..." + MAX_WAIT=480 + WAITED=0 + while [ ${WAITED} -lt ${MAX_WAIT} ]; do + HTTP=$(curl -sk -o /dev/null -w "%{http_code}" "https://${REGISTRY_DOMAIN}/v2/" 2>/dev/null) + if [ "${HTTP}" = "200" ] || [ "${HTTP}" = "401" ] || [ "${HTTP}" = "301" ]; then + echo "Registry is ready (HTTP ${HTTP})." + break + fi + echo "Registry not ready (HTTP ${HTTP}), retrying in 15s... (${WAITED}/${MAX_WAIT}s)" + sleep 15 + WAITED=$((WAITED + 15)) + done + if [ ${WAITED} -ge ${MAX_WAIT} ]; then + echo "ERROR: Registry not ready after ${MAX_WAIT}s, launching pipeline anyway." + fi + fi + cat <<'MANIFEST' | oc create -f - apiVersion: tekton.dev/v1beta1 kind: PipelineRun diff --git a/values-hub.yaml b/values-hub.yaml index c021c512..1119f4bb 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -640,56 +640,56 @@ clusterGroup: argocd.argoproj.io/sync-wave: "51" argoCD: resourceHealthChecks: - - check: | - local hs = {} - if obj.status ~= nil and obj.status.phase ~= nil then - if obj.status.phase == "Bound" then - hs.status = "Healthy" - hs.message = "PVC is bound" - elseif obj.status.phase == "Pending" then - hs.status = "Healthy" - hs.message = "Waiting for first consumer" - elseif obj.status.phase == "Lost" then - hs.status = "Degraded" - hs.message = "PVC is lost" + - check: | + local hs = {} + if obj.status ~= nil and obj.status.phase ~= nil then + if obj.status.phase == "Bound" then + hs.status = "Healthy" + hs.message = "PVC is bound" + elseif obj.status.phase == "Pending" then + hs.status = "Healthy" + hs.message = "Waiting for first consumer" + elseif obj.status.phase == "Lost" then + hs.status = "Degraded" + hs.message = "PVC is lost" + else + hs.status = "Progressing" + hs.message = obj.status.phase + end else hs.status = "Progressing" - hs.message = obj.status.phase + hs.message = "Waiting for PVC status" end - else + return hs + kind: PersistentVolumeClaim + - check: | + local hs = {} hs.status = "Progressing" - hs.message = "Waiting for PVC status" - end - return hs - kind: PersistentVolumeClaim - - check: | - local hs = {} - hs.status = "Progressing" - hs.message = "Waiting for status update." - if obj.status ~= nil then - if obj.status.conditions ~= nil then - for i, condition in ipairs(obj.status.conditions) do - if condition.type == "Done" and condition.status == "True" then - hs.status = "Healthy" - hs.message = condition.message - return hs - end - if condition.type == "Started" and condition.status == "True" then - hs.status = "Progressing" - hs.message = "Realm import is running" - return hs - end - if condition.type == "HasErrors" and condition.status == "True" then - hs.status = "Degraded" - hs.message = condition.message - return hs + hs.message = "Waiting for status update." + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Done" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end + if condition.type == "Started" and condition.status == "True" then + hs.status = "Progressing" + hs.message = "Realm import is running" + return hs + end + if condition.type == "HasErrors" and condition.status == "True" then + hs.status = "Degraded" + hs.message = condition.message + return hs + end end end end - end - return hs - group: k8s.keycloak.org - kind: KeycloakRealmImport + return hs + group: k8s.keycloak.org + kind: KeycloakRealmImport resourceExclusions: | - apiGroups: - internal.open-cluster-management.io From bf010f1b25cecdcd7e3927f820494d9f64c23235 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 10:43:28 -0400 Subject: [PATCH 17/21] fix: quote boolean override values as strings in values-hub.yaml The clustergroup chart requires all override values to be strings. Bare YAML booleans (true/false) cause Helm template error: "wrong type for value; expected string; got bool" Quote compliance.storage.enabled, app.oidc.enabled, and app.spire.enabled values. Signed-off-by: Min Zhang --- values-hub.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/values-hub.yaml b/values-hub.yaml index 1119f4bb..d560d072 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -313,7 +313,7 @@ clusterGroup: # via rawResultStorage in scan-setting.yaml. The explicit PVC causes # ArgoCD 'Progressing' status on storage with WaitForFirstConsumer mode. - name: compliance.storage.enabled - value: false + value: "false" vault: name: vault namespace: vault @@ -530,9 +530,9 @@ clusterGroup: - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) overrides: - name: app.oidc.enabled - value: true + value: "true" - name: app.spire.enabled - value: true + value: "true" - name: app.vault.url value: https://vault.vault.svc.cluster.local:8200 - name: app.vault.role From 9e76da00085fd63fdd2b12a213551421e5017cf9 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 11:30:42 -0400 Subject: [PATCH 18/21] docs: update supply-chain.md with new automation features Add documentation for: - First-install image availability (registry-seed-image best-effort job) - ArgoCD PVC health check for WaitForFirstConsumer storage classes - Automatic pipeline trigger via PostSync hook Job with registry readiness check - restart-qtodo finally task that restarts the deployment after a successful pipeline run - Updated Helm template section to reflect Job wrapper change Signed-off-by: Min Zhang --- docs/supply-chain.md | 82 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 7f1e584a..8dae4832 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -168,13 +168,71 @@ Set `` to match your registry option: > **Note**: Option 2 (BYO/External Registry) does not require `imagePullTrust` because external registries like quay.io and ghcr.io use publicly trusted certificates. +### First-Install Image Availability + +On a fresh install, the configured registry does not yet contain a `qtodo` image because the supply-chain pipeline has not run yet. Without intervention the `qtodo` Deployment would enter `ImagePullBackOff` until the first pipeline completes. + +The `qtodo` chart includes a **registry-seed-image** Job (an ArgoCD `Sync` hook at wave 5) that mirrors the upstream image `quay.io/validatedpatterns/qtodo:latest` into the configured registry before the Deployment is created. This gives the application a working image immediately on first deploy. + +Key design points: + +* **Best-effort** -- the Job wraps all logic in a shell function and unconditionally exits 0. If mirroring fails (e.g., registry not ready, credentials unavailable), it logs a warning but never blocks the ArgoCD sync. The supply-chain pipeline is the authoritative image builder; the seed is only a first-install UX optimization. +* **Registry readiness polling** -- for registries that may not be available immediately (built-in Quay, embedded OpenShift), the Job polls the registry endpoint for up to 8 minutes before attempting the mirror. +* **Credential handling** -- the Job checks for the `qtodo-registry-auth` secret. If not found (e.g., ExternalSecret has not synced yet), it exits gracefully. For the embedded OpenShift registry, it uses the ServiceAccount token directly. +* **Hook cleanup** -- `argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded` ensures old Jobs are cleaned up. + +The seed image is controlled by the `app.seedImage.enabled` flag in the qtodo chart values (enabled by default when `global.registry.enabled` is true). + +### ArgoCD PVC Health Check + +The supply-chain chart creates a `PersistentVolumeClaim` (`qtodo-workspace-source`) for the pipeline workspace. On clusters using a `WaitForFirstConsumer` storage class, this PVC remains in `Pending` state until a pod is scheduled -- which is expected behavior, but ArgoCD reports it as `Progressing`, preventing the application from reaching `Healthy` status. + +A custom `resourceHealthChecks` entry in `values-hub.yaml` teaches ArgoCD to treat `Pending` PVCs with `WaitForFirstConsumer` binding mode as `Healthy`: + +```yaml +resourceHealthChecks: + - kind: PersistentVolumeClaim + check: | + hs = {} + if obj.status ~= nil and obj.status.phase ~= nil then + if obj.status.phase == "Bound" then + hs.status = "Healthy" + hs.message = "PVC is bound" + elseif obj.status.phase == "Pending" then + hs.status = "Healthy" + hs.message = "PVC is pending (WaitForFirstConsumer)" + else + hs.status = "Progressing" + hs.message = "Waiting for PVC" + end + else + hs.status = "Progressing" + hs.message = "Waiting for PVC status" + end + return hs +``` + +This is important because the `PostSync` pipeline launcher hook (see below) only fires once the application is `Healthy`. + ## Automatic approach To automate the application building and certifying process, we will use _Red Hat OpenShift Pipelines_. ZTVP will create a `Pipeline` in our cluster called **qtodo-supply-chain** that will orchestrate the various tasks necessary to build the application from its source code, generate a container image, and publish the resulting image to the defined OCI registry. Within the pipeline, an SBOM containing the build's contents will be generated, binaries and the build attestation will be signed, and the validity of those signatures will be verified. -### How to run the pipeline +### Automatic pipeline trigger + +When the supply-chain application is deployed (or re-synced), a **PostSync** hook Job (`launch-qtodo-pipeline`) automatically creates a `PipelineRun` for the `qtodo-supply-chain` pipeline. This means the pipeline runs on every successful ArgoCD sync without manual intervention. + +The launcher Job includes a **registry readiness check**: before creating the PipelineRun, it polls the configured registry's `/v2/` endpoint for up to 8 minutes, waiting for a healthy HTTP response (200, 401, or 301). This prevents the pipeline from starting before the registry is available -- important on fresh installs where built-in Quay or the embedded OpenShift registry may still be starting up. If the registry is not ready after the timeout, the pipeline is launched anyway. + +The Job uses dedicated RBAC (ServiceAccount, Role, RoleBinding) scoped to creating PipelineRuns in the `layered-zero-trust-hub` namespace. All PostSync resources use `hook-delete-policy: BeforeHookCreation` so old resources are cleaned up on re-sync. + +> **Note**: ArgoCD `resourceExclusions` for `tekton.dev/PipelineRun` prevent direct PipelineRun manifests from being managed as hooks. The Job wrapper sidesteps this by creating the PipelineRun via `oc create` at runtime, outside ArgoCD's resource tracking. + +### How to run the pipeline manually + +The pipeline runs automatically on every sync (see above). If you need to trigger it manually, use one of the methods below. #### Using OpenShift Web Console @@ -227,23 +285,19 @@ oc create -f qtodo-pipeline.yaml #### Using Helm Template -You can also trigger a pipeline run using the Helm template included in the chart. - -**For Built-in Quay Registry:** +> **Note**: The `pipelinerun-qtodo.yaml` template renders a **Job** (the PostSync launcher) rather than a raw PipelineRun. For manual one-off runs, the CLI method above is simpler. The Helm approach is useful for inspecting or customizing the launcher Job. ```shell helm template supply-chain charts/supply-chain \ --set pipelinerun.enabled=true \ --set global.registry.enabled=true \ - --set global.registry.domain=quay-registry-quay-quay-enterprise.apps.example.com \ - --set global.registry.repository=ztvp/qtodo \ - --set global.registry.vaultPath=secret/data/hub/infra/quay/quay-users \ - --set global.registry.passwordVaultKey=quay-user-password \ + --set global.registry.domain=quay.io \ + --set global.registry.repository=your-org/qtodo \ --set global.namespace=layered-zero-trust-hub \ - --show-only templates/pipelinerun-qtodo.yaml | oc create -f - + --show-only templates/pipelinerun-qtodo.yaml ``` -**For BYO/External Registry:** +This renders the launcher Job with its RBAC resources and embedded PipelineRun manifest. To apply it directly: ```shell helm template supply-chain charts/supply-chain \ @@ -251,14 +305,10 @@ helm template supply-chain charts/supply-chain \ --set global.registry.enabled=true \ --set global.registry.domain=quay.io \ --set global.registry.repository=your-org/qtodo \ - --set global.registry.vaultPath=secret/data/hub/infra/registry/registry-user \ - --set global.registry.passwordVaultKey=registry-password \ --set global.namespace=layered-zero-trust-hub \ --show-only templates/pipelinerun-qtodo.yaml | oc create -f - ``` -This renders the PipelineRun template with the correct PVC and secret workspace bindings, then creates it in the cluster. - You can review the current pipeline logs using the [Tekton CLI](https://tekton.dev/docs/cli/). ```shell @@ -293,6 +343,10 @@ The pipeline we have prepared has the following steps: * **qtodo-upload-sbom**. Uploads the generated SBOM file to RHTPA. * **qtodo-verify-image**. Verifies the attestation and the signature attached to the image. +**Finally task:** + +* **restart-qtodo**. Runs after all tasks complete. If `qtodo-verify-image` succeeded, it restarts the `qtodo` Deployment (`oc rollout restart`) so the application picks up the newly built and signed image. This ensures the running application always reflects the latest pipeline output without manual intervention. + ### Inspecting the results #### Openshift Web UI From c30b5bc54dbcb18b51db96552b0b961e8bbdf5af Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 12:27:44 -0400 Subject: [PATCH 19/21] fix: use placeholder repository path for BYO registry option Change BYO registry repository from ztvp/qtodo to your-org/qtodo. quay.io uses org/repo format and users must replace this with their own org. Options 1 and 3 (internal registries) keep ztvp/qtodo since they control the namespace. Signed-off-by: Min Zhang --- values-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-hub.yaml b/values-hub.yaml index d560d072..81973e31 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -33,7 +33,7 @@ spire: # registry: # enabled: true # domain: quay.io -# repository: ztvp/qtodo +# repository: your-org/qtodo # user: your-username # vaultPath: "secret/data/hub/infra/registry/registry-user" # passwordVaultKey: "registry-password" From 5b4d8c83ee15060ab9176aba9ec574b68e0d8d70 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 21:50:33 -0400 Subject: [PATCH 20/21] fix: address PR review feedback - PVC health check: use generic "PVC is pending" message instead of assuming WaitForFirstConsumer binding mode - Disable auto pipeline trigger (pipelinerun.enabled) and seed image mirroring (app.seedImage.enabled) by default; keep templates gated behind flags for future enablement - Guard restart-qtodo task against missing Deployment - Remove auto-trigger and seed image sections from supply-chain docs - Remove pipelinerun/seedImage uncomment logic from gen-byo script Signed-off-by: Min Zhang --- .../templates/tasks/restart-qtodo.yaml | 4 ++ docs/supply-chain.md | 71 +++---------------- .../gen-byo-container-registry-variants.py | 29 ++------ values-hub.yaml | 6 +- 4 files changed, 20 insertions(+), 90 deletions(-) diff --git a/charts/supply-chain/templates/tasks/restart-qtodo.yaml b/charts/supply-chain/templates/tasks/restart-qtodo.yaml index dfc83e71..9fe23387 100644 --- a/charts/supply-chain/templates/tasks/restart-qtodo.yaml +++ b/charts/supply-chain/templates/tasks/restart-qtodo.yaml @@ -13,6 +13,10 @@ spec: script: | #!/usr/bin/env bash set -euo pipefail + if ! oc get deployment/qtodo -n qtodo >/dev/null 2>&1; then + echo "qtodo deployment not found in namespace qtodo -- skipping restart." + exit 0 + fi echo "Restarting qtodo deployment to pull the latest image..." oc rollout restart deployment/qtodo -n qtodo echo "Waiting for rollout to complete..." diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 8dae4832..553dab5d 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -168,26 +168,11 @@ Set `` to match your registry option: > **Note**: Option 2 (BYO/External Registry) does not require `imagePullTrust` because external registries like quay.io and ghcr.io use publicly trusted certificates. -### First-Install Image Availability - -On a fresh install, the configured registry does not yet contain a `qtodo` image because the supply-chain pipeline has not run yet. Without intervention the `qtodo` Deployment would enter `ImagePullBackOff` until the first pipeline completes. - -The `qtodo` chart includes a **registry-seed-image** Job (an ArgoCD `Sync` hook at wave 5) that mirrors the upstream image `quay.io/validatedpatterns/qtodo:latest` into the configured registry before the Deployment is created. This gives the application a working image immediately on first deploy. - -Key design points: - -* **Best-effort** -- the Job wraps all logic in a shell function and unconditionally exits 0. If mirroring fails (e.g., registry not ready, credentials unavailable), it logs a warning but never blocks the ArgoCD sync. The supply-chain pipeline is the authoritative image builder; the seed is only a first-install UX optimization. -* **Registry readiness polling** -- for registries that may not be available immediately (built-in Quay, embedded OpenShift), the Job polls the registry endpoint for up to 8 minutes before attempting the mirror. -* **Credential handling** -- the Job checks for the `qtodo-registry-auth` secret. If not found (e.g., ExternalSecret has not synced yet), it exits gracefully. For the embedded OpenShift registry, it uses the ServiceAccount token directly. -* **Hook cleanup** -- `argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded` ensures old Jobs are cleaned up. - -The seed image is controlled by the `app.seedImage.enabled` flag in the qtodo chart values (enabled by default when `global.registry.enabled` is true). - ### ArgoCD PVC Health Check -The supply-chain chart creates a `PersistentVolumeClaim` (`qtodo-workspace-source`) for the pipeline workspace. On clusters using a `WaitForFirstConsumer` storage class, this PVC remains in `Pending` state until a pod is scheduled -- which is expected behavior, but ArgoCD reports it as `Progressing`, preventing the application from reaching `Healthy` status. +The supply-chain chart creates a `PersistentVolumeClaim` (`qtodo-workspace-source`) for the pipeline workspace. Depending on the storage class, this PVC may remain in `Pending` state until a pod is scheduled -- which is expected behavior, but ArgoCD reports it as `Progressing`, preventing the application from reaching `Healthy` status. -A custom `resourceHealthChecks` entry in `values-hub.yaml` teaches ArgoCD to treat `Pending` PVCs with `WaitForFirstConsumer` binding mode as `Healthy`: +A custom `resourceHealthChecks` entry in `values-hub.yaml` teaches ArgoCD to treat `Pending` PVCs as `Healthy`: ```yaml resourceHealthChecks: @@ -200,7 +185,7 @@ resourceHealthChecks: hs.message = "PVC is bound" elseif obj.status.phase == "Pending" then hs.status = "Healthy" - hs.message = "PVC is pending (WaitForFirstConsumer)" + hs.message = "PVC is pending" else hs.status = "Progressing" hs.message = "Waiting for PVC" @@ -212,27 +197,15 @@ resourceHealthChecks: return hs ``` -This is important because the `PostSync` pipeline launcher hook (see below) only fires once the application is `Healthy`. - -## Automatic approach - -To automate the application building and certifying process, we will use _Red Hat OpenShift Pipelines_. - -ZTVP will create a `Pipeline` in our cluster called **qtodo-supply-chain** that will orchestrate the various tasks necessary to build the application from its source code, generate a container image, and publish the resulting image to the defined OCI registry. Within the pipeline, an SBOM containing the build's contents will be generated, binaries and the build attestation will be signed, and the validity of those signatures will be verified. - -### Automatic pipeline trigger - -When the supply-chain application is deployed (or re-synced), a **PostSync** hook Job (`launch-qtodo-pipeline`) automatically creates a `PipelineRun` for the `qtodo-supply-chain` pipeline. This means the pipeline runs on every successful ArgoCD sync without manual intervention. +## Pipeline -The launcher Job includes a **registry readiness check**: before creating the PipelineRun, it polls the configured registry's `/v2/` endpoint for up to 8 minutes, waiting for a healthy HTTP response (200, 401, or 301). This prevents the pipeline from starting before the registry is available -- important on fresh installs where built-in Quay or the embedded OpenShift registry may still be starting up. If the registry is not ready after the timeout, the pipeline is launched anyway. +To build and certify the application, we will use _Red Hat OpenShift Pipelines_. -The Job uses dedicated RBAC (ServiceAccount, Role, RoleBinding) scoped to creating PipelineRuns in the `layered-zero-trust-hub` namespace. All PostSync resources use `hook-delete-policy: BeforeHookCreation` so old resources are cleaned up on re-sync. +ZTVP creates a `Pipeline` in our cluster called **qtodo-supply-chain** that orchestrates the various tasks necessary to build the application from its source code, generate a container image, and publish the resulting image to the defined OCI registry. Within the pipeline, an SBOM containing the build's contents will be generated, binaries and the build attestation will be signed, and the validity of those signatures will be verified. -> **Note**: ArgoCD `resourceExclusions` for `tekton.dev/PipelineRun` prevent direct PipelineRun manifests from being managed as hooks. The Job wrapper sidesteps this by creating the PipelineRun via `oc create` at runtime, outside ArgoCD's resource tracking. +### How to run the pipeline -### How to run the pipeline manually - -The pipeline runs automatically on every sync (see above). If you need to trigger it manually, use one of the methods below. +Once the supply-chain application has synced in ArgoCD, start the pipeline using one of the methods below. #### Using OpenShift Web Console @@ -283,32 +256,6 @@ Using the previously created definition, start a new execution of the pipeline u oc create -f qtodo-pipeline.yaml ``` -#### Using Helm Template - -> **Note**: The `pipelinerun-qtodo.yaml` template renders a **Job** (the PostSync launcher) rather than a raw PipelineRun. For manual one-off runs, the CLI method above is simpler. The Helm approach is useful for inspecting or customizing the launcher Job. - -```shell -helm template supply-chain charts/supply-chain \ - --set pipelinerun.enabled=true \ - --set global.registry.enabled=true \ - --set global.registry.domain=quay.io \ - --set global.registry.repository=your-org/qtodo \ - --set global.namespace=layered-zero-trust-hub \ - --show-only templates/pipelinerun-qtodo.yaml -``` - -This renders the launcher Job with its RBAC resources and embedded PipelineRun manifest. To apply it directly: - -```shell -helm template supply-chain charts/supply-chain \ - --set pipelinerun.enabled=true \ - --set global.registry.enabled=true \ - --set global.registry.domain=quay.io \ - --set global.registry.repository=your-org/qtodo \ - --set global.namespace=layered-zero-trust-hub \ - --show-only templates/pipelinerun-qtodo.yaml | oc create -f - -``` - You can review the current pipeline logs using the [Tekton CLI](https://tekton.dev/docs/cli/). ```shell @@ -345,7 +292,7 @@ The pipeline we have prepared has the following steps: **Finally task:** -* **restart-qtodo**. Runs after all tasks complete. If `qtodo-verify-image` succeeded, it restarts the `qtodo` Deployment (`oc rollout restart`) so the application picks up the newly built and signed image. This ensures the running application always reflects the latest pipeline output without manual intervention. +* **restart-qtodo**. Runs after all tasks complete. If `qtodo-verify-image` succeeded and the `qtodo` Deployment exists, it restarts the Deployment (`oc rollout restart`) so the application picks up the newly built and signed image. If the Deployment is not yet present (e.g., the pipeline ran before the qtodo application was deployed), the task exits gracefully. ### Inspecting the results diff --git a/scripts/gen-byo-container-registry-variants.py b/scripts/gen-byo-container-registry-variants.py index 1d6cc374..4a586f90 100755 --- a/scripts/gen-byo-container-registry-variants.py +++ b/scripts/gen-byo-container-registry-variants.py @@ -185,21 +185,15 @@ def enable_supply_chain_app(lines, option_num): final.append(line) continue - # Always uncomment RHTAS, RHTPA, and pipelinerun flags - if ( - re.search(r"# - name: rhtas\.enabled", line) - or re.search(r"# - name: rhtpa\.enabled", line) - or re.search(r"# - name: pipelinerun\.enabled", line) + # Always uncomment RHTAS and RHTPA flags + if re.search(r"# - name: rhtas\.enabled", line) or re.search( + r"# - name: rhtpa\.enabled", line ): final.append(uncomment_line(line)) continue if re.search(r"#\s+value:", line) and final: prev = final[-1] - if ( - "rhtas.enabled" in prev - or "rhtpa.enabled" in prev - or "pipelinerun.enabled" in prev - ): + if "rhtas.enabled" in prev or "rhtpa.enabled" in prev: final.append(uncomment_line(line)) continue @@ -387,21 +381,6 @@ def apply_common_supply_chain(lines): prev_re=r"Depends on:", ) - # qtodo override: enable seed image job - new = [] - for idx, line in enumerate(lines): - if re.search(r"# - name: app\.seedImage\.enabled", line): - new.append(uncomment_line(line)) - elif ( - re.search(r'#\s+value: "true"', line) - and new - and "app.seedImage.enabled" in new[-1] - ): - new.append(uncomment_line(line)) - else: - new.append(line) - lines = new - return lines diff --git a/values-hub.yaml b/values-hub.yaml index 81973e31..dae08bf8 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -542,7 +542,7 @@ clusterGroup: # Secure Supply Chain: when global.registry.enabled=true the chart # automatically derives the image from global.registry.domain/repository. # No override needed here. - # Seed image: mirror upstream qtodo image into the configured registry + # Uncomment to seed the registry with the upstream qtodo image # before the supply-chain pipeline runs (avoids ImagePullBackOff on first install) # - name: app.seedImage.enabled # value: "true" @@ -578,7 +578,7 @@ clusterGroup: # # Enable RHTPA SBOM upload # # - name: rhtpa.enabled # # value: "true" - # # Auto-trigger pipeline run on ArgoCD sync + # # Uncomment to auto-trigger a pipeline run on every ArgoCD sync # # - name: pipelinerun.enabled # # value: "true" # @@ -648,7 +648,7 @@ clusterGroup: hs.message = "PVC is bound" elseif obj.status.phase == "Pending" then hs.status = "Healthy" - hs.message = "Waiting for first consumer" + hs.message = "PVC is pending" elseif obj.status.phase == "Lost" then hs.status = "Degraded" hs.message = "PVC is lost" From 019677672b8e3d4e213aa1db47930aaaf565368c Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Fri, 17 Apr 2026 23:28:12 -0400 Subject: [PATCH 21/21] fix: pin clustergroup chart to 0.9.47 Signed-off-by: Min Zhang --- values-global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-global.yaml b/values-global.yaml index c050c5fb..741e09ac 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -9,4 +9,4 @@ main: clusterGroupName: hub multiSourceConfig: enabled: true - clusterGroupChartVersion: "0.9.*" + clusterGroupChartVersion: "0.9.47"