mbp-968: Add support for ACM ManagedClusters

README.md

@@ -26,6 +26,8 @@ The following components are included in the Layered Zero Trust Pattern
   * Secure storage of sensitive assets
 * [External Secrets Operator (ESO)](https://external-secrets.io)
   * Synchronizes secrets stored in HashiCorp Vault with OpenShift
+* [Red Hat Advanced Cluster Management (ACM)](https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.14)
+  * Provides a management control in multi-cluster scenarios
 
 ## Getting Started
 
@@ -41,7 +43,8 @@ Utilize the following steps to prepare your machine and complete any and all pre
 4. [Validated Patterns Tooling](https://validatedpatterns.io/learn/quickstart)
 5. Depending on the characteristics of your cluster, you may need additional hardware resources for Advanced Cluster Management (ACM) component. For a single node cluster you can start with 4 vCPUs, 16 GB of memory and 120 GB of storage. For more detailed information about ACM sizing, please refer to the [official documentation](https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.14/html-single/install/index#sizing-your-cluster).
 
-_NOTE_: The default deployment of this patterns assumes that none of the components associated with the pattern have been deployed previously. Ensure that your OpenShift environment does not include any of the preceding components.
+>[!WARNING]
+> The default deployment of this patterns assumes that none of the components associated with the pattern have been deployed previously. Ensure that your OpenShift environment does not include any of the preceding components.
 
 ### Prepare for Deployment
 
@@ -201,3 +204,53 @@ Switch back to the qtodo application and enter the username and password on the
 Once you have authenticated to RHBK, you will be instructed to change the temporary password and set a more permanent password. Once complete, you will be redirected to the qtodo application verifying the OIDC based authentication functions properly.
 
 Feel free to add new items to the list of todos. By being able to add and remove items from the page, the integration between the Quarkus application and the backend PostgreSQL database using credentials sourced from HashiCorp Vault was successful.
+
+### Importing existing clusters
+
+>[!WARNING]
+> Since ACM chart provisioning functionality uses `ClusterPools` and these technology is limited to Cloud environments, we do not recommend use those configuration settings.
+> Instead, we have enabled the option to import your existing standalone clusters using the **acm-managed-clusters** chart.
+
+The pattern supports importating pre-existing Openshift clusters into the Hub cluster, converting them into **Managed Clusters**.
+
+1. Copy the `kubeconfig` file of the cluster you want to import to your local system.
+
+2. In the `values-secret.yaml` file, add a new secret with the contents of the `kubeconfig` file.
+
+    ```yaml
+      - name: kubeconfig-spoke
+        vaultPrefixes:
+        - hub
+        fields:
+        - name: content
+          path: ~/.kube/kubeconfig-ztvp-spoke
+    ```
+
+3. In the `values-hub.yaml` file, add a new entry in the `clusterGroup.managedClusterGroups` key.
+
+    ```yaml
+      managedClusterGroups:
+        exampleRegion:
+          name: group-one
+          acmlabels:
+            - name: clusterGroup
+              value: group-one
+          helmOverrides:
+            - name: clusterGroup.isHubCluster
+              value: false
+    ```
+
+4. Also in the `values-hub.yaml` file, add your cluster definition in the `acmManagedClusters.clusters` key.
+
+    ```yaml
+    acmManagedClusters:
+      clusters:
+        - name: ztvp-spoke-1
+          clusterGroup: group-one
+          labels:
+            cloud: auto-detect
+            vendor: auto-detect
+          kubeconfigVaultPath: secret/data/hub/kubeconfig-spoke
+    ```
+
+5. Deploy the pattern.

charts/acm-managed-clusters/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: acm-managed-clusters
+description: Import previously provisioned Openshift clusters into an Openshift Hub cluster using ACM
+version: 0.0.1
+keywords:
+  - pattern
+  - acm
+  - managed-clusters

charts/acm-managed-clusters/templates/acm-external-secrets.yaml

@@ -0,0 +1,26 @@
+{{- $clusters := .Values.acmManagedClusters.clusters | default list }}
+{{- range $clusters }}
+---
+apiVersion: "external-secrets.io/v1beta1"
+kind: ExternalSecret
+metadata:
+  name: kubeconfig-{{ .name }}
+  namespace: {{ .name }}
+spec:
+  refreshInterval: 15s
+  secretStoreRef:
+    name: {{ $.Values.global.secretStore.name }}
+    kind: {{ $.Values.global.secretStore.kind }}
+  target:
+    name: auto-import-secret
+    template:
+      type: Opaque
+      data:
+        autoImportRetry: "5"
+        kubeconfig: "{{ `{{ .kubeconfig }}` }}"
+  data:
+  - secretKey: kubeconfig
+    remoteRef:
+      key: {{ .kubeconfigVaultPath }}
+      property: content
+{{- end }}

charts/acm-managed-clusters/templates/acm-managedclusteraddons.yaml

@@ -0,0 +1,13 @@
+{{- $clusters := .Values.acmManagedClusters.clusters | default list }}
+{{- range $cluster := $clusters }}
+{{- range $addon := $.Values.acmManagedClusters.addons | default list }}
+---
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: ManagedClusterAddOn
+metadata:
+  name: {{ $addon }}
+  namespace: {{ $cluster.name }}
+spec:
+  installNamespace: open-cluster-management-agent-addon
+{{- end }}
+{{- end }}

charts/acm-managed-clusters/templates/acm-managedclusterset.yaml

@@ -0,0 +1,11 @@
+{{- $clusters := .Values.acmManagedClusters.clusters | default list }}
+{{- range $clusters }}
+---
+apiVersion: cluster.open-cluster-management.io/v1beta2
+kind: ManagedClusterSet
+metadata:
+  name: {{ .clusterGroup | default $.Values.acmManagedClusters.defaultClusterGroup }}
+  annotations:
+    cluster.open-cluster-management.io/submariner-broker-ns: {{ .clusterGroup | default $.Values.acmManagedClusters.defaultClusterGroup }}-broker
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+{{- end }}

charts/acm-managed-clusters/templates/acm-namespaces.yaml

@@ -0,0 +1,8 @@
+{{- $clusters := .Values.acmManagedClusters.clusters | default list }}
+{{- range $clusters }}
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .name }}
+{{- end }}

charts/acm-managed-clusters/templates/import-cluster.yaml

@@ -0,0 +1,14 @@
+{{- $clusters := .Values.acmManagedClusters.clusters | default list }}
+{{- range $clusters }}
+---
+apiVersion: cluster.open-cluster-management.io/v1
+kind: ManagedCluster
+metadata:
+  name: {{ .name }}
+  labels:
+    clusterGroup: {{ .clusterGroup | default $.Values.acmManagedClusters.defaultClusterGroup }}
+    cluster.open-cluster-management.io/clusterset: {{ .clusterGroup | default $.Values.acmManagedClusters.defaultClusterGroup }}
+    {{- toYaml .labels | nindent 4 }}
+spec:
+  hubAcceptsClient: true
+{{- end }}

charts/acm-managed-clusters/templates/policy-spoke-namespaces.yaml

@@ -0,0 +1,70 @@
+{{- range .Values.clusterGroup.managedClusterGroups }}
+{{- $group := . }}
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: Policy
+metadata:
+  name: {{ $group.name }}-namespace-policy
+  namespace: open-cluster-management
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/compare-options: IgnoreExtraneous
+spec:
+  remediationAction: enforce
+  disabled: false
+  policy-templates:
+    - objectDefinition:
+        apiVersion: policy.open-cluster-management.io/v1
+        kind: ConfigurationPolicy
+        metadata:
+          name: {{ $group.name }}-namespace-config
+        spec:
+          remediationAction: enforce
+          severity: medium
+          namespaceSelector:
+            include:
+              - default
+          object-templates:
+            - complianceType: musthave
+              objectDefinition:
+                kind: Namespace
+                apiVersion: v1
+                metadata:
+                  name: {{ $.Values.global.pattern }}-{{ $group.name }}
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: PlacementBinding
+metadata:
+  name: {{ $group.name }}-namespace-placement-binding
+  namespace: open-cluster-management
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+placementRef:
+  name: {{ $group.name }}-namespace-placement
+  kind: PlacementRule
+  apiGroup: apps.open-cluster-management.io
+subjects:
+  - name: {{ $group.name }}-namespace-policy
+    kind: Policy
+    apiGroup: policy.open-cluster-management.io
+---
+apiVersion: apps.open-cluster-management.io/v1
+kind: PlacementRule
+metadata:
+  name: {{ $group.name }}-namespace-placement
+  namespace: open-cluster-management
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  clusterConditions:
+    - status: 'True'
+      type: ManagedClusterConditionAvailable
+  clusterSelector:
+    matchExpressions:
+    - key: local-cluster
+      operator: NotIn
+      values:
+      - "true"
+    matchLabels:
+      clusterGroup: {{ $group.name }}
+{{- end }}

charts/acm-managed-clusters/values.yaml

@@ -0,0 +1,19 @@
+global:
+  secretStore:
+    kind: ClusterSecretStore
+    name: vault-backend
+
+acmManagedClusters:
+  addons:
+    - config-policy-controller
+    - governance-policy-framework
+    - cert-policy-controller
+    - application-manager
+  clusters: []
+  #  - name: ztvp-spoke-1
+  #    clusterGroup: group-one
+  #    labels:
+  #      cloud: auto-detect
+  #      vendor: auto-detect
+  #    kubeconfigVaultPath: secret/data/hub/kubeconfig-spoke
+  defaultClusterGroup: ztvp

values-hub.yaml

@@ -122,6 +122,14 @@ clusterGroup:
           kind: ManagedClusterInfo
           jsonPointers:
             - /spec/loggingCA
+      # We override the secret store because we are not provisioning clusters
+      overrides:
+        - name: global.secretStore.backend
+          value: none
+    acm-managed-clusters:
+      name: acm-managed-clusters
+      project: hub
+      path: charts/acm-managed-clusters
     compliance-scanning:
       name: compliance-scanning
       namespace: openshift-compliance
@@ -202,8 +210,6 @@ clusterGroup:
         - name: app.vault.secretPath
           value: secret/data/global/qtodo
 
-
-
   imperative:
     # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm
     # The default schedule is every 10 minutes: imperative.schedule
@@ -221,32 +227,6 @@ clusterGroup:
     #   helmOverrides:
     #     - name: clusterGroup.isHubCluster
     #       value: false
-          # Before enabling cluster provisioning, ensure AWS and/or Azure
-          #   credentials and OCP pull secrets are defined in Vault.
-          #   See values-secret.yaml.template
-          #
-          #clusterPools:
-          #  exampleAWSPool:
-          #    name: aws-ap
-          #    openshiftVersion: 4.10.18
-          #    baseDomain: blueprints.rhecoeng.com
-          #    platform:
-          #      aws:
-          #        region: ap-southeast-2
-          #    clusters:
-          #    - One
-          #
-          #  exampleAzurePool:
-          #    name: azure-us
-          #    openshiftVersion: 4.10.18
-          #    baseDomain: blueprints.rhecoeng.com
-          #    platform:
-          #      azure:
-          #        baseDomainResourceGroupName: dojo-dns-zones
-          #        region: eastus
-          #    clusters:
-          #    - Two
-          #    - Three
 #  To have apps in multiple flavors, use namespaces and use helm overrides as appropriate
 #
 #    pipelines:
@@ -296,3 +276,14 @@ clusterGroup:
 #          operator: In
 #          values:
 #            - OpenShift
+
+
+# List of previously provisioned clusters to import and manage from the Hub cluster
+acmManagedClusters:
+  clusters: []
+  #  - name: ztvp-spoke-1
+  #    clusterGroup: group-one
+  #    labels:
+  #      cloud: auto-detect
+  #      vendor: auto-detect
+  #    kubeconfigVaultPath: secret/data/hub/kubeconfig-spoke

values-secret.yaml.template

@@ -98,3 +98,11 @@ secrets:
     - name: developer1-password
       onMissingValue: generate
       vaultPolicy: validatedPatternDefaultPolicy
+
+  # If you are going to import spoke clusters, add here their kubeconfig entries
+  #- name: kubeconfig-spoke
+  #  vaultPrefixes:
+  #  - hub
+  #  fields:
+  #  - name: content
+  #    path: ~/.kube/kubeconfig-ztvp-spoke