From 20035381b38542b000d955381d035b762801c4b0 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 21 Sep 2022 10:18:37 +0200 Subject: [PATCH 1/6] Use upstream csi-cloudscale Helm chart --- class/csi-cloudscale.yml | 19 +- class/defaults.yml | 44 +++- component/main.jsonnet | 110 +-------- .../csi-cloudscale/templates/csi_driver.yaml | 7 + .../csi-cloudscale/templates/daemonset.yaml | 109 +++++++++ .../csi-cloudscale/templates/rbac.yaml} | 1 - .../templates/serviceaccount.yaml | 9 + .../csi-cloudscale/templates/statefulset.yaml | 99 ++++++++ .../csi-cloudscale/10_deployments.yaml | 223 ------------------ 9 files changed, 284 insertions(+), 337 deletions(-) create mode 100644 tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml create mode 100644 tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml rename tests/golden/defaults/csi-cloudscale/csi-cloudscale/{20_rbac.yaml => 01_helm_chart/csi-cloudscale/templates/rbac.yaml} (99%) create mode 100644 tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml create mode 100644 tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml delete mode 100644 tests/golden/defaults/csi-cloudscale/csi-cloudscale/10_deployments.yaml diff --git a/class/csi-cloudscale.yml b/class/csi-cloudscale.yml index 813a4dd..cea51c3 100644 --- a/class/csi-cloudscale.yml +++ b/class/csi-cloudscale.yml @@ -1,15 +1,24 @@ parameters: kapitan: dependencies: - - type: https - source: https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-${csi_cloudscale:version}.yaml - output_path: dependencies/csi-cloudscale/manifests/${csi_cloudscale:version}/deploy.yaml + - type: helm + source: ${csi_cloudscale:charts:csi-cloudscale:source} + chart_name: csi-cloudscale + output_path: ${_base_directory}/helmcharts/csi-cloudscale/${csi_cloudscale:charts:csi-cloudscale:version} compile: - input_paths: - - csi-cloudscale/component/app.jsonnet + - ${_base_directory}/component/app.jsonnet input_type: jsonnet output_path: apps/ - input_paths: - - csi-cloudscale/component/main.jsonnet + - ${_base_directory}/helmcharts/csi-cloudscale/${csi_cloudscale:charts:csi-cloudscale:version} + input_type: helm + helm_values: ${csi_cloudscale:helm_values} + helm_params: + name: csi-cloudscale + namespace: ${csi_cloudscale:namespace} + output_path: csi-cloudscale/01_helm_chart + - input_paths: + - ${_base_directory}/component/main.jsonnet input_type: jsonnet output_path: csi-cloudscale/ diff --git a/class/defaults.yml b/class/defaults.yml index 4c2b648..c25c266 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -1,10 +1,23 @@ parameters: csi_cloudscale: namespace: syn-csi-cloudscale - version: v3.5.0 + + charts: + csi-cloudscale: + source: https://cloudscale-ch.github.io/csi-cloudscale + version: 1.3.3 + + images: + cloudscale-csi-plugin: + registry: quay.io + repository: cloudscalech/cloudscale-csi-plugin + tag: v3.5.0 + api_token: ?{vaultkv:${cluster:tenant}/${cluster:name}/cloudscale/token} + api_token_secret_name: cloudscale fs_type: ext4 driver_daemonset_tolerations: {} + resources: controller: csi-provisioner: @@ -32,3 +45,32 @@ parameters: requests: cpu: 20m memory: 32Mi + + helm_values: + # Set nameOverride to avoid resource names like + # `csi-cloudscale-csi-cloudscale-controller-sa`. + nameOverride: csi-cloudscale + cloudscale: + token: + # Secret managed by the component in `main.jsonnet` + existingSecret: ${csi_cloudscale:api_token_secret_name} + csi: + # Disable chart-managed storageclasses, we manage them directly in + # the component + storageClasses: [] + provisioner: + resources: ${csi_cloudscale:resources:controller:csi-provisioner} + attacher: + resources: ${csi_cloudscale:resources:controller:csi-attacher} + resizer: + resources: ${csi_cloudscale:resources:controller:csi-resizer} + controller: + serviceAccountName: csi-cloudscale-controller-sa + image: ${csi_cloudscale:images:cloudscale-csi-plugin} + resources: ${csi_cloudscale:resources:controller:csi-cloudscale-plugin} + node: + serviceAccountName: csi-cloudscale-node-sa + image: ${csi_cloudscale:images:cloudscale-csi-plugin} + resources: ${csi_cloudscale:resources:csi_driver:csi-cloudscale-plugin} + driverRegistrar: + resources: ${csi_cloudscale:resources:csi_driver:csi-node-driver-registrar} diff --git a/component/main.jsonnet b/component/main.jsonnet index 12d9323..e7eb6b7 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -32,7 +32,7 @@ local storageclasses = [ [ } + config, ] for type in [ 'ssd', 'bulk' ] ]; -local secret = kube.Secret('cloudscale') { +local secret = kube.Secret(params.api_token_secret_name) { metadata+: { namespace: params.namespace, }, @@ -41,18 +41,12 @@ local secret = kube.Secret('cloudscale') { }, }; -local manifests = std.parseJson( - kap.yaml_load_stream('csi-cloudscale/manifests/' + params.version + '/deploy.yaml') -); - local customRBAC = if isOpenshift then [ kube.RoleBinding('csi-hostnetwork') { roleRef_: kube.ClusterRole('system:openshift:scc:hostnetwork'), subjects: [ { kind: 'ServiceAccount', - name: std.filter( - function(obj) obj.kind == 'StatefulSet', manifests - )[0].spec.template.spec.serviceAccount, + name: params.helm_values.controller.serviceAccountName, namespace: params.namespace, } ], }, @@ -60,88 +54,12 @@ local customRBAC = if isOpenshift then [ roleRef_: kube.ClusterRole('system:openshift:scc:privileged'), subjects: [ { kind: 'ServiceAccount', - name: std.filter( - function(obj) obj.kind == 'DaemonSet', manifests - )[0].spec.template.spec.serviceAccount, + name: params.helm_values.node.serviceAccountName, namespace: params.namespace, } ], }, ] else []; -local patch_manifest(object) = - local tolerations = params.driver_daemonset_tolerations; - local resourcesInParams = if object.kind == 'DaemonSet' then - params.resources.csi_driver - else if object.kind == 'StatefulSet' then - params.resources.controller - else - null; - local resources = - if ( - resourcesInParams != null - && ( - std.length(object.spec.template.spec.containers) != - std.length(std.objectFields(resourcesInParams)) - ) - ) then - std.trace( - ( - 'The number of containers in the csi-cloudscale upstream manifest "%s" changed. ' - + 'Please check the default resource requests and limits configured in the component.' - ) % ( - object.metadata.name - ), - resourcesInParams - ) - else - resourcesInParams; - if ( - object.kind == 'DaemonSet' - && object.metadata.name == 'csi-cloudscale-node' - ) then - object { - spec+: { - template+: { - spec+: { - containers: [ - c { - resources+: std.prune( - com.getValueOrDefault(resources, c.name, {}) - ), - } - for c in super.containers - ], - tolerations+: [ - tolerations[t] { - key: t, - } - for t in std.objectFields(tolerations) - ], - }, - }, - }, - } - else if ( - object.kind == 'StatefulSet' - && object.metadata.name == 'csi-cloudscale-controller' - ) then - object { - spec+: { - template+: { - spec+: { - containers: [ - c { - resources+: com.getValueOrDefault(resources, c.name, {}), - } - for c in super.containers - ], - }, - }, - }, - } - else - object; - { [if params.namespace != 'kube-system' then '00_namespace']: kube.Namespace(params.namespace) + if isOpenshift then { metadata+: { @@ -152,27 +70,5 @@ local patch_manifest(object) = } else {}, '01_storageclasses': std.flattenArrays(storageclasses), '02_secret': secret, - '10_deployments': [ - patch_manifest(object) { - metadata+: { - namespace: params.namespace, - }, - } - for object in manifests - if std.setMember(object.kind, std.set([ 'StatefulSet', 'ServiceAccount', 'DaemonSet' ])) - ], - '20_rbac': [ - if std.objectHas(object, 'subjects') then object { - subjects: [ - sub { - namespace: params.namespace, - } - for sub in object.subjects - ], - } - else object - for object in manifests - if std.setMember(object.kind, std.set([ 'ClusterRole', 'ClusterRoleBinding' ])) - ], [if std.length(customRBAC) > 0 then '30_custom_rbac']: customRBAC, } diff --git a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml new file mode 100644 index 0000000..5afd6b4 --- /dev/null +++ b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.cloudscale.ch +spec: + attachRequired: true + podInfoOnMount: true diff --git a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml new file mode 100644 index 0000000..147cccc --- /dev/null +++ b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml @@ -0,0 +1,109 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-cloudscale-node +spec: + selector: + matchLabels: + app: csi-cloudscale-node + template: + metadata: + labels: + app: csi-cloudscale-node + role: csi-cloudscale + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.cloudscale.ch/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/csi.cloudscale.ch /registration/csi.cloudscale.ch-reg.sock + name: csi-node-driver-registrar + resources: + requests: + cpu: null + memory: 32Mi + volumeMounts: + - mountPath: /csi/ + name: plugin-dir + - mountPath: /registration/ + name: registration-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --url=$(CLOUDSCALE_API_URL) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE + value: '125' + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + key: access-token + name: cloudscale + image: quay.io/cloudscalech/cloudscale-csi-plugin:v3.5.0 + imagePullPolicy: IfNotPresent + name: csi-cloudscale-plugin + resources: + limits: + cpu: 1000m + requests: + cpu: 20m + memory: 32Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /tmp + name: tmpfs + hostNetwork: true + priorityClassName: system-node-critical + serviceAccount: csi-cloudscale-node-sa + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.cloudscale.ch + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - emptyDir: + medium: Memory + name: tmpfs diff --git a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/20_rbac.yaml b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml similarity index 99% rename from tests/golden/defaults/csi-cloudscale/csi-cloudscale/20_rbac.yaml rename to tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml index 6e0015c..00440bb 100644 --- a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/20_rbac.yaml +++ b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml @@ -172,7 +172,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: csi-cloudscale-node-driver-registrar-role - namespace: kube-system rules: - apiGroups: - '' diff --git a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml new file mode 100644 index 0000000..c27bbdc --- /dev/null +++ b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-controller-sa +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-node-sa diff --git a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml new file mode 100644 index 0000000..afb2a1f --- /dev/null +++ b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: csi-cloudscale-controller +spec: + replicas: 1 + selector: + matchLabels: + app: csi-cloudscale-controller + serviceName: csi-cloudscale + template: + metadata: + labels: + app: csi-cloudscale-controller + role: csi-cloudscale + spec: + containers: + - args: + - --csi-address=$(ADDRESS) + - --default-fstype=ext4 + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.2.1 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --timeout=30s + - --v=5 + - --handle-volume-inuse-error=false + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.6.0 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --url=$(CLOUDSCALE_API_URL) + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + key: access-token + name: cloudscale + image: quay.io/cloudscalech/cloudscale-csi-plugin:v3.5.0 + imagePullPolicy: IfNotPresent + name: csi-cloudscale-plugin + resources: + limits: + memory: 1Gi + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: csi-cloudscale-controller-sa + volumes: + - emptyDir: {} + name: socket-dir diff --git a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/10_deployments.yaml b/tests/golden/defaults/csi-cloudscale/csi-cloudscale/10_deployments.yaml deleted file mode 100644 index c82adc8..0000000 --- a/tests/golden/defaults/csi-cloudscale/csi-cloudscale/10_deployments.yaml +++ /dev/null @@ -1,223 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cloudscale-controller-sa - namespace: syn-csi-cloudscale ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cloudscale-node-sa - namespace: syn-csi-cloudscale ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: csi-cloudscale-node - namespace: syn-csi-cloudscale -spec: - selector: - matchLabels: - app: csi-cloudscale-node - template: - metadata: - labels: - app: csi-cloudscale-node - role: csi-cloudscale - spec: - containers: - - args: - - --v=5 - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/csi.cloudscale.ch/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.cloudscale.ch /registration/csi.cloudscale.ch-reg.sock - name: csi-node-driver-registrar - resources: - requests: - memory: 32Mi - volumeMounts: - - mountPath: /csi/ - name: plugin-dir - - mountPath: /registration/ - name: registration-dir - - args: - - --endpoint=$(CSI_ENDPOINT) - - --url=$(CLOUDSCALE_API_URL) - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: CLOUDSCALE_API_URL - value: https://api.cloudscale.ch/ - - name: CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE - value: '125' - - name: CLOUDSCALE_ACCESS_TOKEN - valueFrom: - secretKeyRef: - key: access-token - name: cloudscale - image: quay.io/cloudscalech/cloudscale-csi-plugin:v3.5.0 - imagePullPolicy: IfNotPresent - name: csi-cloudscale-plugin - resources: - limits: - cpu: 1000m - requests: - cpu: 20m - memory: 32Mi - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: pods-mount-dir - - mountPath: /dev - name: device-dir - - mountPath: /tmp - name: tmpfs - hostNetwork: true - priorityClassName: system-node-critical - serviceAccount: csi-cloudscale-node-sa - tolerations: [] - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - name: registration-dir - - hostPath: - path: /var/lib/kubelet/plugins/csi.cloudscale.ch - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: pods-mount-dir - - hostPath: - path: /dev - name: device-dir - - emptyDir: - medium: Memory - name: tmpfs ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: csi-cloudscale-controller - namespace: syn-csi-cloudscale -spec: - replicas: 1 - selector: - matchLabels: - app: csi-cloudscale-controller - serviceName: csi-cloudscale - template: - metadata: - labels: - app: csi-cloudscale-controller - role: csi-cloudscale - spec: - containers: - - args: - - --csi-address=$(ADDRESS) - - --default-fstype=ext4 - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-provisioner:v3.2.1 - imagePullPolicy: IfNotPresent - name: csi-provisioner - resources: - requests: - cpu: 20m - memory: 32Mi - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=$(ADDRESS) - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 - imagePullPolicy: IfNotPresent - name: csi-attacher - resources: - requests: - cpu: 20m - memory: 32Mi - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=$(ADDRESS) - - --timeout=30s - - --v=5 - - --handle-volume-inuse-error=false - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-resizer:v1.6.0 - imagePullPolicy: IfNotPresent - name: csi-resizer - resources: - requests: - cpu: 20m - memory: 32Mi - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --endpoint=$(CSI_ENDPOINT) - - --url=$(CLOUDSCALE_API_URL) - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: CLOUDSCALE_API_URL - value: https://api.cloudscale.ch/ - - name: CLOUDSCALE_ACCESS_TOKEN - valueFrom: - secretKeyRef: - key: access-token - name: cloudscale - image: quay.io/cloudscalech/cloudscale-csi-plugin:v3.5.0 - imagePullPolicy: IfNotPresent - name: csi-cloudscale-plugin - resources: - limits: - memory: 1Gi - requests: - cpu: 20m - memory: 32Mi - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - hostNetwork: true - priorityClassName: system-cluster-critical - serviceAccount: csi-cloudscale-controller-sa - volumes: - - emptyDir: {} - name: socket-dir From 8b393fefc89c7fd176a99e4d13448b162dccf1c9 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 21 Sep 2022 10:22:56 +0200 Subject: [PATCH 2/6] Add test case for OpenShift 4 --- .cruft.json | 4 +- .github/workflows/test.yaml | 14 +- Makefile | 16 ++ Makefile.vars.mk | 1 + renovate.json | 2 +- .../csi-cloudscale/apps/csi-cloudscale.yaml | 0 .../csi-cloudscale/00_namespace.yaml | 8 + .../csi-cloudscale/templates/csi_driver.yaml | 7 + .../csi-cloudscale/templates/daemonset.yaml | 107 ++++++++ .../csi-cloudscale/templates/rbac.yaml | 238 ++++++++++++++++++ .../templates/serviceaccount.yaml | 9 + .../csi-cloudscale/templates/statefulset.yaml | 97 +++++++ .../csi-cloudscale/01_storageclasses.yaml | 69 +++++ .../csi-cloudscale/02_secret.yaml | 12 + .../csi-cloudscale/30_custom_rbac.yaml | 31 +++ tests/openshift4.yml | 16 ++ 16 files changed, 626 insertions(+), 5 deletions(-) create mode 100644 tests/golden/openshift4/csi-cloudscale/apps/csi-cloudscale.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/00_namespace.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_storageclasses.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/02_secret.yaml create mode 100644 tests/golden/openshift4/csi-cloudscale/csi-cloudscale/30_custom_rbac.yaml create mode 100644 tests/openshift4.yml diff --git a/.cruft.json b/.cruft.json index d877e32..be861a2 100644 --- a/.cruft.json +++ b/.cruft.json @@ -7,11 +7,11 @@ "name": "csi-cloudscale", "slug": "csi-cloudscale", "parameter_key": "csi_cloudscale", - "test_cases": "defaults", + "test_cases": "defaults openshift4", "add_lib": "n", "add_pp": "n", "add_golden": "y", - "add_matrix": "n", + "add_matrix": "y", "add_go_unit": "n", "copyright_holder": "VSHN AG ", "copyright_year": "2021", diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8d7ac42..8d83020 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,6 +29,11 @@ jobs: args: 'check' test: runs-on: ubuntu-latest + strategy: + matrix: + instance: + - defaults + - openshift4 defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -37,9 +42,14 @@ jobs: with: path: ${{ env.COMPONENT_NAME }} - name: Compile component - run: make test + run: make test -e instance=${{ matrix.instance }} golden: runs-on: ubuntu-latest + strategy: + matrix: + instance: + - defaults + - openshift4 defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -48,4 +58,4 @@ jobs: with: path: ${{ env.COMPONENT_NAME }} - name: Golden diff - run: make golden-diff + run: make golden-diff -e instance=${{ matrix.instance }} diff --git a/Makefile b/Makefile index 8b9ce19..0646f90 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,22 @@ golden-diff: commodore_args += -f tests/$(instance).yml golden-diff: clean .compile ## Diff compile output against the reference version. Review output and run `make gen-golden golden-diff` if this target fails. @git diff --exit-code --minimal --no-index -- tests/golden/$(instance) compiled/ +.PHONY: golden-diff-all +golden-diff-all: recursive_target=golden-diff +golden-diff-all: $(test_instances) ## Run golden-diff for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: gen-golden-all +gen-golden-all: recursive_target=gen-golden +gen-golden-all: $(test_instances) ## Run gen-golden for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: lint_kubent_all +lint_kubent_all: recursive_target=lint_kubent +lint_kubent_all: $(test_instances) ## Lint deprecated Kubernetes API versions for all golden test instances. Will exit on first error. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: $(test_instances) +$(test_instances): + $(MAKE) $(recursive_target) -e instance=$(basename $(@F)) + .PHONY: clean clean: ## Clean the project rm -rf .cache compiled dependencies vendor helmcharts jsonnetfile*.json || true diff --git a/Makefile.vars.mk b/Makefile.vars.mk index 9c126ce..5aa2f61 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -57,3 +57,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) instance ?= defaults +test_instances = tests/defaults.yml tests/openshift4.yml diff --git a/renovate.json b/renovate.json index 1e05b8c..9f6db80 100644 --- a/renovate.json +++ b/renovate.json @@ -6,7 +6,7 @@ ], "postUpgradeTasks": { "commands": [ - "make gen-golden" + "make gen-golden-all" ], "fileFilters": [ "tests/golden/**" ], "executionMode": "update" diff --git a/tests/golden/openshift4/csi-cloudscale/apps/csi-cloudscale.yaml b/tests/golden/openshift4/csi-cloudscale/apps/csi-cloudscale.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/00_namespace.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/00_namespace.yaml new file mode 100644 index 0000000..5eecbcc --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/00_namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + openshift.io/node-selector: '' + labels: + name: syn-csi-cloudscale + name: syn-csi-cloudscale diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml new file mode 100644 index 0000000..5afd6b4 --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/csi_driver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.cloudscale.ch +spec: + attachRequired: true + podInfoOnMount: true diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml new file mode 100644 index 0000000..a30a2a6 --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-cloudscale-node +spec: + selector: + matchLabels: + app: csi-cloudscale-node + template: + metadata: + labels: + app: csi-cloudscale-node + role: csi-cloudscale + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.cloudscale.ch/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/csi.cloudscale.ch /registration/csi.cloudscale.ch-reg.sock + name: csi-node-driver-registrar + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /csi/ + name: plugin-dir + - mountPath: /registration/ + name: registration-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --url=$(CLOUDSCALE_API_URL) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE + value: '125' + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + key: access-token + name: cloudscale + image: quay.io/cloudscalech/cloudscale-csi-plugin:v3.5.0 + imagePullPolicy: IfNotPresent + name: csi-cloudscale-plugin + resources: + requests: + cpu: 20m + memory: 32Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /tmp + name: tmpfs + hostNetwork: true + priorityClassName: system-node-critical + serviceAccount: csi-cloudscale-node-sa + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.cloudscale.ch + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - emptyDir: + medium: Memory + name: tmpfs diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml new file mode 100644 index 0000000..00440bb --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/rbac.yaml @@ -0,0 +1,238 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-cloudscale-provisioner-role +rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-cloudscale-attacher-role +rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-cloudscale-resizer-role +rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-cloudscale-node-driver-registrar-role +rules: + - apiGroups: + - '' + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-cloudscale-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-cloudscale-provisioner-role +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: syn-csi-cloudscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-cloudscale-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-cloudscale-resizer-role +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: syn-csi-cloudscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-cloudscale-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-cloudscale-attacher-role +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: syn-csi-cloudscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-cloudscale-node-driver-registrar-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-cloudscale-node-driver-registrar-role +subjects: + - kind: ServiceAccount + name: csi-cloudscale-node-sa + namespace: syn-csi-cloudscale diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml new file mode 100644 index 0000000..c27bbdc --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-controller-sa +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-node-sa diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml new file mode 100644 index 0000000..4c09e4b --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/statefulset.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: csi-cloudscale-controller +spec: + replicas: 1 + selector: + matchLabels: + app: csi-cloudscale-controller + serviceName: csi-cloudscale + template: + metadata: + labels: + app: csi-cloudscale-controller + role: csi-cloudscale + spec: + containers: + - args: + - --csi-address=$(ADDRESS) + - --default-fstype=ext4 + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.2.1 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --timeout=30s + - --v=5 + - --handle-volume-inuse-error=false + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.6.0 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --url=$(CLOUDSCALE_API_URL) + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + key: access-token + name: cloudscale + image: quay.io/cloudscalech/cloudscale-csi-plugin:v3.5.0 + imagePullPolicy: IfNotPresent + name: csi-cloudscale-plugin + resources: + requests: + cpu: 20m + memory: 32Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: csi-cloudscale-controller-sa + volumes: + - emptyDir: {} + name: socket-dir diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_storageclasses.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_storageclasses.yaml new file mode 100644 index 0000000..e488e51 --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_storageclasses.yaml @@ -0,0 +1,69 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: {} + labels: + name: ssd + name: ssd +parameters: + csi.cloudscale.ch/volume-type: ssd + fsType: ext4 +provisioner: csi.cloudscale.ch +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +--- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: {} + labels: + name: ssd-encrypted + name: ssd-encrypted +parameters: + csi.cloudscale.ch/luks-cipher: aes-xts-plain64 + csi.cloudscale.ch/luks-encrypted: 'true' + csi.cloudscale.ch/luks-key-size: '512' + csi.cloudscale.ch/volume-type: ssd + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + fsType: ext4 +provisioner: csi.cloudscale.ch +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +--- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: {} + labels: + name: bulk + name: bulk +parameters: + csi.cloudscale.ch/volume-type: bulk + fsType: ext4 +provisioner: csi.cloudscale.ch +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +--- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: {} + labels: + name: bulk-encrypted + name: bulk-encrypted +parameters: + csi.cloudscale.ch/luks-cipher: aes-xts-plain64 + csi.cloudscale.ch/luks-encrypted: 'true' + csi.cloudscale.ch/luks-key-size: '512' + csi.cloudscale.ch/volume-type: bulk + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + fsType: ext4 +provisioner: csi.cloudscale.ch +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/02_secret.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/02_secret.yaml new file mode 100644 index 0000000..b5cc8e4 --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/02_secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +data: {} +kind: Secret +metadata: + annotations: {} + labels: + name: cloudscale + name: cloudscale + namespace: syn-csi-cloudscale +stringData: + access-token: t-silent-test-1234/c-green-test-1234/cloudscale/token +type: Opaque diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/30_custom_rbac.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/30_custom_rbac.yaml new file mode 100644 index 0000000..595f9e0 --- /dev/null +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/30_custom_rbac.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: csi-hostnetwork + name: csi-hostnetwork +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:hostnetwork +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: syn-csi-cloudscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: csi-privileged + name: csi-privileged +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:privileged +subjects: + - kind: ServiceAccount + name: csi-cloudscale-node-sa + namespace: syn-csi-cloudscale diff --git a/tests/openshift4.yml b/tests/openshift4.yml new file mode 100644 index 0000000..1d1921c --- /dev/null +++ b/tests/openshift4.yml @@ -0,0 +1,16 @@ +parameters: + kapitan: + dependencies: + - type: https + source: https://raw.githubusercontent.com/projectsyn/component-storageclass/v1.0.0/lib/storageclass.libsonnet + output_path: vendor/lib/storageclass.libsonnet + + storageclass: + defaults: + volumeBindingMode: WaitForFirstConsumer + reclaimPolicy: Delete + classes: {} + defaultClass: '' + + facts: + distribution: openshift4 From 3e95d466ff55f59b8e77c78f7fb258e5051e4960 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 25 Aug 2023 14:54:07 +0200 Subject: [PATCH 3/6] Add postprocessing filter to keep supporting `driver_daemonset_tolerations` parameter We can't simply pass the parameter to the `node.tolerations` Helm value, since that value expects an array of regular tolerations while the component parameter is structured to allow users to overwrite tolerations. Note that the postprocessing filter will retain tolerations configured through the Helm value and will simply append any tolerations configured in `driver_daemonset_tolerations`. --- class/csi-cloudscale.yml | 6 ++++ .../driver_daemonset_tolerations.jsonnet | 29 +++++++++++++++++++ .../csi-cloudscale/templates/daemonset.yaml | 4 +++ tests/openshift4.yml | 6 ++++ 4 files changed, 45 insertions(+) create mode 100644 postprocess/driver_daemonset_tolerations.jsonnet diff --git a/class/csi-cloudscale.yml b/class/csi-cloudscale.yml index cea51c3..0f4b3a3 100644 --- a/class/csi-cloudscale.yml +++ b/class/csi-cloudscale.yml @@ -22,3 +22,9 @@ parameters: - ${_base_directory}/component/main.jsonnet input_type: jsonnet output_path: csi-cloudscale/ + commodore: + postprocess: + filters: + - type: jsonnet + filter: postprocess/driver_daemonset_tolerations.jsonnet + path: csi-cloudscale/01_helm_chart/csi-cloudscale/templates diff --git a/postprocess/driver_daemonset_tolerations.jsonnet b/postprocess/driver_daemonset_tolerations.jsonnet new file mode 100644 index 0000000..d892374 --- /dev/null +++ b/postprocess/driver_daemonset_tolerations.jsonnet @@ -0,0 +1,29 @@ +local com = import 'lib/commodore.libjsonnet'; + +local inv = com.inventory(); +local params = inv.parameters.csi_cloudscale; +local tolerations = params.driver_daemonset_tolerations; + +local chartDir = std.extVar('output_path'); + +com.fixupDir( + chartDir, + function(obj) + if obj.kind == 'DaemonSet' then + obj { + spec+: { + template+: { + spec+: { + [if std.length(tolerations) > 0 then 'tolerations']+: [ + tolerations[name] { + key: name, + } + for name in std.objectFields(tolerations) + ], + }, + }, + }, + } + else + obj +) diff --git a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml index a30a2a6..07395df 100644 --- a/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml +++ b/tests/golden/openshift4/csi-cloudscale/csi-cloudscale/01_helm_chart/csi-cloudscale/templates/daemonset.yaml @@ -86,6 +86,10 @@ spec: hostNetwork: true priorityClassName: system-node-critical serviceAccount: csi-cloudscale-node-sa + tolerations: + - effect: NoSchedule + key: storagenode + operator: Exists volumes: - hostPath: path: /var/lib/kubelet/plugins_registry/ diff --git a/tests/openshift4.yml b/tests/openshift4.yml index 1d1921c..9df4a9a 100644 --- a/tests/openshift4.yml +++ b/tests/openshift4.yml @@ -14,3 +14,9 @@ parameters: facts: distribution: openshift4 + + csi_cloudscale: + driver_daemonset_tolerations: + storagenode: + operator: Exists + effect: NoSchedule From bc5c586ca7bc76e81dd04fc7e3663119b7d78e50 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 25 Aug 2023 15:22:35 +0200 Subject: [PATCH 4/6] Update documentation --- docs/modules/ROOT/pages/index.adoc | 2 +- .../ROOT/pages/references/parameters.adoc | 47 +++++++++++++++---- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 5787d9a..c72cf12 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -13,7 +13,7 @@ See https://github.com/cloudscale-ch/csi-cloudscale#kubernetes-compatibility["Ku == StorageClasses -The following storage classess are set up by this component: +The following storage classes are set up by this component: [cols="2,2a,2"] |=== diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 2180c46..2e7e855 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -14,26 +14,41 @@ In K8s prior to 1.17, the driver won't run in namespaces other than `kube-system This is because the priority class `system-cluster-critical` is only available to pods in namespace `kube-system`. See https://kubernetes.io/docs/concepts/policy/resource-quotas/#limit-priority-class-consumption-by-default[the Kubernetes priority class consumption] documentation for instructions on how to allow the driver to run in a namespace other than `kube-system`. -== `version` +== `images` [horizontal] -type:: string -default:: `v3.5.0` +type:: dictionary +default:: https://github.com/projectsyn/component-csi-cloudscale/blob/master/class/defaults.yml[See `class/defaults.yml`] -Version of the driver to install. -See https://github.com/cloudscale-ch/csi-cloudscale/releases[available versions]. -See https://github.com/cloudscale-ch/csi-cloudscale#kubernetes-compatibility[Kubernetes compatibility] to choose the right version for your cluster. +Container image to use for the cloudscale.ch CSI driver. +See the upstream list of https://github.com/cloudscale-ch/csi-cloudscale/releases[available versions] for supported values. +See the upstream https://github.com/cloudscale-ch/csi-cloudscale#kubernetes-compatibility[Kubernetes compatibility table] to choose the right version for your cluster. + +== `charts` + +[horizontal] +type:: dictionary +default:: https://github.com/projectsyn/component-csi-cloudscale/blob/master/class/defaults.yml[See `class/defaults.yml`] +Helm chart to use to deploy the cloudscale.ch CSI driver. == `api_token` [horizontal] type:: string -default:: Vault reference +default:: `?{vaultkv:${cluster:tenant}/${cluster:name}/cloudscale/token}` -Cloudscale API token to be used by the CSI driver. +cloudscale.ch API token to be used by the CSI driver. This should be a reference to a secret in Vault instead of the plaintext token. +== `api_token_secret_name` + +[horizontal] +type:: string +default:: `cloudscale` + +Name of the Kubernetes secret which is created to hold the API token. +The value of this parameter is also passed to the Helm chart as `helm_values.token.existingSecret` == `fs_type` @@ -55,6 +70,8 @@ Tolerations that should be applied to the CSI node driver daemonset. The component will transform entries in the dictionary to valid Kubernetes `tolerations` entries. The component will reuse the key in the dictionary as value for field `key` in the `tolerations` entry. +NOTE: The component will append tolerations defined in this parameter to tolerations provided through `helm_values.node.tolerations`. + === Example Allow the CSI node driver daemonset to be scheduled on nodes which have a `storagenode` taint. @@ -120,3 +137,17 @@ parameters: limits: memory: 256Mi ---- + +== `helm_values` + +[horizontal] +type:: dictionary +default:: https://github.com/projectsyn/component-csi-cloudscale/blob/master/class/defaults.yml[See `class/defaults.yml`] + +Helm values to use when rendering the CSI driver Helm chart. +See https://github.com/cloudscale-ch/csi-cloudscale/blob/master/charts/csi-cloudscale/values.yaml[the upstream `values.yaml`] for supported values. + +[NOTE] +==== +We explicitly set fields `controller.serviceAccountName` and `node.serviceAccountName` and reference those fields in the component to manage additional RoleBindings on OpenShift 4. +==== From 27b3d79916922aa0580cb29a6f83444348ba06fd Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 25 Aug 2023 15:35:56 +0200 Subject: [PATCH 5/6] Add warning when removed parameter `version` is used --- component/main.jsonnet | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index e7eb6b7..74e7d32 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -60,6 +60,15 @@ local customRBAC = if isOpenshift then [ }, ] else []; +local warnDeprecatedParam(o) = + if std.objectHas(params, 'version') then + std.trace( + 'Component parameter `version` is removed and its value is ignored. Please use parameters `charts` and `images` to override the csi-cloudscale version.', + o + ) + else + o; + { [if params.namespace != 'kube-system' then '00_namespace']: kube.Namespace(params.namespace) + if isOpenshift then { metadata+: { @@ -69,6 +78,6 @@ local customRBAC = if isOpenshift then [ }, } else {}, '01_storageclasses': std.flattenArrays(storageclasses), - '02_secret': secret, + '02_secret': warnDeprecatedParam(secret), [if std.length(customRBAC) > 0 then '30_custom_rbac']: customRBAC, } From 35ddf741b2de397f3f427281bc9ec6f2dfacf248 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 25 Aug 2023 15:43:38 +0200 Subject: [PATCH 6/6] Add migration guide We don't need it ourselves since we don't use parameter `version` anywhere. --- .../ROOT/pages/how-tos/upgrade-3.x-to-4.x.adoc | 12 ++++++++++++ docs/modules/ROOT/partials/nav.adoc | 1 + 2 files changed, 13 insertions(+) create mode 100644 docs/modules/ROOT/pages/how-tos/upgrade-3.x-to-4.x.adoc diff --git a/docs/modules/ROOT/pages/how-tos/upgrade-3.x-to-4.x.adoc b/docs/modules/ROOT/pages/how-tos/upgrade-3.x-to-4.x.adoc new file mode 100644 index 0000000..ea810f7 --- /dev/null +++ b/docs/modules/ROOT/pages/how-tos/upgrade-3.x-to-4.x.adoc @@ -0,0 +1,12 @@ += Upgrade from 3.x to 4.x + +This guide describes the steps to perform an upgrade of the component from version 3.x to 4.x. + +== Migration guide + +Parameter `version` has been removed in component version v4.0.0. +If you use this parameter to deploy a specific version of the CSI driver, you should set parameter `images.cloudscale-csi-plugin.tag` to the desired version. + +Alternatively, you may want to select a specific Helm chart version. +See the https://github.com/cloudscale-ch/csi-cloudscale/releases?q=helm-csi&expanded=true[helm-csi-cloudscale GitHub releases] for available Helm chart versions. +The chart version can be specified in parameter `charts.csi-cloudscale.version`. diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc index d30d749..226fa80 100644 --- a/docs/modules/ROOT/partials/nav.adoc +++ b/docs/modules/ROOT/partials/nav.adoc @@ -3,6 +3,7 @@ .How-to guides * xref:how-tos/upgrade-1.x-to-2.x.adoc[Upgrade 1.x to 2.x] * xref:how-tos/upgrade-2.x-to-3.x.adoc[Upgrade 2.x to 3.x] +* xref:how-tos/upgrade-3.x-to-4.x.adoc[Upgrade 3.x to 4.x] .Technical reference * xref:references/parameters.adoc[Parameters]