diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml new file mode 100644 index 00000000..5dac2d2e --- /dev/null +++ b/.github/workflows/release-chart.yml @@ -0,0 +1,38 @@ +name: Release Charts + +on: + push: + branches: + - master + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3.3 + with: + version: v3.9.3 + + - name: Add repositories + run: | + for dir in $(ls -d charts/*/); do + helm dependency list $dir 2> /dev/null | tail +2 | head -n -1 | awk '{ print "helm repo add " $1 " " $3 }' | while read cmd; do $cmd; done + done + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.4.0 + with: + config: cr.yaml + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index 6ea0da43..1edb61b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # IntelliJ IDEA / GoLand .idea *.iml +values.yaml +charts/csi-cloudscale/charts cmd/cloudscale-csi-plugin/cloudscale-csi-plugin diff --git a/CHANGELOG.md b/CHANGELOG.md index 39667ef3..83d115ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## unreleased +## v3.3.0 - 2022.09.22 +* Package as Helm chart. +* Always set `CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE` in manifest. +* Explicitly set `reclaimPolicy` and `volumeBindingMode` for storage classes to Kubernetes default values. + ## v3.2.1 - 2022.07.12 * Ensure that the device has the expected size in NodeExpandVolume to avoid a race-condition that appeared in testing. * Minor changes to the integration test suite. diff --git a/Makefile b/Makefile index 169b840f..0eb381cb 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ LDFLAGS ?= -X github.com/cloudscale-ch/csi-cloudscale/driver.version=${VERSION} PKG ?= github.com/cloudscale-ch/csi-cloudscale/cmd/cloudscale-csi-plugin VERSION ?= $(shell cat VERSION) +CHART_VERSION ?= $(shell awk '/^version:/ {print $$2}' charts/csi-cloudscale/Chart.yaml) DOCKER_REPO ?= quay.io/cloudscalech/cloudscale-csi-plugin all: check-unused test @@ -30,13 +31,22 @@ bump-version: @(echo ${NEW_VERSION} | grep -E "^v") || ( echo "NEW_VERSION must be a semver ('v' prefix is required)"; exit 1 ) @echo "Bumping VERSION from $(VERSION) to $(NEW_VERSION)" @echo $(NEW_VERSION) > VERSION - @cp deploy/kubernetes/releases/csi-cloudscale-${VERSION}.yaml deploy/kubernetes/releases/csi-cloudscale-${NEW_VERSION}.yaml - @sed -i'' -e 's/${VERSION}/${NEW_VERSION}/g' deploy/kubernetes/releases/csi-cloudscale-${NEW_VERSION}.yaml @sed -i'' -e 's/${VERSION}/${NEW_VERSION}/g' README.md + @sed -i'' -e 's/${VERSION}/${NEW_VERSION}/g' charts/csi-cloudscale/values.yaml + @sed -i'' -e 's/${VERSION:v%=%}/${NEW_VERSION:v%=%}/g' charts/csi-cloudscale/Chart.yaml + @helm template csi-cloudscale -n kube-system --set nameOverride=csi-cloudscale --set renderNamespace=true ./charts/csi-cloudscale > deploy/kubernetes/releases/csi-cloudscale-${NEW_VERSION}.yaml $(eval NEW_DATE = $(shell date +%Y.%m.%d)) @sed -i'' -e 's/## unreleased/## ${NEW_VERSION} - ${NEW_DATE}/g' CHANGELOG.md @ echo '## unreleased\n' | cat - CHANGELOG.md > temp && mv temp CHANGELOG.md - @rm README.md-e CHANGELOG.md-e deploy/kubernetes/releases/csi-cloudscale-${NEW_VERSION}.yaml-e + @rm README.md-e CHANGELOG.md-e charts/csi-cloudscale/Chart.yaml-e charts/csi-cloudscale/values.yaml-e + +.PHONY: bump-chart-version +bump-chart-version: + @[ "${NEW_CHART_VERSION}" ] || ( echo "NEW_CHART_VERSION must be set (ex. make NEW_CHART_VERSION=v1.x.x bump-version)"; exit 1 ) + @(echo ${NEW_CHART_VERSION} | grep -E "^v") || ( echo "NEW_CHART_VERSION must be a semver ('v' prefix is required)"; exit 1 ) + @echo "Bumping CHART_VERSION from $(CHART_VERSION) to $(NEW_CHART_VERSION)" + @sed -i'' -e 's/${CHART_VERSION:v%=%}/${NEW_CHART_VERSION:v%=%}/g' charts/csi-cloudscale/Chart.yaml + @rm charts/csi-cloudscale/Chart.yaml-e .PHONY: compile compile: @@ -84,3 +94,7 @@ vendor: clean: @echo "==> Cleaning releases" @GOOS=${OS} go clean -i -x ./... + +.PHONY: helm-template +helm-template: + @helm template csi-cloudscale -n kube-system --set nameOverride=csi-cloudscale ./charts/csi-cloudscale \ No newline at end of file diff --git a/README.md b/README.md index 35906443..49c85614 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,17 @@ The cloudscale.ch CSI plugin is mostly tested on Kubernetes. Theoretically it should also work on other Container Orchestrators, such as Mesos or Cloud Foundry. Feel free to test it on other COs and give us a feedback. +## TL;DR + +```shell +# Add a cloudscale.ch API token as secret, replace the placeholder string starting with `a05...` with your own secret +$ kubectl -n kube-system create secret generic cloudscale --from-literal=access-token=a05dd2f26b9b9ac2asdas__REPLACE_ME____123cb5d1ec17513e06da +# Add repository +$ helm repo add csi-cloudscale https://cloudscale-ch.github.io/csi-cloudscale +# Install Driver +$ helm install -n kube-system -g csi-cloudscale/csi-cloudscale +``` + ## Volume parameters This plugin supports the following volume parameters (in case of kubernetes: parameters on the @@ -50,7 +61,7 @@ secret `my-pvc-luks-key`. ## Releases The cloudscale.ch CSI plugin follows [semantic versioning](https://semver.org/). -The current version is: **`v3.2.1`**. +The current version is: **`v3.3.0`**. * Bug fixes will be released as a `PATCH` update. * New features (such as CSI spec bumps) will be released as a `MINOR` update. @@ -67,13 +78,13 @@ We recommend using the latest cloudscale.ch CSI driver compatible with your Kube |--------------------|----------------------------------|----------------------------------| | <= 1.16 | | v1.3.1 | | 1.17 | v1.3.1 | v3.0.0 | -| 1.18 | v1.3.1 | v3.2.1 | -| 1.19 | v1.3.1 | v3.2.1 | -| 1.20 | v2.0.0 | v3.2.1 | -| 1.21 | v2.0.0 | v3.2.1 | -| 1.22 | v3.1.0 | v3.2.1 | -| 1.23 | v3.1.0 | v3.2.1 | -| 1.24 | v3.1.0 | v3.2.1 | +| 1.18 | v1.3.1 | v3.3.0 | +| 1.19 | v1.3.1 | v3.3.0 | +| 1.20 | v2.0.0 | v3.3.0 | +| 1.21 | v2.0.0 | v3.3.0 | +| 1.22 | v3.1.0 | v3.3.0 | +| 1.23 | v3.1.0 | v3.3.0 | +| 1.24 | v3.1.0 | v3.3.0 | **Requirements:** @@ -118,19 +129,61 @@ cloudscale Opaque 1 18h #### 2. Deploy the CSI plugin and sidecars: -Before you continue, be sure to checkout to a [tagged -release](https://github.com/cloudscale-ch/csi-cloudscale/releases). -Always use the [latest stable version](https://github.com/cloudscale-ch/csi-cloudscale/releases/latest) -For example, to use the latest stable version (`v3.2.1`) you can execute the following command: +You can install the CSI plugin and sidecars using one of the following methods: + * Helm (requires a Helm installation) + * YAML Manifests (only kubectl required) + + +#### 2a. Using Helm: + +Before you can install the csi-cloudscale chart, you need to add the helm repository: ``` -$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-v3.2.1.yaml +$ helm repo add csi-cloudscale https://cloudscale-ch.github.io/csi-cloudscale ``` -There are also `dev` images available: +Then install the latest stable version: ``` -$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-dev.yaml +$ helm install -n kube-system -g csi-cloudscale/csi-cloudscale +``` + +Advanced users can customize the installation by specifying custom values. +The following table summarizes the most-frequently used parameters. +For a complete list please refer to [values.yaml](./charts/csi-cloudscale/values.yaml) + +| Parameter | Default | Description | +|-------------------------------------|------------------------------|--------------------------------------------------------------------------------------------| +| attacher.resources | `{}` | Resource limits and requests for the attacher side-car. | +| cloudscale.apiUrl | `https://api.cloudscale.ch/` | URL of the cloudscale.ch API. You can almost certainly use the default | +| cloudscale.max_csi_volumes_per_node | `125` | Override [max. Number of CSI Volumes per Node](#Max.-Number-of-CSI-Volumes-per-Node) | +| cloudscale.token.existingSecret | `cloudscale` | Name of the Kubernetes Secret which contains the cloudscale.ch API Token. | +| controller.resources | `{}` | Resource limits and requests for the controller container. | +| controller.serviceAccountName | `null` | Override the controller service account name. | +| driverRegistrar.resources | `{}` | Resource limits and requests for the driverRegistrar side-car. | +| extraDeploy | `[]` | To deploy extra objects together with the driver. | +| nameOverride | `null` | Override the default `{{ .Release.Name }}-csi-cloudscale` name pattern with a custom name. | +| node.resources | `{}` | Resource limits and requests for the node container. | +| node.serviceAccountName | `null` | Override the controller node account name. | +| node.tolerations | `[]` | Set tolerations on the node daemonSet. | +| provisioner.resources | `{}` | Resource limits and requests for the provisioner side-car. | +| resizer.resources | `{}` | Resource limits and requests for the resizer side-car. | + +Note: if you want to test a debug/dev release, you can use the following command: + +``` +$ helm install -g -n kube-system --set controller.image.tag=dev --set node.image.tag=dev ./charts/csi-cloudscale +``` + +#### 2b. Using YAML Manifests: + +Before you continue, be sure to checkout to a [tagged +release](https://github.com/cloudscale-ch/csi-cloudscale/releases). +Always use the [latest stable version](https://github.com/cloudscale-ch/csi-cloudscale/releases/latest) +For example, to use the latest stable version (`v3.3.0`) you can execute the following command: + +``` +$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-v3.3.0.yaml ``` The storage classes `cloudscale-volume-ssd` and `cloudscale-volume-bulk` will be created. The @@ -138,9 +191,6 @@ storage class `cloudscale-volume-ssd` is set to **"default"** for dynamic provis using multiple storage classes you might want to remove the annotation and re-deploy it. This is based on the [recommended mechanism](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md#recommended-mechanism-for-deploying-csi-drivers-on-kubernetes) of deploying CSI drivers on Kubernetes -*Note that the deployment proposal to Kubernetes is still a work in progress and not all of the written -features are implemented. When in doubt, open an issue or ask #sig-storage in [Kubernetes Slack](http://slack.k8s.io)* - #### 3. Test and verify: Create a PersistentVolumeClaim. This makes sure a volume is created and provisioned on your behalf: @@ -260,6 +310,8 @@ env: value: '10' ``` +Or use the `cloudscale.max_csi_volumes_per_node` value of the [Helm chart](#2a-using-helm). + Note that there are currently the following hard-limits per Node: * 26 volumes (including root) for `virtio-blk` (`/dev/vdX`). * 128 volumes (including root) for `virtio-scsi` (`/dev/sdX`). @@ -269,6 +321,26 @@ Note that there are currently the following hard-limits per Node: Requirements: * Go: min `v1.10.x` +* Helm + +Build out the `charts/` directory from the `Chart.lock` file: + +``` +$ helm dependency build charts/csi-cloudscale +``` + +Install the chart from local sources: + +``` +$ helm install -n kube-system -g ./charts/csi-cloudscale +``` + +Useful commands to compare the generated helm chart to the static YAML manifests: + +``` +$ helm template csi-cloudscale --dry-run -n kube-system --set nameOverride=csi-cloudscale charts/csi-cloudscale | kubectl-slice -f - -o deploy/kubernetes/releases/generated +$ kubectl-slice -f deploy/kubernetes/releases/csi-cloudscale-v6.0.0.yaml -o deploy/kubernetes/releases/v3 +``` After making your changes, run the unit tests: @@ -303,6 +375,7 @@ To release a new version bump first the version: ``` $ make NEW_VERSION=vX.Y.Z bump-version +$ make NEW_CHART_VERSION=vX.Y.Z bump-chart-version ``` Make sure everything looks good. Verify that the Kubernetes compatibility matrix is up-to-date. @@ -316,15 +389,15 @@ $ git push origin After it's merged to master, [create a new Github release](https://github.com/cloudscale-ch/csi-cloudscale/releases/new) from -master with the version `v3.2.1` and then publish a new docker build: +master with the version `v3.3.0` and then publish a new docker build: ``` $ git checkout master $ make publish ``` -This will create a binary with version `v3.2.1` and docker image pushed to -`cloudscalech/cloudscale-csi-plugin:v3.2.1` +This will create a binary with version `v3.3.0` and docker image pushed to +`cloudscalech/cloudscale-csi-plugin:v3.3.0` ## Contributing diff --git a/VERSION b/VERSION index 040943e5..b299be97 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v3.2.1 +v3.3.0 diff --git a/charts/csi-cloudscale/Chart.lock b/charts/csi-cloudscale/Chart.lock new file mode 100644 index 00000000..b3c016ca --- /dev/null +++ b/charts/csi-cloudscale/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:c9dc9bde4c38cea08319ae9f4583b58ae3494d9d5ccedfa53fcd8661df127d1f +generated: "2022-08-18T13:40:45.580796+02:00" diff --git a/charts/csi-cloudscale/Chart.yaml b/charts/csi-cloudscale/Chart.yaml new file mode 100644 index 00000000..220869f1 --- /dev/null +++ b/charts/csi-cloudscale/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: csi-cloudscale +description: A Container Storage Interface Driver for cloudscale.ch volumes. +type: application +version: 1.0.0 +appVersion: "3.3.0" +home: https://github.com/cloudscale-ch/csi-cloudscale +sources: + - https://github.com/cloudscale-ch/csi-cloudscale.git +dependencies: + - name: common + version: 1.17.1 + repository: https://charts.bitnami.com/bitnami + diff --git a/charts/csi-cloudscale/templates/_helpers.tpl b/charts/csi-cloudscale/templates/_helpers.tpl new file mode 100644 index 00000000..b2fbfbf2 --- /dev/null +++ b/charts/csi-cloudscale/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* Get Driver Name */}} +{{- define "csi-cloudscale.driver-name" -}} +{{- if .Values.nameOverride -}} + {{ .Values.nameOverride }} +{{- else -}} + {{ .Release.Name }}-csi-cloudscale +{{- end -}} +{{- end -}} + +{{/* Get API Token Name */}} +{{- define "csi-cloudscale.api-token-name" -}} +{{ required "cloudscale.token.existingSecret" .Values.cloudscale.token.existingSecret }} +{{- end -}} + +{{/* Get Controller Service Account Name*/}} +{{- define "csi-cloudscale.controller-service-account-name" -}} +{{- if .Values.controller.serviceAccountName -}} + {{ .Values.controller.serviceAccountName }} +{{- else -}} + {{ include "csi-cloudscale.driver-name" . }}-controller-sa +{{- end -}} +{{- end -}} + +{{/* Get Node Service Account Name*/}} +{{- define "csi-cloudscale.node-service-account-name" -}} +{{- if .Values.node.serviceAccountName -}} + {{ .Values.node.serviceAccountName }} +{{- else -}} + {{ include "csi-cloudscale.driver-name" . }}-node-sa +{{- end -}} +{{- end -}} + +{{/* When renderNamespace is true, include a Namespace definition. This is for emitting old-school YAMLs */}} +{{- define "csi-cloudscale.namespace-in-yaml-manifest" -}} +{{/* See: https://github.com/helm/helm/issues/5465#issuecomment-473942223 */}} +{{- if .Values.renderNamespace -}} +namespace: {{ .Release.Namespace }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/csi-cloudscale/templates/csi_driver.yaml b/charts/csi-cloudscale/templates/csi_driver.yaml new file mode 100644 index 00000000..548192cd --- /dev/null +++ b/charts/csi-cloudscale/templates/csi_driver.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.cloudscale.ch +spec: + attachRequired: true + podInfoOnMount: true diff --git a/charts/csi-cloudscale/templates/daemonset.yaml b/charts/csi-cloudscale/templates/daemonset.yaml new file mode 100644 index 00000000..0879b267 --- /dev/null +++ b/charts/csi-cloudscale/templates/daemonset.yaml @@ -0,0 +1,112 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-node + {{ include "csi-cloudscale.namespace-in-yaml-manifest" . }} +spec: + selector: + matchLabels: + app: csi-cloudscale-node + template: + metadata: + labels: + app: csi-cloudscale-node + role: csi-cloudscale + spec: + priorityClassName: system-node-critical + serviceAccount: {{ include "csi-cloudscale.node-service-account-name" . }} + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: "{{ .Values.driverRegistrar.image.registry }}/{{ .Values.driverRegistrar.image.repository }}:{{ .Values.driverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.driverRegistrar.image.pullPolicy }} + args: + - "--v={{ .Values.driverRegistrar.logLevelVerbosity }}" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.cloudscale.ch /registration/csi.cloudscale.ch-reg.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.cloudscale.ch/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.driverRegistrar.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: csi-cloudscale-plugin + image: "{{ .Values.node.image.registry}}/{{ .Values.node.image.repository }}:{{ .Values.node.image.tag }}" + imagePullPolicy: {{ .Values.node.image.pullPolicy }} + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--url=$(CLOUDSCALE_API_URL)" + {{- with .Values.node.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: CLOUDSCALE_API_URL + value: {{ .Values.cloudscale.apiUrl }} + - name: CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE + value: {{ .Values.cloudscale.max_csi_volumes_per_node | quote }} + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "csi-cloudscale.api-token-name" . }} + key: access-token + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + - name: tmpfs + mountPath: /tmp + {{- with .Values.node.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.cloudscale.ch + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + # to make sure temporary stored luks keys never touch a disk + - name: tmpfs + emptyDir: + medium: Memory diff --git a/charts/csi-cloudscale/templates/extradeploy.yaml b/charts/csi-cloudscale/templates/extradeploy.yaml new file mode 100644 index 00000000..9ac65f9e --- /dev/null +++ b/charts/csi-cloudscale/templates/extradeploy.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraDeploy }} +--- +{{ include "common.tplvalues.render" (dict "value" . "context" $) }} +{{- end }} diff --git a/charts/csi-cloudscale/templates/rbac.yaml b/charts/csi-cloudscale/templates/rbac.yaml new file mode 100644 index 00000000..7013c80a --- /dev/null +++ b/charts/csi-cloudscale/templates/rbac.yaml @@ -0,0 +1,134 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-node-driver-registrar-role + {{ include "csi-cloudscale.namespace-in-yaml-manifest" . }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-provisioner-binding +subjects: + - kind: ServiceAccount + name: {{ include "csi-cloudscale.controller-service-account-name" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "csi-cloudscale.driver-name" . }}-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-resizer-binding +subjects: + - kind: ServiceAccount + name: {{ include "csi-cloudscale.controller-service-account-name" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "csi-cloudscale.driver-name" . }}-resizer-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-attacher-binding +subjects: + - kind: ServiceAccount + name: {{ include "csi-cloudscale.controller-service-account-name" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "csi-cloudscale.driver-name" . }}-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-node-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: {{ include "csi-cloudscale.node-service-account-name" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "csi-cloudscale.driver-name" . }}-node-driver-registrar-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/csi-cloudscale/templates/serviceaccount.yaml b/charts/csi-cloudscale/templates/serviceaccount.yaml new file mode 100644 index 00000000..12ab9ee5 --- /dev/null +++ b/charts/csi-cloudscale/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "csi-cloudscale.controller-service-account-name" . }} + {{ include "csi-cloudscale.namespace-in-yaml-manifest" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "csi-cloudscale.node-service-account-name" . }} + {{ include "csi-cloudscale.namespace-in-yaml-manifest" . }} diff --git a/charts/csi-cloudscale/templates/statefulset.yaml b/charts/csi-cloudscale/templates/statefulset.yaml new file mode 100644 index 00000000..edd25662 --- /dev/null +++ b/charts/csi-cloudscale/templates/statefulset.yaml @@ -0,0 +1,99 @@ +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: {{ include "csi-cloudscale.driver-name" . }}-controller + {{ include "csi-cloudscale.namespace-in-yaml-manifest" . }} +spec: + serviceName: "csi-cloudscale" + selector: + matchLabels: + app: csi-cloudscale-controller + replicas: {{ .Values.controller.replicas }} + template: + metadata: + labels: + app: csi-cloudscale-controller + role: csi-cloudscale + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: {{ include "csi-cloudscale.controller-service-account-name" . }} + containers: + - name: csi-provisioner + image: "{{ .Values.provisioner.image.registry }}/{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--default-fstype=ext4" + - "--v={{ .Values.provisioner.logLevelVerbosity }}" + {{- with .Values.provisioner.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: "{{ .Values.attacher.image.registry }}/{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" + imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.attacher.logLevelVerbosity }}" + {{- with .Values.attacher.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-resizer + image: "{{ .Values.resizer.image.registry }}/{{ .Values.resizer.image.repository }}:{{ .Values.resizer.image.tag }}" + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=30s" + - "--v={{ .Values.resizer.logLevelVerbosity }}" + - "--handle-volume-inuse-error=false" + {{- with .Values.resizer.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.resizer.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-cloudscale-plugin + image: "{{ .Values.controller.image.registry }}/{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--url=$(CLOUDSCALE_API_URL)" + {{- with .Values.controller.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "csi-cloudscale.api-token-name" . }} + key: access-token + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/csi-cloudscale/templates/storageclass.yaml b/charts/csi-cloudscale/templates/storageclass.yaml new file mode 100644 index 00000000..15766a0c --- /dev/null +++ b/charts/csi-cloudscale/templates/storageclass.yaml @@ -0,0 +1,27 @@ +{{- $csi := .Values.csi }} +{{- $provisioner := .Values.provisioner }} +{{- range $storageClass := .Values.csi.storageClasses }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ $storageClass.name }} + {{ include "csi-cloudscale.namespace-in-yaml-manifest" $ }} +{{- if $storageClass.default }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +{{- end }} +provisioner: {{ $storageClass.provisioner | default $provisioner.name }} +allowVolumeExpansion: {{ $storageClass.allowVolumeExpansion | default $csi.allowVolumeExpansion }} +reclaimPolicy: {{ $storageClass.reclaimPolicy | default $csi.reclaimPolicy }} +volumeBindingMode: {{ $storageClass.volumeBindingMode | default $csi.volumeBindingMode }} +parameters: + csi.cloudscale.ch/volume-type: {{ $storageClass.volumeType }} +{{- if $storageClass.luks.enabled }} + csi.cloudscale.ch/luks-encrypted: "true" + csi.cloudscale.ch/luks-cipher: {{ quote $storageClass.luks.cipher }} + csi.cloudscale.ch/luks-key-size: {{ quote $storageClass.luks.keySize }} + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key +{{- end }} +{{- end }} diff --git a/charts/csi-cloudscale/values.yaml b/charts/csi-cloudscale/values.yaml new file mode 100644 index 00000000..827b1de9 --- /dev/null +++ b/charts/csi-cloudscale/values.yaml @@ -0,0 +1,131 @@ +cloudscale: + apiUrl: https://api.cloudscale.ch/ + token: + existingSecret: cloudscale + max_csi_volumes_per_node: 125 + +nameOverride: + +csi: + allowVolumeExpansion: true + reclaimPolicy: Delete + volumeBindingMode: Immediate + storageClasses: + - name: cloudscale-volume-ssd + volumeType: ssd + default: true + luks: + enabled: false + - name: cloudscale-volume-ssd-luks + volumeType: ssd + luks: + enabled: true + cipher: aes-xts-plain64 + keySize: 512 + - name: cloudscale-volume-bulk + volumeType: bulk + luks: + enabled: false + - name: cloudscale-volume-bulk-luks + volumeType: bulk + luks: + enabled: true + cipher: aes-xts-plain64 + keySize: 512 + +provisioner: + name: csi.cloudscale.ch + image: + registry: quay.io + repository: k8scsi/csi-provisioner + tag: v2.0.4 + pullPolicy: IfNotPresent + logLevelVerbosity: 5 + resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +attacher: + image: + registry: quay.io + repository: k8scsi/csi-attacher + tag: v3.0.2 + pullPolicy: IfNotPresent + logLevelVerbosity: "5" + resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +resizer: + image: + registry: quay.io + repository: k8scsi/csi-resizer + tag: v1.0.1 + pullPolicy: IfNotPresent + logLevelVerbosity: "5" + resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +controller: + replicas: 1 + image: + registry: quay.io + repository: cloudscalech/cloudscale-csi-plugin + tag: v3.3.0 + pullPolicy: IfNotPresent + serviceAccountName: + resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +node: + image: + registry: quay.io + repository: cloudscalech/cloudscale-csi-plugin + tag: v3.3.0 + pullPolicy: IfNotPresent + tolerations: [] + serviceAccountName: + resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +driverRegistrar: + image: + registry: quay.io + repository: k8scsi/csi-node-driver-registrar + tag: v2.0.1 + pullPolicy: IfNotPresent + logLevelVerbosity: "5" + resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +extraDeploy: [] + +renderNamespace: false \ No newline at end of file diff --git a/cr.yaml b/cr.yaml new file mode 100644 index 00000000..65d8e669 --- /dev/null +++ b/cr.yaml @@ -0,0 +1,2 @@ +## chart-releaser config +release-name-template: "helm-{{ .Name }}-{{ .Version }}" \ No newline at end of file diff --git a/deploy/README.rst b/deploy/README.rst index 0077799e..e83673e3 100644 --- a/deploy/README.rst +++ b/deploy/README.rst @@ -9,11 +9,13 @@ deploy kubernetes:: python3 -m venv venv . venv/bin/activate + # or requirements-{VERSION}.txt, see https://github.com/kubernetes-sigs/kubespray/blob/master/docs/ansible.md#ansible-python-compatibility pip install -r kubespray/requirements.txt + cd kubespray/ After this you run:: - CLOUDSCALE_TOKEN="foobar" ansible-playbook integration_test.yml -i inventory/hosts.ini + CLOUDSCALE_TOKEN="foobar" ansible-playbook ../integration_test.yml -i inventory/hosts.ini to install kubernetes on cloudscale.ch and run the integration tests. The playbook will also clean up VMs after the test. diff --git a/deploy/integration_test.yml b/deploy/integration_test.yml index 7256bead..eb46dade 100644 --- a/deploy/integration_test.yml +++ b/deploy/integration_test.yml @@ -34,8 +34,9 @@ hostname: "{{ created_servers.results[0].name }}" ansible_ssh_host: "{{ created_servers.results[0].interfaces[0].addresses[0].address }}" groups: - - kube-master + - kube_control_plane - etcd + - k8s_cluster ansible_user: 'root' - name: Register nodes in inventory @@ -43,13 +44,14 @@ hostname: "{{ item.name }}" ansible_ssh_host: "{{ item.interfaces[0].addresses[0].address }}" groups: - - kube-node + - kube_node + - k8s_cluster ansible_user: 'root' loop: ["{{ created_servers.results[1] }}", "{{ created_servers.results[2] }}"] # this is not needed for the plugin or the integration tests to work, but helps if you want # to take a closer look at the LUKS volumes with SSH on a node -- hosts: k8s-cluster +- hosts: k8s_cluster tags: [install-cryptsetup] tasks: - name: "Install cryptsetup on nodes" @@ -60,7 +62,7 @@ tags: [install-kubernetes] import_playbook: kubespray/cluster.yml -- hosts: kube-master +- hosts: kube_control_plane vars: cloudscale_api_token: "{{ lookup('env','CLOUDSCALE_TOKEN') }}" secret_file: 'secret.yml' @@ -87,13 +89,24 @@ debug: msg: "{{ secret_output.stdout_lines }}" - - name: Copy default cloudscale-csi config to kubernetes + # It would be nice to install helm using the tasks/roles in kubepsray, + # but I could not get it working. + - name: Get Helm installer + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + dest: "./get_helm.sh" + mode: '0700' + + - name: Install Helm + shell: "./get_helm.sh" + + - name: Copy chart copy: - src: "kubernetes/releases/csi-cloudscale-{{ version }}.yaml" - dest: "{{ config_file }}" + src: "../charts/csi-cloudscale" + dest: "charts" - - name: Apply default cloudscale-csi config to kubernetes - shell: "kubectl apply -f {{ config_file }}" + - name: Install csi-driver + shell: "helm install -g -n kube-system --set controller.image.tag={{ version }} --set node.image.tag={{ version }} ./charts/csi-cloudscale" - name: Copy kubernetes config to localhost fetch: diff --git a/deploy/inventory/hosts.ini b/deploy/inventory/hosts.ini index 1e5c1b1c..38416399 100644 --- a/deploy/inventory/hosts.ini +++ b/deploy/inventory/hosts.ini @@ -1,17 +1,4 @@ -# ## Configure 'ip' variable to bind kubernetes services on a -# ## different ip than the default iface -# ## We should set etcd_member_name for etcd cluster. The node that is not a etcd member do not need to set the value, or can set the empty string value. -# node1 ansible_host=95.54.0.12 # ip=10.3.0.1 etcd_member_name=etcd1 -# node2 ansible_host=95.54.0.13 # ip=10.3.0.2 etcd_member_name=etcd2 -# node3 ansible_host=95.54.0.14 # ip=10.3.0.3 etcd_member_name=etcd3 -# node4 ansible_host=95.54.0.15 # ip=10.3.0.4 etcd_member_name=etcd4 -# node5 ansible_host=95.54.0.16 # ip=10.3.0.5 etcd_member_name=etcd5 -# node6 ansible_host=95.54.0.17 # ip=10.3.0.6 etcd_member_name=etcd6 - -# ## configure a bastion host if your nodes are not directly reachable -# bastion ansible_host=x.x.x.x ansible_user=some_user - -[kube-master] +[kube_control_plane] # node1 # node2 @@ -20,13 +7,13 @@ # node2 # node3 -[kube-node] +[kube_node] # node2 # node3 # node4 # node5 # node6 -[k8s-cluster:children] -kube-master -kube-node +[k8s_cluster:children] +kube_node +kube_control_plane diff --git a/deploy/kubernetes/releases/csi-cloudscale-dev.yaml b/deploy/kubernetes/releases/csi-cloudscale-v3.3.0.yaml similarity index 84% rename from deploy/kubernetes/releases/csi-cloudscale-dev.yaml rename to deploy/kubernetes/releases/csi-cloudscale-v3.3.0.yaml index 33b94601..c571e113 100644 --- a/deploy/kubernetes/releases/csi-cloudscale-dev.yaml +++ b/deploy/kubernetes/releases/csi-cloudscale-v3.3.0.yaml @@ -1,34 +1,21 @@ -# Copyright cloudscale.ch -# Copyright 2020 DigitalOcean -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Install the CSI Driver. This simplifies driver discovery and enables us to -# customize Kubernetes behavior -# https://kubernetes-csi.github.io/docs/csi-driver-object.html -apiVersion: storage.k8s.io/v1 -kind: CSIDriver +--- +# Source: csi-cloudscale/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount metadata: - name: csi.cloudscale.ch -spec: - attachRequired: true - podInfoOnMount: true - + name: csi-cloudscale-controller-sa + namespace: kube-system --- - -kind: StorageClass +# Source: csi-cloudscale/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-node-sa + namespace: kube-system +--- +# Source: csi-cloudscale/templates/storageclass.yaml apiVersion: storage.k8s.io/v1 +kind: StorageClass metadata: name: cloudscale-volume-ssd namespace: kube-system @@ -36,30 +23,21 @@ metadata: storageclass.kubernetes.io/is-default-class: "true" provisioner: csi.cloudscale.ch allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate parameters: csi.cloudscale.ch/volume-type: ssd - --- - -kind: StorageClass +# Source: csi-cloudscale/templates/storageclass.yaml apiVersion: storage.k8s.io/v1 -metadata: - name: cloudscale-volume-bulk - namespace: kube-system -provisioner: csi.cloudscale.ch -allowVolumeExpansion: true -parameters: - csi.cloudscale.ch/volume-type: bulk - ---- - kind: StorageClass -apiVersion: storage.k8s.io/v1 metadata: name: cloudscale-volume-ssd-luks namespace: kube-system provisioner: csi.cloudscale.ch allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate parameters: csi.cloudscale.ch/volume-type: ssd csi.cloudscale.ch/luks-encrypted: "true" @@ -67,16 +45,30 @@ parameters: csi.cloudscale.ch/luks-key-size: "512" csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key - --- - +# Source: csi-cloudscale/templates/storageclass.yaml +apiVersion: storage.k8s.io/v1 kind: StorageClass +metadata: + name: cloudscale-volume-bulk + namespace: kube-system +provisioner: csi.cloudscale.ch +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + csi.cloudscale.ch/volume-type: bulk +--- +# Source: csi-cloudscale/templates/storageclass.yaml apiVersion: storage.k8s.io/v1 +kind: StorageClass metadata: name: cloudscale-volume-bulk-luks namespace: kube-system provisioner: csi.cloudscale.ch allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate parameters: csi.cloudscale.ch/volume-type: bulk csi.cloudscale.ch/luks-encrypted: "true" @@ -84,108 +76,8 @@ parameters: csi.cloudscale.ch/luks-key-size: "512" csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key - ---- - -############################################## -########### ############ -########### Controller plugin ############ -########### ############ -############################################## - -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-cloudscale-controller - namespace: kube-system -spec: - serviceName: "csi-cloudscale" - selector: - matchLabels: - app: csi-cloudscale-controller - replicas: 1 - template: - metadata: - labels: - app: csi-cloudscale-controller - role: csi-cloudscale - spec: - hostNetwork: true - priorityClassName: system-cluster-critical - serviceAccount: csi-cloudscale-controller-sa - containers: - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v2.0.4 - args: - - "--csi-address=$(ADDRESS)" - - "--default-fstype=ext4" - - "--v=5" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "Always" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v3.0.2 - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "Always" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:v1.0.1 - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=30s" - - "--v=5" - # cloudscale.ch volumes support online resize. - - "--handle-volume-inuse-error=false" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-cloudscale-plugin - image: quay.io/cloudscalech/cloudscale-csi-plugin:dev - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--url=$(CLOUDSCALE_API_URL)" - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: CLOUDSCALE_API_URL - value: https://api.cloudscale.ch/ - - name: CLOUDSCALE_ACCESS_TOKEN - valueFrom: - secretKeyRef: - name: cloudscale - key: access-token - imagePullPolicy: "Always" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - volumes: - - name: socket-dir - emptyDir: {} - ---- - -kind: ServiceAccount -apiVersion: v1 -metadata: - name: csi-cloudscale-controller-sa - namespace: kube-system - --- +# Source: csi-cloudscale/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -218,24 +110,8 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] - --- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-cloudscale-provisioner-binding -subjects: - - kind: ServiceAccount - name: csi-cloudscale-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-cloudscale-provisioner-role - apiGroup: rbac.authorization.k8s.io - ---- -# Attacher must be able to work with PVs, nodes and VolumeAttachments +# Source: csi-cloudscale/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -256,24 +132,8 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-cloudscale-attacher-binding -subjects: - - kind: ServiceAccount - name: csi-cloudscale-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-cloudscale-attacher-role - apiGroup: rbac.authorization.k8s.io - --- - -# Resizer must be able to work with PVCs, PVs, SCs. +# Source: csi-cloudscale/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -291,9 +151,33 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - --- - +# Source: csi-cloudscale/templates/rbac.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-node-driver-registrar-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +# Source: csi-cloudscale/templates/rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: csi-cloudscale/templates/rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -306,15 +190,36 @@ roleRef: kind: ClusterRole name: csi-cloudscale-resizer-role apiGroup: rbac.authorization.k8s.io - --- - -######################################## -########### ############ -########### Node plugin ############ -########### ############ -######################################## - +# Source: csi-cloudscale/templates/rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: csi-cloudscale/templates/rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-node-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-node-driver-registrar-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: csi-cloudscale/templates/daemonset.yaml kind: DaemonSet apiVersion: apps/v1 metadata: @@ -335,7 +240,8 @@ spec: hostNetwork: true containers: - name: csi-node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 + image: "quay.io/k8scsi/csi-node-driver-registrar:v2.0.1" + imagePullPolicy: IfNotPresent args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -359,7 +265,8 @@ spec: - name: registration-dir mountPath: /registration/ - name: csi-cloudscale-plugin - image: quay.io/cloudscalech/cloudscale-csi-plugin:dev + image: "quay.io/cloudscalech/cloudscale-csi-plugin:v3.3.0" + imagePullPolicy: IfNotPresent args : - "--endpoint=$(CSI_ENDPOINT)" - "--url=$(CLOUDSCALE_API_URL)" @@ -368,12 +275,13 @@ spec: value: unix:///csi/csi.sock - name: CLOUDSCALE_API_URL value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE + value: "125" - name: CLOUDSCALE_ACCESS_TOKEN valueFrom: secretKeyRef: name: cloudscale key: access-token - imagePullPolicy: "Always" securityContext: privileged: true capabilities: @@ -411,38 +319,96 @@ spec: - name: tmpfs emptyDir: medium: Memory - --- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cloudscale-node-sa - namespace: kube-system - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +# Source: csi-cloudscale/templates/statefulset.yaml +kind: StatefulSet +apiVersion: apps/v1 metadata: - name: csi-cloudscale-node-driver-registrar-role + name: csi-cloudscale-controller namespace: kube-system -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - +spec: + serviceName: "csi-cloudscale" + selector: + matchLabels: + app: csi-cloudscale-controller + replicas: 1 + template: + metadata: + labels: + app: csi-cloudscale-controller + role: csi-cloudscale + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: csi-cloudscale-controller-sa + containers: + - name: csi-provisioner + image: "quay.io/k8scsi/csi-provisioner:v2.0.4" + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--default-fstype=ext4" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: "quay.io/k8scsi/csi-attacher:v3.0.2" + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-resizer + image: "quay.io/k8scsi/csi-resizer:v1.0.1" + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=30s" + - "--v=5" + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-cloudscale-plugin + image: "quay.io/cloudscalech/cloudscale-csi-plugin:v3.3.0" + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--url=$(CLOUDSCALE_API_URL)" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: cloudscale + key: access-token + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} --- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +# Source: csi-cloudscale/templates/csi_driver.yaml +apiVersion: storage.k8s.io/v1 +kind: CSIDriver metadata: - name: csi-cloudscale-node-driver-registrar-binding -subjects: - - kind: ServiceAccount - name: csi-cloudscale-node-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-cloudscale-node-driver-registrar-role - apiGroup: rbac.authorization.k8s.io + name: csi.cloudscale.ch +spec: + attachRequired: true + podInfoOnMount: true diff --git a/driver/driver_test.go b/driver/driver_test.go index e9b8c05d..63300aa4 100644 --- a/driver/driver_test.go +++ b/driver/driver_test.go @@ -230,7 +230,7 @@ func (f FakeVolumeServiceOperations) Update(ctx context.Context, volumeID string } volumesCount := getVolumesPerServer(f, serverUUID) - if volumesCount >= defaultMaxVolumesPerNode { + if volumesCount >= fallbackMaxVolumesPerNode { return &cloudscale.ErrorResponse{ StatusCode: 400, Message: map[string]string{"detail": "Due to internal limitations, it is currently not possible to attach more than 128 volumes"}, diff --git a/driver/node.go b/driver/node.go index 9886cc86..5940cd4d 100644 --- a/driver/node.go +++ b/driver/node.go @@ -48,7 +48,7 @@ const ( // - 1 for root // - 1 for /var/lib/docker // - 1 additional volume outside of CSI - defaultMaxVolumesPerNode = 125 + fallbackMaxVolumesPerNode = 125 volumeModeBlock = "block" volumeModeFilesystem = "filesystem" @@ -330,7 +330,7 @@ func getEnvAsInt(key string, fallback int64) int64 { func (d *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { d.log.WithField("method", "node_get_info").Info("node get info called") - maxVolumesPerNode := getEnvAsInt("CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE", defaultMaxVolumesPerNode) + maxVolumesPerNode := getEnvAsInt("CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE", fallbackMaxVolumesPerNode) return &csi.NodeGetInfoResponse{ NodeId: d.serverId,