From 03e1b78dd4de7e932fb395f15238e6982c2848e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Ga=C3=9Fner?= <jan.gassner@plusserver.com>
Date: Mon, 21 Dec 2020 09:34:00 +0100
Subject: [PATCH] Added option for st2packs-PersistentVolumes to allow for
 seamless custom pack-updates without job-impact

---
 CHANGELOG.md                          |   2 +-
 README.md                             |  30 +++++-
 templates/_helpers.tpl                |  54 ++++++++++
 templates/deployments.yaml            | 150 +++-----------------------
 templates/jobs.yaml                   |   7 +-
 templates/persistentvolumeclaims.yaml |  32 ++++++
 values.yaml                           |  22 +++-
 7 files changed, 149 insertions(+), 148 deletions(-)
 create mode 100644 templates/persistentvolumeclaims.yaml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e1e646ec..c13f33ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,7 @@
 # Changelog
 
 ## In Development
-
+* Add option for st2packs-PersistentVolumes to allow for seamless custom pack-updates without job-impact (#160) (by @moonrail)
 
 ## v0.40.0
 * Switch st2 version to `v3.4dev` as a new latest development version (#157)
diff --git a/README.md b/README.md
index d35d9ddb..47194656 100644
--- a/README.md
+++ b/README.md
@@ -182,7 +182,7 @@ As any other Helm dependency, it's possible to further configure it for specific
 ## Install custom st2 packs in the cluster
 In distributed environment of the Kubernetes cluster `st2 pack install` won’t work.
 Instead, you need to bake the packs into a custom docker image, push it to a private or public docker registry and reference that image in Helm values.
-Helm chart will take it from there, sharing `/opt/stackstorm/{packs,virtualenvs}` via a sidecar container in pods which require access to the packs.
+Helm chart will take it from there, sharing `/opt/stackstorm/{packs,virtualenvs}` via a sidecar container or alternatively shared PersistentVolume in pods which require access to the packs.
 
 ### Building st2packs image
 For your convenience, we created a new `st2-pack-install <pack1> <pack2> <pack3>` utility and included it in a container that will help to install custom packs during the Docker build process without relying on live DB and MQ connection.
@@ -213,6 +213,34 @@ kubectl create secret docker-registry st2packs-auth --docker-server=<your-regist
 ```
 Once secret created, reference its name in helm value: `st2.packs.image.pullSecret`.
 
+### Share custom st2packs via PersistentVolume
+By using a PersistentVolume instead of sidecar container (the default) for custom st2packs, the actionrunner, sensor & api-containers will not be restarted on redeploying st2packs oder config maps.
+
+For this you have to uncomment following section in values.yaml and provide your specific requirements:
+```yaml
+st2:
+  packs:
+    persistentVolumes:
+      packs:
+        resources:
+          requests:
+        # If your packs bring a lot of files, this value may require a raise
+            storage: 512Mi
+        # If your kubernetes cluster does not have a default storageClass set, you can provide a specific one here
+#        storageClassName: 'your_provider'
+      venvs:
+        resources:
+          requests:
+        # If your packs require a lot of packages in their virtual environments, this value may require a raise
+            storage: 1Gi
+        # If your kubernetes cluster does not have a default storageClass set, you can provide a specific one here
+#        storageClassName: 'your_provider'
+```
+Don't forget running Helm upgrade to apply new changes.
+
+The only container, that will be replaced on upgrade, is `job-st2-register-content`.
+
+This container will update Packs & VirtualEnvironments in PersistentVolumes & register them and any given pack configs in StackStorm, without any impact on running executions.
 
 ## Tips & Tricks
 Grab all logs for entire StackStorm cluster with dependent services in Helm release:
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 9374d825..6586ea18 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -71,3 +71,57 @@ Create the name of the stackstorm-ha service account to use
 {{ $mongo_fullname }}-{{ $index0 }}.{{ $mongo_fullname }}{{ if ne $index1 $replicas }},{{ end }}
   {{- end -}}
 {{- end -}}
+
+# For custom st2packs-Container reduce duplicity by defining it here once
+{{- define "packs-volumes" -}}
+{{- if .Values.st2.packs.image.repository }}
+{{- if .Values.st2.packs.persistentVolumes }}
+- name: st2-packs-vol
+  persistentVolumeClaim:
+    claimName: 'st2-packs-pvol'
+- name: st2-virtualenvs-vol
+  persistentVolumeClaim:
+    claimName: 'st2-venv-pvol'
+{{- else }}
+- name: st2-packs-vol
+  emptyDir: {}
+- name: st2-virtualenvs-vol
+  emptyDir: {}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+# For custom st2packs-initContainers reduce duplicity by defining them here once
+{{- define "packs-initContainers" -}}
+# Merge packs and virtualenvs from st2api with those from the st2.packs image
+# Custom packs
+- name: st2-custom-packs
+  image: "{{ .Values.st2.packs.image.repository }}/{{ .Values.st2.packs.image.name }}:{{ .Values.st2.packs.image.tag }}"
+  imagePullPolicy: {{ .Values.st2.packs.image.pullPolicy | quote }}
+  volumeMounts:
+  - name: st2-packs-vol
+    mountPath: /opt/stackstorm/packs-shared
+  - name: st2-virtualenvs-vol
+    mountPath: /opt/stackstorm/virtualenvs-shared
+  command:
+    - 'sh'
+    - '-ec'
+    - |
+      /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
+      /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
+# System packs
+- name: st2-system-packs
+  image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
+  imagePullPolicy: {{ .Values.image.pullPolicy }}
+  volumeMounts:
+  - name: st2-packs-vol
+    mountPath: /opt/stackstorm/packs-shared
+  - name: st2-virtualenvs-vol
+    mountPath: /opt/stackstorm/virtualenvs-shared
+  command:
+    - 'sh'
+    - '-ec'
+    - |
+      /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
+      /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
+{{- end -}}
diff --git a/templates/deployments.yaml b/templates/deployments.yaml
index c1aed4bc..fdb662df 100644
--- a/templates/deployments.yaml
+++ b/templates/deployments.yaml
@@ -158,39 +158,9 @@ spec:
       {{- if .Values.image.pullSecret }}
       - name: {{ .Values.image.pullSecret }}
       {{- end }}
-      {{- if .Values.st2.packs.image.repository }}
+      {{- if and .Values.st2.packs.image.repository (not .Values.st2.packs.persistentVolumes) }}
       initContainers:
-      # Merge packs and virtualenvs from st2api with those from the st2.packs image
-      # Custom packs
-      - name: st2-custom-packs
-        image: "{{ .Values.st2.packs.image.repository }}/{{ .Values.st2.packs.image.name }}:{{ .Values.st2.packs.image.tag }}"
-        imagePullPolicy: {{ .Values.st2.packs.image.pullPolicy | quote }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
-      # System packs
-      - name: st2-system-packs
-        image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
+{{- include "packs-initContainers" . | indent 6 }}
       {{- end }}
       containers:
       - name: st2api{{ template "enterpriseSuffix" . }}
@@ -242,10 +212,7 @@ spec:
           configMap:
             name: {{ .Release.Name }}-st2-config
         {{- if .Values.st2.packs.image.repository }}
-        - name: st2-packs-vol
-          emptyDir: {}
-        - name: st2-virtualenvs-vol
-          emptyDir: {}
+{{- include "packs-volumes" . | indent 8 }}
         {{- end }}
     {{- with .Values.st2api.nodeSelector }}
       nodeSelector:
@@ -923,37 +890,9 @@ spec:
       {{- if $.Values.image.pullSecret }}
       - name: {{ $.Values.image.pullSecret }}
       {{- end }}
-      {{- if $.Values.st2.packs.image.repository }}
+      {{- if and $.Values.st2.packs.image.repository (not $.Values.st2.packs.persistentVolumes) }}
       initContainers:
-      # Merge packs and virtualenvs from st2sensorcontainer with those from the st2.packs image
-      # Custom packs
-      - name: st2-custom-packs
-        image: "{{ $.Values.st2.packs.image.repository }}/{{ $.Values.st2.packs.image.name }}:{{ $.Values.st2.packs.image.tag }}"
-        imagePullPolicy: {{ $.Values.st2.packs.image.pullPolicy | quote }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
-      # System packs
-      - name: st2-system-packs
-        image: "{{ template "imageRepository" $ }}/st2actionrunner{{ template "enterpriseSuffix" $ }}:{{ $.Chart.AppVersion }}"
-        imagePullPolicy: {{ $.Values.image.pullPolicy }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
+{{- include "packs-initContainers" $ | indent 6 }}
       {{- end }}
       containers:
       - name: st2sensorcontainer{{ template "hyphenPrefix" .name }}{{ template "enterpriseSuffix" $ }}
@@ -1018,10 +957,7 @@ spec:
           configMap:
             name: {{ $.Release.Name }}-st2-config
         {{- if $.Values.st2.packs.image.repository }}
-        - name: st2-packs-vol
-          emptyDir: {}
-        - name: st2-virtualenvs-vol
-          emptyDir: {}
+{{- include "packs-volumes" $ | indent 8 }}
         {{- end }}
     {{- with .nodeSelector }}
       nodeSelector:
@@ -1092,37 +1028,9 @@ spec:
       {{- if .Values.image.pullSecret }}
       - name: {{ .Values.image.pullSecret }}
       {{- end }}
-      {{- if .Values.st2.packs.image.repository }}
+      {{- if and .Values.st2.packs.image.repository (not .Values.st2.packs.persistentVolumes) }}
       initContainers:
-      # Merge packs and virtualenvs from st2actionrunner with those from the st2.packs image
-      # Custom packs
-      - name: st2-custom-packs
-        image: "{{ .Values.st2.packs.image.repository }}/{{ .Values.st2.packs.image.name }}:{{ .Values.st2.packs.image.tag }}"
-        imagePullPolicy: {{ .Values.st2.packs.image.pullPolicy | quote }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
-      # System packs
-      - name: st2-system-packs
-        image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
+{{- include "packs-initContainers" . | indent 6 }}
       {{- end }}
       containers:
       - name: st2actionrunner{{ template "enterpriseSuffix" . }}
@@ -1183,10 +1091,7 @@ spec:
               # 0400 file permission
               mode: 256
         {{- if .Values.st2.packs.image.repository }}
-        - name: st2-packs-vol
-          emptyDir: {}
-        - name: st2-virtualenvs-vol
-          emptyDir: {}
+{{- include "packs-volumes" . | indent 8 }}
         {{- end }}
     {{- with .Values.st2actionrunner.nodeSelector }}
       nodeSelector:
@@ -1333,36 +1238,8 @@ spec:
       - name: {{ .Values.image.pullSecret }}
       {{- end }}
       initContainers:
-      {{- if .Values.st2.packs.image.repository }}
-      # Merge packs and virtualenvs from st2actionrunner with those from the st2.packs image
-      # Custom packs
-      - name: st2-custom-packs
-        image: "{{ .Values.st2.packs.image.repository }}/{{ .Values.st2.packs.image.name }}:{{ .Values.st2.packs.image.tag }}"
-        imagePullPolicy: {{ .Values.st2.packs.image.pullPolicy | quote }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
-      # System packs
-      - name: st2-system-packs
-        image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        volumeMounts:
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs-shared
-        command:
-          - 'sh'
-          - '-ec'
-          - |
-            /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
-            /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
+      {{- if and .Values.st2.packs.image.repository (not .Values.st2.packs.persistentVolumes) }}
+{{- include "packs-initContainers" . | indent 6 }}
       {{- end }}
       # Sidecar container for generating st2client config with st2 username & password pair and sharing produced file with the main container
       - name: generate-st2client-config
@@ -1486,10 +1363,7 @@ spec:
               # 0400 file permission
               mode: 256
         {{- if .Values.st2.packs.image.repository }}
-        - name: st2-packs-vol
-          emptyDir: {}
-        - name: st2-virtualenvs-vol
-          emptyDir: {}
+{{- include "packs-volumes" . | indent 8 }}
         {{- end }}
 
 {{ if .Values.st2chatops.enabled -}}
diff --git a/templates/jobs.yaml b/templates/jobs.yaml
index c5e4e808..22220cd0 100644
--- a/templates/jobs.yaml
+++ b/templates/jobs.yaml
@@ -403,10 +403,5 @@ spec:
         - name: st2-pack-configs-vol
           configMap:
             name: {{ .Release.Name }}-st2-pack-configs
-        {{- if .Values.st2.packs.image.repository }}
-        - name: st2-packs-vol
-          emptyDir: {}
-        - name: st2-virtualenvs-vol
-          emptyDir: {}
-        {{- end }}
+{{- include "packs-volumes" $ | indent 8}}
       restartPolicy: OnFailure
diff --git a/templates/persistentvolumeclaims.yaml b/templates/persistentvolumeclaims.yaml
new file mode 100644
index 00000000..76a315e3
--- /dev/null
+++ b/templates/persistentvolumeclaims.yaml
@@ -0,0 +1,32 @@
+# This is only used, when user wants to share packs & venvs via persistentVolumes
+{{- if and .Values.st2.packs.image.repository .Values.st2.packs.persistentVolumes }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: st2-packs-pvol
+spec:
+  accessModes:
+    - ReadWriteMany
+  volumeMode: Filesystem
+  resources:
+{{ toYaml .Values.st2.packs.persistentVolumes.packs.resources | indent 4 }}
+{{- if .Values.st2.packs.persistentVolumes.packs.storageClassName }}
+  storageClassName: "{{ .Values.st2.packs.persistentVolumes.packs.storageClassName }}"
+{{- end }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: st2-venv-pvol
+spec:
+  accessModes:
+    - ReadWriteMany
+  volumeMode: Filesystem
+  resources:
+{{ toYaml .Values.st2.packs.persistentVolumes.venvs.resources | indent 4 }}
+{{- if .Values.st2.packs.persistentVolumes.venvs.storageClassName }}
+  storageClassName: "{{ .Values.st2.packs.persistentVolumes.venvs.storageClassName }}"
+{{- end }}
+{{- end }}
diff --git a/values.yaml b/values.yaml
index 1d6ac142..3fe7dae4 100644
--- a/values.yaml
+++ b/values.yaml
@@ -96,13 +96,31 @@ st2:
         ---
         # example core pack config yaml
 
+    # Custom packs are shared via two persistent volumes across all containers
+    # Here you can customize these volumes, to accomodate to your specific requirements
+#    persistentVolumes:
+#      packs:
+#        resources:
+#          requests:
+        # If your packs bring a lot of files, this value may require a raise
+#            storage: 512Mi
+        # If your kubernetes cluster does not have a default storageClass set, you can provide a specific one here
+#        storageClassName: 'your_provider'
+#      venvs:
+#        resources:
+#          requests:
+        # If your packs require a lot of packages in their virtual environments, this value may require a raise
+#            storage: 1Gi
+        # If your kubernetes cluster does not have a default storageClass set, you can provide a specific one here
+#        storageClassName: 'your_provider'
+
     # Custom packs image settings. The repository, name, tag and pullPolicy for this image
     # are specified below.
     image:
       # Uncomment the following block to make the custom packs image available to the necessary pods
-      #repository: your-remote-docker-registry.io
+      # repository: index.docker.io/stackstorm
       name: st2packs
-      tag: latest
+      tag: example
       pullPolicy: IfNotPresent
       # Optional name of the imagePullSecret if your custom packs image is hosted by a private Docker registry behind the auth
       #pullSecret: st2packs-auth