From 19b45bc758291d6e87686815ae15f37a4d85026b Mon Sep 17 00:00:00 2001 From: jacopo binosi Date: Mon, 30 Jan 2023 14:10:36 +0000 Subject: [PATCH 1/3] add helm template --- k8s/helm/coolbeans/.helmignore | 23 +++ k8s/helm/coolbeans/Chart.yaml | 24 +++ k8s/helm/coolbeans/templates/NOTES.txt | 22 +++ k8s/helm/coolbeans/templates/_helpers.tpl | 93 +++++++++ .../templates/cluster-node/configmap.yaml | 18 ++ .../coolbeans/templates/cluster-node/hpa.yaml | 28 +++ .../cluster-node/persistent-volume.yaml | 30 +++ .../templates/cluster-node/service.yaml | 18 ++ .../templates/cluster-node/statefulset.yaml | 176 ++++++++++++++++++ .../coolbeans/templates/proxy/deployment.yaml | 93 +++++++++ k8s/helm/coolbeans/templates/proxy/hpa.yaml | 28 +++ .../templates/proxy/service-external-lb.yaml | 22 +++ .../coolbeans/templates/proxy/service.yaml | 16 ++ .../coolbeans/templates/serviceaccount.yaml | 12 ++ .../templates/tests/test-connection.yaml | 15 ++ k8s/helm/coolbeans/values.yaml | 94 ++++++++++ 16 files changed, 712 insertions(+) create mode 100644 k8s/helm/coolbeans/.helmignore create mode 100644 k8s/helm/coolbeans/Chart.yaml create mode 100644 k8s/helm/coolbeans/templates/NOTES.txt create mode 100644 k8s/helm/coolbeans/templates/_helpers.tpl create mode 100644 k8s/helm/coolbeans/templates/cluster-node/configmap.yaml create mode 100644 k8s/helm/coolbeans/templates/cluster-node/hpa.yaml create mode 100644 k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml create mode 100644 k8s/helm/coolbeans/templates/cluster-node/service.yaml create mode 100644 k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml create mode 100644 k8s/helm/coolbeans/templates/proxy/deployment.yaml create mode 100644 k8s/helm/coolbeans/templates/proxy/hpa.yaml create mode 100644 k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml create mode 100644 k8s/helm/coolbeans/templates/proxy/service.yaml create mode 100644 k8s/helm/coolbeans/templates/serviceaccount.yaml create mode 100644 k8s/helm/coolbeans/templates/tests/test-connection.yaml create mode 100644 k8s/helm/coolbeans/values.yaml diff --git a/k8s/helm/coolbeans/.helmignore b/k8s/helm/coolbeans/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s/helm/coolbeans/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/helm/coolbeans/Chart.yaml b/k8s/helm/coolbeans/Chart.yaml new file mode 100644 index 0000000..1befb97 --- /dev/null +++ b/k8s/helm/coolbeans/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: coolbeans +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v0.1.10" diff --git a/k8s/helm/coolbeans/templates/NOTES.txt b/k8s/helm/coolbeans/templates/NOTES.txt new file mode 100644 index 0000000..b24a560 --- /dev/null +++ b/k8s/helm/coolbeans/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "coolbeans.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "coolbeans.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "coolbeans.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "coolbeans.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/k8s/helm/coolbeans/templates/_helpers.tpl b/k8s/helm/coolbeans/templates/_helpers.tpl new file mode 100644 index 0000000..97dc70f --- /dev/null +++ b/k8s/helm/coolbeans/templates/_helpers.tpl @@ -0,0 +1,93 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "coolbeans.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "coolbeans.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "coolbeans.fullname.proxy" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s-proxy" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "coolbeans.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "coolbeans.labels" -}} +helm.sh/chart: {{ include "coolbeans.chart" . }} +{{ include "coolbeans.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.context/name: {{ include "coolbeans.chart" . }} +{{- end }} + +{{- define "coolbeans.labels.proxy" -}} +helm.sh/chart: {{ include "coolbeans.chart" . }} +{{ include "coolbeans.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.context/name: {{ include "coolbeans.chart" . }}-proxy +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "coolbeans.selectorLabels" -}} +app.kubernetes.io/name: {{ include "coolbeans.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.context/name: {{ include "coolbeans.chart" . }} +{{- end }} + +{{- define "coolbeans.selectorLabels.proxy" -}} +app.kubernetes.io/name: {{ include "coolbeans.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.context/name: {{ include "coolbeans.chart" . }}-proxy +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coolbeans.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "coolbeans.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/helm/coolbeans/templates/cluster-node/configmap.yaml b/k8s/helm/coolbeans/templates/cluster-node/configmap.yaml new file mode 100644 index 0000000..44baec0 --- /dev/null +++ b/k8s/helm/coolbeans/templates/cluster-node/configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "coolbeans.fullname" . }}-config + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +data: + BEANSTALKD_PORT: "11300" + PEERS_ADDRS: "coolbeans-0.coolbeans.{{ .Release.Namespace }}.svc.cluster.local:11000,coolbeans-1.coolbeans.{{ .Release.Namespace }}.svc.cluster.local:11000,coolbeans-2.coolbeans.{{ .Release.Namespace }}.svc.cluster.local:11000" + BOOTSTRAP_NODE_ID: "coolbeans-0" + COOLBEANS_RAFT_PORT: "21000" + COOLBEANS_RPC_PORT: "11000" + DATA_DIR: "/root/data" + ADDR_SUFFIX: "coolbeans.{{ .Release.Namespace }}.svc.cluster.local" + SNAPSHOT_THRESHOLD: "100" + TRAILING_LOG_COUNT: "10240" + SNAPSHOT_INTERVAL_SECS: "30" + MAX_JOB_SIZE_BYTES: "65535" diff --git a/k8s/helm/coolbeans/templates/cluster-node/hpa.yaml b/k8s/helm/coolbeans/templates/cluster-node/hpa.yaml new file mode 100644 index 0000000..a37e082 --- /dev/null +++ b/k8s/helm/coolbeans/templates/cluster-node/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.clusterNode.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "coolbeans.fullname" . }} + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "coolbeans.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml b/k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml new file mode 100644 index 0000000..18724d5 --- /dev/null +++ b/k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ebs-beanstalk-data +spec: + accessModes: + - ReadWriteOnce + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1e/vol-01f4c6fc104ca275b + capacity: + storage: 15Gi + claimRef: + apiVersion: v1 + kind: PersistentVolumeClaim + name: beanstalk-data + namespace: system-team-admin-ondemand + resourceVersion: "80830523" + uid: 5fbf66e5-3030-4158-ad4c-209d00c0879a + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: topology.ebs.csi.aws.com/zone + operator: In + values: + - us-east-1e + persistentVolumeReclaimPolicy: Retain + storageClassName: gp3-us-east-1e + volumeMode: Filesystem diff --git a/k8s/helm/coolbeans/templates/cluster-node/service.yaml b/k8s/helm/coolbeans/templates/cluster-node/service.yaml new file mode 100644 index 0000000..0a68809 --- /dev/null +++ b/k8s/helm/coolbeans/templates/cluster-node/service.yaml @@ -0,0 +1,18 @@ + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "coolbeans.fullname" . }} + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +spec: + clusterIP: None + ports: + - name: grpc-server + port: 11000 + targetPort: 11000 + - name: raft-tcp + port: 21000 + targetPort: 21000 + selector: + {{- include "coolbeans.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml b/k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml new file mode 100644 index 0000000..0a7d41d --- /dev/null +++ b/k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml @@ -0,0 +1,176 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "coolbeans.fullname" . }} + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +spec: + serviceName: {{ include "coolbeans.fullname" . }} + podManagementPolicy: "Parallel" + {{- if not .Values.clusterNode.autoscaling.enabled }} + replicas: {{ .Values.clusterNode.replicaCount }} + {{- end }} + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: {{ .Values.clusterNode.replicaCount }} + selector: + matchLabels: + {{- include "coolbeans.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "coolbeans.labels" . | nindent 4 }} + spec: + {{- if or .Values.topologySpreadConstraints.zone .Values.topologySpreadConstraints.hostname }} + topologySpreadConstraints: + {{- if .Values.topologySpreadConstraints.zone }} + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "coolbeans.selectorLabels" . | nindent 12 }} + {{- end }} + {{- if .Values.topologySpreadConstraints.hostname }} + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "coolbeans.selectorLabels" . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 30 + serviceAccountName: {{ include "coolbeans.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: COOLBEANS_RAFT_PORT + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: COOLBEANS_RAFT_PORT + - name: COOLBEANS_RPC_PORT + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: COOLBEANS_RPC_PORT + - name: PEERS_ADDRS + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: PEERS_ADDRS + - name: BOOTSTRAP_NODE_ID + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: BOOTSTRAP_NODE_ID + - name: DATA_DIR + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: DATA_DIR + - name: ADDR_SUFFIX + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: ADDR_SUFFIX + - name: SNAPSHOT_THRESHOLD + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: SNAPSHOT_THRESHOLD + - name: TRAILING_LOG_COUNT + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: TRAILING_LOG_COUNT + - name: SNAPSHOT_INTERVAL_SECS + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: SNAPSHOT_INTERVAL_SECS + command: ["/root/coolbeans", "cluster-node", + "--node-id", "$(NODE_ID)", + "--root-dir", "$(DATA_DIR)", + "--bootstrap-node-id", "$(BOOTSTRAP_NODE_ID)", + "--raft-listen-addr", "$(POD_IP):$(COOLBEANS_RAFT_PORT)", + "--raft-advertized-addr", "$(NODE_ID).$(ADDR_SUFFIX):$(COOLBEANS_RAFT_PORT)", + "--node-listen-addr", ":$(COOLBEANS_RPC_PORT)", + "--node-peer-addrs", "$(PEERS_ADDRS)", + ## By default all logs are fsynced to disk (bolt-db) + ## By default fsync is called for every commit to the disk log + # "--no-fsync", + ## + ## Here, all the logs are in memory & not in bolt-db, but snapshots as usual are on + ## disk. You can tune the snapshot threshold, trailing-log-count etc.. + # "--no-disk-log" + # "--snapshot-threshold", "$(SNAPSHOT_THRESHOLD)", + # "--trailing-log-count", "$(TRAILING_LOG_COUNT)", + # "--snapshot-interval-secs", "$(SNAPSHOT_INTERVAL_SECS)", + {{- if .Values.clusterNode.prometheus.enabled }} + "--prometheus-addr", "{{- .Values.clusterNode.prometheus.endpoint -}}" + {{- end }} + ] + ports: + - containerPort: 11000 + name: grpc-server + - containerPort: 21000 + name: raft-tcp + resources: + {{- toYaml .Values.clusterNode.resources | nindent 12 }} + {{- if .Values.clusterNode.persistance.enabled}} + volumeMounts: + - name: data + mountPath: /root/data + {{- end }} + readinessProbe: + exec: + command: ["/root/coolbeans", "--verbose", "cluster-client", "is_leader", "--node-addr", "localhost:11000"] + initialDelaySeconds: 30 + periodSeconds: 15 + {{- if .Values.clusterNode.persistance.enabled}} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{- .Values.clusterNode.persistance.storage -}} + {{- end }} diff --git a/k8s/helm/coolbeans/templates/proxy/deployment.yaml b/k8s/helm/coolbeans/templates/proxy/deployment.yaml new file mode 100644 index 0000000..abdf8f6 --- /dev/null +++ b/k8s/helm/coolbeans/templates/proxy/deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "coolbeans.fullname.proxy" . }} + labels: + {{- include "coolbeans.labels.proxy" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "coolbeans.selectorLabels.proxy" . | nindent 6 }} + {{- if not .Values.proxy.autoscaling.enabled }} + replicas: {{ .Values.proxy.replicaCount }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "coolbeans.labels.proxy" . | nindent 4 }} + spec: + {{- if or .Values.topologySpreadConstraints.zone .Values.topologySpreadConstraints.hostname }} + topologySpreadConstraints: + {{- if .Values.topologySpreadConstraints.zone }} + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "coolbeans.selectorLabels.proxy" . | nindent 12 }} + {{- end }} + {{- if .Values.topologySpreadConstraints.hostname }} + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "coolbeans.selectorLabels.proxy" . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "coolbeans.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ "/root/coolbeans", + "beanstalkd", + "--listen-port", "$(BEANSTALKD_PORT)", + "--upstream-addrs", "$(PEERS_ADDRS)", + "--max-job-size", "$(MAX_JOB_SIZE_BYTES)"] + ports: + - containerPort: 11300 + name: beanstalkd-tcp + resources: + {{- toYaml .Values.proxy.resources | nindent 12 }} + env: + - name: BEANSTALKD_PORT + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: BEANSTALKD_PORT + - name: PEERS_ADDRS + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: PEERS_ADDRS + - name: MAX_JOB_SIZE_BYTES + valueFrom: + configMapKeyRef: + name: coolbeans-config + key: MAX_JOB_SIZE_BYTES + restartPolicy: Always \ No newline at end of file diff --git a/k8s/helm/coolbeans/templates/proxy/hpa.yaml b/k8s/helm/coolbeans/templates/proxy/hpa.yaml new file mode 100644 index 0000000..fd90b68 --- /dev/null +++ b/k8s/helm/coolbeans/templates/proxy/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.proxy.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "coolbeans.fullname.proxy" . }} + labels: + {{- include "coolbeans.labels.proxy" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "coolbeans.fullname.proxy" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml b/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml new file mode 100644 index 0000000..af2a1f2 --- /dev/null +++ b/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml @@ -0,0 +1,22 @@ +{{- if .Values.proxy.service.externalLoadBalancer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "coolbeans.fullname.proxy" . }}-elb + labels: + {{- include "coolbeans.labels.proxy" . | nindent 4 }} + {{- with .Values.proxy.service.externalLoadBalancer.annotations }} + annotations: + {{- . | toYaml | nindent 4 }} + {{- end }} +spec: +spec: + type: Loadbalancer + ports: + - port: {{ .Values.proxy.service.port }} + targetPort: {{ .Values.proxy.service.port }} + protocol: TCP + name: beanstalk-tcp + selector: + {{- include "coolbeans.selectorLabels.proxy" . | nindent 4 }} +{{- end }} diff --git a/k8s/helm/coolbeans/templates/proxy/service.yaml b/k8s/helm/coolbeans/templates/proxy/service.yaml new file mode 100644 index 0000000..4173e6d --- /dev/null +++ b/k8s/helm/coolbeans/templates/proxy/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "coolbeans.fullname.proxy" . }} + labels: + {{- include "coolbeans.labels.proxy" . | nindent 4 }} +spec: +spec: + type: {{ .Values.proxy.service.type }} + ports: + - port: {{ .Values.proxy.service.port }} + targetPort: {{ .Values.proxy.service.port }} + protocol: TCP + name: beanstalk-tcp + selector: + {{- include "coolbeans.selectorLabels.proxy" . | nindent 4 }} diff --git a/k8s/helm/coolbeans/templates/serviceaccount.yaml b/k8s/helm/coolbeans/templates/serviceaccount.yaml new file mode 100644 index 0000000..c87d0cb --- /dev/null +++ b/k8s/helm/coolbeans/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "coolbeans.serviceAccountName" . }} + labels: + {{- include "coolbeans.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/k8s/helm/coolbeans/templates/tests/test-connection.yaml b/k8s/helm/coolbeans/templates/tests/test-connection.yaml new file mode 100644 index 0000000..48c0423 --- /dev/null +++ b/k8s/helm/coolbeans/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "coolbeans.fullname" . }}-test-connection" + labels: + {{- include "coolbeans.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "coolbeans.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/k8s/helm/coolbeans/values.yaml b/k8s/helm/coolbeans/values.yaml new file mode 100644 index 0000000..88d6a9e --- /dev/null +++ b/k8s/helm/coolbeans/values.yaml @@ -0,0 +1,94 @@ +# Default values for coolbeans. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +topologySpreadConstraints: + zone: false + hostname: false + +image: + repository: 1xyz/coolbeans + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v0.1.10 + +clusterNode: + replicaCount: 3 + autoscaling: + enabled: false + minReplicas: 3 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + resources: + limits: + cpu: "2000m" + memory: 2Gi + requests: + cpu: "500m" + memory: 1Gi + persistance: + enabled: false + storage: 100Gi + createPersistentVolume: false + prometheus: + enabled: false + # endpoint: prom-endpoint:prom-port + endpoint: "" + +proxy: + replicaCount: 2 + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + resources: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "64Mi" + cpu: "250m" + service: + type: ClusterIP + port: 11300 + externalLoadBalancer: + enabled: false + annotations: {} + # service.beta.kubernetes.io/aws-load-balancer-internal: "true" + # service.beta.kubernetes.io/aws-load-balancer-scheme: "internal" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + From 3ec2c6ae8ffc8d75f4bbcb8bf858c446adaff094 Mon Sep 17 00:00:00 2001 From: jacopo binosi Date: Mon, 30 Jan 2023 14:11:41 +0000 Subject: [PATCH 2/3] add helm template --- .../cluster-node/0-namespace.yaml | 0 .../cluster-node/1-configmap.yaml | 0 .../cluster-node/2-service.yaml | 0 .../cluster-node/3-statefulset.yaml | 0 .../prometheus/0-configmap.yaml | 0 .../prometheus/1-deployment.yaml | 0 .../proxy/deployment.yaml | 0 k8s/poc-static-resources/setup.sh | 20 +++++++++++++++++ .../poc-static-resources/shutdown.sh | 0 .../workload/consumer.yaml | 0 .../workload/producer.yaml | 0 setup.sh | 22 ------------------- 12 files changed, 20 insertions(+), 22 deletions(-) rename k8s/{ => poc-static-resources}/cluster-node/0-namespace.yaml (100%) rename k8s/{ => poc-static-resources}/cluster-node/1-configmap.yaml (100%) rename k8s/{ => poc-static-resources}/cluster-node/2-service.yaml (100%) rename k8s/{ => poc-static-resources}/cluster-node/3-statefulset.yaml (100%) rename k8s/{ => poc-static-resources}/prometheus/0-configmap.yaml (100%) rename k8s/{ => poc-static-resources}/prometheus/1-deployment.yaml (100%) rename k8s/{ => poc-static-resources}/proxy/deployment.yaml (100%) create mode 100755 k8s/poc-static-resources/setup.sh rename shutdown.sh => k8s/poc-static-resources/shutdown.sh (100%) rename k8s/{ => poc-static-resources}/workload/consumer.yaml (100%) rename k8s/{ => poc-static-resources}/workload/producer.yaml (100%) delete mode 100755 setup.sh diff --git a/k8s/cluster-node/0-namespace.yaml b/k8s/poc-static-resources/cluster-node/0-namespace.yaml similarity index 100% rename from k8s/cluster-node/0-namespace.yaml rename to k8s/poc-static-resources/cluster-node/0-namespace.yaml diff --git a/k8s/cluster-node/1-configmap.yaml b/k8s/poc-static-resources/cluster-node/1-configmap.yaml similarity index 100% rename from k8s/cluster-node/1-configmap.yaml rename to k8s/poc-static-resources/cluster-node/1-configmap.yaml diff --git a/k8s/cluster-node/2-service.yaml b/k8s/poc-static-resources/cluster-node/2-service.yaml similarity index 100% rename from k8s/cluster-node/2-service.yaml rename to k8s/poc-static-resources/cluster-node/2-service.yaml diff --git a/k8s/cluster-node/3-statefulset.yaml b/k8s/poc-static-resources/cluster-node/3-statefulset.yaml similarity index 100% rename from k8s/cluster-node/3-statefulset.yaml rename to k8s/poc-static-resources/cluster-node/3-statefulset.yaml diff --git a/k8s/prometheus/0-configmap.yaml b/k8s/poc-static-resources/prometheus/0-configmap.yaml similarity index 100% rename from k8s/prometheus/0-configmap.yaml rename to k8s/poc-static-resources/prometheus/0-configmap.yaml diff --git a/k8s/prometheus/1-deployment.yaml b/k8s/poc-static-resources/prometheus/1-deployment.yaml similarity index 100% rename from k8s/prometheus/1-deployment.yaml rename to k8s/poc-static-resources/prometheus/1-deployment.yaml diff --git a/k8s/proxy/deployment.yaml b/k8s/poc-static-resources/proxy/deployment.yaml similarity index 100% rename from k8s/proxy/deployment.yaml rename to k8s/poc-static-resources/proxy/deployment.yaml diff --git a/k8s/poc-static-resources/setup.sh b/k8s/poc-static-resources/setup.sh new file mode 100755 index 0000000..9469fde --- /dev/null +++ b/k8s/poc-static-resources/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if ! [ -x "$(command -v kubectl)" ]; then + echo 'Error: kubectl not found in path.' >&2 + exit 1 +fi + +for f in cluster-node/*.yaml; do + echo "kubectl -n coolbeans apply -f $f" + kubectl -n coolbeans apply -f $f +done + +echo "Apply proxy/deployment.yaml" +kubectl -n coolbeans apply -f proxy/deployment.yaml + +echo "Installing a Prometheus service to scrape metrics" +for f in prometheus/*.yaml; do + echo "kubectl -n coolbeans apply -f $f" + kubectl -n coolbeans apply -f $f +done diff --git a/shutdown.sh b/k8s/poc-static-resources/shutdown.sh similarity index 100% rename from shutdown.sh rename to k8s/poc-static-resources/shutdown.sh diff --git a/k8s/workload/consumer.yaml b/k8s/poc-static-resources/workload/consumer.yaml similarity index 100% rename from k8s/workload/consumer.yaml rename to k8s/poc-static-resources/workload/consumer.yaml diff --git a/k8s/workload/producer.yaml b/k8s/poc-static-resources/workload/producer.yaml similarity index 100% rename from k8s/workload/producer.yaml rename to k8s/poc-static-resources/workload/producer.yaml diff --git a/setup.sh b/setup.sh deleted file mode 100755 index ca439a7..0000000 --- a/setup.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -if ! [ -x "$(command -v kubectl)" ]; then - echo 'Error: kubectl not found in path.' >&2 - exit 1 -fi - -for f in k8s/cluster-node/*.yaml -do - echo "kubectl apply -f $f" - kubectl apply -f $f -done - -echo "Apply proxy/deployment.yaml" -kubectl apply -f k8s/proxy/deployment.yaml - -echo "Installing a Prometheus service to scrape metrics" -for f in k8s/prometheus/*.yaml -do - echo "kubectl apply -f $f" - kubectl apply -f $f -done From 7d4413386469986c0f24826398bba2295a6b3160 Mon Sep 17 00:00:00 2001 From: cimxm Date: Wed, 8 Feb 2023 09:04:08 +0000 Subject: [PATCH 3/3] Update helm template --- k8s/helm/coolbeans/templates/NOTES.txt | 22 ---- k8s/helm/coolbeans/templates/_helpers.tpl | 32 +++--- .../templates/cluster-node/configmap.yaml | 18 --- .../cluster-node/persistent-volume.yaml | 30 ----- .../templates/cluster-node/service.yaml | 18 --- .../coolbeans/templates/node/configmap.yaml | 20 ++++ .../templates/{cluster-node => node}/hpa.yaml | 4 +- k8s/helm/coolbeans/templates/node/pdr.yaml | 17 +++ .../coolbeans/templates/node/service.yaml | 21 ++++ .../{cluster-node => node}/statefulset.yaml | 104 ++++++++++-------- .../coolbeans/templates/proxy/deployment.yaml | 14 +-- k8s/helm/coolbeans/templates/proxy/hpa.yaml | 4 +- .../templates/proxy/service-external-lb.yaml | 9 +- .../coolbeans/templates/proxy/service.yaml | 9 +- .../templates/tests/test-connection.yaml | 15 --- k8s/helm/coolbeans/templates/tests/test.yaml | 51 +++++++++ k8s/helm/coolbeans/values.yaml | 29 +++-- 17 files changed, 223 insertions(+), 194 deletions(-) delete mode 100644 k8s/helm/coolbeans/templates/NOTES.txt delete mode 100644 k8s/helm/coolbeans/templates/cluster-node/configmap.yaml delete mode 100644 k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml delete mode 100644 k8s/helm/coolbeans/templates/cluster-node/service.yaml create mode 100644 k8s/helm/coolbeans/templates/node/configmap.yaml rename k8s/helm/coolbeans/templates/{cluster-node => node}/hpa.yaml (89%) create mode 100644 k8s/helm/coolbeans/templates/node/pdr.yaml create mode 100644 k8s/helm/coolbeans/templates/node/service.yaml rename k8s/helm/coolbeans/templates/{cluster-node => node}/statefulset.yaml (55%) delete mode 100644 k8s/helm/coolbeans/templates/tests/test-connection.yaml create mode 100644 k8s/helm/coolbeans/templates/tests/test.yaml diff --git a/k8s/helm/coolbeans/templates/NOTES.txt b/k8s/helm/coolbeans/templates/NOTES.txt deleted file mode 100644 index b24a560..0000000 --- a/k8s/helm/coolbeans/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "coolbeans.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "coolbeans.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "coolbeans.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "coolbeans.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/k8s/helm/coolbeans/templates/_helpers.tpl b/k8s/helm/coolbeans/templates/_helpers.tpl index 97dc70f..1909a94 100644 --- a/k8s/helm/coolbeans/templates/_helpers.tpl +++ b/k8s/helm/coolbeans/templates/_helpers.tpl @@ -23,18 +23,18 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} -{{- define "coolbeans.fullname.proxy" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s-proxy" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} +# {{- define "coolbeans.fullname.proxy" -}} +# {{- if .Values.fullnameOverride }} +# {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +# {{- else }} +# {{- $name := default .Chart.Name .Values.nameOverride }} +# {{- if contains $name .Release.Name }} +# {{- .Release.Name | trunc 63 | trimSuffix "-" }} +# {{- else }} +# {{- printf "%s-%s-proxy" .Release.Name $name | trunc 63 | trimSuffix "-" }} +# {{- end }} +# {{- end }} +# {{- end }} {{/* Create chart name and version as used by the chart label. @@ -53,17 +53,15 @@ helm.sh/chart: {{ include "coolbeans.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -app.context/name: {{ include "coolbeans.chart" . }} {{- end }} {{- define "coolbeans.labels.proxy" -}} helm.sh/chart: {{ include "coolbeans.chart" . }} -{{ include "coolbeans.selectorLabels" . }} +{{ include "coolbeans.selectorLabels.proxy" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -app.context/name: {{ include "coolbeans.chart" . }}-proxy {{- end }} {{/* @@ -72,13 +70,13 @@ Selector labels {{- define "coolbeans.selectorLabels" -}} app.kubernetes.io/name: {{ include "coolbeans.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -app.context/name: {{ include "coolbeans.chart" . }} +app.kubernetes.io/component: node {{- end }} {{- define "coolbeans.selectorLabels.proxy" -}} app.kubernetes.io/name: {{ include "coolbeans.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -app.context/name: {{ include "coolbeans.chart" . }}-proxy +app.kubernetes.io/component: proxy {{- end }} {{/* diff --git a/k8s/helm/coolbeans/templates/cluster-node/configmap.yaml b/k8s/helm/coolbeans/templates/cluster-node/configmap.yaml deleted file mode 100644 index 44baec0..0000000 --- a/k8s/helm/coolbeans/templates/cluster-node/configmap.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "coolbeans.fullname" . }}-config - labels: - {{- include "coolbeans.labels" . | nindent 4 }} -data: - BEANSTALKD_PORT: "11300" - PEERS_ADDRS: "coolbeans-0.coolbeans.{{ .Release.Namespace }}.svc.cluster.local:11000,coolbeans-1.coolbeans.{{ .Release.Namespace }}.svc.cluster.local:11000,coolbeans-2.coolbeans.{{ .Release.Namespace }}.svc.cluster.local:11000" - BOOTSTRAP_NODE_ID: "coolbeans-0" - COOLBEANS_RAFT_PORT: "21000" - COOLBEANS_RPC_PORT: "11000" - DATA_DIR: "/root/data" - ADDR_SUFFIX: "coolbeans.{{ .Release.Namespace }}.svc.cluster.local" - SNAPSHOT_THRESHOLD: "100" - TRAILING_LOG_COUNT: "10240" - SNAPSHOT_INTERVAL_SECS: "30" - MAX_JOB_SIZE_BYTES: "65535" diff --git a/k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml b/k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml deleted file mode 100644 index 18724d5..0000000 --- a/k8s/helm/coolbeans/templates/cluster-node/persistent-volume.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ebs-beanstalk-data -spec: - accessModes: - - ReadWriteOnce - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1e/vol-01f4c6fc104ca275b - capacity: - storage: 15Gi - claimRef: - apiVersion: v1 - kind: PersistentVolumeClaim - name: beanstalk-data - namespace: system-team-admin-ondemand - resourceVersion: "80830523" - uid: 5fbf66e5-3030-4158-ad4c-209d00c0879a - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: topology.ebs.csi.aws.com/zone - operator: In - values: - - us-east-1e - persistentVolumeReclaimPolicy: Retain - storageClassName: gp3-us-east-1e - volumeMode: Filesystem diff --git a/k8s/helm/coolbeans/templates/cluster-node/service.yaml b/k8s/helm/coolbeans/templates/cluster-node/service.yaml deleted file mode 100644 index 0a68809..0000000 --- a/k8s/helm/coolbeans/templates/cluster-node/service.yaml +++ /dev/null @@ -1,18 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "coolbeans.fullname" . }} - labels: - {{- include "coolbeans.labels" . | nindent 4 }} -spec: - clusterIP: None - ports: - - name: grpc-server - port: 11000 - targetPort: 11000 - - name: raft-tcp - port: 21000 - targetPort: 21000 - selector: - {{- include "coolbeans.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/k8s/helm/coolbeans/templates/node/configmap.yaml b/k8s/helm/coolbeans/templates/node/configmap.yaml new file mode 100644 index 0000000..52c085e --- /dev/null +++ b/k8s/helm/coolbeans/templates/node/configmap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "coolbeans.fullname" . }} + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +data: + PEERS_ADDRS: "{{ include "coolbeans.fullname" . }}-node-0.{{ include "coolbeans.fullname" . }}-node.{{ .Release.Namespace }}.svc.cluster.local:11000,{{ include "coolbeans.fullname" . }}-node-1.{{ include "coolbeans.fullname" . }}-node.{{ .Release.Namespace }}.svc.cluster.local:11000,{{ include "coolbeans.fullname" . }}-node-2.{{ include "coolbeans.fullname" . }}-node.{{ .Release.Namespace }}.svc.cluster.local:11000" + BOOTSTRAP_NODE_ID: "{{ include "coolbeans.fullname" . }}-node-0" + ADDR_SUFFIX: "{{ include "coolbeans.fullname" . }}-node.{{ .Release.Namespace }}.svc.cluster.local" + BEANSTALKD_PORT: "{{ .Values.clusterNode.config.beanstalkd_port | default "11300" }}" + COOLBEANS_RAFT_PORT: "{{ .Values.clusterNode.config.coolbeans_raft_port | default "21000" }}" + COOLBEANS_RPC_PORT: "{{ .Values.clusterNode.config.coolbeans_rpc_port | default "11000" }}" + DATA_DIR: "{{ .Values.clusterNode.config.data_dir | default "/root/data" }}" + SNAPSHOT_THRESHOLD: "{{ .Values.clusterNode.config.snapshot_threshold | default "100" }}" + TRAILING_LOG_COUNT: "{{ .Values.clusterNode.config.trailing_log_count | default "10240" }}" + SNAPSHOT_INTERVAL_SECS: "{{ .Values.clusterNode.config.snapshot_interval_secs | default "30" }}" + MAX_JOB_SIZE_BYTES: "{{ .Values.clusterNode.config.max_job_size_bytes | default "65535" }}" + PROMETHEUS_PORT: "{{ .Values.clusterNode.config.prometheus_port | default "2020" }}" + RETAIN_SNAPSHOT_COUNT: "{{ .Values.clusterNode.config.retain_snapshot_count | default "3" }}" diff --git a/k8s/helm/coolbeans/templates/cluster-node/hpa.yaml b/k8s/helm/coolbeans/templates/node/hpa.yaml similarity index 89% rename from k8s/helm/coolbeans/templates/cluster-node/hpa.yaml rename to k8s/helm/coolbeans/templates/node/hpa.yaml index a37e082..4d2cbf7 100644 --- a/k8s/helm/coolbeans/templates/cluster-node/hpa.yaml +++ b/k8s/helm/coolbeans/templates/node/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "coolbeans.fullname" . }} + name: {{ include "coolbeans.fullname" . }}-node labels: {{- include "coolbeans.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "coolbeans.fullname" . }} + name: {{ include "coolbeans.fullname" . }}-node minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: diff --git a/k8s/helm/coolbeans/templates/node/pdr.yaml b/k8s/helm/coolbeans/templates/node/pdr.yaml new file mode 100644 index 0000000..0729f44 --- /dev/null +++ b/k8s/helm/coolbeans/templates/node/pdr.yaml @@ -0,0 +1,17 @@ +{{ if or (eq (.Values.clusterNode.replicaCount | float64) 3.0) (eq (.Values.clusterNode.replicaCount | float64) 5.0) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "coolbeans.fullname" . }}-node + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +spec: +{{ if eq (.Values.clusterNode.replicaCount | float64) 3.0 }} + minAvailable: 2 +{{ else if eq (.Values.clusterNode.replicaCount | float64) 5.0 }} + minAvailable: 3 +{{ end }} + selector: + matchLabels: + {{- include "coolbeans.selectorLabels" . | nindent 6 }} +{{ end }} diff --git a/k8s/helm/coolbeans/templates/node/service.yaml b/k8s/helm/coolbeans/templates/node/service.yaml new file mode 100644 index 0000000..4cb6735 --- /dev/null +++ b/k8s/helm/coolbeans/templates/node/service.yaml @@ -0,0 +1,21 @@ + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "coolbeans.fullname" . }}-node + labels: + {{- include "coolbeans.labels" . | nindent 4 }} +spec: + clusterIP: None + ports: + - name: grpc-server + port: {{ .Values.clusterNode.config.coolbeans_rpc_port | default "11000" }} + targetPort: grpc-server + - name: raft-tcp + port: {{ .Values.clusterNode.config.coolbeans_raft_port | default "21000" }} + targetPort: raft-tcp + - name: metrics + port: {{ .Values.clusterNode.config.prometheus_port | default "2020" }} + targetPort: metrics + selector: + {{- include "coolbeans.selectorLabels" . | nindent 4 }} diff --git a/k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml b/k8s/helm/coolbeans/templates/node/statefulset.yaml similarity index 55% rename from k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml rename to k8s/helm/coolbeans/templates/node/statefulset.yaml index 0a7d41d..d0801e9 100644 --- a/k8s/helm/coolbeans/templates/cluster-node/statefulset.yaml +++ b/k8s/helm/coolbeans/templates/node/statefulset.yaml @@ -1,19 +1,19 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "coolbeans.fullname" . }} + name: {{ include "coolbeans.fullname" . }}-node labels: {{- include "coolbeans.labels" . | nindent 4 }} spec: - serviceName: {{ include "coolbeans.fullname" . }} + serviceName: {{ include "coolbeans.fullname" . }}-node podManagementPolicy: "Parallel" {{- if not .Values.clusterNode.autoscaling.enabled }} replicas: {{ .Values.clusterNode.replicaCount }} {{- end }} updateStrategy: type: RollingUpdate - rollingUpdate: - partition: {{ .Values.clusterNode.replicaCount }} + # rollingUpdate: + # partition: {{ .Values.clusterNode.replicaCount }} selector: matchLabels: {{- include "coolbeans.selectorLabels" . | nindent 6 }} @@ -24,7 +24,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "coolbeans.labels" . | nindent 4 }} + {{ toYaml .Values.podLabels | nindent 8 }} + {{- include "coolbeans.labels" . | nindent 8 }} spec: {{- if or .Values.topologySpreadConstraints.zone .Values.topologySpreadConstraints.hostname }} topologySpreadConstraints: @@ -83,94 +84,103 @@ spec: - name: COOLBEANS_RAFT_PORT valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: COOLBEANS_RAFT_PORT - name: COOLBEANS_RPC_PORT valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: COOLBEANS_RPC_PORT - name: PEERS_ADDRS valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: PEERS_ADDRS - name: BOOTSTRAP_NODE_ID valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: BOOTSTRAP_NODE_ID - name: DATA_DIR valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: DATA_DIR - name: ADDR_SUFFIX valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: ADDR_SUFFIX - name: SNAPSHOT_THRESHOLD valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: SNAPSHOT_THRESHOLD - name: TRAILING_LOG_COUNT valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: TRAILING_LOG_COUNT - name: SNAPSHOT_INTERVAL_SECS valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: SNAPSHOT_INTERVAL_SECS - command: ["/root/coolbeans", "cluster-node", - "--node-id", "$(NODE_ID)", - "--root-dir", "$(DATA_DIR)", - "--bootstrap-node-id", "$(BOOTSTRAP_NODE_ID)", - "--raft-listen-addr", "$(POD_IP):$(COOLBEANS_RAFT_PORT)", - "--raft-advertized-addr", "$(NODE_ID).$(ADDR_SUFFIX):$(COOLBEANS_RAFT_PORT)", - "--node-listen-addr", ":$(COOLBEANS_RPC_PORT)", - "--node-peer-addrs", "$(PEERS_ADDRS)", - ## By default all logs are fsynced to disk (bolt-db) - ## By default fsync is called for every commit to the disk log - # "--no-fsync", - ## - ## Here, all the logs are in memory & not in bolt-db, but snapshots as usual are on - ## disk. You can tune the snapshot threshold, trailing-log-count etc.. - # "--no-disk-log" - # "--snapshot-threshold", "$(SNAPSHOT_THRESHOLD)", - # "--trailing-log-count", "$(TRAILING_LOG_COUNT)", - # "--snapshot-interval-secs", "$(SNAPSHOT_INTERVAL_SECS)", - {{- if .Values.clusterNode.prometheus.enabled }} - "--prometheus-addr", "{{- .Values.clusterNode.prometheus.endpoint -}}" - {{- end }} - ] + - name: MAX_JOB_SIZE_BYTES + valueFrom: + configMapKeyRef: + name: {{ include "coolbeans.fullname" . }} + key: MAX_JOB_SIZE_BYTES + command: [ + "/root/coolbeans", "cluster-node", + "--node-id", "$(NODE_ID)", + "--root-dir", "$(DATA_DIR)", + "--bootstrap-node-id", "$(BOOTSTRAP_NODE_ID)", + "--raft-listen-addr", "$(POD_IP):$(COOLBEANS_RAFT_PORT)", + "--raft-advertized-addr", "$(NODE_ID).$(ADDR_SUFFIX):$(COOLBEANS_RAFT_PORT)", + "--node-listen-addr", ":$(COOLBEANS_RPC_PORT)", + "--node-peer-addrs", "$(PEERS_ADDRS)", + ## By default all logs are fsynced to disk (bolt-db) + ## By default fsync is called for every commit to the disk log + # "--no-fsync", + ## + ## Here, all the logs are in memory & not in bolt-db, but snapshots as usual are on + ## disk. You can tune the snapshot threshold, trailing-log-count etc.. + # "--no-disk-log" + "--retain-snapshot-count", "{{ .Values.clusterNode.config.retain_snapshot_count | default "3" }}", + "--restore-timeout-secs", "600", + "--snapshot-threshold", "$(SNAPSHOT_THRESHOLD)", + "--trailing-log-count", "$(TRAILING_LOG_COUNT)", + "--snapshot-interval-secs", "$(SNAPSHOT_INTERVAL_SECS)", + "--prometheus-addr", "127.0.0.1:{{ .Values.clusterNode.config.prometheus_port | default "2020" }}" + ] ports: - - containerPort: 11000 + - containerPort: {{ .Values.clusterNode.config.coolbeans_rpc_port | default "11000" }} name: grpc-server - - containerPort: 21000 + - containerPort: {{ .Values.clusterNode.config.coolbeans_raft_port | default "21000" }} name: raft-tcp + - containerPort: {{ .Values.clusterNode.config.prometheus_port | default "2020" }} + name: metrics resources: {{- toYaml .Values.clusterNode.resources | nindent 12 }} {{- if .Values.clusterNode.persistance.enabled}} volumeMounts: - - name: data - mountPath: /root/data + - name: {{ include "coolbeans.fullname" . }}-node + mountPath: {{ .Values.clusterNode.config.data_dir | default "/root/data" }} {{- end }} - readinessProbe: - exec: - command: ["/root/coolbeans", "--verbose", "cluster-client", "is_leader", "--node-addr", "localhost:11000"] - initialDelaySeconds: 30 - periodSeconds: 15 + # readinessProbe: + # exec: + # command: ["/root/coolbeans", "--verbose", "cluster-client", "is_leader", "--node-addr", "localhost:11000"] + # initialDelaySeconds: 30 + # periodSeconds: 15 {{- if .Values.clusterNode.persistance.enabled}} volumeClaimTemplates: - metadata: - name: data + name: {{ include "coolbeans.fullname" . }}-node spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.clusterNode.persistance.storageClassName }} resources: requests: - storage: {{- .Values.clusterNode.persistance.storage -}} + storage: {{ .Values.clusterNode.persistance.storage }} {{- end }} diff --git a/k8s/helm/coolbeans/templates/proxy/deployment.yaml b/k8s/helm/coolbeans/templates/proxy/deployment.yaml index abdf8f6..31ba527 100644 --- a/k8s/helm/coolbeans/templates/proxy/deployment.yaml +++ b/k8s/helm/coolbeans/templates/proxy/deployment.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "coolbeans.fullname.proxy" . }} + name: {{ include "coolbeans.fullname" . }}-proxy labels: {{- include "coolbeans.labels.proxy" . | nindent 4 }} spec: @@ -18,7 +18,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "coolbeans.labels.proxy" . | nindent 4 }} + {{- include "coolbeans.labels.proxy" . | nindent 8 }} spec: {{- if or .Values.topologySpreadConstraints.zone .Values.topologySpreadConstraints.hostname }} topologySpreadConstraints: @@ -70,24 +70,24 @@ spec: "--upstream-addrs", "$(PEERS_ADDRS)", "--max-job-size", "$(MAX_JOB_SIZE_BYTES)"] ports: - - containerPort: 11300 - name: beanstalkd-tcp + - containerPort: {{ .Values.clusterNode.config.beanstalkd_port | default "11300" }} + name: beanstalkd resources: {{- toYaml .Values.proxy.resources | nindent 12 }} env: - name: BEANSTALKD_PORT valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: BEANSTALKD_PORT - name: PEERS_ADDRS valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: PEERS_ADDRS - name: MAX_JOB_SIZE_BYTES valueFrom: configMapKeyRef: - name: coolbeans-config + name: {{ include "coolbeans.fullname" . }} key: MAX_JOB_SIZE_BYTES restartPolicy: Always \ No newline at end of file diff --git a/k8s/helm/coolbeans/templates/proxy/hpa.yaml b/k8s/helm/coolbeans/templates/proxy/hpa.yaml index fd90b68..29af2e1 100644 --- a/k8s/helm/coolbeans/templates/proxy/hpa.yaml +++ b/k8s/helm/coolbeans/templates/proxy/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "coolbeans.fullname.proxy" . }} + name: {{ include "coolbeans.fullname" . }}-proxy labels: {{- include "coolbeans.labels.proxy" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "coolbeans.fullname.proxy" . }} + name: {{ include "coolbeans.fullname" . }}-proxy minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: diff --git a/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml b/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml index af2a1f2..87a038c 100644 --- a/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml +++ b/k8s/helm/coolbeans/templates/proxy/service-external-lb.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "coolbeans.fullname.proxy" . }}-elb + name: {{ include "coolbeans.fullname" . }}-proxy labels: {{- include "coolbeans.labels.proxy" . | nindent 4 }} {{- with .Values.proxy.service.externalLoadBalancer.annotations }} @@ -10,13 +10,12 @@ metadata: {{- . | toYaml | nindent 4 }} {{- end }} spec: -spec: - type: Loadbalancer + type: LoadBalancer ports: - port: {{ .Values.proxy.service.port }} - targetPort: {{ .Values.proxy.service.port }} + targetPort: beanstalkd protocol: TCP - name: beanstalk-tcp + name: beanstalkd selector: {{- include "coolbeans.selectorLabels.proxy" . | nindent 4 }} {{- end }} diff --git a/k8s/helm/coolbeans/templates/proxy/service.yaml b/k8s/helm/coolbeans/templates/proxy/service.yaml index 4173e6d..8819a64 100644 --- a/k8s/helm/coolbeans/templates/proxy/service.yaml +++ b/k8s/helm/coolbeans/templates/proxy/service.yaml @@ -1,16 +1,17 @@ +{{- if not .Values.proxy.service.externalLoadBalancer.enabled }} apiVersion: v1 kind: Service metadata: - name: {{ include "coolbeans.fullname.proxy" . }} + name: {{ include "coolbeans.fullname" . }}-proxy labels: {{- include "coolbeans.labels.proxy" . | nindent 4 }} -spec: spec: type: {{ .Values.proxy.service.type }} ports: - port: {{ .Values.proxy.service.port }} - targetPort: {{ .Values.proxy.service.port }} + targetPort: beanstalkd protocol: TCP - name: beanstalk-tcp + name: beanstalkd selector: {{- include "coolbeans.selectorLabels.proxy" . | nindent 4 }} +{{- end }} diff --git a/k8s/helm/coolbeans/templates/tests/test-connection.yaml b/k8s/helm/coolbeans/templates/tests/test-connection.yaml deleted file mode 100644 index 48c0423..0000000 --- a/k8s/helm/coolbeans/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "coolbeans.fullname" . }}-test-connection" - labels: - {{- include "coolbeans.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "coolbeans.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/k8s/helm/coolbeans/templates/tests/test.yaml b/k8s/helm/coolbeans/templates/tests/test.yaml new file mode 100644 index 0000000..faeec3e --- /dev/null +++ b/k8s/helm/coolbeans/templates/tests/test.yaml @@ -0,0 +1,51 @@ +{{- if .Values.test.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "coolbeans.fullname" . }}-test-producer + labels: + jobgroup: {{ include "coolbeans.fullname" . }}-test-producer +spec: + template: + metadata: + name: {{ include "coolbeans.fullname" . }}-test-producer + namespace: coolbeans + labels: + jobgroup: {{ include "coolbeans.fullname" . }}-test-producer + spec: + containers: + - name: {{ include "coolbeans.fullname" . }}-test-producer + image: 1xyz/jellybeans-workload:master-ebe38f7 + command: [ + "/root/jellybeans-workload", "producer", + "--addr", "{{ include "coolbeans.fullname" . }}-proxy.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.clusterNode.config.beanstalkd_port | default "11300" }}", + "--count", "10000" + ] + restartPolicy: Never + backoffLimit: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: consume-batch-0 + namespace: coolbeans + labels: + jobgroup: consumer +spec: + template: + metadata: + name: consumer + namespace: coolbeans + labels: + jobgroup: consumer + spec: + containers: + - name: consumer + image: 1xyz/jellybeans-workload:master-ebe38f7 + command: [ + "/root/jellybeans-workload", "consumer", + "--addr", "{{ include "coolbeans.fullname" . }}-proxy.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.clusterNode.config.beanstalkd_port | default "11300" }}" + ] + restartPolicy: Never + backoffLimit: 1 +{{- end }} diff --git a/k8s/helm/coolbeans/values.yaml b/k8s/helm/coolbeans/values.yaml index 88d6a9e..2254994 100644 --- a/k8s/helm/coolbeans/values.yaml +++ b/k8s/helm/coolbeans/values.yaml @@ -2,6 +2,7 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +podLabels: {} podAnnotations: {} podSecurityContext: {} @@ -47,13 +48,24 @@ clusterNode: cpu: "500m" memory: 1Gi persistance: - enabled: false - storage: 100Gi - createPersistentVolume: false - prometheus: - enabled: false + enabled: true + storage: 1Gi + storageClassName: "" + # prometheus: + # enabled: false + # endpoint: "" # endpoint: prom-endpoint:prom-port - endpoint: "" + config: + beanstalkd_port: "11300" + coolbeans_raft_port: "21000" + coolbeans_rpc_port: "11000" + data_dir: "/root/data" + snapshot_threshold: "100" + trailing_log_count: "10240" + snapshot_interval_secs: "30" + max_job_size_bytes: "65535" + prometheus_port: "2020" + retain_snapshot_count: "3" proxy: replicaCount: 2 @@ -85,10 +97,13 @@ fullnameOverride: "" serviceAccount: # Specifies whether a service account should be created - create: true + create: false # Annotations to add to the service account annotations: {} # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" +test: + enabled: false +