From 3f6b4ea551466d31e5bd8a7f328b9506e303593c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Denis=20Krienb=C3=BChl?= <denis.krienbuehl@cloudscale.ch>
Date: Fri, 11 Aug 2023 17:22:48 +0200
Subject: [PATCH] Lay the foundation for integration tests

---
 .gitignore                                    |  1 +
 Makefile                                      |  2 +-
 README.md                                     |  8 ++
 .../main.go                                   |  3 +
 helpers/cleanup                               | 11 +++
 helpers/run-in-test-cluster                   | 87 +++++++++++++++++++
 6 files changed, 111 insertions(+), 1 deletion(-)
 create mode 100755 helpers/cleanup
 create mode 100755 helpers/run-in-test-cluster

diff --git a/.gitignore b/.gitignore
index bff2d76..ad190a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 *.iml
+k8test
diff --git a/Makefile b/Makefile
index 3b0ee60..86281e5 100644
--- a/Makefile
+++ b/Makefile
@@ -8,4 +8,4 @@ lint:
 	golangci-lint run
 
 test:
-	go test -coverpkg=./pkg/... ./pkg/...
+	go test -race -coverpkg=./pkg/... ./pkg/...
diff --git a/README.md b/README.md
index ca2c9cd..36788eb 100644
--- a/README.md
+++ b/README.md
@@ -7,3 +7,11 @@ Provides the following features:
 * Automatically provisions load balancers for [`LoadBalancer` services](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
 * Enriches `Node` metadata with information from our cloud.
 * Updates `Node` state depending on the state of the underlying VM.
+
+## Test Cluster
+
+To test the code on a vanilla Kubernetes cluster, you can use `helpers/run-in-test-cluster`. Note that you need a `CLOUDSCALE_API_TOKEN` for this to work, and this may incur costs on your side.
+
+To clean the cluster up, run `helpers/cleanup`.
+
+This is based on [k8test](https://github.com/cloudscale-ch/k8test), our in-house Kubernetes integration test utility.
diff --git a/cmd/cloudscale-cloud-controller-manager/main.go b/cmd/cloudscale-cloud-controller-manager/main.go
index 5e5519b..58e00e6 100644
--- a/cmd/cloudscale-cloud-controller-manager/main.go
+++ b/cmd/cloudscale-cloud-controller-manager/main.go
@@ -62,5 +62,8 @@ func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
 				"allow-untagged-cloud option")
 		}
 	}
+
+	klog.Info("cloudscale-cloud-controller-manager initialized")
+
 	return cloud
 }
diff --git a/helpers/cleanup b/helpers/cleanup
new file mode 100755
index 0000000..badb0de
--- /dev/null
+++ b/helpers/cleanup
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+#
+# Ensures that any Kubernetes cluster is cleaned up
+#
+set -euo pipefail
+
+if ! test -f k8test/cluster/inventory.yml; then
+  exit 0
+fi
+
+k8test/playbooks/destroy-cluster.yml -i k8test/cluster/inventory.yml
diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster
new file mode 100755
index 0000000..33454a3
--- /dev/null
+++ b/helpers/run-in-test-cluster
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+#
+# Ensures that a Kubernetes test cluster is present and updated
+#
+set -euo pipefail
+
+# Prepares k8test with an existing virtual env, or a newly created on
+function ensure-k8test() {
+  if ! test -d k8test; then
+    git clone git@github.com:cloudscale-ch/k8test.git
+    git -C k8test switch denis/init
+  fi
+
+  if [[ "${VIRTUAL_ENV:-}" == "" ]]; then
+
+    if ! test -d k8test/venv; then
+      python3 -m venv k8test/venv
+    fi
+
+    # shellcheck source=/dev/null
+    source k8test/venv/bin/activate
+  fi
+
+  if ! command -v poetry > /dev/null; then
+    pip install poetry
+  fi
+
+  if ! command -v ansible > /dev/null; then
+    poetry install --directory k8test
+  fi
+}
+
+# Prints a random zone
+function random-zone() {
+  arr[0]="rma1"
+  arr[1]="lpg1"
+
+  rand=$((RANDOM % 2))
+  echo "${arr[$rand]}"
+}
+
+# Launches the test cluster, if there's no inventory yet
+function ensure-inventory() {
+  if ! test -d k8test/cluster; then
+    mkdir k8test/cluster
+  fi
+
+  if ! test -f k8test/cluster/ssh.pub; then
+    ssh-keygen -t ed25519 -N '' -f k8test/cluster/ssh
+  fi
+
+  if ! test -f k8test/cluster/inventory.yml; then
+    k8test/playbooks/create-cluster.yml \
+      -e zone="$(random-zone)" \
+      -e ssh_key=k8test/cluster/ssh.pub \
+      -e control_count=2 \
+      -e worker_count=2
+
+    # Those won't really change between runs, so update them during install
+    k8test/playbooks/update-secrets.yml \
+      -i k8test/cluster/inventory.yml
+  fi
+}
+
+# Build the latest container each time
+function build-container() {
+  k8test/playbooks/build-container.yml \
+    -i k8test/cluster/inventory.yml \
+    -e dockerfile=./Dockerfile \
+    -e tag=quay.io/cloudscalech/cloudscale-cloud-controller-manager:test \
+    -e extra='--build-arg=VERSION=test'
+}
+
+# Re-apply the resources each time
+function apply-resources() {
+  sed 's/controller-manager:latest/controller-manager:test/g' \
+    deploy/kubernetes/releases/latest.yml > /tmp/ccm.yml
+
+  export KUBECONFIG=k8test/cluster/admin.conf
+  kubectl delete -f /tmp/ccm.yml || true
+  kubectl apply -f /tmp/ccm.yml
+}
+
+ensure-k8test
+ensure-inventory
+build-container
+apply-resources