From 968a37c17ae277fd5a054506adde8e2d21d8de27 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Mon, 10 Oct 2022 17:27:09 +0000 Subject: [PATCH 01/17] X-Smart-Branch-Parent: origin/master From 39e21c663ebac384ce8b736e816452928789f619 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Thu, 6 Oct 2022 17:28:08 +0000 Subject: [PATCH 02/17] RS-574: Render without save --- Makefile | 4 +++- chart/infra-server/templates/demo/secrets.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d5a113c60..3f13486c3 100644 --- a/Makefile +++ b/Makefile @@ -183,11 +183,13 @@ render-local: clean-render exit 1; \ fi @mkdir -p chart-rendered + gsutil cat gs://infra-configuration/latest/configuration/development-values.yaml \ + gs://infra-configuration/latest/configuration/development-values-from-files.yaml | \ helm template chart/infra-server \ --output-dir chart-rendered \ --set deployment="local" \ --set tag="$(TAG)" \ - --values chart/infra-server/configuration/development-values.yaml + --values - .PHONY: render-development render-development: clean-render diff --git a/chart/infra-server/templates/demo/secrets.yaml b/chart/infra-server/templates/demo/secrets.yaml index 4d8ed2e69..d3dd980cb 100644 --- a/chart/infra-server/templates/demo/secrets.yaml +++ b/chart/infra-server/templates/demo/secrets.yaml @@ -11,7 +11,7 @@ metadata: data: # Service account used for provisioning demo infrastructure. google-credentials.json: |- - {{- include "require-file" (list "demo/demo-provisioner.json" .) | b64enc | nindent 4 }} + {{ .Values.demo_demo_provisioner_json }} # Service account used for demo GCR integration. google-scanner-credentials.json: |- From fd2280a7157d4ecb4a764c2a2bb97df9d292bec5 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Thu, 6 Oct 2022 23:43:01 +0000 Subject: [PATCH 03/17] combine config files into a single yaml --- Makefile | 38 ++++++++++++---- chart/infra-server/templates/_helpers.tpl | 9 ---- .../infra-server/templates/argo/secrets.yaml | 2 +- .../templates/demo-certifier.yaml | 2 +- .../infra-server/templates/demo/secrets.yaml | 4 +- chart/infra-server/templates/gke/secrets.yaml | 2 +- .../templates/openshift-4/secrets.yaml | 4 +- .../templates/openshift/secrets.yaml | 2 +- .../templates/qa-demo/secrets.yaml | 4 +- chart/infra-server/templates/secrets.yaml | 6 +-- scripts/add-PR-comment-for-deploy-to-dev.sh | 8 ++-- scripts/create-consolidated-values.sh | 43 +++++++++++++++++++ scripts/lib.sh | 12 ++++++ 13 files changed, 101 insertions(+), 35 deletions(-) create mode 100755 scripts/create-consolidated-values.sh create mode 100755 scripts/lib.sh diff --git a/Makefile b/Makefile index 3f13486c3..659b13580 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,13 @@ configuration-upload: @echo "Uploading configuration to gs://infra-configuration/latest/" gsutil -m cp -R chart/infra-server/configuration "gs://infra-configuration/latest/" +# Combines configuration/{development,production} files into single helm value.yaml files +# (configuration/{development,production}-values-from-files.yaml) that can be used in template +# rendering. +.PHONY: create-consolidated-values +create-consolidated-values: + @./scripts/create-consolidated-values.sh + .PHONY: push push: docker push us.gcr.io/stackrox-infra/infra-server:$(TAG) | cat @@ -176,20 +183,19 @@ clean-render: @rm -rf chart-rendered .PHONY: render-local -render-local: clean-render +render-local: clean-render create-consolidated-values @if [[ ! -e chart/infra-server/configuration ]]; then \ echo chart/infra-server/configuration is absent. Try:; \ echo make configuration-download; \ exit 1; \ fi @mkdir -p chart-rendered - gsutil cat gs://infra-configuration/latest/configuration/development-values.yaml \ - gs://infra-configuration/latest/configuration/development-values-from-files.yaml | \ helm template chart/infra-server \ --output-dir chart-rendered \ --set deployment="local" \ --set tag="$(TAG)" \ - --values - + --values chart/infra-server/configuration/development-values.yaml \ + --values chart/infra-server/configuration/development-values-from-files.yaml .PHONY: render-development render-development: clean-render @@ -198,7 +204,8 @@ render-development: clean-render --output-dir chart-rendered \ --set deployment="development" \ --set tag="$(TAG)" \ - --values chart/infra-server/configuration/development-values.yaml + --values chart/infra-server/configuration/development-values.yaml \ + --values chart/infra-server/configuration/development-values-from-files.yaml .PHONY: render-production render-production: clean-render @@ -207,7 +214,8 @@ render-production: clean-render --output-dir chart-rendered \ --set deployment="production" \ --set tag="$(TAG)" \ - --values chart/infra-server/configuration/production-values.yaml + --values chart/infra-server/configuration/production-values.yaml \ + --values chart/infra-server/configuration/production-values-from-files.yaml dev_context = gke_stackrox-infra_us-west2_infra-development prod_context = gke_stackrox-infra_us-west2_infra-production @@ -215,8 +223,8 @@ this_context = $(shell kubectl config current-context) kcdev = kubectl --context $(dev_context) kcprod = kubectl --context $(prod_context) -.PHONY: install-local -install-local: +.PHONY: install-local-common +install-local-common: @if [[ "$(this_context)" == "$(dev_context)" ]]; then \ echo Your kube context is set to development infra, should be a local cluster; \ exit 1; \ @@ -234,9 +242,23 @@ install-local: -f chart-rendered/infra-server/templates/namespace.yaml; \ sleep 10; \ fi + +.PHONY: install-local +install-local: install-local-common kubectl apply -R \ -f chart-rendered/infra-server +.PHONY: install-local-without-write +install-local-without-write: install-local-common + gsutil cat gs://infra-configuration/latest/configuration/development-values.yaml \ + gs://infra-configuration/latest/configuration/development-values-from-files.yaml | \ + helm template chart/infra-server \ + --set deployment="local" \ + --set tag="$(TAG)" \ + --values - | \ + kubectl apply -R \ + -f - + .PHONY: diff-development diff-development: render-development $(kcdev) diff -R \ diff --git a/chart/infra-server/templates/_helpers.tpl b/chart/infra-server/templates/_helpers.tpl index 7780a359a..1e307ff2a 100644 --- a/chart/infra-server/templates/_helpers.tpl +++ b/chart/infra-server/templates/_helpers.tpl @@ -1,12 +1,3 @@ -{{- define "require-file" }} - {{- $context := (last .) -}} - {{- $filename := (first .) -}} - {{- $full_filename := (printf "configuration/%s/%s" (required "A valid .Values.environment entry is required!" $context.Values.environment) $filename) -}} - {{- if not ($context.Files.Get $full_filename) -}} - {{- fail (printf "Failed to locate the file %q." $full_filename) -}} - {{- end -}} - {{ printf "%s" ($context.Files.Get $full_filename) }} -{{- end }} {{- define "docker-io-pull-secret" }} {{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.pullSecrets.docker.registry (printf "%s:%s" .Values.pullSecrets.docker.username .Values.pullSecrets.docker.password | b64enc) | b64enc }} diff --git a/chart/infra-server/templates/argo/secrets.yaml b/chart/infra-server/templates/argo/secrets.yaml index dbf170246..3dd57a04d 100644 --- a/chart/infra-server/templates/argo/secrets.yaml +++ b/chart/infra-server/templates/argo/secrets.yaml @@ -38,6 +38,6 @@ metadata: data: credentials.json: |- - {{- include "require-file" (list "google-calendar-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.google_calendar_credentials_json is undefined" .Values.google_calendar_credentials_json }} --- diff --git a/chart/infra-server/templates/demo-certifier.yaml b/chart/infra-server/templates/demo-certifier.yaml index 0a4beaec2..d3025e296 100644 --- a/chart/infra-server/templates/demo-certifier.yaml +++ b/chart/infra-server/templates/demo-certifier.yaml @@ -10,7 +10,7 @@ metadata: data: google-credentials.json: |- - {{- include "require-file" (list "demo/demo-cert-bot.json" .) | b64enc | nindent 4 }} + {{ required ".Values.demo__demo_cert_bot_json is undefined" .Values.demo__demo_cert_bot_json }} --- diff --git a/chart/infra-server/templates/demo/secrets.yaml b/chart/infra-server/templates/demo/secrets.yaml index d3dd980cb..60a138896 100644 --- a/chart/infra-server/templates/demo/secrets.yaml +++ b/chart/infra-server/templates/demo/secrets.yaml @@ -11,11 +11,11 @@ metadata: data: # Service account used for provisioning demo infrastructure. google-credentials.json: |- - {{ .Values.demo_demo_provisioner_json }} + {{ required ".Values.demo__demo_provisioner_json is undefined" .Values.demo__demo_provisioner_json }} # Service account used for demo GCR integration. google-scanner-credentials.json: |- - {{- include "require-file" (list "demo/google-scanner-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.demo__google_scanner_credentials_json is undefined" .Values.demo__google_scanner_credentials_json }} --- diff --git a/chart/infra-server/templates/gke/secrets.yaml b/chart/infra-server/templates/gke/secrets.yaml index 1e51fe32e..5b97699db 100644 --- a/chart/infra-server/templates/gke/secrets.yaml +++ b/chart/infra-server/templates/gke/secrets.yaml @@ -10,6 +10,6 @@ metadata: data: google-credentials.json: |- - {{- include "require-file" (list "gke/gke-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.gke__gke_credentials_json is undefined" .Values.gke__gke_credentials_json }} --- diff --git a/chart/infra-server/templates/openshift-4/secrets.yaml b/chart/infra-server/templates/openshift-4/secrets.yaml index 78b0dbac9..3bce61ba5 100644 --- a/chart/infra-server/templates/openshift-4/secrets.yaml +++ b/chart/infra-server/templates/openshift-4/secrets.yaml @@ -8,7 +8,7 @@ metadata: data: google-credentials.json: |- - {{- include "require-file" (list "openshift-4/google-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.openshift_4__google_credentials_json is undefined" .Values.openshift_4__google_credentials_json }} --- @@ -23,4 +23,4 @@ metadata: data: REDHAT_PULL_SECRET: |- - {{- include "require-file" (list "openshift-4/redhat-pull-secret.json" .) | b64enc | nindent 4 }} + {{ required ".Values.openshift_4__redhat_pull_secret_json is undefined" .Values.openshift_4__redhat_pull_secret_json }} diff --git a/chart/infra-server/templates/openshift/secrets.yaml b/chart/infra-server/templates/openshift/secrets.yaml index 0202c5cc2..42ef2129b 100644 --- a/chart/infra-server/templates/openshift/secrets.yaml +++ b/chart/infra-server/templates/openshift/secrets.yaml @@ -8,4 +8,4 @@ metadata: data: google-credentials.json: |- - {{- include "require-file" (list "openshift/google-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.openshift__google_credentials_json is undefined" .Values.openshift__google_credentials_json }} diff --git a/chart/infra-server/templates/qa-demo/secrets.yaml b/chart/infra-server/templates/qa-demo/secrets.yaml index 2e71ef4ab..f70d5edef 100644 --- a/chart/infra-server/templates/qa-demo/secrets.yaml +++ b/chart/infra-server/templates/qa-demo/secrets.yaml @@ -11,8 +11,8 @@ metadata: data: # Service account used for provisioning demo infrastructure. google-credentials.json: |- - {{- include "require-file" (list "qa-demo/qa-demo-provisioner.json" .) | b64enc | nindent 4 }} + {{ required ".Values.qa_demo__qa_demo_provisioner_json is undefined" .Values.qa_demo__qa_demo_provisioner_json }} # Service account used for demo GCR integration. google-scanner-credentials.json: |- - {{- include "require-file" (list "demo/google-scanner-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.demo__google_scanner_credentials_json is undefined" .Values.demo__google_scanner_credentials_json }} diff --git a/chart/infra-server/templates/secrets.yaml b/chart/infra-server/templates/secrets.yaml index a1bed441d..b0e3b46c3 100644 --- a/chart/infra-server/templates/secrets.yaml +++ b/chart/infra-server/templates/secrets.yaml @@ -11,10 +11,10 @@ data: # Service account used for fetching Google Calendar events as well as # generating GCS signed URLs. google-credentials.json: |- - {{- include "require-file" (list "google-calendar-credentials.json" .) | b64enc | nindent 4 }} + {{ required ".Values.google_calendar_credentials_json is undefined" .Values.google_calendar_credentials_json }} oidc.yaml: |- - {{- tpl (include "require-file" (list "oidc.yaml" .)) . | b64enc | nindent 4 }} + {{- tpl (required ".Values.oidc_yaml | b64dec) . | b64enc | nindent 4 is undefined" .Values.oidc_yaml | b64dec) . | b64enc | nindent 4 }} cert.pem: |- {{- .Files.Get "static/tls-cert.pem" | b64enc | nindent 4 }} @@ -23,7 +23,7 @@ data: {{- .Files.Get "static/tls-key.pem" | b64enc | nindent 4 }} infra.yaml: |- - {{- include "require-file" (list "infra.yaml" .) | b64enc | nindent 4 }} + {{ required ".Values.infra_yaml is undefined" .Values.infra_yaml }} flavors.yaml: |- {{- .Files.Get "static/flavors.yaml" | b64enc | nindent 4 }} diff --git a/scripts/add-PR-comment-for-deploy-to-dev.sh b/scripts/add-PR-comment-for-deploy-to-dev.sh index 93da2450d..15756a154 100755 --- a/scripts/add-PR-comment-for-deploy-to-dev.sh +++ b/scripts/add-PR-comment-for-deploy-to-dev.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" +source "$ROOT/scripts/lib.sh" + set -euo pipefail add_PR_comment_for_deploy_to_dev() { @@ -48,9 +51,4 @@ EOT hub-comment -type deploy -template-file "$tmpfile" } -die() { - echo >&2 "$@" - exit 1 -} - add_PR_comment_for_deploy_to_dev "$@" diff --git a/scripts/create-consolidated-values.sh b/scripts/create-consolidated-values.sh new file mode 100755 index 000000000..fa5c41913 --- /dev/null +++ b/scripts/create-consolidated-values.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" +source "$ROOT/scripts/lib.sh" + +set -euo pipefail + +create_consolidated_values() { + if [[ "$#" -ne 1 ]]; then + die "missing args. usage: create_consolidated_values " + fi + local environment="$1" + + info "Creating a combined values file for chart/infra-server/configuration/$environment files" + + if [[ ! -e "$ROOT/chart/infra-server/configuration" ]]; then + die "chart/infra-server/configuration is missing. Download the configuration with 'make configuration-download'" + fi + + local values_file="$ROOT/chart/infra-server/configuration/$environment-values-from-files.yaml" + rm -f "$values_file" + + pushd "$ROOT/chart/infra-server/configuration/$environment" > /dev/null + shopt -s globstar nullglob + for cfg_file in **; do + if [[ -d "$cfg_file" ]]; then + continue + fi + if [[ "$cfg_file" =~ (README|DS_Store) ]]; then + continue + fi + + local helm_safe_key="${cfg_file//[.-]/_}" + helm_safe_key="${helm_safe_key////__}" + + echo "$helm_safe_key: $(base64 -w0 < "$cfg_file")" >> "$values_file" + echo >> "$values_file" + done + popd > /dev/null +} + +create_consolidated_values "development" +create_consolidated_values "production" diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100755 index 000000000..2b072f753 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +info() { + echo "INFO: $(date): $*" +} + +die() { + echo >&2 "$@" + exit 1 +} From 9c3f9b6b3b9716298598a4560f7282e9fb20def9 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 00:30:10 +0000 Subject: [PATCH 04/17] up to cluster connect --- .github/workflows/PR.yaml | 45 ++++++++++++++++++++- Makefile | 21 +++++++--- scripts/add-PR-comment-for-deploy-to-dev.sh | 29 ++++++++++--- scripts/create-consolidated-values.sh | 6 +++ 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 9d65216a5..d72175e61 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -10,6 +10,9 @@ defaults: run: working-directory: go/src/github.com/stackrox/infra +env: + CLUSTER_NAME: infra-pr-${{ github.event.pull_request.number }} + jobs: lint: @@ -31,7 +34,7 @@ jobs: wait: true token: ${{ secrets.INFRA_TOKEN }} - comment-for-dev-deploy: + comment-on-PR: needs: - build-and-push - create-dev-cluster @@ -51,4 +54,42 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.ROBOT_ROX_GITHUB_TOKEN }} run: | - ./scripts/add-PR-comment-for-deploy-to-dev.sh "${{ github.event.pull_request.html_url }}" "infra-pr-${{ github.event.pull_request.number }}" + ./scripts/add-PR-comment-for-deploy-to-dev.sh "${{ github.event.pull_request.html_url }}" "$CLUSTER_NAME" + + deploy-to-dev-cluster: + needs: + - build-and-push + - create-dev-cluster + if: needs.create-dev-cluster.outputs.status == '2' || needs.create-dev-cluster.outputs.status == 'CREATED' + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.3.46 + env: + INFRA_TOKEN: ${{secrets.INFRA_TOKEN}} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + path: go/src/github.com/stackrox/infra + + - name: Download production infractl + run: | + mkdir -p bin + curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \ + | jq -r ".result.fileChunk" \ + | base64 -d \ + > bin/production-infractl + chmod +x bin/production-infractl + + - name: Download artifacts + run: | + mkdir artifacts + bin/production-infractl artifacts "$CLUSTER_NAME" -d artifacts + + - name: Connect to dev cluster + run: | + KUBECONFIG=artifacts/kubeconfig + kubectl get nodes -o wide diff --git a/Makefile b/Makefile index 659b13580..3f76aac7d 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,16 @@ export GO111MODULE=on .PHONY: all all: image -TAG=$(shell git describe --tags --abbrev=10 --dirty --long) +TAG=$(shell git describe --tags --abbrev=10 --long) .PHONY: tag tag: @echo $(TAG) +IMAGE=us.gcr.io/stackrox-infra/infra-server:$(TAG) +.PHONY: image-name +image-name: + @echo $(IMAGE) + ########### ## Build ## ########### @@ -50,7 +55,7 @@ image: server cli ui clean-image @cp bin/infractl-darwin-amd64 image/static/downloads @cp bin/infractl-darwin-arm64 image/static/downloads @cp bin/infractl-linux-amd64 image/static/downloads - docker build -t us.gcr.io/stackrox-infra/infra-server:$(TAG) image + docker build -t $(IMAGE) image .PHONY: clean-image clean-image: @@ -176,7 +181,7 @@ create-consolidated-values: .PHONY: push push: - docker push us.gcr.io/stackrox-infra/infra-server:$(TAG) | cat + docker push $(IMAGE) | cat .PHONY: clean-render clean-render: @@ -198,7 +203,7 @@ render-local: clean-render create-consolidated-values --values chart/infra-server/configuration/development-values-from-files.yaml .PHONY: render-development -render-development: clean-render +render-development: clean-render create-consolidated-values @mkdir -p chart-rendered helm template chart/infra-server \ --output-dir chart-rendered \ @@ -208,7 +213,7 @@ render-development: clean-render --values chart/infra-server/configuration/development-values-from-files.yaml .PHONY: render-production -render-production: clean-render +render-production: clean-render create-consolidated-values @mkdir -p chart-rendered helm template chart/infra-server \ --output-dir chart-rendered \ @@ -259,6 +264,11 @@ install-local-without-write: install-local-common kubectl apply -R \ -f - +.PHONY: local-data-dev-cycle +local-data-dev-cycle: render-local install-local + @sleep 5 + kubectl -n infra delete pods -l app=infra-server + .PHONY: diff-development diff-development: render-development $(kcdev) diff -R \ @@ -342,6 +352,7 @@ update-version: # i.e. nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 & .PHONY: pull-infractl-from-dev-server pull-infractl-from-dev-server: + @mkdir -p bin @rm -f bin/infractl set -o pipefail; \ curl --retry 3 --insecure --silent --show-error --fail --location https://localhost:8443/v1/cli/linux/amd64/upgrade \ diff --git a/scripts/add-PR-comment-for-deploy-to-dev.sh b/scripts/add-PR-comment-for-deploy-to-dev.sh index 15756a154..6d87bd2a0 100755 --- a/scripts/add-PR-comment-for-deploy-to-dev.sh +++ b/scripts/add-PR-comment-for-deploy-to-dev.sh @@ -16,22 +16,21 @@ add_PR_comment_for_deploy_to_dev() { export DEV_CLUSTER_NAME="$2" + IMAGE_NAME="$(make image-name)" + export IMAGE_NAME + local tmpfile tmpfile=$(mktemp) cat > "$tmpfile" <<- EOT A single node development cluster ({{.Env.DEV_CLUSTER_NAME}}) was allocated in production infra for this PR. +CI will attempt to deploy {{.Env.IMAGE_NAME}} to it. + :electric_plug: You can **connect** to this cluster with: \`\`\` gcloud container clusters get-credentials {{.Env.DEV_CLUSTER_NAME}} --zone us-central1-a --project srox-temp-dev-test \`\`\` -:rocket: And then **deploy** your development infra-server with: -\`\`\` -make render-local -make install-local -\`\`\` - :hammer_and_wrench: And pull **infractl** from the deployed dev infra-server with: \`\`\` nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 & @@ -46,6 +45,24 @@ bin/infractl -k -e localhost:8443 whoami :warning: ***Any clusters that you start using your dev infra instance should have a lifespan shorter then the development cluster instance. Otherwise they will not be destroyed when the dev infra instance ceases to exist when the development cluster is deleted.*** :warning: + +### Further Development + +If you make changes, you can commit and push and CI will take care of updating the development cluster. + +:rocket: If you only modify configuration (chart/infra-server/configuration) or templates (chart/infra-server/{static,templates}), +you can have a faster update by: + +\`\`\` +make render-local +make install-local +\`\`\` + +Or in 1 command that also restarts the infra-server: +\`\`\` +make local-data-dev-cycle +\`\`\` + EOT hub-comment -type deploy -template-file "$tmpfile" diff --git a/scripts/create-consolidated-values.sh b/scripts/create-consolidated-values.sh index fa5c41913..d1908c7ca 100755 --- a/scripts/create-consolidated-values.sh +++ b/scripts/create-consolidated-values.sh @@ -20,6 +20,12 @@ create_consolidated_values() { local values_file="$ROOT/chart/infra-server/configuration/$environment-values-from-files.yaml" rm -f "$values_file" + { + echo "# This is a helm values file that combines the contents of the $environment configuration files." + echo "# It is updated by each render-* make target. Changes made here will be lost." + echo + } >> "$values_file" + pushd "$ROOT/chart/infra-server/configuration/$environment" > /dev/null shopt -s globstar nullglob for cfg_file in **; do From 976f997e1bc6fd103ad916326306f8a4c041931b Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 00:51:54 +0000 Subject: [PATCH 05/17] export --- .github/workflows/PR.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index d72175e61..6609eb5d3 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -91,5 +91,5 @@ jobs: - name: Connect to dev cluster run: | - KUBECONFIG=artifacts/kubeconfig + export KUBECONFIG=artifacts/kubeconfig kubectl get nodes -o wide From 62f1444ec1f4d26c94cbf023e3c2485befcb712b Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 01:20:16 +0000 Subject: [PATCH 06/17] like rox --- .github/workflows/PR.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 6609eb5d3..4acb3a7b5 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -62,8 +62,6 @@ jobs: - create-dev-cluster if: needs.create-dev-cluster.outputs.status == '2' || needs.create-dev-cluster.outputs.status == 'CREATED' runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.3.46 env: INFRA_TOKEN: ${{secrets.INFRA_TOKEN}} From 16e60493637faee7832370e66e64dbde2112e7ce Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 17:40:00 +0000 Subject: [PATCH 07/17] GCP auth --- .github/workflows/PR.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 4acb3a7b5..dc17bf437 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -62,6 +62,8 @@ jobs: - create-dev-cluster if: needs.create-dev-cluster.outputs.status == '2' || needs.create-dev-cluster.outputs.status == 'CREATED' runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.3.46 env: INFRA_TOKEN: ${{secrets.INFRA_TOKEN}} @@ -87,6 +89,11 @@ jobs: mkdir artifacts bin/production-infractl artifacts "$CLUSTER_NAME" -d artifacts + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + credentials_json: ${{ secrets.GCP_INFRA_CI_AUTOMATION_SA }} + - name: Connect to dev cluster run: | export KUBECONFIG=artifacts/kubeconfig From 98b29fefd12ef1c9893f98d3a08d27442c74baa7 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 18:19:25 +0000 Subject: [PATCH 08/17] do the install --- .github/workflows/PR.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index dc17bf437..9fc78e031 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -94,7 +94,8 @@ jobs: with: credentials_json: ${{ secrets.GCP_INFRA_CI_AUTOMATION_SA }} - - name: Connect to dev cluster + - name: Deploy infra to dev cluster run: | export KUBECONFIG=artifacts/kubeconfig kubectl get nodes -o wide + make install-local-without-write From 6467582f178f93b305eab2570c9bf96218e7f2a2 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 18:35:24 +0000 Subject: [PATCH 09/17] it is not rendered to FS --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3f76aac7d..965e8701a 100644 --- a/Makefile +++ b/Makefile @@ -244,7 +244,7 @@ install-local-common: fi @if ! kubectl get ns infra; then \ kubectl apply \ - -f chart-rendered/infra-server/templates/namespace.yaml; \ + -f chart/infra-server/templates/namespace.yaml; \ sleep 10; \ fi From 3685305653e8d26a552bb0802cec332423b17797 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 18:36:01 +0000 Subject: [PATCH 10/17] hide misleading errors --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 965e8701a..2e16ba606 100644 --- a/Makefile +++ b/Makefile @@ -238,11 +238,11 @@ install-local-common: echo Your kube context is set to production infra, should be a local cluster; \ exit 1; \ fi - @if ! kubectl get ns argo; then \ + @if ! kubectl get ns argo 2> /dev/null; then \ kubectl create namespace argo; \ kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.3.9/install.yaml; \ fi - @if ! kubectl get ns infra; then \ + @if ! kubectl get ns infra 2> /dev/null; then \ kubectl apply \ -f chart/infra-server/templates/namespace.yaml; \ sleep 10; \ From 4ff1a790c9e50e776f7fd3eb0fc3dd47fe9a8e2f Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 18:42:33 +0000 Subject: [PATCH 11/17] stackrox/actions/infra/create-cluster does not provide status --- .github/workflows/PR.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 9fc78e031..076128f48 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -60,7 +60,6 @@ jobs: needs: - build-and-push - create-dev-cluster - if: needs.create-dev-cluster.outputs.status == '2' || needs.create-dev-cluster.outputs.status == 'CREATED' runs-on: ubuntu-latest container: image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.3.46 From cd018cb96213d4525bfe5fa380682df13cb591e6 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 19:03:26 +0000 Subject: [PATCH 12/17] try it all --- .github/workflows/PR.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 076128f48..1119778db 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -94,7 +94,12 @@ jobs: credentials_json: ${{ secrets.GCP_INFRA_CI_AUTOMATION_SA }} - name: Deploy infra to dev cluster + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GCP_INFRA_CI_AUTOMATION_SA}} run: | + gcloud auth activate-service-account --key-file <(echo "$GOOGLE_APPLICATION_CREDENTIALS") + gcloud auth list + export KUBECONFIG=artifacts/kubeconfig kubectl get nodes -o wide make install-local-without-write From 91994a2c03991e52c58be7437d0260e9c78f5435 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 19:33:23 +0000 Subject: [PATCH 13/17] dedupe --- .github/workflows/PR.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 1119778db..7e7805a6e 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -88,11 +88,6 @@ jobs: mkdir artifacts bin/production-infractl artifacts "$CLUSTER_NAME" -d artifacts - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v0 - with: - credentials_json: ${{ secrets.GCP_INFRA_CI_AUTOMATION_SA }} - - name: Deploy infra to dev cluster env: GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GCP_INFRA_CI_AUTOMATION_SA}} From 262d3c5ced24258c7f9a922c41060100ea25a233 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 19:44:05 +0000 Subject: [PATCH 14/17] more verification --- .github/workflows/PR.yaml | 28 ++++++++++++++++++++++++++-- Makefile | 4 ++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 7e7805a6e..30e21b26f 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -88,7 +88,7 @@ jobs: mkdir artifacts bin/production-infractl artifacts "$CLUSTER_NAME" -d artifacts - - name: Deploy infra to dev cluster + - name: Deploy infra to dev cluster and check the deployment env: GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GCP_INFRA_CI_AUTOMATION_SA}} run: | @@ -96,5 +96,29 @@ jobs: gcloud auth list export KUBECONFIG=artifacts/kubeconfig - kubectl get nodes -o wide make install-local-without-write + + nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 & + sleep 5 + make pull-infractl-from-dev-server + + version="$(bin/infractl -k -e localhost:8443 version --json)" + echo "$version" + client="$(echo $version | jq -r '.Client.Version')" + server="$(echo $version | jq -r '.Server.Version')" + if [[ "$client" == "$server" ]]; then + echo "Client and server versions match" + else + echo "Client and server versions are mismatched" + exit 1 + fi + + tag="$(make tag)" + if [[ "$client" == "$tag" ]]; then + echo "Infra and make tag match" + else + echo "Infra and make tag are mismatched" + exit 1 + fi + + kill %1 diff --git a/Makefile b/Makefile index 2e16ba606..0b8bbabd9 100644 --- a/Makefile +++ b/Makefile @@ -263,9 +263,13 @@ install-local-without-write: install-local-common --values - | \ kubectl apply -R \ -f - + # Bounce the infra-server to ensure proper update + @sleep 5 + kubectl -n infra delete pods -l app=infra-server .PHONY: local-data-dev-cycle local-data-dev-cycle: render-local install-local + # Bounce the infra-server to ensure proper update @sleep 5 kubectl -n infra delete pods -l app=infra-server From af068bca99d7d8b5b882e4d91d5d748251eca121 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 20:12:25 +0000 Subject: [PATCH 15/17] update dev-cycle comments --- DEPLOYMENT.md | 64 +-------------------- README.md | 15 ++++- scripts/add-PR-comment-for-deploy-to-dev.sh | 10 +--- 3 files changed, 15 insertions(+), 74 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 851508a55..fea0ff03b 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,67 +1,5 @@ # Infra Deployment -## Deploy to an adhoc development cluster - -For example one created with `infractl create gke-default`. - -To deploy to such a cluster simply: - -``` -make deploy-local -``` - -The infra server should start and argo should deploy. - -``` -$ kubectl -n infra get pods -NAME READY STATUS RESTARTS AGE -infra-server-deployment-5c6cfb69c-54k6x 1/1 Running 0 11s -$ kubectl -n argo get pods -NAME READY STATUS RESTARTS AGE -argo-server-58bf6d4f79-cc96j 1/1 Running 1 95s -workflow-controller-6487cc4688-cdbfz 1/1 Running 0 95s -``` - -To connect to the infra-server run a proxy: - -``` -kubectl -n infra port-forward svc/infra-server-service 8443:8443 -``` - -Then use *safari* to connect to the UI if needed. (note: chrome will not accept -the infra self-signed cert). - -Or the locally compiled infractl binary: - -``` -bin/infractl-darwin-amd64 -k -e localhost:8443 whoami -``` - -### Notes - -For clusters created in the `srox-temp-dev-test` to be able to pull images from -the `stackrox-infra` `us.gcr.io` and `gcr.io` registries, the -`srox-temp-dev-test` default compute service account requires *Storage Object Viewer* access to -`artifacts.stackrox-infra.appspot.com` and -`us.artifacts.stackrox-infra.appspot.com`. - -For other clusters e.g. `docker-desktop` image pull secrets will work after the -deployment has created the namespaces. e.g. - -``` -kubectl create secret docker-registry infra-us-gcr-access --docker-server=us.gcr.io --docker-username=_json_key \ - --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com -kubectl create secret docker-registry infra-gcr-access --docker-server=gcr.io --docker-username=_json_key \ - --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com -kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "infra-gcr-access"},{"name": "infra-us-gcr-access"}]}' - -kubectl -n infra create secret docker-registry infra-us-gcr-access --docker-server=us.gcr.io --docker-username=_json_key \ - --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com -kubectl -n infra create secret docker-registry infra-gcr-access --docker-server=gcr.io --docker-username=_json_key \ - --docker-password="$(cat chart/infra-server/configuration/production/gke/gke-credentials.json)" --docker-email=infra@stackrox.com -kubectl -n infra patch serviceaccount default -p '{"imagePullSecrets": [{"name": "infra-gcr-access"},{"name": "infra-us-gcr-access"}]}' -``` - ## Production and Staging Clusters To work with either of the clusters in `project=stackrox-infra` you will need to either be a member of the `team-automation` group or have someone add you as a project owner. @@ -154,7 +92,7 @@ correct tooling installed with: `make image push` -### Development +### Development/Staging To render a copy of the charts (for inspection), run: diff --git a/README.md b/README.md index ac412f3e0..5264742b5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,16 @@ ## Development -Infra (the server) and infractl (the cli) are written in Go, and use gRPC for client-server communication. +Infra (the server) and infractl (the cli) are written in Go, and use gRPC for +client-server communication. The UI uses a React/Typescript/Yarn toolchain (see +(ui/README.md)[ui/README.md]). + +While a development workflow can be achieved using a locally installed +toolchain, it is also possible to rely on CI. CI will lint, build and push the +infra server. And then deploy it to a development cluster created using the +production infra deployment. A +(comment)[https://github.com/stackrox/infra/pull/711#issuecomment-1270457578] +will appear on PRs with more detail. ### Regenerate Go bindings from protos @@ -32,11 +41,11 @@ correct tooling installed with: ## Deployment -For additional information on how this service is deployed, please refer to the [deployment instructions](https://github.com/stackrox/infra/blob/master/DEPLOYMENT.md). +For additional information on how this service is deployed, please refer to the [deployment instructions](DEPLOYMENT.md). ## Runbook -For additional information on how to debug and remediate issues with the deployed service, please refer to the [runbook instructions](https://github.com/stackrox/infra/blob/master/TROUBLESHOOTING.md). +For additional information on how to debug and remediate issues with the deployed service, please refer to the [runbook instructions](TROUBLESHOOTING.md). [circleci-badge]: https://circleci.com/gh/stackrox/infra.svg?style=shield&circle-token=afa342906b658b5349c68b70fa82fd85d1422212 [circleci-link]: https://circleci.com/gh/stackrox/infra diff --git a/scripts/add-PR-comment-for-deploy-to-dev.sh b/scripts/add-PR-comment-for-deploy-to-dev.sh index 6d87bd2a0..8cc5d6bb9 100755 --- a/scripts/add-PR-comment-for-deploy-to-dev.sh +++ b/scripts/add-PR-comment-for-deploy-to-dev.sh @@ -48,17 +48,11 @@ ceases to exist when the development cluster is deleted.*** :warning: ### Further Development -If you make changes, you can commit and push and CI will take care of updating the development cluster. +:coffee: If you make changes, you can commit and push and CI will take care of updating the development cluster. :rocket: If you only modify configuration (chart/infra-server/configuration) or templates (chart/infra-server/{static,templates}), -you can have a faster update by: +you can get a faster update with: -\`\`\` -make render-local -make install-local -\`\`\` - -Or in 1 command that also restarts the infra-server: \`\`\` make local-data-dev-cycle \`\`\` From f2fe15c9d733e6e1427936c74efee8f2d8513145 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Fri, 7 Oct 2022 21:33:01 +0000 Subject: [PATCH 16/17] let the browser handle newlines --- scripts/add-PR-comment-for-deploy-to-dev.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/add-PR-comment-for-deploy-to-dev.sh b/scripts/add-PR-comment-for-deploy-to-dev.sh index 8cc5d6bb9..3e92702d8 100755 --- a/scripts/add-PR-comment-for-deploy-to-dev.sh +++ b/scripts/add-PR-comment-for-deploy-to-dev.sh @@ -42,16 +42,13 @@ make pull-infractl-from-dev-server bin/infractl -k -e localhost:8443 whoami \`\`\` -:warning: ***Any clusters that you start using your dev infra instance should have a lifespan shorter -then the development cluster instance. Otherwise they will not be destroyed when the dev infra instance -ceases to exist when the development cluster is deleted.*** :warning: +:warning: ***Any clusters that you start using your dev infra instance should have a lifespan shorter then the development cluster instance. Otherwise they will not be destroyed when the dev infra instance ceases to exist when the development cluster is deleted.*** :warning: ### Further Development :coffee: If you make changes, you can commit and push and CI will take care of updating the development cluster. -:rocket: If you only modify configuration (chart/infra-server/configuration) or templates (chart/infra-server/{static,templates}), -you can get a faster update with: +:rocket: If you only modify configuration (chart/infra-server/configuration) or templates (chart/infra-server/{static,templates}), you can get a faster update with: \`\`\` make local-data-dev-cycle From 694d32f7c36e958875ea7a561cb0e9b6ee1228b2 Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Mon, 10 Oct 2022 17:17:57 +0000 Subject: [PATCH 17/17] set in env --- .github/workflows/PR.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 30e21b26f..e12dd5725 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -91,11 +91,11 @@ jobs: - name: Deploy infra to dev cluster and check the deployment env: GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GCP_INFRA_CI_AUTOMATION_SA}} + KUBECONFIG: artifacts/kubeconfig run: | gcloud auth activate-service-account --key-file <(echo "$GOOGLE_APPLICATION_CREDENTIALS") gcloud auth list - export KUBECONFIG=artifacts/kubeconfig make install-local-without-write nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 &