From 0bdbcacd084e203aba92c8f361eefc459095af9c Mon Sep 17 00:00:00 2001 From: Dejan K Date: Thu, 16 Oct 2025 18:44:04 +0200 Subject: [PATCH 1/5] ci: improve cache missing handling --- .semaphore/semaphore.yml | 50 +++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 88dec25..ce86efb 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -12,6 +12,11 @@ queue: name: Testing scope: project +global_job_config: + prologue: + commands: + - unset SEMAPHORE_CACHE_BACKEND + blocks: - name: Install dependencies dependencies: [] @@ -28,7 +33,7 @@ blocks: - nvm install - node --version - npm --version - - cache restore + - cache restore || echo "Cache restore skipped; continuing without cache" - npm install - cache store @@ -70,7 +75,7 @@ blocks: - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install python3-venv -fuy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" - checkout - - cache restore venv-$SEMAPHORE_GIT_BRANCH-$(checksum requirements.txt) + - cache restore venv-$SEMAPHORE_GIT_BRANCH-$(checksum requirements.txt) || echo "Cache restore skipped; continuing without cache" - make ansible.lint - cache store venv-$SEMAPHORE_GIT_BRANCH-$(checksum requirements.txt) venv - name: Unit tests @@ -79,7 +84,8 @@ blocks: - nvm install - node --version - npm --version - - cache restore + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - npm test - name: Build @@ -92,7 +98,7 @@ blocks: prologue: commands: - checkout - - cache restore venv-$SEMAPHORE_GIT_BRANCH-$(checksum requirements.txt) + - cache restore venv-$SEMAPHORE_GIT_BRANCH-$(checksum requirements.txt) || echo "Cache restore skipped; continuing without cache" - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install python3-venv -fuy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" - curl -sL https://releases.hashicorp.com/packer/1.7.9/packer_1.7.9_linux_amd64.zip -o /tmp/packer_1.7.9_linux_amd64.zip @@ -122,7 +128,9 @@ blocks: - name: Bootstrap stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - ./ci/create-execution-policy-and-bootstrap.sh $AWS_ACCOUNT_ID $AWS_DEFAULT_REGION - name: Linux Focal launch @@ -136,7 +144,9 @@ blocks: - name: Launch stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - ./ci/create-ssm-param.sh s1-agent-aws-stack-linux-token $TOKEN - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-focal-config.json npm run deploy:ci @@ -151,7 +161,9 @@ blocks: - name: Launch stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - ./ci/create-ssm-param.sh s1-agent-aws-stack-linux-jammy-token $TOKEN_JAMMY - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-jammy-config.json npm run deploy:ci @@ -166,7 +178,9 @@ blocks: - name: Launch stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - ./ci/create-ssm-param.sh s1-agent-aws-stack-linux-noble-token $TOKEN_NOBLE - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-noble-config.json npm run deploy:ci @@ -181,7 +195,9 @@ blocks: - name: Launch stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - ./ci/create-ssm-param.sh s1-agent-aws-stack-windows-token $TOKEN - SEMAPHORE_AGENT_STACK_CONFIG=./ci/windows-config.json npm run deploy:ci @@ -281,20 +297,28 @@ after_pipeline: - name: Destroy Linux Focal stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-focal-config.json npm run destroy:ci - name: Destroy Linux Jammy stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-jammy-config.json npm run destroy:ci - name: Destroy Linux Noble stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-noble-config.json npm run destroy:ci - name: Destroy Windows stack commands: - checkout - - cache restore + - nvm install + - cache restore || echo "Cache restore skipped; continuing without cache" + - if [ ! -d node_modules ]; then echo "node_modules missing; running npm ci"; npm ci; fi - SEMAPHORE_AGENT_STACK_CONFIG=./ci/windows-config.json npm run destroy:ci From 8ebbc6b55c5f142ca625c961d4193f39d43164c7 Mon Sep 17 00:00:00 2001 From: Dejan K Date: Thu, 16 Oct 2025 18:53:06 +0200 Subject: [PATCH 2/5] fix(Makefile): update security toolbox setup and use sparse checkout --- Makefile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4733a4c..29183bf 100644 --- a/Makefile +++ b/Makefile @@ -27,23 +27,29 @@ SYSTEMD_RESTART_SECONDS=1800 VERSION=$(shell cat package.json | jq -r '.version') HASH=$(shell find Makefile packer/$(PACKER_OS) -type f -exec md5sum "{}" + | awk '{print $$1}' | sort | md5sum | awk '{print $$1}') -SECURITY_TOOLBOX_BRANCH ?= master -SECURITY_TOOLBOX_TMP_DIR ?= /tmp/security-toolbox +MONOREPO_TMP_DIR ?= /tmp/monorepo +SECURITY_TOOLBOX_TMP_DIR ?= $(MONOREPO_TMP_DIR)/security-toolbox +SECURITY_TOOLBOX_BRANCH ?= main check.prepare: - rm -rf $(SECURITY_TOOLBOX_TMP_DIR) - git clone git@github.com:renderedtext/security-toolbox.git $(SECURITY_TOOLBOX_TMP_DIR) && (cd $(SECURITY_TOOLBOX_TMP_DIR) && git checkout $(SECURITY_TOOLBOX_BRANCH) && cd -) + rm -rf $(MONOREPO_TMP_DIR) + git clone --depth 1 --filter=blob:none --sparse https://github.com/semaphoreio/semaphore $(MONOREPO_TMP_DIR) && \ + cd $(MONOREPO_TMP_DIR) && \ + git config core.sparseCheckout true && \ + git sparse-checkout init --cone && \ + git sparse-checkout set security-toolbox && \ + git checkout $(SECURITY_TOOLBOX_BRANCH) && cd - check.static: check.prepare docker run -it -v $$(pwd):/app \ -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ - registry.semaphoreci.com/ruby:2.7 \ + registry.semaphoreci.com/ruby:3 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/code --language js -d' check.deps: check.prepare docker run -it -v $$(pwd):/app \ -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ - registry.semaphoreci.com/ruby:2.7 \ + registry.semaphoreci.com/ruby:3 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language js -d' venv.execute: From a62cf5da42f6df555bac3b51039098692eb8280b Mon Sep 17 00:00:00 2001 From: Dejan K Date: Thu, 16 Oct 2025 20:33:04 +0200 Subject: [PATCH 3/5] fix(Makefile): add PIP_BREAK_SYSTEM_PACKAGES environment variable to Docker commands --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 29183bf..9194699 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,14 @@ check.prepare: check.static: check.prepare docker run -it -v $$(pwd):/app \ -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ + -e PIP_BREAK_SYSTEM_PACKAGES=1 \ registry.semaphoreci.com/ruby:3 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/code --language js -d' check.deps: check.prepare docker run -it -v $$(pwd):/app \ -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ + -e PIP_BREAK_SYSTEM_PACKAGES=1 \ registry.semaphoreci.com/ruby:3 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language js -d' From ac08fde5809d87a96a8087b761552426c200d4f8 Mon Sep 17 00:00:00 2001 From: Dejan K Date: Fri, 17 Oct 2025 08:55:19 +0200 Subject: [PATCH 4/5] ci: remove unsetting CACHE BACKEND for testing --- .semaphore/semaphore.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ce86efb..aa2ae14 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -12,11 +12,6 @@ queue: name: Testing scope: project -global_job_config: - prologue: - commands: - - unset SEMAPHORE_CACHE_BACKEND - blocks: - name: Install dependencies dependencies: [] From a3bf7bc4a2da5a77c2723a8def2746ba98d9cd86 Mon Sep 17 00:00:00 2001 From: Dejan K Date: Fri, 17 Oct 2025 13:03:25 +0200 Subject: [PATCH 5/5] ci: remove unnecessary secrets and prologue commands from Security checks block --- .semaphore/semaphore.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index aa2ae14..7ce6019 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -36,13 +36,9 @@ blocks: dependencies: - Install dependencies task: - secrets: - - name: security-toolbox-shared-read-access prologue: commands: - checkout - - mv ~/.ssh/security-toolbox ~/.ssh/id_rsa - - sudo chmod 600 ~/.ssh/id_rsa epilogue: always: commands: