From b2febe71eea4dbe2030a2cb79a079b78862944a4 Mon Sep 17 00:00:00 2001 From: Dejan K Date: Mon, 20 Oct 2025 19:36:17 +0200 Subject: [PATCH 1/2] feat: add support for custom base AMI in Packer configuration --- Makefile | 3 +++ README.md | 14 ++++++++++++++ ci/build-ami.sh | 7 ++++++- packer/linux/ubuntu.pkr.hcl | 32 +++++++++++++++++++++++--------- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 9194699..4cb22df 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ AGENT_VERSION=v2.2.16 TOOLBOX_VERSION=v1.20.5 PACKER_OS=linux UBUNTU_VERSION=focal +SOURCE_AMI?= # Set Ubuntu name and version number based on UBUNTU_VERSION ifeq ($(UBUNTU_VERSION),focal) @@ -89,6 +90,7 @@ packer.validate.linux: -var "instance_type=$(AMI_INSTANCE_TYPE)" \ -var "ubuntu_name=$(UBUNTU_NAME)" \ -var "ubuntu_version=$(UBUNTU_VERSION_NUMBER)" \ + -var "source_ami=$(SOURCE_AMI)" \ .' packer.validate.windows: @@ -148,6 +150,7 @@ packer.build.linux: -var "instance_type=$(AMI_INSTANCE_TYPE)" \ -var "ubuntu_name=$(UBUNTU_NAME)" \ -var "ubuntu_version=$(UBUNTU_VERSION_NUMBER)" \ + -var "source_ami=$(SOURCE_AMI)" \ .' packer.build.windows: diff --git a/README.md b/README.md index 749d616..05eda90 100644 --- a/README.md +++ b/README.md @@ -12,3 +12,17 @@ This project is a CDK application used to deploy a fleet of Semaphore agents in - Control the size of your agent instances and of your agent pool Check out the [docs](https://docs.semaphoreci.com/ci-cd-environment/aws-support). + +## Custom base AMI + +Set the `SOURCE_AMI` environment variable when running the Packer targets if you need to pin the build to a specific Ubuntu kernel. For example: + +``` +SOURCE_AMI=ami-0123456789abcdef0 make packer.build PACKER_OS=linux UBUNTU_VERSION=noble +``` + +The helper script `ci/build-ami.sh` also accepts the desired AMI ID as an optional third argument so it can be used in the same way: + +``` +./ci/build-ami.sh ubuntu-noble x86_64 ami-0123456789abcdef0 +``` diff --git a/ci/build-ami.sh b/ci/build-ami.sh index adc3071..533bb21 100755 --- a/ci/build-ami.sh +++ b/ci/build-ami.sh @@ -15,6 +15,11 @@ if [[ -z "${arch}" ]]; then exit 1 fi +source_ami=$3 +if [[ -n "${source_ami}" ]]; then + export SOURCE_AMI=${source_ami} +fi + packer_os=linux ubuntu_version="" @@ -44,4 +49,4 @@ if [[ ${images} == "0" ]]; then fi else echo "Image with name ${image_name} already exists. Not building anything." -fi \ No newline at end of file +fi diff --git a/packer/linux/ubuntu.pkr.hcl b/packer/linux/ubuntu.pkr.hcl index 588ed35..06569b9 100644 --- a/packer/linux/ubuntu.pkr.hcl +++ b/packer/linux/ubuntu.pkr.hcl @@ -51,6 +51,15 @@ variable "ubuntu_version" { default = "20.04" } +variable "source_ami" { + type = string + default = null +} + +locals { + use_custom_source_ami = var.source_ami != null && var.source_ami != "" +} + packer { required_plugins { amazon = { @@ -74,17 +83,22 @@ source "amazon-ebs" "ubuntu" { Hash = "${var.hash}" } - source_ami_filter { - most_recent = true + source_ami = local.use_custom_source_ami ? var.source_ami : null + + dynamic "source_ami_filter" { + for_each = local.use_custom_source_ami ? [] : [true] + content { + most_recent = true - // Canonical's ownerId: https://ubuntu.com/server/docs/cloud-images/amazon-ec2 - owners = ["099720109477"] + // Canonical's ownerId: https://ubuntu.com/server/docs/cloud-images/amazon-ec2 + owners = ["099720109477"] - filters = { - name = "ubuntu/images/*ubuntu-${var.ubuntu_name}-${var.ubuntu_version}-*" - architecture = "${var.arch}" - root-device-type = "ebs" - virtualization-type = "hvm" + filters = { + name = "ubuntu/images/*ubuntu-${var.ubuntu_name}-${var.ubuntu_version}-*" + architecture = "${var.arch}" + root-device-type = "ebs" + virtualization-type = "hvm" + } } } } From c8f048a553ca8d70ec405ea4a931e6849548f414 Mon Sep 17 00:00:00 2001 From: Dejan K Date: Mon, 20 Oct 2025 20:15:11 +0200 Subject: [PATCH 2/2] feat: add comprehensive AGENTS.md and documentation for LLM agents to describe project structure, build commands, and testing --- AGENTS.md | 37 ++++++++++++++++++++++++++++++ DOCUMENTATION.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 AGENTS.md create mode 100644 DOCUMENTATION.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5bbac3f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,37 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `bin/aws-semaphore-agent.js` boots the CDK app and passes stack parameters. +- `lib/` hosts stack logic and helpers for AMI hashes, GitHub keys, and SSH key rotation. +- `lambdas/` packages runtime handlers (`agent-scaler`, `az-rebalance-suspender`, `ssh-keys-updater`) deployed with the stack. +- `packer/` contains OS-specific templates and Ansible roles; reuse `ci/build-ami.sh` for repeatable builds. +- Supporting automation lives in `ci/`, `goss/`, and the Jest suite under `test/`. + +## Triage & Deep Dives +- Use `DOCUMENTATION.md` for architecture context, dependency maps, and troubleshooting cues before triaging incidents or planning new work. + +## Build, Test, and Development Commands +- Install dependencies with `npm install` and export AWS credentials before invoking CDK. +- `npm run synth` emits the CloudFormation template into `cdk.out` for fast validation. +- `npm run diff` compares the stack with AWS; attach its output to infrastructure PRs. +- `npm run deploy` (or `npm run deploy:ci`) pushes the stack; follow with `npm run destroy` for clean teardown. +- AMIs are baked via `make packer.build PACKER_OS=linux UBUNTU_VERSION=noble SOURCE_AMI=ami-xxxx` once `make packer.validate` succeeds. + +## Coding Style & Naming Conventions +- JavaScript sources use 2-space indentation, single quotes, and `const`/`let` semantics; mirror the patterns in `lib/*.js`. +- Export CommonJS modules via `module.exports` and match exports to filenames (for example `github-keys.js` exposes `githubKeys`). +- Keep new folders lowercase-hyphenated and favor small helpers over inline logic to retain declarative stacks. + +## Testing Guidelines +- Use Jest (`npm test`) with files suffixed `.test.js` under `test/`; follow the structure in `argument-store.test.js`. +- Mock AWS services with CDK assertions or spies and avoid live AWS calls in unit suites. +- Add regression tests for every Lambda or stack behaviour change and run them locally before pushing. + +## Commit & Pull Request Guidelines +- Follow the conventional commit style already in history (`feat(packer/linux): support jammy`) to signal scope and intent. +- Squash noisy work-in-progress commits before review; keep messages imperative and under 72 characters. +- PRs should include a succinct summary, relevant `npm run diff` or Packer logs, linked issues, and note any operational impact. + +## Security & Configuration Tips +- Never commit AWS credentials or Semaphore tokens; reference parameters managed in SSM or KMS instead. +- Keep `execution-policy.json` and IAM statements least-privileged, and request review when adding new permissions or dedicated hosts. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..33e29a6 --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,59 @@ +# Repository Reference + +## Core Purpose +- CDK app that provisions Semaphore self-hosted agent fleets on AWS. +- Manages autoscaling EC2 pools, operational Lambdas, AMI build tooling, and supporting scripts. + +## CDK Application Layout +- Entry point `bin/aws-semaphore-agent.js` instantiates `AwsSemaphoreAgentStack` and injects runtime parameters via `ArgumentStore`. +- `lib/aws-semaphore-agent-stack.js` composes the stack: EC2 Auto Scaling Group, Launch Template, SSM parameters, IAM roles/policies, CloudWatch alarms, custom resources, and Lambda integrations. +- `lib/argument-store.js` normalises configuration, applies defaults, and validates required environment variables (e.g., endpoint, VPC settings, overprovision strategy). +- Supporting modules: + - `lib/dynamic-ssh-keys-updater.js` provisions an SSM parameter and scheduled Lambda to refresh GitHub SSH keys. + - `lib/github-keys.js` caches GitHub SSH keys locally to reduce API calls during synthesis. + - `lib/ami-hash.js` fingerprints Packer templates to detect AMI drift in the stack. + +## Lambda Functions +- `lambdas/agent-scaler/app.js` polls Semaphore occupancy metrics and adjusts the Auto Scaling Group size; publishes metrics to CloudWatch. +- `lambdas/az-rebalance-suspender/app.js` custom resource that suspends the `AZRebalance` process to keep macOS dedicated hosts stable. +- `lambdas/ssh-keys-updater/app.js` daily job that mirrors GitHub SSH keys into the SSM parameter backing agent instances. +- Lambdas share patterns: Node.js 18 runtimes, AWS SDK v3 clients with aggressive timeouts, JSON logging, and explicit retries left to CloudWatch rules. + +## Configuration & Parameters +- Runtime settings come from environment variables or a JSON file referenced by `SEMAPHORE_AGENT_STACK_CONFIG`. +- Mandatory variables: `SEMAPHORE_AGENT_STACK_NAME`, `SEMAPHORE_AGENT_TOKEN_PARAMETER_NAME`, plus either `SEMAPHORE_ENDPOINT` or `SEMAPHORE_ORGANIZATION`. +- Defaults cover instance type, scaling limits, OS family, storage sizing, IPv6, and SSH ingress toggles. +- IAM/KMS/SSM identifiers determine permissions for agent tokens, cache buckets, and managed policies. + +## AMI Build Tooling +- `packer/linux`, `packer/windows`, `packer/macos` define HCL templates, scripts, and Ansible roles for baking agent images. +- `ci/build-ami.sh` wraps packer invocations; `lib/ami-hash.js` ties template changes to CDK asset updates. +- Make targets (`make packer.validate`, `make packer.build`) orchestrate packer init/validate/build per OS. +- `goss/` provides OS-specific health checks that run inside packer builds or post-provision. + +## Scripts & Utilities +- `ci/create-execution-policy-and-bootstrap.sh`, `ci/create-ssm-param.sh`, and `ci/delete-old-images.sh` help bootstrap AWS environments and clean stale AMIs. +- `execution-policy.json` captures baseline IAM permissions to run the stack. +- `bin/aws-semaphore-agent.js` applies optional tags supplied via `SEMAPHORE_AGENT_TAGS`. + +## Testing & Validation +- Jest specs live in `test/` and use CDK assertions to verify template outputs (IAM roles, SSM parameters, Launch Templates). +- Run `npm test` before modifying infrastructure primitives; add targeted tests for new resources or argument validations. +- Goss manifests validate baked AMIs; integrate via packer pipelines when altering base images. + +## Deployment Workflow +- Install dependencies: `npm install`. +- Synth & inspect: `npm run synth`, `npm run diff`. +- Deploy & clean: `npm run deploy` / `npm run destroy`. CI variants (`deploy:ci`, `destroy:ci`) skip approval prompts. +- AMI lifecycle: `make packer.validate PACKER_OS=`, then `make packer.build ...` with versioned inputs (e.g., `UBUNTU_VERSION=noble`, `SOURCE_AMI=ami-xxxxx`). + +## Troubleshooting Notes +- CDK synth failures usually trace back to missing environment variables; reference defaults in `lib/argument-store.js`. +- GitHub API rate limits: clear `.gh_ssh_keys_*` cache files or wait for expiry when switching networks. +- Autoscaling stuck: inspect CloudWatch metrics published by `agent-scaler` and ensure the SSM token parameter exists and is decryptable. +- Mac dedicated hosts require `SEMAPHORE_AGENT_MAC_DEDICATED_HOSTS` and correct `SEMAPHORE_AGENT_MAC_FAMILY` to avoid AZ placement errors. + +## Useful References +- AWS SDK v3 clients are initialised with 1s timeouts—extend carefully if diagnosing latency-sensitive regions. +- Stack outputs live in `cdk.out`; share `npm run diff` snapshots for PR context. +- For new regions or accounts, rerun `cdk bootstrap` (`npm run bootstrap`) with the target environment.