diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 7f3b4fd..88dec25 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,9 +99,15 @@ blocks: - unzip /tmp/packer_1.7.9_linux_amd64.zip -d /tmp/ - sudo mv /tmp/packer /usr/local/bin/ jobs: - - name: Build Linux AMI + - name: Build Linux Focal AMI commands: - ./ci/build-ami.sh ubuntu-focal x86_64 + - name: Build Linux Jammy AMI + commands: + - ./ci/build-ami.sh ubuntu-jammy x86_64 + - name: Build Linux Noble AMI + commands: + - ./ci/build-ami.sh ubuntu-noble x86_64 - name: Build Windows AMI commands: - ./ci/build-ami.sh windows x86_64 @@ -119,7 +125,7 @@ blocks: - cache restore - ./ci/create-execution-policy-and-bootstrap.sh $AWS_ACCOUNT_ID $AWS_DEFAULT_REGION - - name: Linux launch + - name: Linux Focal launch dependencies: - Bootstrap task: @@ -132,7 +138,37 @@ blocks: - checkout - cache restore - ./ci/create-ssm-param.sh s1-agent-aws-stack-linux-token $TOKEN - - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-config.json npm run deploy:ci + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-focal-config.json npm run deploy:ci + + - name: Linux Jammy launch + dependencies: + - Bootstrap + task: + secrets: + - name: s1-agent-aws-stack-linux-token + - name: agent-aws-stack-tester-credentials + jobs: + - name: Launch stack + commands: + - checkout + - cache restore + - ./ci/create-ssm-param.sh s1-agent-aws-stack-linux-jammy-token $TOKEN_JAMMY + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-jammy-config.json npm run deploy:ci + + - name: Linux Noble launch + dependencies: + - Bootstrap + task: + secrets: + - name: s1-agent-aws-stack-linux-token + - name: agent-aws-stack-tester-credentials + jobs: + - name: Launch stack + commands: + - checkout + - cache restore + - ./ci/create-ssm-param.sh s1-agent-aws-stack-linux-noble-token $TOKEN_NOBLE + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-noble-config.json npm run deploy:ci - name: Windows launch dependencies: @@ -149,9 +185,9 @@ blocks: - ./ci/create-ssm-param.sh s1-agent-aws-stack-windows-token $TOKEN - SEMAPHORE_AGENT_STACK_CONFIG=./ci/windows-config.json npm run deploy:ci - - name: Linux tests + - name: Linux Focal tests dependencies: - - Linux launch + - Linux Focal launch execution_time_limit: minutes: 10 task: @@ -165,10 +201,55 @@ blocks: jobs: - name: Run goss tests commands: + - if [ "$(grep VERSION_ID /etc/os-release | cut -d= -f2 | tr -d '"' | cut -d. -f1-2)" != "20.04" ]; then exit 1; fi + - sudo curl -L https://github.com/aelsabbahy/goss/releases/latest/download/goss-linux-amd64 -o /usr/local/bin/goss + - sudo chmod +rx /usr/local/bin/goss + - checkout + - goss -g goss/goss.yaml --vars goss/vars-focal.yml validate --max-concurrent 1 --format junit > results.xml + + - name: Linux Jammy tests + dependencies: + - Linux Jammy launch + execution_time_limit: + minutes: 10 + task: + agent: + machine: + type: s1-agent-aws-stack-linux-jammy + epilogue: + always: + commands: + - if [ -f results.xml ]; then test-results publish results.xml; fi + jobs: + - name: Run goss tests + commands: + - if [ "$(grep VERSION_ID /etc/os-release | cut -d= -f2 | tr -d '"' | cut -d. -f1-2)" != "22.04" ]; then exit 1; fi - sudo curl -L https://github.com/aelsabbahy/goss/releases/latest/download/goss-linux-amd64 -o /usr/local/bin/goss - sudo chmod +rx /usr/local/bin/goss - checkout - - goss -g goss/goss.yaml --vars goss/vars.yml validate --max-concurrent 1 --format junit > results.xml + - goss -g goss/goss.yaml --vars goss/vars-jammy.yml validate --max-concurrent 1 --format junit > results.xml + + - name: Linux Noble tests + dependencies: + - Linux Noble launch + execution_time_limit: + minutes: 10 + task: + agent: + machine: + type: s1-agent-aws-stack-linux-noble + epilogue: + always: + commands: + - if [ -f results.xml ]; then test-results publish results.xml; fi + jobs: + - name: Run goss tests + commands: + - if [ "$(grep VERSION_ID /etc/os-release | cut -d= -f2 | tr -d '"' | cut -d. -f1-2)" != "24.04" ]; then exit 1; fi + - sudo curl -L https://github.com/aelsabbahy/goss/releases/latest/download/goss-linux-amd64 -o /usr/local/bin/goss + - sudo chmod +rx /usr/local/bin/goss + - checkout + - goss -g goss/goss.yaml --vars goss/vars-noble.yml validate --max-concurrent 1 --format junit > results.xml - name: Windows tests dependencies: @@ -197,13 +278,23 @@ after_pipeline: - name: Submit Reports commands: - test-results gen-pipeline-report - - name: Destroy Linux stack + - name: Destroy Linux Focal stack + commands: + - checkout + - cache restore + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-focal-config.json npm run destroy:ci + - name: Destroy Linux Jammy stack + commands: + - checkout + - cache restore + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-jammy-config.json npm run destroy:ci + - name: Destroy Linux Noble stack commands: - checkout - cache restore - - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-config.json npm run destroy:ci + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-noble-config.json npm run destroy:ci - name: Destroy Windows stack commands: - checkout - cache restore - - SEMAPHORE_AGENT_STACK_CONFIG=./ci/windows-config.json npm run destroy:ci \ No newline at end of file + - SEMAPHORE_AGENT_STACK_CONFIG=./ci/windows-config.json npm run destroy:ci diff --git a/Makefile b/Makefile index 5da7b19..4733a4c 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,23 @@ AMI_INSTANCE_TYPE=t2.micro AGENT_VERSION=v2.2.16 TOOLBOX_VERSION=v1.20.5 PACKER_OS=linux +UBUNTU_VERSION=focal + +# Set Ubuntu name and version number based on UBUNTU_VERSION +ifeq ($(UBUNTU_VERSION),focal) + UBUNTU_NAME=focal + UBUNTU_VERSION_NUMBER=20.04 +else ifeq ($(UBUNTU_VERSION),noble) + UBUNTU_NAME=noble + UBUNTU_VERSION_NUMBER=24.04 +else ifeq ($(UBUNTU_VERSION),jammy) + UBUNTU_NAME=jammy + UBUNTU_VERSION_NUMBER=22.04 +else + UBUNTU_NAME=focal + UBUNTU_VERSION_NUMBER=20.04 +endif + INSTALL_ERLANG=true SYSTEMD_RESTART_SECONDS=1800 VERSION=$(shell cat package.json | jq -r '.version') @@ -62,6 +79,8 @@ packer.validate.linux: -var "install_erlang=$(INSTALL_ERLANG)" \ -var "systemd_restart_seconds=$(SYSTEMD_RESTART_SECONDS)" \ -var "instance_type=$(AMI_INSTANCE_TYPE)" \ + -var "ubuntu_name=$(UBUNTU_NAME)" \ + -var "ubuntu_version=$(UBUNTU_VERSION_NUMBER)" \ .' packer.validate.windows: @@ -119,6 +138,8 @@ packer.build.linux: -var "install_erlang=$(INSTALL_ERLANG)" \ -var "systemd_restart_seconds=$(SYSTEMD_RESTART_SECONDS)" \ -var "instance_type=$(AMI_INSTANCE_TYPE)" \ + -var "ubuntu_name=$(UBUNTU_NAME)" \ + -var "ubuntu_version=$(UBUNTU_VERSION_NUMBER)" \ .' packer.build.windows: diff --git a/ci/build-ami.sh b/ci/build-ami.sh index b2afaf2..adc3071 100755 --- a/ci/build-ami.sh +++ b/ci/build-ami.sh @@ -16,8 +16,13 @@ if [[ -z "${arch}" ]]; then fi packer_os=linux +ubuntu_version="" + if [[ ${os} == "windows" ]]; then packer_os=windows +elif [[ ${os} == *"ubuntu-"* ]]; then + # Extract ubuntu version from os parameter (e.g., ubuntu-noble -> noble) + ubuntu_version=${os#*ubuntu-} fi version=$(cat package.json | jq -r '.version') @@ -28,9 +33,15 @@ response=$(aws ec2 describe-images --filters "Name=name,Values=${image_name}") images=$(echo $response | jq '.Images' | jq length) if [[ ${images} == "0" ]]; then echo "No images published with name ${image_name}. Creating it..." - make packer.init - make packer.validate PACKER_OS=${packer_os} - make packer.build PACKER_OS=${packer_os} + if [[ -n "${ubuntu_version}" ]]; then + make packer.init PACKER_OS=${packer_os} UBUNTU_VERSION=${ubuntu_version} + make packer.validate PACKER_OS=${packer_os} UBUNTU_VERSION=${ubuntu_version} + make packer.build PACKER_OS=${packer_os} UBUNTU_VERSION=${ubuntu_version} + else + make packer.init PACKER_OS=${packer_os} + make packer.validate PACKER_OS=${packer_os} + make packer.build PACKER_OS=${packer_os} + fi else echo "Image with name ${image_name} already exists. Not building anything." fi \ No newline at end of file diff --git a/ci/create-execution-policy-and-bootstrap.sh b/ci/create-execution-policy-and-bootstrap.sh index b57924a..2f86d91 100755 --- a/ci/create-execution-policy-and-bootstrap.sh +++ b/ci/create-execution-policy-and-bootstrap.sh @@ -65,6 +65,6 @@ echo "Bootstrapping application..." # For the bootstrap part, it doesn't really matter which stack # we are deploying, only that the CDK required resources are bootstrapped, # so we just use the linux config for this. -SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-config.json npm run bootstrap -- aws://${aws_account_id}/${aws_region} \ +SEMAPHORE_AGENT_STACK_CONFIG=./ci/linux-focal-config.json npm run bootstrap -- aws://${aws_account_id}/${aws_region} \ --cloudformation-execution-policies "${policy_arn}" \ --verbose \ No newline at end of file diff --git a/ci/linux-config.json b/ci/linux-focal-config.json similarity index 78% rename from ci/linux-config.json rename to ci/linux-focal-config.json index 33f31c3..84f3828 100644 --- a/ci/linux-config.json +++ b/ci/linux-focal-config.json @@ -4,5 +4,6 @@ "SEMAPHORE_AGENT_TOKEN_PARAMETER_NAME": "s1-agent-aws-stack-linux-token", "SEMAPHORE_AGENT_USE_DYNAMIC_SCALING": "true", "SEMAPHORE_AGENT_DISCONNECT_AFTER_JOB": "false", - "SEMAPHORE_AGENT_ASG_MAX_SIZE": "1" + "SEMAPHORE_AGENT_ASG_MAX_SIZE": "1", + "SEMAPHORE_AGENT_OS": "ubuntu-focal" } diff --git a/ci/linux-jammy-config.json b/ci/linux-jammy-config.json new file mode 100644 index 0000000..319656a --- /dev/null +++ b/ci/linux-jammy-config.json @@ -0,0 +1,9 @@ +{ + "SEMAPHORE_AGENT_STACK_NAME": "agent-aws-stack-linux-jammy", + "SEMAPHORE_ORGANIZATION": "semaphore", + "SEMAPHORE_AGENT_TOKEN_PARAMETER_NAME": "s1-agent-aws-stack-linux-jammy-token", + "SEMAPHORE_AGENT_USE_DYNAMIC_SCALING": "true", + "SEMAPHORE_AGENT_DISCONNECT_AFTER_JOB": "false", + "SEMAPHORE_AGENT_ASG_MAX_SIZE": "1", + "SEMAPHORE_AGENT_OS": "ubuntu-jammy" +} diff --git a/ci/linux-noble-config.json b/ci/linux-noble-config.json new file mode 100644 index 0000000..3a106b1 --- /dev/null +++ b/ci/linux-noble-config.json @@ -0,0 +1,9 @@ +{ + "SEMAPHORE_AGENT_STACK_NAME": "agent-aws-stack-linux-noble", + "SEMAPHORE_ORGANIZATION": "semaphore", + "SEMAPHORE_AGENT_TOKEN_PARAMETER_NAME": "s1-agent-aws-stack-linux-noble-token", + "SEMAPHORE_AGENT_USE_DYNAMIC_SCALING": "true", + "SEMAPHORE_AGENT_DISCONNECT_AFTER_JOB": "false", + "SEMAPHORE_AGENT_ASG_MAX_SIZE": "1", + "SEMAPHORE_AGENT_OS": "ubuntu-noble" +} diff --git a/goss/goss.yaml b/goss/goss.yaml index e84bd9c..3c5d42f 100644 --- a/goss/goss.yaml +++ b/goss/goss.yaml @@ -113,6 +113,7 @@ command: docker-compose: exit-status: 0 exec: "docker compose version" + timeout: 30000 python: exit-status: 0 exec: "python --version" diff --git a/goss/vars.yml b/goss/vars-focal.yml similarity index 100% rename from goss/vars.yml rename to goss/vars-focal.yml diff --git a/goss/vars-jammy.yml b/goss/vars-jammy.yml new file mode 100644 index 0000000..f4f8810 --- /dev/null +++ b/goss/vars-jammy.yml @@ -0,0 +1,3 @@ +user: "semaphore" +awscli: "2.4.9" +python: "3.10.12" diff --git a/goss/vars-noble.yml b/goss/vars-noble.yml new file mode 100644 index 0000000..f83da6d --- /dev/null +++ b/goss/vars-noble.yml @@ -0,0 +1,3 @@ +user: "semaphore" +awscli: "2.4.9" +python: "3.12.3" diff --git a/lib/ami-hash.js b/lib/ami-hash.js index e636d9e..d650e3d 100644 --- a/lib/ami-hash.js +++ b/lib/ami-hash.js @@ -2,7 +2,7 @@ const path = require("path"); const { execSync } = require('child_process'); function hash(os) { - packerOs = os == "ubuntu-focal" ? "linux" : os; + packerOs = os.startsWith("ubuntu-") ? "linux" : os; hash = execSync(`find Makefile packer/${packerOs} -type f -exec md5sum "{}" + | awk '{print $1}' | sort | md5sum | awk '{print $1}'`, { cwd: path.resolve(__dirname, '../') }); diff --git a/package-lock.json b/package-lock.json index 5fd9aa0..579c1f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "aws-semaphore-agent", - "version": "0.7.1", + "version": "0.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aws-semaphore-agent", - "version": "0.7.1", + "version": "0.8.0", "dependencies": { "aws-cdk": "^2.164.1", "aws-cdk-lib": "^2.164.1", diff --git a/package.json b/package.json index ec2811b..6ac736d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aws-semaphore-agent", - "version": "0.7.1", + "version": "0.8.0", "bin": { "aws-semaphore-agent": "bin/aws-semaphore-agent.js" }, diff --git a/packer/linux/ansible/roles/python/tasks/main.yml b/packer/linux/ansible/roles/python/tasks/main.yml index b95e958..3d7cc7b 100644 --- a/packer/linux/ansible/roles/python/tasks/main.yml +++ b/packer/linux/ansible/roles/python/tasks/main.yml @@ -10,8 +10,9 @@ pkg: - python3-pip -- name: Install botocore - ansible.builtin.pip: - name: - - boto3 - - botocore +- name: Install botocore and boto3 + ansible.builtin.apt: + pkg: + - python3-boto3 + - python3-botocore + state: present diff --git a/packer/linux/ansible/ubuntu-jammy.yml b/packer/linux/ansible/ubuntu-jammy.yml new file mode 100644 index 0000000..c037ccc --- /dev/null +++ b/packer/linux/ansible/ubuntu-jammy.yml @@ -0,0 +1,18 @@ +--- +- name: Ubuntu Jammy 22.04 + hosts: all + vars: + container_user: semaphore + become: true + become_user: root + roles: + - upgrade + - system_tools + - python + - erlang + - awscli + - cloudwatch_agent + - docker + - users + - agent + - cleanup diff --git a/packer/linux/ansible/ubuntu-noble.yml b/packer/linux/ansible/ubuntu-noble.yml new file mode 100644 index 0000000..fe69b29 --- /dev/null +++ b/packer/linux/ansible/ubuntu-noble.yml @@ -0,0 +1,18 @@ +--- +- name: Ubuntu Noble 24.04 + hosts: all + vars: + container_user: semaphore + become: true + become_user: root + roles: + - upgrade + - system_tools + - python + - erlang + - awscli + - cloudwatch_agent + - docker + - users + - agent + - cleanup diff --git a/packer/linux/ubuntu-focal.pkr.hcl b/packer/linux/ubuntu.pkr.hcl similarity index 81% rename from packer/linux/ubuntu-focal.pkr.hcl rename to packer/linux/ubuntu.pkr.hcl index 2bc3b87..588ed35 100644 --- a/packer/linux/ubuntu-focal.pkr.hcl +++ b/packer/linux/ubuntu.pkr.hcl @@ -41,6 +41,16 @@ variable "systemd_restart_seconds" { default = "1800" } +variable "ubuntu_name" { + type = string + default = "focal" +} + +variable "ubuntu_version" { + type = string + default = "20.04" +} + packer { required_plugins { amazon = { @@ -51,7 +61,7 @@ packer { } source "amazon-ebs" "ubuntu" { - ami_name = "${var.ami_prefix}-${var.stack_version}-ubuntu-focal-${var.arch}-${var.hash}" + ami_name = "${var.ami_prefix}-${var.stack_version}-ubuntu-${var.ubuntu_name}-${var.arch}-${var.hash}" region = "${var.region}" instance_type = "${var.instance_type}" ssh_username = "ubuntu" @@ -71,7 +81,7 @@ source "amazon-ebs" "ubuntu" { owners = ["099720109477"] filters = { - name = "ubuntu/images/*ubuntu-focal-20.04-*" + name = "ubuntu/images/*ubuntu-${var.ubuntu_name}-${var.ubuntu_version}-*" architecture = "${var.arch}" root-device-type = "ebs" virtualization-type = "hvm" @@ -80,14 +90,14 @@ source "amazon-ebs" "ubuntu" { } build { - name = "semaphore-agent-ubuntu-focal" + name = "semaphore-agent-ubuntu-${var.ubuntu_name}" sources = [ "source.amazon-ebs.ubuntu" ] provisioner "ansible" { - playbook_file = "ansible/ubuntu-focal.yml" + playbook_file = "ansible/ubuntu-${var.ubuntu_name}.yml" user = "ubuntu" use_proxy = false extra_arguments = [