feat(linux): add package hold support during Ubuntu AMI upgrades

.semaphore/semaphore.yml

@@ -104,6 +104,10 @@ blocks:
             - ./ci/build-ami.sh ubuntu-jammy x86_64
         - name: Build Linux Noble AMI
           commands:
+            # Enable package hold patterns and extra verbosity for
+            # Ansible during the noble AMI build, for testing purposes.
+            - export APT_HOLD_PATTERNS=linux-image*,linux-headers*
+            - export ANSIBLE_VERBOSITY=1
             - ./ci/build-ami.sh ubuntu-noble x86_64
         - name: Build Windows AMI
           commands:

Makefile

@@ -7,6 +7,7 @@ TOOLBOX_VERSION=v1.20.5
 PACKER_OS=linux
 UBUNTU_VERSION=focal
 SOURCE_AMI?=
+APT_HOLD_PATTERNS?=
 
 # Set Ubuntu name and version number based on UBUNTU_VERSION
 ifeq ($(UBUNTU_VERSION),focal)
@@ -77,7 +78,9 @@ packer.validate:
 packer.validate.linux:
 	$(MAKE) venv.execute COMMAND='\
 		cd packer/linux && \
-		packer validate \
+		env \
+			ANSIBLE_VERBOSITY=$(ANSIBLE_VERBOSITY) \
+			packer validate \
 			-var "stack_version=v$(VERSION)" \
 			-var "agent_version=$(AGENT_VERSION)" \
 			-var "toolbox_version=$(TOOLBOX_VERSION)" \
@@ -91,6 +94,7 @@ packer.validate.linux:
 			-var "ubuntu_name=$(UBUNTU_NAME)" \
 			-var "ubuntu_version=$(UBUNTU_VERSION_NUMBER)" \
 			-var "source_ami=$(SOURCE_AMI)" \
+			-var "apt_hold_patterns=$$(APT_HOLD_PATTERNS)" \
 			.'
 
 packer.validate.windows:
@@ -137,7 +141,9 @@ packer.build:
 packer.build.linux:
 	$(MAKE) venv.execute COMMAND='\
 		cd packer/linux && \
-		packer build \
+		env \
+			ANSIBLE_VERBOSITY=$(ANSIBLE_VERBOSITY) \
+			packer build \
 			-var "stack_version=v$(VERSION)" \
 			-var "agent_version=$(AGENT_VERSION)" \
 			-var "toolbox_version=$(TOOLBOX_VERSION)" \
@@ -151,6 +157,7 @@ packer.build.linux:
 			-var "ubuntu_name=$(UBUNTU_NAME)" \
 			-var "ubuntu_version=$(UBUNTU_VERSION_NUMBER)" \
 			-var "source_ami=$(SOURCE_AMI)" \
+			-var "apt_hold_patterns=$${APT_HOLD_PATTERNS}" \
 			.'
 
 packer.build.windows:

README.md

@@ -21,6 +21,14 @@ Set the `SOURCE_AMI` environment variable when running the Packer targets if you
 SOURCE_AMI=ami-0123456789abcdef0 make packer.build PACKER_OS=linux UBUNTU_VERSION=noble
 ```
 
+To keep specific packages from being upgraded during the image bake, provide one or more glob patterns via `APT_HOLD_PATTERNS`. Patterns can be comma -separated; the following holds every installed `linux-image*` or `linux-headers*` package before the `dist-upgrade` step runs:
+
+```
+APT_HOLD_PATTERNS=linux-image*,linux-headers* make packer.build PACKER_OS=linux UBUNTU_VERSION=noble
+```
+
+Extra Ansible logging can be enabled by exporting `ANSIBLE_VERBOSITY` (0–4). For example, `ANSIBLE_VERBOSITY=1 make packer.build ...` will surface the resolved hold patterns and other informational output from the provisioning playbook.
+
 The helper script `ci/build-ami.sh` also accepts the desired AMI ID as an optional third argument so it can be used in the same way:
 
 ```

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aws-semaphore-agent",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "bin": {
     "aws-semaphore-agent": "bin/aws-semaphore-agent.js"
   },

packer/linux/ansible/group_vars/all.yml

@@ -12,3 +12,4 @@ erlang_gpg_key_checksum: "84df2e5fd80d464c3eb9acd2f751b2f6a723438200915dd50fbf12
 erlang_gpg_key_path: /usr/share/keyrings/rabbitmq-erlang.E495BB49CC4BBE5B.asc
 cloudwatch_agent_url: https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
 systemd_restart_seconds: 300
+apt_hold_patterns: []

packer/linux/ansible/roles/upgrade/tasks/main.yml

@@ -2,6 +2,83 @@
 - name: Run apt update
   ansible.builtin.command: apt -y update
 
+- name: Resolve packages requested for hold
+  ansible.builtin.shell: |
+    python3 - <<'PY'
+    import fnmatch
+    import json
+    import subprocess
+
+    raw = json.loads('''{{ apt_hold_patterns | default([]) | to_json }}''')
+
+    def normalize(value):
+        if isinstance(value, str):
+            stripped = value.strip()
+            if not stripped:
+                return []
+            if stripped.startswith('['):
+                try:
+                    parsed = json.loads(stripped)
+                except json.JSONDecodeError:
+                    pass
+                else:
+                    return [
+                        str(item).strip()
+                        for item in parsed
+                        if str(item).strip()
+                    ]
+            stripped = stripped.replace(',', ' ')
+            return [
+                part for part in (segment.strip() for segment in stripped.split())
+                if part
+            ]
+        if isinstance(value, list):
+            return [
+                str(item).strip()
+                for item in value
+                if str(item).strip()
+            ]
+        return []
+
+    patterns = normalize(raw)
+    packages = []
+    if patterns:
+        installed = subprocess.check_output(
+            ['dpkg-query', '-W', '-f=${Package}\\n'],
+            text=True,
+        ).splitlines()
+        seen = set()
+        for pattern in patterns:
+            for pkg in installed:
+                if fnmatch.fnmatch(pkg, pattern) and pkg not in seen:
+                    packages.append(pkg)
+                    seen.add(pkg)
+    print(json.dumps({"patterns": patterns, "packages": packages}))
+    PY
+  args:
+    executable: /bin/bash
+  register: apt_hold_resolution
+  changed_when: false
+
+- name: Cache package hold data
+  ansible.builtin.set_fact:
+    apt_hold_data: "{{ apt_hold_resolution.stdout | from_json }}"
+
+- name: Show resolved package holds
+  ansible.builtin.debug:
+    msg:
+      patterns: "{{ apt_hold_data.patterns }}"
+      packages: "{{ apt_hold_data.packages }}"
+  when:
+    - (apt_hold_data.patterns | length > 0) or (apt_hold_data.packages | length > 0)
+    - (ansible_verbosity | default(0) | int) >= 1
+
+- name: Hold requested packages prior to upgrade
+  when: apt_hold_data.packages | length > 0
+  ansible.builtin.command:
+    argv: "{{ ['apt-mark', 'hold'] + apt_hold_data.packages }}"
+  register: apt_hold_command
+
 - name: Run apt dist-upgrade
   ansible.builtin.command: apt -fuy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade
   environment:

packer/linux/ubuntu.pkr.hcl

@@ -41,6 +41,11 @@ variable "systemd_restart_seconds" {
   default = "1800"
 }
 
+variable "apt_hold_patterns" {
+  type    = string
+  default = ""
+}
+
 variable "ubuntu_name" {
   type    = string
   default = "focal"
@@ -121,6 +126,7 @@ build {
       "-e toolbox_version=${var.toolbox_version}",
       "-e install_erlang=${var.install_erlang}",
       "-e systemd_restart_seconds=${var.systemd_restart_seconds}",
+      "-e apt_hold_patterns=${var.apt_hold_patterns}",
     ]
   }
 }