From ec5ada404edbe89691724142ccf4316168081280 Mon Sep 17 00:00:00 2001 From: yashdeep Date: Wed, 28 Jan 2026 19:37:27 +0530 Subject: [PATCH 1/4] fix(packer/linux): disable unattended-upgrades to prevent job interruptions Disable unattended-upgrades and apt timers to prevent mid-job interruptions. Library upgrades (e.g., libglib2.0) trigger libc-bin processing which causes systemd to restart services, sending SIGTERM to semaphore-agent and killing running CI jobs. - Disable apt-daily.timer and apt-daily-upgrade.timer - Stop unattended-upgrades service - Configure apt to disable automatic upgrades - Add documentation explaining the change --- .../ansible/roles/system_tools/tasks/main.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packer/linux/ansible/roles/system_tools/tasks/main.yml b/packer/linux/ansible/roles/system_tools/tasks/main.yml index 39e7cd6..b660bc8 100644 --- a/packer/linux/ansible/roles/system_tools/tasks/main.yml +++ b/packer/linux/ansible/roles/system_tools/tasks/main.yml @@ -23,3 +23,45 @@ dest: /usr/local/bin/yq mode: '0755' force: true + +# =========================================================== +# Disable unattended-upgrades to prevent job interruptions +# Unattended-upgrades can trigger systemd service restarts +# (e.g., libglib2.0 upgrades trigger libc-bin processing) +# which sends SIGTERM to all services including semaphore-agent +# =========================================================== +- name: Stop and disable apt timers + ansible.builtin.systemd: + name: "{{ item }}" + enabled: no + state: stopped + loop: + - apt-daily.timer + - apt-daily-upgrade.timer + ignore_errors: yes + +- name: Stop unattended-upgrades service if running + ansible.builtin.systemd: + name: unattended-upgrades + state: stopped + ignore_errors: yes + +- name: Configure apt to disable automatic upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "0"; + APT::Periodic::Unattended-Upgrade "0"; + APT::Periodic::Download-Upgradeable-Packages "0"; + APT::Periodic::AutocleanInterval "0"; + mode: '0644' + +- name: Add documentation comment for disabled unattended-upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/99-semaphore-ci-note + content: | + // Unattended upgrades disabled for Semaphore CI agents + // to prevent mid-job interruptions from library upgrades + // (e.g., libglib2.0) triggering systemd service restarts + // which send SIGTERM to semaphore-agent and kill running jobs + mode: '0644' From 655898872f4e55c48391008c75af37585564c926 Mon Sep 17 00:00:00 2001 From: yashdeep Date: Tue, 3 Feb 2026 17:11:29 +0530 Subject: [PATCH 2/4] refactor(packer/linux): simplify unattended-upgrades disable Replace multi-step approach (stopping services, disabling timers, writing config files) with single package purge. Also add purge at instance launch to ensure consistency between baked AMI and running instances. --- .../ansible/roles/agent/files/start-agent.sh | 3 ++ .../ansible/roles/system_tools/tasks/main.yml | 46 +++---------------- 2 files changed, 9 insertions(+), 40 deletions(-) diff --git a/packer/linux/ansible/roles/agent/files/start-agent.sh b/packer/linux/ansible/roles/agent/files/start-agent.sh index d6ae6f7..7ece8ae 100755 --- a/packer/linux/ansible/roles/agent/files/start-agent.sh +++ b/packer/linux/ansible/roles/agent/files/start-agent.sh @@ -213,6 +213,9 @@ region=$(curl \ --location "http://169.254.169.254/latest/meta-data/placement/region" ) +echo "Removing unattended-upgrades package..." +sudo apt-get remove -y --purge unattended-upgrades || true + # The parameters required for the agent configuration are stored in an SSM parameter. # We need to fetch them before proceeding with anything else. echo "Fetching agent params from SSM parameter '$agent_config_param_name'..." diff --git a/packer/linux/ansible/roles/system_tools/tasks/main.yml b/packer/linux/ansible/roles/system_tools/tasks/main.yml index b660bc8..9ad50db 100644 --- a/packer/linux/ansible/roles/system_tools/tasks/main.yml +++ b/packer/linux/ansible/roles/system_tools/tasks/main.yml @@ -24,44 +24,10 @@ mode: '0755' force: true -# =========================================================== -# Disable unattended-upgrades to prevent job interruptions -# Unattended-upgrades can trigger systemd service restarts -# (e.g., libglib2.0 upgrades trigger libc-bin processing) -# which sends SIGTERM to all services including semaphore-agent -# =========================================================== -- name: Stop and disable apt timers - ansible.builtin.systemd: - name: "{{ item }}" - enabled: no - state: stopped - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - ignore_errors: yes - -- name: Stop unattended-upgrades service if running - ansible.builtin.systemd: +- name: Remove unattended-upgrades package + ansible.builtin.apt: name: unattended-upgrades - state: stopped - ignore_errors: yes - -- name: Configure apt to disable automatic upgrades - ansible.builtin.copy: - dest: /etc/apt/apt.conf.d/20auto-upgrades - content: | - APT::Periodic::Update-Package-Lists "0"; - APT::Periodic::Unattended-Upgrade "0"; - APT::Periodic::Download-Upgradeable-Packages "0"; - APT::Periodic::AutocleanInterval "0"; - mode: '0644' - -- name: Add documentation comment for disabled unattended-upgrades - ansible.builtin.copy: - dest: /etc/apt/apt.conf.d/99-semaphore-ci-note - content: | - // Unattended upgrades disabled for Semaphore CI agents - // to prevent mid-job interruptions from library upgrades - // (e.g., libglib2.0) triggering systemd service restarts - // which send SIGTERM to semaphore-agent and kill running jobs - mode: '0644' + state: absent + purge: yes + tags: + - disable_unattended From 99567d1e470ec9acf8ca3efc57d3a3a70488f0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kutryj?= Date: Tue, 3 Feb 2026 12:53:07 +0100 Subject: [PATCH 3/4] toil: add context for future maintenance --- packer/linux/ansible/roles/agent/files/start-agent.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packer/linux/ansible/roles/agent/files/start-agent.sh b/packer/linux/ansible/roles/agent/files/start-agent.sh index 7ece8ae..6dd7110 100755 --- a/packer/linux/ansible/roles/agent/files/start-agent.sh +++ b/packer/linux/ansible/roles/agent/files/start-agent.sh @@ -213,6 +213,7 @@ region=$(curl \ --location "http://169.254.169.254/latest/meta-data/placement/region" ) +# Remove unattended-upgrades to prevent mid-job interruptions from automatic updates echo "Removing unattended-upgrades package..." sudo apt-get remove -y --purge unattended-upgrades || true From cd1e961e2401282173866d178cf5a5e9cb36081e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kutryj?= Date: Tue, 3 Feb 2026 13:23:12 +0100 Subject: [PATCH 4/4] Change purge option from 'yes' to 'true' --- packer/linux/ansible/roles/system_tools/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/linux/ansible/roles/system_tools/tasks/main.yml b/packer/linux/ansible/roles/system_tools/tasks/main.yml index 9ad50db..dfe1be2 100644 --- a/packer/linux/ansible/roles/system_tools/tasks/main.yml +++ b/packer/linux/ansible/roles/system_tools/tasks/main.yml @@ -28,6 +28,6 @@ ansible.builtin.apt: name: unattended-upgrades state: absent - purge: yes + purge: true tags: - disable_unattended