From efc5d22374a87551fb3eef1055313df25a988108 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 14 Apr 2023 14:30:38 +0200 Subject: [PATCH 01/41] Fixing linting so CI can work. --- .ansible-lint | 10 +++++++--- roles/cache_clear/cache_clear-opcache/tasks/main.yml | 8 ++++---- roles/cli/cachetool/tasks/main.yml | 7 +++---- .../database_apply-drupal7/tasks/ctools.yml | 2 +- .../database_apply-drupal7/tasks/features.yml | 2 +- .../database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- .../database_backup-mysql/tasks/revert-dump.yml | 2 +- roles/deploy_code/tasks/cleanup.yml | 2 +- .../database_sync/database_sync-mysql/tasks/sync.yml | 4 ++-- 10 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 7a9eb05b..c9e7b608 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,4 +1,8 @@ skip_list: - - '204' - - '301' - - '701' + - '204' # Lines should be no longer than 160 chars + - '301' # Commands should not change things if nothing needs doing + - '701' # 701 No 'galaxy_info' found + - unnamed-task + - risky-file-permissions + - no-jinja-nesting + - command-instead-of-shell # some Drupal 7 drush commands require shell diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index 428e9053..8f3b4bf6 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Get latest php installed ansible.builtin.shell: - cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1' + cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1' register: _php_version - name: Set cachetool adapter. @@ -9,16 +9,16 @@ _cachetool_adapter: "{{ cache_clear_opcache.adapter | default('--fcgi=127.0.0.1:90' + _php_version.stdout | replace('.','')) }}" - name: Clear opcache. - command: + ansible.builtin.command: cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n opcache:reset" when: cache_clear_opcache.clear_opcache - name: Clear apcu. - command: + ansible.builtin.command: cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear" when: cache_clear_opcache.clear_apcu - name: Clear stats. - command: + ansible.builtin.command: cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n stat:clear" when: cache_clear_opcache.clear_stat diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 2e2a49d1..591e2309 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -14,6 +14,9 @@ - deploy_operation == 'deploy' - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility + when: + - deploy_operation == 'deploy' + - cachetool.version | length == 0 block: - name: Get latest php installed ansible.builtin.shell: @@ -69,10 +72,6 @@ when: - _php_version_str.stdout | int < 71 - when: - - deploy_operation == 'deploy' - - cachetool.version | length == 0 - - name: "Download the specified {{ cachetool.version }} cachetool version installer." ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index 8102d3d6..4dcbcfff 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -2,7 +2,7 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Ctools module is enabled. ansible.builtin.shell: - cmd: "{{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" + cmd: "set -o pipefail && {{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" register: ctools_enabled diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index f55795dc..c3597bec 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -1,7 +1,7 @@ --- # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Features module is enabled. - ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" + ansible.builtin.shell: "set -o pipefail && cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" register: features_enabled - name: Revert Drupal configuration from Features. diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 80a6e5ef..333a768b 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -23,6 +23,6 @@ run_once: true - name: Take a database dump. - ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index f7f109bc..815213ed 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -22,6 +22,6 @@ # run_once: true - name: Populate new database. - ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index f914ee78..2b57d911 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,5 +1,5 @@ --- - name: Revert database from dump. - ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" when: previous_build_number > 0 run_once: true diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 27878342..6dbbc0fe 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -127,7 +127,7 @@ - name: Check if we have a mount already. ansible.builtin.shell: - cmd: "mount | grep {{ deploy_base_path }}" + cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" failed_when: false register: _deploy_code_mount_check when: diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 29100ab5..142e857a 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -61,7 +61,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - database.source.fresh_db @@ -133,7 +133,7 @@ cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - name: Repopulate database from dump. - ansible.builtin.shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" - name: Remove tmp dump file. ansible.builtin.file: From 1cb5a6df03eb326ac2f9e2a9046f0e1ab979b365 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 14 Apr 2023 14:38:57 +0200 Subject: [PATCH 02/41] More linting fixes. --- .ansible-lint | 1 + roles/cli/drush/tasks/main.yml | 2 +- .../database_apply-drupal7/tasks/main.yml | 6 ++++-- .../database_apply-drupal8/tasks/main.yml | 14 ++++++++------ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index c9e7b608..b6256e0b 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -6,3 +6,4 @@ skip_list: - risky-file-permissions - no-jinja-nesting - command-instead-of-shell # some Drupal 7 drush commands require shell + - command-instead-of-module diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml index 5c73be51..af5a5d10 100644 --- a/roles/cli/drush/tasks/main.yml +++ b/roles/cli/drush/tasks/main.yml @@ -16,7 +16,7 @@ register: drush_global_directory - name: Check if installed Drush version match. - ansible.builtin.shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" + ansible.builtin.shell: "set -o pipefail && {{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" register: drush_global_version when: - deploy_operation == 'deploy' diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 6b745c3b..d64364c4 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -12,8 +12,10 @@ when: previous_build_number == 0 or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. - ansible.builtin.shell: - cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + ansible.builtin.file: + path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + state: directory + mode: '0755' with_items: "{{ drupal.sites }}" loop_control: loop_var: site diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 16f0417b..4dc66a3a 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Fix file permissions for settings.php. - file: + ansible.builtin.file: state: file path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" owner: "{{ www_user }}" @@ -14,7 +14,7 @@ - previous_build_number == 0 - name: Install Drupal. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -25,8 +25,10 @@ when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. - shell: - cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + ansible.builtin.file: + path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + state: directory + mode: '0755' with_items: "{{ drupal.sites }}" loop_control: loop_var: site @@ -48,7 +50,7 @@ when: previous_build_number > 0 - name: Apply Drupal database updates. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -59,7 +61,7 @@ when: site.config_import_command != 'deploy' - name: Import configuration. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" From 7d9463d3b2d4c3ad9dd17b0c84622a73bed6055d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 14 Apr 2023 15:14:24 +0200 Subject: [PATCH 03/41] Downgrading Ubuntu due to Docker issues with ce-dev and latest. --- .github/workflows/ce-deploy-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml index 141fc8e7..c6429842 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-test.yml @@ -10,7 +10,7 @@ jobs: # Name the Job name: Run tests against Ansible code base # Set the type of machine to run on - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: # Checks out a copy of your repository on the ubuntu-latest machine From da0bb084bc42df85831cd48c8e765af20dee7d11 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 14 Apr 2023 15:16:18 +0200 Subject: [PATCH 04/41] Making pipefail shell commands use /bin/bash. --- roles/cache_clear/cache_clear-opcache/tasks/main.yml | 2 ++ roles/database_apply/database_apply-drupal7/tasks/ctools.yml | 2 ++ .../database_apply/database_apply-drupal7/tasks/features.yml | 2 ++ .../database_backup-mysql/tasks/deploy-dump.yml | 2 ++ .../database_backup-mysql/tasks/deploy-rolling.yml | 2 ++ .../database_backup-mysql/tasks/revert-dump.yml | 2 ++ roles/deploy_code/tasks/cleanup.yml | 2 ++ roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 4 ++++ 8 files changed, 18 insertions(+) diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index 8f3b4bf6..97f91ed4 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -2,6 +2,8 @@ - name: Get latest php installed ansible.builtin.shell: cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1' + args: + executable: /bin/bash register: _php_version - name: Set cachetool adapter. diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index 4dcbcfff..518996ab 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -4,6 +4,8 @@ ansible.builtin.shell: cmd: "set -o pipefail && {{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + args: + executable: /bin/bash register: ctools_enabled - name: Revert Drupal configuration from Ctools. diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index c3597bec..2b4357eb 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -2,6 +2,8 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Features module is enabled. ansible.builtin.shell: "set -o pipefail && cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" + args: + executable: /bin/bash register: features_enabled - name: Revert Drupal configuration from Features. diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 333a768b..9f6f6959 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -24,5 +24,7 @@ - name: Take a database dump. ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + args: + executable: /bin/bash when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index 815213ed..aec45596 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -23,5 +23,7 @@ - name: Populate new database. ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + args: + executable: /bin/bash when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 2b57d911..80898e7d 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,5 +1,7 @@ --- - name: Revert database from dump. ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + args: + executable: /bin/bash when: previous_build_number > 0 run_once: true diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 6dbbc0fe..db2bb9bb 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -128,6 +128,8 @@ - name: Check if we have a mount already. ansible.builtin.shell: cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" + args: + executable: /bin/bash failed_when: false register: _deploy_code_mount_check when: diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 142e857a..c695982c 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -62,6 +62,8 @@ - name: Take a dump from source database. ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + args: + executable: /bin/bash delegate_to: "{{ database.source.host }}" when: - database.source.fresh_db @@ -134,6 +136,8 @@ - name: Repopulate database from dump. ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + args: + executable: /bin/bash - name: Remove tmp dump file. ansible.builtin.file: From c24f6a373bec0e589a72f59370ad296d0b5a6763 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 14 Apr 2023 15:20:29 +0200 Subject: [PATCH 05/41] Adding pipefail code to cachetool installer block. --- roles/cli/cachetool/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 591e2309..c0cbce61 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -20,7 +20,9 @@ block: - name: Get latest php installed ansible.builtin.shell: - cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"' + cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"' + args: + executable: /bin/bash register: _php_version_str - name: Download latest cachetool installer if PHP is 8.1 or newer. From d69de84aa3b26e5028869e05bc4dadc90407748a Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 14 Apr 2023 19:12:43 +0200 Subject: [PATCH 06/41] Adding executable for drush install checker. --- roles/cli/drush/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml index af5a5d10..0836c91f 100644 --- a/roles/cli/drush/tasks/main.yml +++ b/roles/cli/drush/tasks/main.yml @@ -17,6 +17,8 @@ - name: Check if installed Drush version match. ansible.builtin.shell: "set -o pipefail && {{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" + args: + executable: /bin/bash register: drush_global_version when: - deploy_operation == 'deploy' From 92107be571ed7777f09edead77e4bbb23c573d8c Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 19 Apr 2023 19:21:38 +0200 Subject: [PATCH 07/41] Fixing D7 ctools behaviour to match features handling. --- docs/roles/_init.md | 5 +++++ roles/_init/README.md | 5 +++++ roles/_init/defaults/main.yml | 2 +- roles/database_apply/database_apply-drupal7/tasks/ctools.yml | 2 +- roles/database_apply/database_apply-drupal7/tasks/main.yml | 3 ++- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 1c7e835e..9cb9e024 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -12,6 +12,7 @@ These variables **must** be set in the `deploy/common.yml` file, at least. # this will ensure defaults to other roles too. # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: @@ -23,6 +24,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" + revert_ctools_command: "ctools-export-revert --all" # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com @@ -35,6 +37,9 @@ mautic: image_path: "media/images" force_install: false bin_directory: "/home/{{ deploy_user }}/.bin" +# Number of dumps/db to look up for cleanup. +cleanup_history_depth: 50 + ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index 1c7e835e..9cb9e024 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -12,6 +12,7 @@ These variables **must** be set in the `deploy/common.yml` file, at least. # this will ensure defaults to other roles too. # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: @@ -23,6 +24,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" + revert_ctools_command: "ctools-export-revert --all" # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com @@ -35,6 +37,9 @@ mautic: image_path: "media/images" force_install: false bin_directory: "/home/{{ deploy_user }}/.bin" +# Number of dumps/db to look up for cleanup. +cleanup_history_depth: 50 + ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 2602afd2..9a955f3e 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -15,7 +15,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" - revert_ctools: true + revert_ctools_command: "ctools-export-revert --all" # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index 518996ab..608e1a2f 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -10,6 +10,6 @@ - name: Revert Drupal configuration from Ctools. ansible.builtin.shell: - cmd: "{{ drush_bin }} -y ctools-export-revert --all" + cmd: "{{ drush_bin }} -y {{ site.revert_ctools_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: ctools_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index d64364c4..4f476253 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -54,4 +54,5 @@ loop_control: loop_var: site when: - - site.revert_ctools + - previous_build_number > 0 + - site.revert_ctools_command | length > 0 From bb65b1212a7a1224d030c35b6aac9580e3d604cc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 26 Apr 2023 09:51:45 +0200 Subject: [PATCH 08/41] Jinja2 template looking for settings.php using wrong path. --- .../config_generate/config_generate-drupal7/tasks/settings.yml | 2 +- .../config_generate/config_generate-drupal8/tasks/settings.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f8e3b55e..c49d2d63 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -19,6 +19,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index ebf182e8..aeb4f56d 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -23,6 +23,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" From 0b26394cbf8ed340771f0383f8f24e29175e5a42 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 May 2023 10:27:45 +0200 Subject: [PATCH 09/41] Fixing config_generate for Drupal. --- docs/roles/sync/database_sync/database_sync-mysql.md | 1 + .../config_generate/config_generate-drupal7/tasks/settings.yml | 2 +- .../config_generate/config_generate-drupal8/tasks/settings.yml | 2 +- roles/sync/database_sync/database_sync-mysql/README.md | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md index cf7e5311..cbe1a353 100644 --- a/docs/roles/sync/database_sync/database_sync-mysql.md +++ b/docs/roles/sync/database_sync/database_sync-mysql.md @@ -6,6 +6,7 @@ Sync MySQL databases between environments. --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index c49d2d63..55d9dee7 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -19,6 +19,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ playbook_dir }}/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index aeb4f56d..ebf182e8 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -23,6 +23,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md index cf7e5311..cbe1a353 100644 --- a/roles/sync/database_sync/database_sync-mysql/README.md +++ b/roles/sync/database_sync/database_sync-mysql/README.md @@ -6,6 +6,7 @@ Sync MySQL databases between environments. --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. From 00331c2a94a291ef7a15d9a1f3bcc2df56b7ff4f Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 May 2023 10:27:57 +0200 Subject: [PATCH 10/41] Making host checking more robust. --- scripts/_common.sh | 6 +++--- scripts/build.sh | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 63e89bb9..74ba9a60 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -156,9 +156,6 @@ ansible_host_check(){ if [ -n "$TARGET_DEPLOY_HOST" ]; then ANSIBLE_BIN=$(command -v ansible-playbook) ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml" - if [ "$DRY_RUN" = "yes" ]; then - ANSIBLE_CMD="$ANSIBLE_CMD --check" - fi if [ "$VERBOSE" = "yes" ]; then ANSIBLE_CMD="$ANSIBLE_CMD -vvvv" fi @@ -167,6 +164,9 @@ ansible_host_check(){ fi $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS" return $? + # No host to check provided, just return a clean exit code. + else + return 0 fi } diff --git a/scripts/build.sh b/scripts/build.sh index 233ca764..fa9a1211 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -95,8 +95,15 @@ fi # Get Ansible defaults. get_ansible_defaults_vars -# Optionally carry out a host check. +# Optionally carry out a host check if --host is provided. +ANSIBLE_HOST_CHECK_RESULT=1 ansible_host_check +ANSIBLE_HOST_CHECK_RESULT=$? +# Exit early if host not found. +if [ -n "$ANSIBLE_HOST_CHECK_RESULT" ] && [ "$ANSIBLE_HOST_CHECK_RESULT" != 0 ]; then + echo "ce-deploy failed to find the host. Aborting." + exit 1 +fi # From this point on, we want to trigger the "revert" if anything fails. ANSIBLE_BUILD_RESULT=1 @@ -113,5 +120,5 @@ if [ -n "$ANSIBLE_BUILD_RESULT" ] && [ "$ANSIBLE_BUILD_RESULT" = 0 ]; then exit 0 fi # Failed somehow. Normally unreachable in strict mode. -echo "Something went wrong. Please fill a bug report against ce-deploy." +echo "Something went unexpectedly wrong with ce-deploy. Please file a bug report - https://github.com/codeenigma/ce-deploy/issues/new" exit 1 \ No newline at end of file From 2bbb925bd1029e84fabba1ada016f22b4be13ab0 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 May 2023 10:53:18 +0200 Subject: [PATCH 11/41] Removing variable declaration that will break host check. --- scripts/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index fa9a1211..93f36887 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -96,7 +96,6 @@ fi get_ansible_defaults_vars # Optionally carry out a host check if --host is provided. -ANSIBLE_HOST_CHECK_RESULT=1 ansible_host_check ANSIBLE_HOST_CHECK_RESULT=$? # Exit early if host not found. From 0f7a89deecd1a823163bae0739bc858c447b6b08 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 May 2023 13:57:55 +0200 Subject: [PATCH 12/41] Allow us to use deploy_code with a completely custom build and do nothing. --- roles/deploy_code/deploy_code-custom/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 roles/deploy_code/deploy_code-custom/tasks/main.yml diff --git a/roles/deploy_code/deploy_code-custom/tasks/main.yml b/roles/deploy_code/deploy_code-custom/tasks/main.yml new file mode 100644 index 00000000..6f34c92f --- /dev/null +++ b/roles/deploy_code/deploy_code-custom/tasks/main.yml @@ -0,0 +1,3 @@ +--- + +# Nothing to do here. \ No newline at end of file From f23a77b3952b11dc17261be52b08da82973bd26a Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 12 May 2023 14:46:33 +0200 Subject: [PATCH 13/41] Make PHP cachetool install optional. --- docs/roles/_init.md | 1 + roles/_init/README.md | 1 + roles/_init/defaults/main.yml | 1 + roles/_init/tasks/main.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 9cb9e024..08ffe301 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -39,6 +39,7 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index 9cb9e024..08ffe301 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -39,6 +39,7 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9a955f3e..ac06e2a7 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -30,3 +30,4 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a0d90571..c054e8fb 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -124,3 +124,4 @@ - name: Ensure we have a cachetool binary. ansible.builtin.import_role: name: cli/cachetool + when: install_php_cachetool From b9557495000aa7c6e44fff6910cd087600614d8c Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 May 2023 15:43:40 +0200 Subject: [PATCH 14/41] Fixing bug introduced by failed_when - need to check for return code on mount points. --- roles/deploy_code/tasks/cleanup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index db2bb9bb..b3a4f132 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -150,7 +150,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - deploy_code.services | length > 0 - name: Stop any services that might be keeping the loop device busy. @@ -166,7 +166,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - deploy_code.services | length > 0 - name: Unmount existing SquashFS image. @@ -177,7 +177,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - name: Mount new SquashFS image. ansible.builtin.command: From a54569c447fa6aa90994293f50dd95c5a166ace1 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 25 May 2023 15:32:03 +0200 Subject: [PATCH 15/41] Moving where opcache clears get called so they can be excluded if necessary. --- roles/_meta/deploy-drupal8/tasks/main.yml | 4 ++-- roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 ----- roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 ----- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml index 6ce31d4f..3e66875b 100644 --- a/roles/_meta/deploy-drupal8/tasks/main.yml +++ b/roles/_meta/deploy-drupal8/tasks/main.yml @@ -20,14 +20,14 @@ name: database_backup - import_role: name: config_generate +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: database_apply - import_role: name: sanitize/admin_creds - import_role: name: live_symlink -- import_role: - name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal8 - import_role: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 4f476253..3012032f 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -26,11 +26,6 @@ name: "cache_clear/cache_clear-{{ project_type }}" when: previous_build_number > 0 -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 4dc66a3a..c649826c 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -44,11 +44,6 @@ - previous_build_number > 0 - site.config_import_command != 'deploy' -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From a43c7597bdc7ab328cadcce4ce933fcb7ae1245d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 21 Jun 2023 12:37:03 +0200 Subject: [PATCH 16/41] Fixing linting errors around white space. --- roles/cache_clear/cache_clear-drupal8/tasks/main.yml | 1 - roles/database_apply/database_apply-drupal8/tasks/main.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml index e9524b81..5289f254 100644 --- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -15,4 +15,3 @@ ansible.builtin.debug: msg: "{{ _drush_output }}" when: drupal.drush_verbose_output - diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index e2fecd17..c423fad9 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -85,4 +85,3 @@ ansible.builtin.debug: msg: "{{ _drush_output }}" when: drupal.drush_verbose_output - From 6d8b72cbae071664fbe4523f81ce56fa5be0e316 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 10 Jul 2023 11:09:28 +0200 Subject: [PATCH 17/41] The unarchive module requires 'dest' to be a directory. --- .../database_backup/database_backup-mysql/tasks/revert-dump.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 38522af3..4fee5ceb 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -2,7 +2,7 @@ - name: Unpack dump file. ansible.builtin.unarchive: src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" - dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + dest: "/tmp" remote_src: true run_once: true when: previous_build_number > 0 From 749664ed979a7dad7c1d76119b56769cb20bd7db Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 19 Jul 2023 14:04:43 +0200 Subject: [PATCH 18/41] Fixing Drupal example templates. --- ce-dev/ansible/common.yml | 2 +- ce-dev/ansible/examples/drupal8/deploy.yml | 33 +++++++++++++--- ce-dev/ansible/examples/drupal9/deploy.yml | 13 ++++--- ce-dev/ansible/examples/localgov/deploy.yml | 42 ++++++++++++++++----- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/ce-dev/ansible/common.yml b/ce-dev/ansible/common.yml index 98babe50..2cffc05e 100644 --- a/ce-dev/ansible/common.yml +++ b/ce-dev/ansible/common.yml @@ -1,5 +1,5 @@ # Common vars for local development. -is_local: yes +is_local: true ce_deploy: username: "ce-dev" own_repository: "https://github.com/codeenigma/ce-deploy.git" diff --git a/ce-dev/ansible/examples/drupal8/deploy.yml b/ce-dev/ansible/examples/drupal8/deploy.yml index 87c6efad..7c8def57 100644 --- a/ce-dev/ansible/examples/drupal8/deploy.yml +++ b/ce-dev/ansible/examples/drupal8/deploy.yml @@ -37,14 +37,35 @@ pre_tasks: # You can safely remove these steps once you have a working composer.json. - name: Download composer file. - get_url: - url: https://raw.githubusercontent.com/drupal/recommended-project/8.8.x/composer.json + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/8.9.x/composer.json dest: "{{ deploy_path }}/composer.json" - force: no + force: false + - name: Adding composer/installers plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.composer/installers true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-composer-scaffold plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-composer-scaffold true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-project-message plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-project-message true + working_dir: "{{ deploy_path }}" + - name: Adding dealerdirect/phpcodesniffer-composer-installer plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true + working_dir: "{{ deploy_path }}" - name: Install drush. - command: - cmd: composer require drush/drush - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: drush/drush:10.* + working_dir: "{{ deploy_path }}" roles: - _init # Sets some variables the deploy scripts rely on. - composer # Composer install step. diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml index deeef9ee..b7579ac4 100644 --- a/ce-dev/ansible/examples/drupal9/deploy.yml +++ b/ce-dev/ansible/examples/drupal9/deploy.yml @@ -37,14 +37,15 @@ pre_tasks: # You can safely remove these steps once you have a working composer.json. - name: Download composer file. - get_url: - url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.5.x/composer.json dest: "{{ deploy_path }}/composer.json" - force: no + force: false - name: Install drush. - command: - cmd: composer require drush/drush:11.* - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: drush/drush:11.* + working_dir: "{{ deploy_path }}" roles: - _init # Sets some variables the deploy scripts rely on. - composer # Composer install step. diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml index 9844206b..7f746f87 100644 --- a/ce-dev/ansible/examples/localgov/deploy.yml +++ b/ce-dev/ansible/examples/localgov/deploy.yml @@ -37,18 +37,40 @@ pre_tasks: # You can safely remove these steps once you have a working composer.json. - name: Download composer file. - get_url: - url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json - dest: "{{ deploy_path }}/composer.json" - force: false + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.1.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: false + - name: Adding composer/installers plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.composer/installers true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-composer-scaffold plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-composer-scaffold true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-project-message plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-project-message true + working_dir: "{{ deploy_path }}" + - name: Adding dealerdirect/phpcodesniffer-composer-installer plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true + working_dir: "{{ deploy_path }}" - name: Install drush. - command: - cmd: composer require drush/drush:11.* - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: drush/drush:10.* + working_dir: "{{ deploy_path }}" - name: Install localgov. - command: - cmd: composer require localgovdrupal/localgov - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: localgovdrupal/localgov + working_dir: "{{ deploy_path }}" roles: - _init # Sets some variables the deploy scripts rely on. - composer # Composer install step. From bd101466325c093e750cb63ad44fa400987e5160 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 19 Jul 2023 14:40:00 +0200 Subject: [PATCH 19/41] Logically splitting tests and removing LGD because it's broken. --- .github/workflows/ce-deploy-lint.yml | 28 ++++++++ ...oy-test.yml => ce-deploy-publish-docs.yml} | 64 ++++--------------- .github/workflows/ce-deploy-test-drupal8.yml | 44 +++++++++++++ .github/workflows/ce-deploy-test-drupal9.yml | 44 +++++++++++++ 4 files changed, 128 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/ce-deploy-lint.yml rename .github/workflows/{ce-deploy-test.yml => ce-deploy-publish-docs.yml} (61%) create mode 100644 .github/workflows/ce-deploy-test-drupal8.yml create mode 100644 .github/workflows/ce-deploy-test-drupal9.yml diff --git a/.github/workflows/ce-deploy-lint.yml b/.github/workflows/ce-deploy-lint.yml new file mode 100644 index 00000000..283cd3b9 --- /dev/null +++ b/.github/workflows/ce-deploy-lint.yml @@ -0,0 +1,28 @@ +name: Linting + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + run-linting: + # Name the Job + name: Linting + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Linter checks. + - name: Run linters. + if: ${{ always() }} + run: | + sudo apt-get update + sudo apt-get install -y ansible-lint yamllint shellcheck + find ./roles -name "*.yml" | xargs ansible-lint + yamllint ./roles + cd scripts && shellcheck *.sh diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-publish-docs.yml similarity index 61% rename from .github/workflows/ce-deploy-test.yml rename to .github/workflows/ce-deploy-publish-docs.yml index b93a6d3e..45148600 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-publish-docs.yml @@ -1,14 +1,18 @@ -name: Run tests and build docs +name: Publish documentation # Run this workflow every time a new commit pushed to your repository -on: pull_request +on: + pull_request: + types: [closed] + branches: + - 1.x jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided - run-tests: + publish-docs: # Name the Job - name: Run tests against Ansible code base + name: Publish ce-deploy documentation # Set the type of machine to run on runs-on: ubuntu-20.04 @@ -17,15 +21,11 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - # Linter checks. - - name: Run linters. - if: ${{ always() }} + # Configures global Git variables for committing + - name: Configure Git run: | - sudo apt-get update - sudo apt-get install -y ansible-lint yamllint shellcheck - find ./roles -name "*.yml" | xargs ansible-lint - yamllint ./roles - cd scripts && shellcheck *.sh + git config --global user.email "sysadm@codeenigma.com" + git config --global user.name "Code Enigma CI" # Installs the ce-dev stack - name: Install ce-dev @@ -41,46 +41,6 @@ jobs: rm -Rf mkcert curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux - # Configures global Git variables for committing - - name: Configure Git - run: | - git config --global user.email "sysadm@codeenigma.com" - git config --global user.name "Code Enigma CI" - - # Uses the ce-dev stack to run a test Drupal deploy - - name: Run a test ce-dev deploy with Drupal 8 - run: | - ce-dev create -p test -t drupal8 -d ~/test - cd ~/test - ce-dev init - ce-dev start - ce-dev provision - ce-dev deploy - curl https://www.test.local - shell: bash - - - name: Run a test ce-dev deploy with Drupal 9 - run: | - ce-dev create -p testnine -t drupal9 -d ~/testnine - cd ~/testnine - ce-dev init - ce-dev start - ce-dev provision - ce-dev deploy - curl https://www.testnine.local - shell: bash - - - name: Run a test ce-dev deploy with Localgov - run: | - ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov - cd ~/testlocalgov - ce-dev init - ce-dev start - ce-dev provision - ce-dev deploy - curl https://www.testlocalgov.local - shell: bash - # Builds the table of contents for the docs - name: Documentation (build table of contents) if: ${{ github.event.pull_request.base.ref == '1.x' }} diff --git a/.github/workflows/ce-deploy-test-drupal8.yml b/.github/workflows/ce-deploy-test-drupal8.yml new file mode 100644 index 00000000..ec9909a9 --- /dev/null +++ b/.github/workflows/ce-deploy-test-drupal8.yml @@ -0,0 +1,44 @@ +name: Run test for Drupal 8 + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + run-tests: + # Name the Job + name: Run Drupal 8 test build + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + run: | + cd /tmp + wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test Drupal deploy + - name: Run a test ce-dev deploy with Drupal 8 + run: | + ce-dev create -p test -t drupal8 -d ~/test + cd ~/test + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.test.local + shell: bash diff --git a/.github/workflows/ce-deploy-test-drupal9.yml b/.github/workflows/ce-deploy-test-drupal9.yml new file mode 100644 index 00000000..afaaca7a --- /dev/null +++ b/.github/workflows/ce-deploy-test-drupal9.yml @@ -0,0 +1,44 @@ +name: Run test for Drupal 9 + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + run-tests: + # Name the Job + name: Run Drupal 9 test build + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + run: | + cd /tmp + wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test Drupal deploy + - name: Run a test ce-dev deploy with Drupal 9 + run: | + ce-dev create -p test -t drupal9 -d ~/test + cd ~/test + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.test.local + shell: bash From 273eacd639e8b2c17bb55f8ddcdcfbff332c5edc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 19 Jul 2023 14:41:12 +0200 Subject: [PATCH 20/41] Adding extra plugin LGD needs. --- ce-dev/ansible/examples/localgov/deploy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml index 7f746f87..09db8ead 100644 --- a/ce-dev/ansible/examples/localgov/deploy.yml +++ b/ce-dev/ansible/examples/localgov/deploy.yml @@ -61,6 +61,11 @@ command: config arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true working_dir: "{{ deploy_path }}" + - name: Adding cweagans/composer-patches plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.cweagans/composer-patches true + working_dir: "{{ deploy_path }}" - name: Install drush. community.general.composer: command: require @@ -78,4 +83,4 @@ - config_generate # Generates settings.php # - sync/database_sync # Grab database from a remote server. - database_apply # Run drush updb and config import. - - _exit # Some common housekeeping. \ No newline at end of file + - _exit # Some common housekeeping. From 54036ead7dcf46eec7275a90961546773cebf0d8 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 8 Aug 2023 16:04:46 +0200 Subject: [PATCH 21/41] Clearing stat and apcu cache for PHP by default. --- roles/cache_clear/cache_clear-opcache/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index 222325ce..e851d795 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -7,6 +7,6 @@ cache_clear_opcache: # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true - clear_apcu: false - clear_stat: false + clear_apcu: true + clear_stat: true # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined From 07e2c2179334fbc48503b78b87528417f6415ff4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 22 Sep 2023 17:50:29 +0200 Subject: [PATCH 22/41] Adding some perms handling for Drupal 8+ config sync where deploy and web user are different. --- .../database_apply-drupal8/tasks/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 7faa03c7..1935ac79 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -13,6 +13,21 @@ - www_user != deploy_user - previous_build_number == 0 +- name: Fix file permissions for config directory. + ansible.builtin.file: + state: directory + path: "{{ deploy_path }}/{{ site.config_sync_directory }}" + owner: "{{ www_user }}" + group: "{{ www_user }}" + mode: '0775' # in this case often our deploy user is in the web user group and will need to be able to manipulate config + become: true + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - www_user != deploy_user + - previous_build_number == 0 + - name: Install Drupal. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" From 48bb1879a2ab072b774e8be32ff88165d317babc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 22 Sep 2023 17:55:27 +0200 Subject: [PATCH 23/41] Rolling back APCu cache by default as the package must be installed. --- roles/cache_clear/cache_clear-opcache/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index e851d795..fd75afb9 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -7,6 +7,6 @@ cache_clear_opcache: # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true - clear_apcu: true + clear_apcu: false clear_stat: true # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined From 37298ea6d3dfd3a6e75b8000977ecd86ac706b9e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 25 Sep 2023 13:24:17 +0200 Subject: [PATCH 24/41] Making the remount squashfs code more defensive by using a block. --- roles/deploy_code/tasks/cleanup.yml | 122 ++++++++++++++-------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index b3a4f132..9811a6d8 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -1,4 +1,18 @@ --- +- name: Build warning. + ansible.builtin.debug: + msg: "### WARNING - if you are using a server terminal exit NOW! You have 20 seconds to comply..." + run_once: true + when: + - deploy_code.mount_type == "squashfs" + +- name: Give people 20 seconds to exit terminal sessions. + ansible.builtin.pause: + seconds: 20 + run_once: true + when: + - deploy_code.mount_type == "squashfs" + - name: Ensure codebase is writable. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" @@ -137,72 +151,58 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" -- name: Reload any services that might be keeping the loop device busy. - ansible.builtin.service: - name: "{{ www_service }}" - state: reloaded - with_items: "{{ deploy_code.services }}" - loop_control: - loop_var: www_service - become: true - when: - - deploy_code.service_action == "reload" - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check.rc == 0 - - deploy_code.services | length > 0 - -- name: Stop any services that might be keeping the loop device busy. - ansible.builtin.service: - name: "{{ www_service }}" - state: stopped - with_items: "{{ deploy_code.services }}" - loop_control: - loop_var: www_service - become: true - when: - - deploy_code.service_action == "stop" - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check.rc == 0 - - deploy_code.services | length > 0 - -- name: Unmount existing SquashFS image. - ansible.builtin.command: - cmd: "umount {{ deploy_base_path }}" - become: true +- name: Unmount and remount squashfs images. when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - _deploy_code_mount_check.rc == 0 - -- name: Mount new SquashFS image. - ansible.builtin.command: - cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" - become: true - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Start any services we stopped. - ansible.builtin.service: - name: "{{ www_service }}" - state: started - with_items: "{{ deploy_code.services }}" - loop_control: - loop_var: www_service - become: true - when: - - deploy_code.service_action == "stop" - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded - - deploy_code.services | length > 0 + block: + - name: Reload any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: reloaded + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "reload" + - deploy_code.services | length > 0 + + - name: Stop any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: stopped + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "stop" + - deploy_code.services | length > 0 + + - name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount {{ deploy_base_path }}" + become: true + + - name: Mount new SquashFS image. + ansible.builtin.command: + cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" + become: true + + - name: Start any services we stopped. + ansible.builtin.service: + name: "{{ www_service }}" + state: started + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "stop" + - deploy_code.services | length > 0 - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: From 3e8a252922c1349bb16518b78ad70d0ab104a702 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 25 Sep 2023 17:49:17 +0200 Subject: [PATCH 25/41] Trying with 'throttle' keyword to restrict to one machine at a time. --- roles/deploy_code/tasks/cleanup.yml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 9811a6d8..18588e9c 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -139,25 +139,21 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" -- name: Check if we have a mount already. - ansible.builtin.shell: - cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" - args: - executable: /bin/bash - failed_when: false - register: _deploy_code_mount_check - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - name: Unmount and remount squashfs images. when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check.rc == 0 + throttle: 1 # execute one machine at a time block: + - name: Check if we have a mount already. + ansible.builtin.shell: + cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" + args: + executable: /bin/bash + failed_when: false + register: _deploy_code_mount_check + - name: Reload any services that might be keeping the loop device busy. ansible.builtin.service: name: "{{ www_service }}" @@ -167,6 +163,7 @@ loop_var: www_service become: true when: + - _deploy_code_mount_check.rc == 0 - deploy_code.service_action == "reload" - deploy_code.services | length > 0 @@ -179,6 +176,7 @@ loop_var: www_service become: true when: + - _deploy_code_mount_check.rc == 0 - deploy_code.service_action == "stop" - deploy_code.services | length > 0 @@ -186,6 +184,8 @@ ansible.builtin.command: cmd: "umount {{ deploy_base_path }}" become: true + when: + - _deploy_code_mount_check.rc == 0 - name: Mount new SquashFS image. ansible.builtin.command: @@ -201,6 +201,7 @@ loop_var: www_service become: true when: + - _deploy_code_mount_check.rc == 0 - deploy_code.service_action == "stop" - deploy_code.services | length > 0 From 63fd83a334000bd8c83e7da89855d0e050f2c512 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 25 Sep 2023 18:01:29 +0200 Subject: [PATCH 26/41] Sadly 'throttle' seems to work on a task level only, not a block level. --- roles/deploy_code/tasks/cleanup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 18588e9c..76c7e810 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -144,7 +144,6 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - throttle: 1 # execute one machine at a time block: - name: Check if we have a mount already. ansible.builtin.shell: From 0ea98abceaf0edd9a19269d65b6d87225cf9db9f Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 27 Sep 2023 13:10:11 +0200 Subject: [PATCH 27/41] Adding hash salt generation for Drupal 8+. --- .../config_generate-drupal8/tasks/settings.yml | 4 ++++ .../config_generate-drupal8/templates/settings.php.j2 | 2 ++ 2 files changed, 6 insertions(+) diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index 6f698ba5..4cbf8f5e 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -18,6 +18,10 @@ mode: 0775 become: "{{ false if www_user == deploy_user else true }}" +- name: Generate or retrieve hash salt value. + ansible.builtin.set_fact: + _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name + '_' + build_type + '/drupal/hash_salt', length=74, chars=['ascii_letters', 'digits']) }}" + - name: Generates settings.php file. ansible.builtin.template: src: "{{ item }}" diff --git a/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 b/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 index e5d43204..895f654f 100644 --- a/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 +++ b/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 @@ -23,3 +23,5 @@ $settings['file_public_path'] = '{{ build_public_file_path }}'; $config_directories['sync'] = '{{ build_config_sync_directory }}'; // Drupal 8.8 $settings['config_sync_directory'] = '{{ build_config_sync_directory }}'; + +$settings['hash_salt'] = '{{ _drupal_hash_salt }}'; From 5a04f52c93c14fe3071f0af0a7a5e86bb78c8920 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 27 Sep 2023 13:49:26 +0200 Subject: [PATCH 28/41] Trying a different syntax for passing params to the password plugin. --- .../config_generate/config_generate-drupal8/tasks/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index 4cbf8f5e..79e0ad5e 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -20,7 +20,7 @@ - name: Generate or retrieve hash salt value. ansible.builtin.set_fact: - _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name + '_' + build_type + '/drupal/hash_salt', length=74, chars=['ascii_letters', 'digits']) }}" + _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name + '_' + build_type + '/drupal/hash_salt length=74 chars=ascii_letters,digits') }}" - name: Generates settings.php file. ansible.builtin.template: From fb8d5fa30beafed6b705a8d4317ced69ede537ed Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 31 Oct 2023 18:19:54 +0100 Subject: [PATCH 29/41] Forgot the constants import in plugin. --- plugins/callback/fail_on_no_hosts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/callback/fail_on_no_hosts.py b/plugins/callback/fail_on_no_hosts.py index 862ae9a4..e022c0a4 100644 --- a/plugins/callback/fail_on_no_hosts.py +++ b/plugins/callback/fail_on_no_hosts.py @@ -1,5 +1,6 @@ import sys +from ansible import constants as C from ansible.plugins.callback import CallbackBase DOCUMENTATION = ''' From bbcb52473acbc8ae65538aee599f0c1bcea5e2ac Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 2 Nov 2023 12:58:10 +0100 Subject: [PATCH 30/41] Reinstating the SOPS plugin - we *do* need it. --- plugins/README.md | 16 ++++ plugins/vars/sops_vars.py | 163 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 plugins/README.md create mode 100644 plugins/vars/sops_vars.py diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 00000000..58e31070 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,16 @@ +# Ansible Plugins +This directory contains extra plugins for Ansible. + +## Vars plugins +Here we include a plugin for handling SOPS decryption. + +## Callback plugins +Here we have a small custom override that fails builds if there are no matching hosts found, to avoid `ce-deploy` incrementing the track file when it didn't actually run because of a host issue. + +## Enabling plugins +To use these plugins you need to find the `# set plugin path directories here` section of `ansible.cfg` which should be kept in your `ce-deploy-config` repository. Add the paths to the plugin directories to enable the plugins, e.g. + +``` +vars_plugins = /home/deploy/ce-deploy/plugins/vars:/usr/share/ansible/plugins/vars +callback_plugins = /home/deploy/ce-deploy/plugins/callback:/usr/share/ansible/plugins/callback +``` diff --git a/plugins/vars/sops_vars.py b/plugins/vars/sops_vars.py new file mode 100644 index 00000000..9193d61c --- /dev/null +++ b/plugins/vars/sops_vars.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2019, Arduino, srl +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# +############################################# + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + vars: sops_vars + author: Edoardo Tenani (@endorama) + version_added: "2.10" + short_description: Loading sops-encrypted vars files + description: + - Load encrypted YAML files into correspondind groups/hosts in group_vars/ and host_vars/ directories. + - Files are encrypted prior to reading, making this plugin an effective companion to host_group_vars plugin. + - Files are restricted to .sops.yaml, .sops.yml, .sops.json extensions. + - Hidden files are ignored. + options: + _valid_extensions: + default: [".sops.yml", ".sops.yaml", ".sops.json"] + description: + - "Check all of these extensions when looking for 'variable' files which should be YAML or JSON or vaulted versions of these." + - 'This affects vars_files, include_vars, inventory and vars plugins among others.' + type: list +''' + +import os +from ansible import constants as C +from ansible.errors import AnsibleParserError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.vars import BaseVarsPlugin +from ansible.inventory.host import Host +from ansible.inventory.group import Group +from ansible.utils.vars import combine_vars +from ansible.errors import AnsibleError +from subprocess import Popen, PIPE +from ansible.utils.display import Display +display = Display() + +FOUND = {} +DEFAULT_VALID_EXTENSIONS = [".sops.yaml", ".sops.yml", ".sops.json"] + +# From https://github.com/mozilla/sops/blob/master/cmd/sops/codes/codes.go +# Should be manually updated +sops_error_codes = { + 1: "SopsErrorGeneric", + 2: "SopsCouldNotReadInputFile", + 3: "SopsCouldNotWriteOutputFile", + 4: "SopsErrorDumpingTree", + 5: "SopsErrorReadingConfig", + 6: "SopsErrorInvalidKMSEncryptionContextFormat", + 7: "SopsErrorInvalidSetFormat", + 8: "SopsErrorConflictingParameters", + 21: "SopsErrorEncryptingMac", + 23: "SopsErrorEncryptingTree", + 24: "SopsErrorDecryptingMac", + 25: "SopsErrorDecryptingTree", + 49: "SopsCannotChangeKeysFromNonExistentFile", + 51: "SopsMacMismatch", + 52: "SopsMacNotFound", + 61: "SopsConfigFileNotFound", + 85: "SopsKeyboardInterrupt", + 91: "SopsInvalidTreePathFormat", + 100: "SopsNoFileSpecified", + 128: "SopsCouldNotRetrieveKey", + 111: "SopsNoEncryptionKeyFound", + 200: "SopsFileHasNotBeenModified", + 201: "SopsNoEditorFound", + 202: "SopsFailedToCompareVersions", + 203: "SopsFileAlreadyEncrypted" +} + + +class SopsError(AnsibleError): + ''' extend AnsibleError class with sops specific informations ''' + + def __init__(self, filename, exit_code, message,): + exception_name = sops_error_codes[exit_code] + message = "error with file %s: %s exited with code %d: %s" % (filename, exception_name, exit_code, message) + super(SopsError, self).__init__(message=message) + + +def decrypt_with_sops(filename): + display.vvvv(u"sops --decrypt %s" % filename) + + # Run sops directly as python module is deprecated + process = Popen(["sops", "--decrypt", filename], stdout=PIPE, stderr=PIPE) + (output, err) = process.communicate() + exit_code = process.wait() + + # DO NOT display output + # is the decrypted secret and would easily end in logs :) + # if output: + # display.vvvv(output) + + # sops logs always to stderr ( stdout is used for file content ) + if err: + display.vvvv(err) + + if exit_code > 0: + if exit_code in sops_error_codes.keys(): + raise SopsError(filename, exit_code, err) + else: + raise AnsibleError(message=err) + + return output + + +class VarsModule(BaseVarsPlugin): + + def get_vars(self, loader, path, entities, cache=True): + ''' parses the inventory file ''' + + if not isinstance(entities, list): + entities = [entities] + + super(VarsModule, self).get_vars(loader, path, entities) + + data = {} + for entity in entities: + if isinstance(entity, Host): + subdir = 'host_vars' + elif isinstance(entity, Group): + subdir = 'group_vars' + else: + raise AnsibleParserError("Supplied entity must be Host or Group, got %s instead" % (type(entity))) + + # avoid 'chroot' type inventory hostnames /path/to/chroot + if not entity.name.startswith(os.path.sep): + try: + found_files = [] + # load vars + b_opath = os.path.realpath(to_bytes(os.path.join(self._basedir, subdir))) + opath = to_text(b_opath) + key = '%s.%s' % (entity.name, opath) + self._display.vvvv("key: %s" % (key)) + if cache and key in FOUND: + found_files = FOUND[key] + else: + # no need to do much if path does not exist for basedir + if os.path.exists(b_opath): + if os.path.isdir(b_opath): + self._display.debug("\tprocessing dir %s" % opath) + found_files = loader.find_vars_files(opath, entity.name) + found_files = [file_path for file_path in found_files + if any(file_path.endswith(extension) for extension in DEFAULT_VALID_EXTENSIONS)] + FOUND[key] = found_files + else: + self._display.warning("Found %s that is not a directory, skipping: %s" % (subdir, opath)) + + for found in found_files: + file_content = decrypt_with_sops(found) + new_data = loader.load(file_content) + if new_data: # ignore empty files + data = combine_vars(data, new_data) + + except Exception as e: + raise AnsibleParserError(to_native(e)) + + return data \ No newline at end of file From 344348c73776b6eacdc5ec260fa90ed09d3aa5da Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 7 Nov 2023 16:42:08 +0100 Subject: [PATCH 31/41] Making online/offline tasks use current live code/database. --- roles/_init/tasks/drupal7.yml | 8 ++++++-- roles/_init/tasks/drupal8.yml | 6 +++++- .../maintenance_mode-drupal-core/tasks/offline.yml | 4 ++-- .../maintenance_mode-drupal-core/tasks/online.yml | 8 ++++---- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml index 133b3d73..b1781c1b 100644 --- a/roles/_init/tasks/drupal7.yml +++ b/roles/_init/tasks/drupal7.yml @@ -1,8 +1,12 @@ --- -- name: Define Drush path. +- name: Define path to drush for this build. set_fact: drush_bin: "{{ drush_bin | default('{{ bin_directory }}/drush.phar') }}" -- name: Ensure we have Drush binary. +- name: Define path to drush for currently live build. # usually the same as drush_bin for Drupal 7 + set_fact: + drush_live_bin: "{{ drush_live_bin | default('{{ bin_directory }}/drush.phar') }}" + +- name: Ensure we have a drush binary. import_role: name: cli/drush diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml index ebe63b82..3792c489 100644 --- a/roles/_init/tasks/drupal8.yml +++ b/roles/_init/tasks/drupal8.yml @@ -1,4 +1,8 @@ --- -- name: Define Drush path. +- name: Define path to drush for this build. set_fact: drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}" + +- name: Define path to drush for currently live build. + set_fact: + drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}" \ No newline at end of file diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 2d6a9310..78314024 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -1,7 +1,7 @@ --- - name: Enable maintenance mode. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -13,7 +13,7 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Enable maintenance mode D7. ansible.builtin.shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 1" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 6b127f89..2598481a 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,9 +1,9 @@ --- - name: Disable maintenance mode. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" args: - chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" when: @@ -12,8 +12,8 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Disable maintenance mode D7. ansible.builtin.shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" - chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 0" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" when: From d364d9c976a09f68598e42963e14886984769be7 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 9 Nov 2023 10:26:34 +0100 Subject: [PATCH 32/41] We need build paths to bring the site back online if it's a squashfs build. --- .../maintenance_mode-drupal-core/tasks/online.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 2598481a..6b127f89 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,9 +1,9 @@ --- - name: Disable maintenance mode. ansible.builtin.command: - cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" args: - chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" when: @@ -12,8 +12,8 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Disable maintenance mode D7. ansible.builtin.shell: - cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 0" - chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" when: From 6175dff0bdabce1483fdc8259ce38dda91a79d0a Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 13 Nov 2023 17:05:05 +0100 Subject: [PATCH 33/41] Ensuring _common.sh includes /etc/profile.d for Python venv path. --- scripts/_common.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/_common.sh b/scripts/_common.sh index 74ba9a60..7c7c7659 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -21,19 +21,26 @@ BUILD_WORKSPACE_BASE="$OWN_DIR/build" DRY_RUN="no" VERBOSE="no" BOTO_PROFILE="" +# Ensure build workspace exists. if [ ! -d "$BUILD_WORKSPACE_BASE" ]; then mkdir "$BUILD_WORKSPACE_BASE" fi BUILD_TMP_DIR=$(mktemp -d -p "$BUILD_WORKSPACE_BASE") +# Ensure ce-deploy data directory exists. ANSIBLE_DATA_DIR="$OWN_DIR/data" if [ ! -d "$ANSIBLE_DATA_DIR" ]; then mkdir "$ANSIBLE_DATA_DIR" fi +# Ensure directory for build track files exists. BUILD_TRACK_DIR="$OWN_DIR/track" if [ ! -d "$BUILD_TRACK_DIR" ]; then mkdir "$BUILD_TRACK_DIR" fi ANSIBLE_LOCATION=$(command -v ansible) +# Load the contents of profile.d in case we added items to $PATH there. +for f in /etc/profile.d/*; do + . "$f" +done # Parse options arguments. parse_options(){ while [ "${1:-}" ]; do From 4ec522d69ab32d6a942cdb2f14f0f57510cafcdf Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 13 Nov 2023 17:21:47 +0100 Subject: [PATCH 34/41] Making linter happy. --- scripts/_common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/_common.sh b/scripts/_common.sh index 7c7c7659..04e14c18 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -39,6 +39,7 @@ fi ANSIBLE_LOCATION=$(command -v ansible) # Load the contents of profile.d in case we added items to $PATH there. for f in /etc/profile.d/*; do +# shellcheck source=/dev/null . "$f" done # Parse options arguments. From 9f5731dc26339d7bb254922b0a348f506c107949 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 16 Nov 2023 17:29:09 +0100 Subject: [PATCH 35/41] Using the drush_live_bin var in sync jobs for Drupal 8+. --- roles/deploy_container/defaults/main.yml | 2 +- .../cache_clear-drupal8/tasks/main.yml | 16 ++-------------- .../database_apply-drupal8/tasks/main.yml | 18 +++--------------- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml index be178aca..f01dcb15 100644 --- a/roles/deploy_container/defaults/main.yml +++ b/roles/deploy_container/defaults/main.yml @@ -68,7 +68,7 @@ deploy_container: launch_type: FARGATE network_mode: awsvpc #volumes: [] # list of additional volumes to attach - target_group_name: example # 32 character limit + target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen target_group_protocol: http target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile target_group_wait_timeout: 200 # how long to wait for target group events to complete diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml index 431366cf..0f116adc 100644 --- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,16 +1,8 @@ --- -# Drupal 8 ships drush with the website code so we need the previous build in the path. -- name: Stash the drush_bin variable. - ansible.builtin.set_fact: - _drush_bin_deploy: "{{ drush_bin }}" - -- name: Update location of drush for the sync cache clear command. - ansible.builtin.set_fact: - drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush" - +# Drupal 8 ships drush with the website code so to use the drush_live_bin variable - name: Clear Drupal cache. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y cr" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -18,7 +10,3 @@ loop_control: loop_var: site run_once: true - -- name: Restore the drush_bin variable. - ansible.builtin.set_fact: - drush_bin: "{{ _drush_bin_deploy }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index d13b0129..297e476b 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -1,16 +1,8 @@ --- -# Drupal 8 ships drush with the website code so we need the previous build in the path. -- name: Stash the drush_bin variable. - ansible.builtin.set_fact: - _drush_bin_deploy: "{{ drush_bin }}" - -- name: Update location of drush for the update/config import commands. - ansible.builtin.set_fact: - drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush" - +# Drupal 8 ships drush with the website code so to use the drush_live_bin variable - name: Apply Drupal database updates. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y updb" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -27,7 +19,7 @@ # This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run. - name: Import configuration. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -47,7 +39,3 @@ - name: Clear the cache. ansible.builtin.include_role: name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" - -- name: Restore the drush_bin variable. - ansible.builtin.set_fact: - drush_bin: "{{ _drush_bin_deploy }}" From 0b773b2573cec53534982a7a4712e0c75a39fdee Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 24 Nov 2023 13:34:27 +0100 Subject: [PATCH 36/41] Fixing database dump revert. --- .../database_backup-mysql/tasks/revert-dump.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 08059e30..6834e45c 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,9 +1,9 @@ --- +# unarchive module does not support archives not in tar.gz format - name: Unpack dump file. - ansible.builtin.unarchive: - src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz" - dest: "/tmp" - remote_src: true + ansible.builtin.command: + cmd: "gunzip {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz /tmp/{{ database.database }}-{{ previous_build_number }}.sql" + creates: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" run_once: true when: previous_build_number > 0 From 631f7639e683b0b5fb7a8572e88c02da3b54dd9b Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 24 Nov 2023 13:45:41 +0100 Subject: [PATCH 37/41] Fixing gunzip command. --- .../database_backup/database_backup-mysql/tasks/revert-dump.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 6834e45c..ab729217 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -2,7 +2,7 @@ # unarchive module does not support archives not in tar.gz format - name: Unpack dump file. ansible.builtin.command: - cmd: "gunzip {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz /tmp/{{ database.database }}-{{ previous_build_number }}.sql" + cmd: "gunzip {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz" creates: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" run_once: true when: previous_build_number > 0 From e9b0ab41d43d37df654b5d8f02b1da1a7dcda889 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 24 Nov 2023 16:52:47 +0100 Subject: [PATCH 38/41] Having to do a double step to unpack SQL file in /tmp. --- .../database_backup-mysql/tasks/revert-dump.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index ab729217..6040c73a 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,9 +1,16 @@ --- +- name: Move dump file to local disk. + ansible.builtin.copy: + remote_src: true + src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz" + dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql.gz" + run_once: true + when: previous_build_number > 0 + # unarchive module does not support archives not in tar.gz format - name: Unpack dump file. ansible.builtin.command: - cmd: "gunzip {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz" - creates: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + cmd: "gunzip /tmp/{{ database.database }}-{{ previous_build_number }}.sql.gz" run_once: true when: previous_build_number > 0 From 75dd6082c39477e2a9c97988980f3c42016813bc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 10:50:48 +0100 Subject: [PATCH 39/41] Load bash profiles before executing a build. --- scripts/_common.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/_common.sh b/scripts/_common.sh index 04e14c18..0d99858f 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -36,6 +36,13 @@ BUILD_TRACK_DIR="$OWN_DIR/track" if [ ! -d "$BUILD_TRACK_DIR" ]; then mkdir "$BUILD_TRACK_DIR" fi +# Load the contents of profile.d in case we added items to $PATH there. +if [ ! -z "$(ls -A /etc/profile.d)" ]; then + for f in /etc/profile.d/*; do + # shellcheck source=/dev/null + . "$f" + done +fi ANSIBLE_LOCATION=$(command -v ansible) # Load the contents of profile.d in case we added items to $PATH there. for f in /etc/profile.d/*; do From f7bc80029d8772400c335e867ef43af2629a14af Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 10:54:55 +0100 Subject: [PATCH 40/41] Fixing linting error, use -n instead of ! -z. --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 0d99858f..a34ff141 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -37,7 +37,7 @@ if [ ! -d "$BUILD_TRACK_DIR" ]; then mkdir "$BUILD_TRACK_DIR" fi # Load the contents of profile.d in case we added items to $PATH there. -if [ ! -z "$(ls -A /etc/profile.d)" ]; then +if [ -n "$(ls -A /etc/profile.d)" ]; then for f in /etc/profile.d/*; do # shellcheck source=/dev/null . "$f" From d052df38daeee50ae6cb7fb42376fda6bdb9c047 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 15 Jan 2024 13:32:13 +0100 Subject: [PATCH 41/41] Adding support for specifying path to Ansible and path to Python interpreter. --- scripts/_common.sh | 21 ++++++++++++++++++++- scripts/build.sh | 2 ++ scripts/cleanup.sh | 2 ++ scripts/deploy.sh | 2 ++ scripts/revert.sh | 2 ++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index a34ff141..07aa7368 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -14,6 +14,8 @@ PREVIOUS_BUILD_NUMBER="" CURRENT_BUILD_NUMBER="" ANSIBLE_EXTRA_VARS="" ANSIBLE_DEFAULT_EXTRA_VARS="" +ANSIBLE_PATH="" +PYTHON_INTERPRETER="" BUILD_WORKSPACE="" BUILD_TRACK_FILE="" BUILD_ID="" @@ -103,6 +105,14 @@ parse_options(){ shift BOTO_PROFILE="$1" ;; + "--ansible-path") + shift + ANSIBLE_PATH="$1" + ;; + "--python-interpreter") + shift + PYTHON_INTERPRETER="$1" + ;; "--build-id") shift BUILD_ID="$1" @@ -128,7 +138,11 @@ get_build_workspace(){ # Common extra-vars to pass to Ansible. get_ansible_defaults_vars(){ - ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}" + if [ -n "$PYTHON_INTERPRETER" ]; then + ANSIBLE_DEFAULT_EXTRA_VARS="{ansible_python_interpreter: $PYTHON_INTERPRETER, _ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}" + else + ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}" + fi } # Fetch previous build number from track file. @@ -169,6 +183,11 @@ cleanup_build_tmp_dir(){ # Call Ansible playbook to ensure host exists. ansible_host_check(){ if [ -n "$TARGET_DEPLOY_HOST" ]; then + if [ -z "$ANSIBLE_PATH" ]; then + ANSIBLE_BIN=$(command -v ansible-playbook) + else + ANSIBLE_BIN="$ANSIBLE_PATH/ansible-playbook" + fi ANSIBLE_BIN=$(command -v ansible-playbook) ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml" if [ "$VERBOSE" = "yes" ]; then diff --git a/scripts/build.sh b/scripts/build.sh index 93f36887..a29ec43b 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -21,6 +21,8 @@ usage(){ echo 'Available options:' echo '--host: Valid Ansible hostname, if you want to run a host check. Can also be a group name.' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output' diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 9f0e50cf..abe24675 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -20,6 +20,8 @@ usage(){ echo '' echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output' diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 341af1ce..66ab7fe3 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -20,6 +20,8 @@ usage(){ echo '' echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output' diff --git a/scripts/revert.sh b/scripts/revert.sh index a10655a3..7b36b42b 100755 --- a/scripts/revert.sh +++ b/scripts/revert.sh @@ -20,6 +20,8 @@ usage(){ echo '' echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output'