From d7054493976fa9fb7299bdbdaa77be6b495fcada Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Mon, 10 Jan 2022 13:29:49 +0000 Subject: [PATCH 01/10] Change location of nightly backup script and delegate the cron that runs it to the deploy server, if required (for ASGs). --- .../cron_database_backup-mysql/tasks/main.yml | 9 +++++++++ .../cron_database_backup-mysql/tasks/setup.yml | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml index 04789df8..13966493 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml @@ -3,6 +3,15 @@ # screwing the daily backup when using rolling db, we could # add a keep mechanism for backup scripts, like for the dumps themselves. # Nice to have more than anything. +- name: Setup PATH in crontab. + cron: + name: PATH + env: true + job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + when: + - drupal.defer is defined + - drupal.defer + - include_tasks: setup.yml vars: database: database diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index bdc1534c..40b0c47e 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -25,12 +25,26 @@ - name: Create backup script. template: src: "regular-backups.sh.j2" - dest: "/home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" mode: 0700 +- name: Define backup cron job command. + set_fact: + _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + +- name: Define backup cron job command if deferred (ASG). + set_fact: + _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\"" + when: + - drupal.defer is defined + - drupal.defer + - drupal.defer_target is defined + - drupal.defer_target | length > 0 + - name: Setup regular backup for MySQL. cron: name: "cron_mysql_{{ database.host }}_{{ database.original.database }}" minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" - job: "/bin/sh /home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + job: "{{ _backup_cron_job_command }}" + delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" From 6e290241171161881ad8901b43592ad085cbac8d Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Mon, 10 Jan 2022 16:35:33 +0000 Subject: [PATCH 02/10] Set up nightly backup crons in separate files in /etc/cron.d --- .../cron_database_backup-mysql/tasks/setup.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index 40b0c47e..406eca8c 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -47,4 +47,6 @@ minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" job: "{{ _backup_cron_job_command }}" + cron_file: "{{ project_name }}-{{ build_type }}" + user: "{{ deploy_user }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" From 0fb99c628249e101be9edc15bafe53a3e7af24d7 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Wed, 12 Jan 2022 16:54:56 +0000 Subject: [PATCH 03/10] Can't put site cron files in /etc/cron.d because the deploy user doesn't have perms. --- .../cron_database_backup-mysql/tasks/setup.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index 406eca8c..40b0c47e 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -47,6 +47,4 @@ minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" job: "{{ _backup_cron_job_command }}" - cron_file: "{{ project_name }}-{{ build_type }}" - user: "{{ deploy_user }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" From 5f7be712c587d6cfb82b55106f23398872224110 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Thu, 13 Jan 2022 15:57:37 +0000 Subject: [PATCH 04/10] Try and add the ability to sync a site using a nightly backup instead of a fresh DB dump. --- .../database_sync-mysql/defaults/main.yml | 5 +++++ .../database_sync-mysql/tasks/sync.yml | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index c1d7b259..2157bc35 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -15,6 +15,11 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 81e9b2ce..195c1c75 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -33,7 +33,20 @@ - name: Take a dump from source database. shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" - when: not database.source.type == 'dump' + when: + - not database.source.type == 'dump' + - database.source.fresh_db + +- name: Register source database host. + shell: "grep 'host' {{ database.source.credentials_file }} | cut -d= -f2" + register: mysql_sync_source_database_host + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Copy a nightly backup for the source database. + shell: "cp -L {{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host.stdout }}/{{ project_name }}_{{ build_type }} {{ mysql_sync_source_dump_path }}" + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db - name: Register tmp target dump name. set_fact: From c940fbbea92ca7990ade461e640967c3eca8f9f0 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 14 Jan 2022 10:27:43 +0000 Subject: [PATCH 05/10] Use Ansible modules to look up RDS host and to copy the nightly backup into place. --- .../database_sync/database_sync-mysql/tasks/sync.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 195c1c75..93cfad96 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -38,13 +38,16 @@ - database.source.fresh_db - name: Register source database host. - shell: "grep 'host' {{ database.source.credentials_file }} | cut -d= -f2" - register: mysql_sync_source_database_host + set_fact: + mysql_sync_source_database_host: "{{ lookup('ini', 'host section=client file={{ database.source.credentials_file }}') }}" delegate_to: "{{ database.source.host }}" when: not database.source.fresh_db - name: Copy a nightly backup for the source database. - shell: "cp -L {{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host.stdout }}/{{ project_name }}_{{ build_type }} {{ mysql_sync_source_dump_path }}" + ansible.builtin.copy: + src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ project_name }}_{{ build_type }}" + dest: "{{ mysql_sync_source_dump_path }}" + remote_src: true delegate_to: "{{ database.source.host }}" when: not database.source.fresh_db From 369661925eb66f37d677e3b8308253b0a3583b61 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 14 Jan 2022 10:58:01 +0000 Subject: [PATCH 06/10] Delegate PATH setup in db backup cron to localhost. --- .../cron_database_backup-mysql/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml index 13966493..ffd14157 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml @@ -8,6 +8,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer From 0f790567ac08747bdf746b3c6ea52cb56afb7ac1 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 14 Jan 2022 12:16:59 +0000 Subject: [PATCH 07/10] Shell bad. Command good. But makes it convoluted. Oh well. --- .../database_sync/database_sync-mysql/tasks/sync.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 93cfad96..633b6d61 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -37,9 +37,16 @@ - not database.source.type == 'dump' - database.source.fresh_db +- name: Find source database host. + ansible.builtin.command: + cmd: "grep 'host' {{ database.source.credentials_file }}" + register: mysql_host_info_grep + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + - name: Register source database host. set_fact: - mysql_sync_source_database_host: "{{ lookup('ini', 'host section=client file={{ database.source.credentials_file }}') }}" + mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}" delegate_to: "{{ database.source.host }}" when: not database.source.fresh_db From 74fff5b52c37da2502a2e3f992e9e6f17431fc87 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 14 Jan 2022 12:42:16 +0000 Subject: [PATCH 08/10] Used wrong database name in source database copy. --- roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 633b6d61..44e6f7e9 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -52,7 +52,7 @@ - name: Copy a nightly backup for the source database. ansible.builtin.copy: - src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ project_name }}_{{ build_type }}" + src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}" dest: "{{ mysql_sync_source_dump_path }}" remote_src: true delegate_to: "{{ database.source.host }}" From 99e3f5fb36fb6b1753363cefd42f6b6785ce34f1 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 14 Jan 2022 15:59:46 +0000 Subject: [PATCH 09/10] Try and disable the ReplaceUnhealthy auto scale process during syncs. --- .../database_sync-mysql/defaults/main.yml | 4 ++ .../database_sync-mysql/tasks/sync.yml | 54 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 2157bc35..48e3e4a0 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -20,6 +20,8 @@ mysql_sync: # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. # This var is only used when fresh_db is set to "false". dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -30,3 +32,5 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 44e6f7e9..b8182bc3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -1,4 +1,40 @@ --- +- name: Get database source host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_source_database_host_info + delegate_to: "{{ database.source.host }}" + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host. +- name: Get database target host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_target_database_host_info + when: + - database.target.asg is defined + - database.target.asg | length > 0 + +- name: Disable ReplaceUnhealthy autoscale process on source ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Disable ReplaceUnhealthy autoscale process on target ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 + - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" @@ -118,3 +154,21 @@ path: "{{ mysql_sync_target_dump_path }}" state: absent when: not database.target.type == 'dump' + +- name: Enable all autoscale processes on source ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Enable all autoscale processes on target ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.target.asg }} --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 From bcab81ebed5569777145bb0c32be1b8298447b8a Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Tue, 18 Jan 2022 13:36:27 +0000 Subject: [PATCH 10/10] Fix Drupal cron roles when deferring to deploy server. --- roles/cron/cron_drupal7/tasks/main.yml | 1 + roles/cron/cron_drupal8/tasks/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal8/tasks/main.yml +++ b/roles/cron/cron_drupal8/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer