Defer nightly backups, disable ASG processes during syncs and run syncs with backup

roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml

@@ -3,6 +3,16 @@
 # screwing the daily backup when using rolling db, we could
 # add a keep mechanism for backup scripts, like for the dumps themselves.
 # Nice to have more than anything.
+- name: Setup PATH in crontab.
+  cron:
+    name: PATH
+    env: true
+    job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
+  delegate_to: localhost
+  when:
+    - drupal.defer is defined
+    - drupal.defer
+
 - include_tasks: setup.yml
   vars:
     database: database

roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml

@@ -25,12 +25,26 @@
 - name: Create backup script.
   template:
     src: "regular-backups.sh.j2"
-    dest: "/home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
+    dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
     mode: 0700
 
+- name: Define backup cron job command.
+  set_fact:
+    _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
+
+- name: Define backup cron job command if deferred (ASG).
+  set_fact:
+    _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\""
+  when:
+    - drupal.defer is defined
+    - drupal.defer
+    - drupal.defer_target is defined
+    - drupal.defer_target | length > 0
+
 - name: Setup regular backup for MySQL.
   cron:
     name: "cron_mysql_{{ database.host }}_{{ database.original.database }}"
     minute: "{{ _cron_mysql_backup_minute }}"
     hour: "{{ _cron_mysql_backup_hour }}"
-    job: "/bin/sh /home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
+    job: "{{ _backup_cron_job_command }}"
+  delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"

roles/cron/cron_drupal7/tasks/main.yml

@@ -4,6 +4,7 @@
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
+  delegate_to: localhost
   when:
     - drupal.defer is defined
     - drupal.defer

roles/cron/cron_drupal8/tasks/main.yml

@@ -4,6 +4,7 @@
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
+  delegate_to: localhost
   when:
     - drupal.defer is defined
     - drupal.defer

roles/sync/database_sync/database_sync-mysql/defaults/main.yml

@@ -15,6 +15,13 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
+        # Whether or not use to create a fresh database backup or use a nightly one.
+        fresh_db: true
+        # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default.
+        # This var is only used when fresh_db is set to "false".
+        dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
+        # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
@@ -25,3 +32,5 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuilddev
+        # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""

roles/sync/database_sync/database_sync-mysql/tasks/sync.yml

@@ -1,4 +1,40 @@
 ---
+- name: Get database source host region.
+  amazon.aws.ec2_metadata_facts:
+  register: mysql_sync_source_database_host_info
+  delegate_to: "{{ database.source.host }}"
+  when:
+    - database.source.asg is defined
+    - database.source.asg | length > 0
+    - database.source.fresh_db is defined
+    - database.source.fresh_db
+
+# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host.
+- name: Get database target host region.
+  amazon.aws.ec2_metadata_facts:
+  register: mysql_sync_target_database_host_info
+  when:
+    - database.target.asg is defined
+    - database.target.asg | length > 0
+
+- name: Disable ReplaceUnhealthy autoscale process on source ASG.
+  ansible.builtin.command: >
+    aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.source.asg is defined
+    - database.source.asg | length > 0
+    - database.source.fresh_db is defined
+    - database.source.fresh_db
+
+- name: Disable ReplaceUnhealthy autoscale process on target ASG.
+  ansible.builtin.command: >
+    aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.target.asg is defined
+    - database.target.asg | length > 0
+
 - name: Register remote dump name (from database).
   set_fact:
     mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2"
@@ -33,7 +69,30 @@
 - name: Take a dump from source database.
   shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
   delegate_to: "{{ database.source.host }}"
-  when: not database.source.type == 'dump'
+  when:
+    - not database.source.type == 'dump'
+    - database.source.fresh_db
+
+- name: Find source database host.
+  ansible.builtin.command:
+    cmd: "grep 'host' {{ database.source.credentials_file }}"
+  register: mysql_host_info_grep
+  delegate_to: "{{ database.source.host }}"
+  when: not database.source.fresh_db
+
+- name: Register source database host.
+  set_fact:
+    mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}"
+  delegate_to: "{{ database.source.host }}"
+  when: not database.source.fresh_db
+
+- name: Copy a nightly backup for the source database.
+  ansible.builtin.copy:
+    src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}"
+    dest: "{{ mysql_sync_source_dump_path }}"
+    remote_src: true
+  delegate_to: "{{ database.source.host }}"
+  when: not database.source.fresh_db
 
 - name: Register tmp target dump name.
   set_fact:
@@ -95,3 +154,21 @@
     path: "{{ mysql_sync_target_dump_path }}"
     state: absent
   when: not database.target.type == 'dump'
+
+- name: Enable all autoscale processes on source ASG.
+  ansible.builtin.command: >
+    aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.source.asg is defined
+    - database.source.asg | length > 0
+    - database.source.fresh_db is defined
+    - database.source.fresh_db
+
+- name: Enable all autoscale processes on target ASG.
+  ansible.builtin.command: >
+    aws autoscaling resume-processes --auto-scaling-group-name {{ database.target.asg }} --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.target.asg is defined
+    - database.target.asg | length > 0