From 45d639914b5b5ffbf5a2e3f86eaf4901dc21672f Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 30 May 2024 20:41:19 +0300 Subject: [PATCH 1/4] r69347-db-import-from-deploy-server --- .../database_sync-mysql/defaults/main.yml | 4 +- .../database_sync-mysql/tasks/sync.yml | 211 ++++++++++-------- 2 files changed, 119 insertions(+), 96 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 98ac19bf..84f38e42 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -13,9 +13,11 @@ mysql_sync: credentials_file: "/home/{{ deploy_user }}/.mysql.creds" # This can be of types: # - rolling: (database backups). In that case we'll need build parameters.@todo - # - fixed: "fixed" database name + # - fixed: "fixed" database name # currently this var means the same as "dump" # - dump: Use an existing dump. In that case, the "database" variable is the absolute file path. + # This parameter is ignored if var 'path_on_deploy_server' is defined. type: fixed + #path_on_deploy_server: "" # full path to the compressed dump file on the deploy server, overrides "type" variable for source DB when defined. # For "rolling builds", so we can compute the database name. build_id: mybuildprod # Whether or not use to create a fresh database backup or use a nightly one. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 41266982..fbf82ade 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -1,42 +1,38 @@ --- -- name: Get database source host region. - amazon.aws.ec2_metadata_facts: - register: mysql_sync_source_database_host_info - delegate_to: "{{ database.source.host }}" - when: - - database.source.asg is defined - - database.source.asg | length > 0 - - database.source.fresh_db is defined - - database.source.fresh_db - -# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host. -- name: Get database target host region. - amazon.aws.ec2_metadata_facts: - register: mysql_sync_target_database_host_info - when: - - database.target.asg is defined - - database.target.asg | length > 0 - -# @TODO - the autoscaling_group module can do this - https://docs.ansible.com/ansible/latest/collections/amazon/aws/autoscaling_group_module.html -- name: Disable ReplaceUnhealthy autoscale process on source ASG. - ansible.builtin.command: > - aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} - delegate_to: localhost +- name: Prepare source ASG. + block: + - name: Get database source host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_source_database_host_info + delegate_to: "{{ database.source.host }}" + + # @TODO - the autoscaling_group module can do this - https://docs.ansible.com/ansible/latest/collections/amazon/aws/autoscaling_group_module.html + - name: Disable ReplaceUnhealthy autoscale process on source ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost when: - database.source.asg is defined - database.source.asg | length > 0 - database.source.fresh_db is defined - database.source.fresh_db - -- name: Disable ReplaceUnhealthy autoscale process on target ASG. - ansible.builtin.command: > - aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} - delegate_to: localhost + - database.source.path_on_deploy_server is not defined + +- name: Prepare target ASG. + block: + # This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host. + - name: Get database target host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_target_database_host_info + + - name: Disable ReplaceUnhealthy autoscale process on target ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost when: - database.target.asg is defined - database.target.asg | length > 0 - - name: Register bzip2 archive type vars. ansible.builtin.set_fact: archive_file_type: "bz2" @@ -54,6 +50,8 @@ - name: Register remote dump name (from database). ansible.builtin.set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.{{ archive_file_type }}" + when: + - database.source.path_on_deploy_server is not defined - name: Get source last known good build number. ansible.builtin.command: @@ -64,46 +62,60 @@ - "{{ database.source.build_id }}" register: mysql_sync_source_build_number delegate_to: localhost - when: database.source.type == 'rolling' + when: + - database.source.type == 'rolling' - name: Register source database name. ansible.builtin.set_fact: mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}" - when: database.source.type == 'rolling' + when: + - database.source.type == 'rolling' - name: Register source database name. ansible.builtin.set_fact: mysql_sync_source_database: "{{ database.source.database }}" - when: not database.source.type == 'rolling' - -- name: Take a dump from source database. - ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}" - args: - executable: /bin/bash - delegate_to: "{{ database.source.host }}" when: - - database.source.fresh_db - -- name: Find source database host. - ansible.builtin.command: - cmd: "grep 'host' {{ database.source.credentials_file }}" - register: mysql_host_info_grep - delegate_to: "{{ database.source.host }}" - when: not database.source.fresh_db - -- name: Register source database host. - set_fact: - mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}" - delegate_to: "{{ database.source.host }}" - when: not database.source.fresh_db - -- name: Copy a nightly backup for the source database. - ansible.builtin.copy: - src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}" - dest: "{{ mysql_sync_source_dump_path }}" - remote_src: true + - not database.source.type == 'rolling' + - database.source.path_on_deploy_server is not defined + +- name: Fetch the source database. + block: + - name: Take a dump from source database. + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}" + args: + executable: /bin/bash + when: + - database.source.fresh_db + + - name: Find source database host. + ansible.builtin.command: + cmd: "grep 'host' {{ database.source.credentials_file }}" + register: mysql_host_info_grep + when: + - not database.source.fresh_db + + - name: Register source database host. + set_fact: + mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}" + when: + - not database.source.fresh_db + + - name: Copy a nightly backup for the source database. + ansible.builtin.copy: + src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}" + dest: "{{ mysql_sync_source_dump_path }}" + remote_src: true + when: + - not database.source.fresh_db + + - name: Fetch dump file. + ansible.builtin.fetch: + src: "{{ mysql_sync_source_dump_path }}" + dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" + flat: true delegate_to: "{{ database.source.host }}" - when: not database.source.fresh_db + when: + - database.source.path_on_deploy_server is not defined - name: Register tmp target dump name. ansible.builtin.set_fact: @@ -122,29 +134,34 @@ - "{{ database.target.build_id }}" register: mysql_sync_target_build_number delegate_to: localhost - when: database.target.type == 'rolling' + when: + - database.target.type == 'rolling' - name: Register target rolling database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}" - when: database.target.type == 'rolling' + when: + - database.target.type == 'rolling' - name: Register target static database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}" - when: not database.target.type == 'rolling' - -- name: Fetch dump file. - ansible.builtin.fetch: - src: "{{ mysql_sync_source_dump_path }}" - dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" - flat: true - delegate_to: "{{ database.source.host }}" + when: + - not database.target.type == 'rolling' - name: Copy dump file to destination. ansible.builtin.copy: src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" dest: "{{ mysql_sync_target_dump_path }}" + when: + - database.source.path_on_deploy_server is not defined + +- name: Copy dump file from deploy server to destination. + ansible.builtin.copy: + src: "{{ database.source.path_on_deploy_server }}" + dest: "{{ mysql_sync_target_dump_path }}" + when: + - database.source.path_on_deploy_server is defined - name: Unpack dump file. ansible.builtin.shell: "{{ archival_command }} -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" @@ -174,32 +191,36 @@ path: "{{ mysql_sync_target_dump_unpacked_path }}" state: absent -- name: Delete temporary dump file on source. - ansible.builtin.file: - path: "{{ mysql_sync_source_dump_path }}" - state: absent - delegate_to: "{{ database.source.host }}" - -- name: Delete temporary dump file on deploy server. - ansible.builtin.file: - path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}" - state: absent - delegate_to: localhost - when: - - mysql_sync.cleanup - with_items: - - ".bz2" - - ".gz" - -- name: Enable all autoscale processes on source ASG. - ansible.builtin.command: > - aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} - delegate_to: localhost - when: - - database.source.asg is defined - - database.source.asg | length > 0 - - database.source.fresh_db is defined - - database.source.fresh_db +- name: Clean up source. + block: + - name: Delete temporary dump file on source. + ansible.builtin.file: + path: "{{ mysql_sync_source_dump_path }}" + state: absent + delegate_to: "{{ database.source.host }}" + + - name: Delete temporary dump file on deploy server. + ansible.builtin.file: + path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}" + state: absent + delegate_to: localhost + when: + - mysql_sync.cleanup + with_items: + - ".bz2" + - ".gz" + + - name: Enable all autoscale processes on source ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + when: + - database.source.path_on_deploy_server is not defined - name: Enable all autoscale processes on target ASG. ansible.builtin.command: > From 25490bfff6b736f9701ebd4ecd3129d04a667ea4 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 31 May 2024 13:00:51 +0300 Subject: [PATCH 2/4] r69347-db-import-from-deploy-server --- .../database_sync-mysql/tasks/sync.yml | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index fbf82ade..01f2ca52 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -53,23 +53,24 @@ when: - database.source.path_on_deploy_server is not defined -- name: Get source last known good build number. - ansible.builtin.command: - argv: - - "/bin/sh" - - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh" - - "--build-id" - - "{{ database.source.build_id }}" - register: mysql_sync_source_build_number - delegate_to: localhost - when: - - database.source.type == 'rolling' +- name: Detect rolling DB name on source. + block: + - name: Get source last known good build number. + ansible.builtin.command: + argv: + - "/bin/sh" + - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh" + - "--build-id" + - "{{ database.source.build_id }}" + register: mysql_sync_source_build_number + delegate_to: localhost -- name: Register source database name. - ansible.builtin.set_fact: - mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}" + - name: Register source database name. + ansible.builtin.set_fact: + mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}" when: - database.source.type == 'rolling' + - database.source.path_on_deploy_server is not defined - name: Register source database name. ansible.builtin.set_fact: From 21ebc5dab86eab51b7f6ac292499f2974ce6f198 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 31 May 2024 14:21:08 +0300 Subject: [PATCH 3/4] r69347-db-import-from-deploy-server --- roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 01f2ca52..3904650a 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -69,6 +69,7 @@ ansible.builtin.set_fact: mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}" when: + - database.source.type is defined - database.source.type == 'rolling' - database.source.path_on_deploy_server is not defined @@ -76,6 +77,7 @@ ansible.builtin.set_fact: mysql_sync_source_database: "{{ database.source.database }}" when: + - database.source.type is defined - not database.source.type == 'rolling' - database.source.path_on_deploy_server is not defined From 7b7d454e698d1930825a4950898837fd665628c2 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 31 May 2024 17:53:15 +0300 Subject: [PATCH 4/4] typo fixes and tidying up --- .../database_sync/database_sync-mysql/defaults/main.yml | 4 +++- .../sync/database_sync/database_sync-mysql/tasks/sync.yml | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 84f38e42..359f9e55 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -17,7 +17,6 @@ mysql_sync: # - dump: Use an existing dump. In that case, the "database" variable is the absolute file path. # This parameter is ignored if var 'path_on_deploy_server' is defined. type: fixed - #path_on_deploy_server: "" # full path to the compressed dump file on the deploy server, overrides "type" variable for source DB when defined. # For "rolling builds", so we can compute the database name. build_id: mybuildprod # Whether or not use to create a fresh database backup or use a nightly one. @@ -27,6 +26,9 @@ mysql_sync: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. asg: "" + # Uncomment and specify full path if you want to use your own dump file on the deploy server (gzip compressed). + # In this case it's the only variable you need in 'database.source'. + #path_on_deploy_server: "/home/{{ deploy_user }}/compressed_database_dump.gz" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 3904650a..0ae03be9 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -117,7 +117,7 @@ dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" flat: true delegate_to: "{{ database.source.host }}" - when: + when: - database.source.path_on_deploy_server is not defined - name: Register tmp target dump name. @@ -156,14 +156,14 @@ ansible.builtin.copy: src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" dest: "{{ mysql_sync_target_dump_path }}" - when: + when: - database.source.path_on_deploy_server is not defined - name: Copy dump file from deploy server to destination. ansible.builtin.copy: src: "{{ database.source.path_on_deploy_server }}" dest: "{{ mysql_sync_target_dump_path }}" - when: + when: - database.source.path_on_deploy_server is defined - name: Unpack dump file. @@ -222,7 +222,7 @@ - database.source.asg | length > 0 - database.source.fresh_db is defined - database.source.fresh_db - when: + when: - database.source.path_on_deploy_server is not defined - name: Enable all autoscale processes on target ASG.