diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 16139d8f..3d47469a 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -31,6 +31,7 @@ - [Data backups](/roles/database_backup) - [MySQL backups](/roles/database_backup/database_backup-mysql) - [Deploy](/roles/deploy_code) + - [Deploy container](/roles/deploy_container) - [Init](/roles/_init) - [LHCI run](/roles/lhci_run) - ["Meta"](/roles/_meta) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 9cb9e024..edf1f5d3 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -15,6 +15,7 @@ deploy_user: "deploy" # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: + drush_verbose_output: false sites: - folder: "default" public_files: "sites/default/files" @@ -39,6 +40,7 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md new file mode 100644 index 00000000..8f7dca28 --- /dev/null +++ b/docs/roles/deploy_container.md @@ -0,0 +1,101 @@ +# Deploy container +Step that deploys the codebase in a Docker container image. Requires Docker and the `community.docker` collection for Ansible to be installed on your deploy server. You will also need to add a `docker` group and make sure your local deploy user is in that group, for example: + +``` +sudo groupadd docker +sudo usermod -aG docker deploy +``` + +This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles. + +AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the managed AWS `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers. + + + + + +## Default variables +```yaml +--- +deploy_container: + container_name: example + container_tag: latest # tag will take format container_name:container_tag + container_force_build: true # force Docker to build and tag a new image + docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name + docker_registry_user: example + docker_registry_pass: asdf1234 + docker_base_command: "docker image build" + docker_build_dir: "{{ _ce_deploy_build_dir }}" + dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name + environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template + # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached + aws_ecr: + enabled: false # set to true if using AWS ECR + region: eu-west-1 + aws_profile: example + # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached + # Note, you can if you wish make more restrictive roles and policies + aws_ecs: + enabled: false + region: eu-west-1 + aws_profile: example + tags: {} + domain_name: www.example.com + route_53: + zone: example.com + aws_profile: example2 # might not be the same account + vpc_name: example + #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use + subnets: # list of public subnet names + - example-dev-a + - example-dev-b + security_groups: [] # list of security groups, accepts names or IDs + cluster_name: example + family_name: example + task_definition_revision: "" # integer, but must be presented as a string for Jinja2 + task_count: 1 + task_minimum_count: 1 + task_maximum_count: 4 + # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html + service_autoscale_metric_type: ECSServiceAverageCPUUtilization + service_autoscale_up_cooldown: 120 + service_autoscale_down_cooldown: 120 + service_autoscale_target_value: 70 # the value to trigger a scaling event at + execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable + containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers + cpu: 512 # these values can be set globally or per container + memory: 1024 + launch_type: FARGATE + network_mode: awsvpc + #volumes: [] # list of additional volumes to attach + target_group_name: example # 32 character limit + target_group_protocol: http + target_group_port: 80 + target_group_wait_timeout: 200 # how long to wait for target group events to complete + targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service + #- Id: 10.0.0.2 + # Port: 80 + # AvailabilityZone: all + health_check: + protocol: http + path: / + response_codes: "200" + # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached + acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm + create_cert: false + extra_domains: [] # list of Subject Alternative Name domains and zones + ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager + elb_security_groups: [] # default SG is used if none provided - module supports names or IDs + elb_http_port: 80 + elb_https_port: 443 + elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + elb_listener_http_rules: [] + elb_listener_https_rules: [] + # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html + elb_listeners: [] + elb_idle_timeout: 60 + elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses). + +``` + + diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md index cf7e5311..cbe1a353 100644 --- a/docs/roles/sync/database_sync/database_sync-mysql.md +++ b/docs/roles/sync/database_sync/database_sync-mysql.md @@ -6,6 +6,7 @@ Sync MySQL databases between environments. --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. diff --git a/roles/_init/README.md b/roles/_init/README.md index 9cb9e024..edf1f5d3 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -15,6 +15,7 @@ deploy_user: "deploy" # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: + drush_verbose_output: false sites: - folder: "default" public_files: "sites/default/files" @@ -39,6 +40,7 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9a955f3e..57ee7e44 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -6,6 +6,7 @@ deploy_user: "deploy" # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: + drush_verbose_output: false sites: - folder: "default" public_files: "sites/default/files" @@ -30,3 +31,4 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a0d90571..c054e8fb 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -124,3 +124,4 @@ - name: Ensure we have a cachetool binary. ansible.builtin.import_role: name: cli/cachetool + when: install_php_cachetool diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml index 6ce31d4f..3e66875b 100644 --- a/roles/_meta/deploy-drupal8/tasks/main.yml +++ b/roles/_meta/deploy-drupal8/tasks/main.yml @@ -20,14 +20,14 @@ name: database_backup - import_role: name: config_generate +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: database_apply - import_role: name: sanitize/admin_creds - import_role: name: live_symlink -- import_role: - name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal8 - import_role: diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml index 5b780086..e9524b81 100644 --- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Clear Drupal cache. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -9,3 +9,10 @@ loop_control: loop_var: site run_once: true + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output + diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f8e3b55e..55d9dee7 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -19,6 +19,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ playbook_dir }}/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 4f476253..3012032f 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -26,11 +26,6 @@ name: "cache_clear/cache_clear-{{ project_type }}" when: previous_build_number > 0 -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 4dc66a3a..e2fecd17 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -23,6 +23,12 @@ loop_control: loop_var: site when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output - name: Fix permissions on Drupal directory. ansible.builtin.file: @@ -44,11 +50,6 @@ - previous_build_number > 0 - site.config_import_command != 'deploy' -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" @@ -59,6 +60,12 @@ loop_control: loop_var: site when: site.config_import_command != 'deploy' + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output - name: Import configuration. ansible.builtin.command: @@ -72,3 +79,10 @@ when: - previous_build_number > 0 - site.config_import_command + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output + diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 80898e7d..38522af3 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,7 +1,22 @@ --- +- name: Unpack dump file. + ansible.builtin.unarchive: + src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + remote_src: true + run_once: true + when: previous_build_number > 0 + - name: Revert database from dump. - ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql" args: executable: /bin/bash + run_once: true when: previous_build_number > 0 + +- name: Delete unpacked dump file. + ansible.builtin.file: + path: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + state: absent run_once: true + when: previous_build_number > 0 diff --git a/roles/deploy_code/deploy_code-custom/tasks/main.yml b/roles/deploy_code/deploy_code-custom/tasks/main.yml new file mode 100644 index 00000000..6f34c92f --- /dev/null +++ b/roles/deploy_code/deploy_code-custom/tasks/main.yml @@ -0,0 +1,3 @@ +--- + +# Nothing to do here. \ No newline at end of file diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index db2bb9bb..b3a4f132 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -150,7 +150,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - deploy_code.services | length > 0 - name: Stop any services that might be keeping the loop device busy. @@ -166,7 +166,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - deploy_code.services | length > 0 - name: Unmount existing SquashFS image. @@ -177,7 +177,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - name: Mount new SquashFS image. ansible.builtin.command: diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md new file mode 100644 index 00000000..8f7dca28 --- /dev/null +++ b/roles/deploy_container/README.md @@ -0,0 +1,101 @@ +# Deploy container +Step that deploys the codebase in a Docker container image. Requires Docker and the `community.docker` collection for Ansible to be installed on your deploy server. You will also need to add a `docker` group and make sure your local deploy user is in that group, for example: + +``` +sudo groupadd docker +sudo usermod -aG docker deploy +``` + +This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles. + +AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the managed AWS `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers. + + + + + +## Default variables +```yaml +--- +deploy_container: + container_name: example + container_tag: latest # tag will take format container_name:container_tag + container_force_build: true # force Docker to build and tag a new image + docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name + docker_registry_user: example + docker_registry_pass: asdf1234 + docker_base_command: "docker image build" + docker_build_dir: "{{ _ce_deploy_build_dir }}" + dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name + environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template + # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached + aws_ecr: + enabled: false # set to true if using AWS ECR + region: eu-west-1 + aws_profile: example + # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached + # Note, you can if you wish make more restrictive roles and policies + aws_ecs: + enabled: false + region: eu-west-1 + aws_profile: example + tags: {} + domain_name: www.example.com + route_53: + zone: example.com + aws_profile: example2 # might not be the same account + vpc_name: example + #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use + subnets: # list of public subnet names + - example-dev-a + - example-dev-b + security_groups: [] # list of security groups, accepts names or IDs + cluster_name: example + family_name: example + task_definition_revision: "" # integer, but must be presented as a string for Jinja2 + task_count: 1 + task_minimum_count: 1 + task_maximum_count: 4 + # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html + service_autoscale_metric_type: ECSServiceAverageCPUUtilization + service_autoscale_up_cooldown: 120 + service_autoscale_down_cooldown: 120 + service_autoscale_target_value: 70 # the value to trigger a scaling event at + execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable + containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers + cpu: 512 # these values can be set globally or per container + memory: 1024 + launch_type: FARGATE + network_mode: awsvpc + #volumes: [] # list of additional volumes to attach + target_group_name: example # 32 character limit + target_group_protocol: http + target_group_port: 80 + target_group_wait_timeout: 200 # how long to wait for target group events to complete + targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service + #- Id: 10.0.0.2 + # Port: 80 + # AvailabilityZone: all + health_check: + protocol: http + path: / + response_codes: "200" + # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached + acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm + create_cert: false + extra_domains: [] # list of Subject Alternative Name domains and zones + ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager + elb_security_groups: [] # default SG is used if none provided - module supports names or IDs + elb_http_port: 80 + elb_https_port: 443 + elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + elb_listener_http_rules: [] + elb_listener_https_rules: [] + # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html + elb_listeners: [] + elb_idle_timeout: 60 + elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses). + +``` + + diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml new file mode 100644 index 00000000..7605f796 --- /dev/null +++ b/roles/deploy_container/defaults/main.yml @@ -0,0 +1,79 @@ +--- +deploy_container: + container_name: example + container_tag: latest # tag will take format container_name:container_tag + container_force_build: true # force Docker to build and tag a new image + docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name + docker_registry_user: example + docker_registry_pass: asdf1234 + docker_base_command: "docker image build" + docker_build_dir: "{{ _ce_deploy_build_dir }}" + dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name + environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template + # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached + aws_ecr: + enabled: false # set to true if using AWS ECR + region: eu-west-1 + aws_profile: example + # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached + # Note, you can if you wish make more restrictive roles and policies + aws_ecs: + enabled: false + region: eu-west-1 + aws_profile: example + tags: {} + domain_name: www.example.com + route_53: + zone: example.com + aws_profile: example2 # might not be the same account + vpc_name: example + #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use + subnets: # list of public subnet names + - example-dev-a + - example-dev-b + security_groups: [] # list of security groups, accepts names or IDs + cluster_name: example + family_name: example + task_definition_revision: "" # integer, but must be presented as a string for Jinja2 + task_count: 1 + task_minimum_count: 1 + task_maximum_count: 4 + # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html + service_autoscale_metric_type: ECSServiceAverageCPUUtilization + service_autoscale_up_cooldown: 120 + service_autoscale_down_cooldown: 120 + service_autoscale_target_value: 70 # the value to trigger a scaling event at + execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable + containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers + cpu: 512 # these values can be set globally or per container + memory: 1024 + launch_type: FARGATE + network_mode: awsvpc + #volumes: [] # list of additional volumes to attach + target_group_name: example # 32 character limit + target_group_protocol: http + target_group_port: 80 + target_group_wait_timeout: 200 # how long to wait for target group events to complete + targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service + #- Id: 10.0.0.2 + # Port: 80 + # AvailabilityZone: all + health_check: + protocol: http + path: / + response_codes: "200" + # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached + acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm + create_cert: false + extra_domains: [] # list of Subject Alternative Name domains and zones + ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager + elb_security_groups: [] # default SG is used if none provided - module supports names or IDs + elb_http_port: 80 + elb_https_port: 443 + elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + elb_listener_http_rules: [] + elb_listener_https_rules: [] + # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html + elb_listeners: [] + elb_idle_timeout: 60 + elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses). diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml new file mode 100644 index 00000000..34866aa1 --- /dev/null +++ b/roles/deploy_container/tasks/main.yml @@ -0,0 +1,351 @@ +--- +# Build and ship a container image +- name: Create Dockerfile from template. + ansible.builtin.template: + src: "{{ deploy_container.dockerfile_template }}" + dest: "{{ deploy_container.docker_build_dir }}/Dockerfile" + delegate_to: localhost + +- name: Set Docker registry username and password. + ansible.builtin.set_fact: + _docker_registry_username: "{{ deploy_container.docker_registry_user }}" + _docker_registry_password: "{{ deploy_container.docker_registry_pass }}" + delegate_to: localhost + +- name: Fetch AWS ECR registry login token. # token valid for 12 hours + ansible.builtin.command: + cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + register: _docker_registry_ecr_token + +- name: Set AWS ECR registry password. + ansible.builtin.set_fact: + _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + +- name: Set AWS ECR registry username. + ansible.builtin.set_fact: + _docker_registry_username: "AWS" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + +- name: Remove Docker credentials file. + ansible.builtin.file: + state: absent + path: "/home/{{ deploy_user }}/.docker/config.json" + delegate_to: localhost + +- name: Log into Docker registry. + community.docker.docker_login: + registry_url: "{{ deploy_container.docker_registry_url }}" + username: "{{ _docker_registry_username }}" + password: "{{ _docker_registry_password }}" + reauthorize: true + delegate_to: localhost + +- name: Build and push container image. + community.docker.docker_image: + build: + path: "{{ deploy_container.docker_build_dir }}" + name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}" + tag: "{{ deploy_container.container_tag | default('latest') }}" + push: true + source: build + force_source: "{{ deploy_container.container_force_build }}" + force_tag: "{{ deploy_container.container_force_build }}" + delegate_to: localhost + +# Fetch the ACM role from ce-provision +- name: Ensure the aws_acm directory exists. + ansible.builtin.file: + path: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}" + state: directory + mode: '0755' + delegate_to: localhost + with_items: + - tasks + - defaults + +- name: Fetch the aws_acm files. + ansible.builtin.get_url: + url: "https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/{{ item }}/main.yml" + dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}/main.yml" + delegate_to: localhost + with_items: + - tasks + - defaults + +- name: Fetch the aws_acm tasks. + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/tasks/main.yml + dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/tasks/main.yml" + delegate_to: localhost + +# Gather all network information +- name: Gather VPC information. + amazon.aws.ec2_vpc_net_info: + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + region: "{{ deploy_container.aws_ecs.region }}" + filters: + "tag:Name": "{{ deploy_container.aws_ecs.vpc_name }}" + register: _aws_ecs_cluster_vpc + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.vpc_name is defined + - deploy_container.aws_ecs.vpc_name | length > 0 + +- name: Set the VPC id from name. + ansible.builtin.set_fact: + _aws_ecs_cluster_vpc_id: "{{ _aws_ecs_cluster_vpc.vpcs[0].vpc_id }}" + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.vpc_name is defined + - deploy_container.aws_ecs.vpc_name | length > 0 + +- name: Use provided VPC id. + ansible.builtin.set_fact: + _aws_ecs_cluster_vpc_id: "{{ deploy_container.aws_ecs.vpc_id }}" + when: + - deploy_container.aws_ecs.enabled + - (deploy_container.aws_ecs.vpc_name is not defined or deploy_container.aws_ecs.vpc_name | length < 0) + +- name: Reset subnets lists. + ansible.builtin.set_fact: + _aws_ecs_cluster_public_subnets_ids: [] + when: deploy_container.aws_ecs.enabled + +- name: Construct list of public subnet IDs. + ansible.builtin.include_tasks: subnet.yml + with_items: "{{ deploy_container.aws_ecs.subnets }}" + loop_control: + loop_var: subnet + when: deploy_container.aws_ecs.enabled + +# Construct AWS supporting assets +- name: Create task definition. + community.aws.ecs_taskdefinition: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + family: "{{ deploy_container.aws_ecs.family_name }}" + execution_role_arn: "{{ deploy_container.aws_ecs.execution_role_arn }}" + containers: "{{ deploy_container.aws_ecs.containers }}" + launch_type: "{{ deploy_container.aws_ecs.launch_type }}" + cpu: "{{ deploy_container.aws_ecs.cpu | default(omit) }}" + memory: "{{ deploy_container.aws_ecs.memory | default(omit) }}" + state: present + network_mode: "{{ deploy_container.aws_ecs.network_mode }}" + volumes: "{{ deploy_container.aws_ecs.volumes | default(omit) }}" + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Create a target group with IP address targets. + community.aws.elb_target_group: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + protocol: "{{ deploy_container.aws_ecs.target_group_protocol }}" + port: "{{ deploy_container.aws_ecs.target_group_port }}" + vpc_id: "{{ _aws_ecs_cluster_vpc_id }}" + health_check_protocol: "{{ deploy_container.aws_ecs.health_check.protocol }}" + health_check_path: "{{ deploy_container.aws_ecs.health_check.path }}" + successful_response_codes: "{{ deploy_container.aws_ecs.health_check.response_codes }}" + target_type: ip + targets: "{{ deploy_container.aws_ecs.targets }}" + state: present + wait_timeout: "{{ deploy_container.aws_ecs.target_group_wait_timeout }}" + wait: true + register: _aws_ecs_target_group + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Create SSL certificate for load balancer. + ansible.builtin.include_role: + name: aws_acm + vars: + aws_acm: + region: "{{ deploy_container.aws_ecs.region }}" + aws_profile: "{{ deploy_container.aws_ecs.aws_profile }}" + tags: "{{ deploy_container.aws_ecs.tags }}" + export: false + domain_name: "{{ deploy_container.aws_ecs.domain_name }}" + extra_domains: "{{ deploy_container.aws_ecs.acm.extra_domains }}" + route_53: + aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}" + zone: "{{ deploy_container.aws_ecs.route_53.zone }}" + when: + - deploy_container.aws_ecs.acm.create_cert + - deploy_container.aws_ecs.enabled + +- name: Default to provided SSL certificate ARN. + ansible.builtin.set_fact: + _ssl_certificate_ARN: "{{ deploy_container.aws_ecs.ssl_certificate_ARN }}" + when: deploy_container.aws_ecs.enabled + +- name: If provided, override SSL certificate ARN with the one received from ACM. + ansible.builtin.set_fact: + _ssl_certificate_ARN: "{{ aws_acm_certificate_arn }}" + when: + - deploy_container.aws_ecs.acm.create_cert + - deploy_container.aws_ecs.enabled + +- name: Define default ALB listeners. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners_http: + Protocol: HTTP + Port: "{{ deploy_container.aws_ecs.elb_http_port }}" + DefaultActions: + - Type: forward + TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" + Rules: "{{ deploy_container.aws_ecs.elb_listener_http_rules }}" + _aws_ecs_cluster_listeners_redirect: + Protocol: HTTP + Port: "{{ deploy_container.aws_ecs.elb_http_port }}" + DefaultActions: + - Type: redirect + RedirectConfig: + Protocol: HTTPS + Host: "#{host}" + Query: "#{query}" + Path: "/#{path}" + Port: "{{ deploy_container.aws_ecs.elb_https_port }}" + StatusCode: HTTP_301 + _aws_ecs_cluster_listeners_https: + Protocol: HTTPS + Port: "{{ deploy_container.aws_ecs.elb_https_port }}" + SslPolicy: "{{ deploy_container.aws_ecs.elb_ssl_policy }}" + Certificates: + - CertificateArn: "{{ _ssl_certificate_ARN }}" + DefaultActions: + - Type: forward + TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" + Rules: "{{ deploy_container.aws_ecs.elb_listener_https_rules }}" + when: deploy_container.aws_ecs.enabled + +- name: Add HTTP listeners. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_http ] }}" + when: + - _ssl_certificate_ARN | length < 1 + - deploy_container.aws_ecs.enabled + +- name: Add HTTPS Listener. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_redirect, _aws_ecs_cluster_listeners_https ] }}" + when: + - _ssl_certificate_ARN | length > 1 + - deploy_container.aws_ecs.enabled + +- name: Add custom Listeners. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners: "{{ _aws_ecs_cluster_listeners + deploy_container.aws_ecs.elb_listeners }}" + when: + - deploy_container.aws_ecs.elb_listeners is defined + - deploy_container.aws_ecs.elb_listeners | length + - deploy_container.aws_ecs.enabled + +- name: Create an ALB. + amazon.aws.elb_application_lb: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + state: present + tags: "{{ deploy_container.aws_ecs.tags }}" + subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}" + security_groups: "{{ deploy_container.aws_ecs.elb_security_groups }}" + listeners: "{{ _aws_ecs_cluster_listeners }}" + idle_timeout: "{{ deploy_container.aws_ecs.elb_idle_timeout }}" + ip_address_type: "{{ deploy_container.aws_ecs.elb_ip_address_type }}" + register: _aws_ecs_cluster_alb + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Set task definition name. + ansible.builtin.set_fact: + _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}" + when: deploy_container.aws_ecs.enabled + +- name: Set task definition revision if applicable. + ansible.builtin.set_fact: + _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}:{{ deploy_container.aws_ecs.task_definition_revision }}" + when: + - deploy_container.aws_ecs.task_definition_revision | length > 0 + - deploy_container.aws_ecs.enabled + +- name: Create ECS service. + community.aws.ecs_service: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: present + name: "{{ deploy_container.aws_ecs.family_name }}" + cluster: "{{ deploy_container.aws_ecs.cluster_name }}" + task_definition: "{{ _aws_ecs_service_task_definition }}" + desired_count: "{{ deploy_container.aws_ecs.task_count }}" + launch_type: "{{ deploy_container.aws_ecs.launch_type }}" + platform_version: LATEST + load_balancers: # see https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_LoadBalancer.html + - containerName: "{{ deploy_container.container_name }}" + containerPort: "{{ deploy_container.aws_ecs.target_group_port }}" + targetGroupArn: "{{ _aws_ecs_target_group.target_group_arn }}" + network_configuration: + subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}" + security_groups: "{{ deploy_container.aws_ecs.security_groups }}" + assign_public_ip: true # must be true for now - details: https://stackoverflow.com/a/66802973 + tags: "{{ deploy_container.aws_ecs.tags }}" + wait: true + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Create target tracking scaling policy for ECS service. + community.aws.application_autoscaling_policy: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: present + policy_name: "{{ deploy_container.aws_ecs.family_name }}" + service_namespace: ecs + resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}" + scalable_dimension: ecs:service:DesiredCount + minimum_tasks: "{{ deploy_container.aws_ecs.task_minimum_count }}" + maximum_tasks: "{{ deploy_container.aws_ecs.task_maximum_count }}" + policy_type: TargetTrackingScaling + target_tracking_scaling_policy_configuration: + PredefinedMetricSpecification: + PredefinedMetricType: "{{ deploy_container.aws_ecs.service_autoscale_metric_type }}" + ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_up_cooldown }}" + ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_down_cooldown }}" + DisableScaleIn: false + TargetValue: "{{ deploy_container.aws_ecs.service_autoscale_target_value }}" + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Initialise the domains loop var with main domain entry DNS settings. + ansible.builtin.set_fact: + _aws_ecs_cluster_dns_all_domains: + - domain: "{{ deploy_container.aws_ecs.domain_name }}" + zone: "{{ deploy_container.aws_ecs.route_53.zone }}" + aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}" + when: deploy_container.aws_ecs.enabled + +- name: Add extra_domains so we can loop through DNS records. + ansible.builtin.set_fact: + _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}" + loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}" + when: + - deploy_container.aws_ecs.acm.extra_domains | length > 0 + - deploy_container.aws_ecs.enabled + +- name: Add DNS records in Route 53. + amazon.aws.route53: + state: present + profile: "{{ item.aws_profile }}" + zone: "{{ item.zone }}" + record: "{{ item.domain }}" + type: CNAME + value: "{{ _aws_ecs_cluster_alb.dns_name }}" + overwrite: true + loop: "{{ _aws_ecs_cluster_dns_all_domains }}" + when: + - deploy_container.aws_ecs.route_53.zone | length > 0 + - deploy_container.aws_ecs.enabled \ No newline at end of file diff --git a/roles/deploy_container/tasks/subnet.yml b/roles/deploy_container/tasks/subnet.yml new file mode 100644 index 00000000..a0701a36 --- /dev/null +++ b/roles/deploy_container/tasks/subnet.yml @@ -0,0 +1,13 @@ +- name: Gather public subnet information. + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + region: "{{ deploy_container.aws_ecs.region }}" + filters: + vpc-id: "{{ _aws_ecs_cluster_vpc_id }}" + tag:Name: "{{ subnet }}" + register: _aws_ecs_cluster_public_subnet + delegate_to: localhost + +- name: Add public subnet to the list. + ansible.builtin.set_fact: + _aws_ecs_cluster_public_subnets_ids: "{{ _aws_ecs_cluster_public_subnets_ids + [ _aws_ecs_cluster_public_subnet.subnets[0].subnet_id ] }}" diff --git a/roles/deploy_container/templates/example.j2 b/roles/deploy_container/templates/example.j2 new file mode 100644 index 00000000..5a155197 --- /dev/null +++ b/roles/deploy_container/templates/example.j2 @@ -0,0 +1,7 @@ +# Basic Dockerfile example +FROM debian:bullseye-slim +MAINTAINER sysadm@codeenigma.com + +RUN apt-get update +RUN apt-get install -y nginx +CMD ["echo","Image created"] \ No newline at end of file diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md index cf7e5311..cbe1a353 100644 --- a/roles/sync/database_sync/database_sync-mysql/README.md +++ b/roles/sync/database_sync/database_sync-mysql/README.md @@ -6,6 +6,7 @@ Sync MySQL databases between environments. --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 0ab99e98..88143a7b 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,6 +1,8 @@ --- mysql_sync: - mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here. + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes. + archival_method: "bzip2" # oprions are "bzip2" or "gzip". databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index c695982c..ac346444 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -35,9 +35,24 @@ - database.target.asg is defined - database.target.asg | length > 0 + +- name: Register bzip2 archive type vars. + ansible.builtin.set_fact: + archive_file_type: "bz2" + archival_command: "bzip2" + when: + - mysql_sync.archival_method == 'bzip2' + +- name: Register gunzip archive type vars. + ansible.builtin.set_fact: + archive_file_type: "gz" + archival_command: "gzip" + when: + - mysql_sync.archival_method == 'gzip' + - name: Register remote dump name (from database). ansible.builtin.set_fact: - mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" + mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.{{ archive_file_type }}" - name: Get source last known good build number. ansible.builtin.command: @@ -61,7 +76,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}" args: executable: /bin/bash delegate_to: "{{ database.source.host }}" @@ -91,7 +106,11 @@ - name: Register tmp target dump name. ansible.builtin.set_fact: - mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" + mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.{{ archive_file_type }}" + +- name: Register tmp unpacked target dump name. + ansible.builtin.set_fact: + mysql_sync_target_dump_unpacked_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql" - name: Get target last known good build number. ansible.builtin.command: @@ -104,12 +123,12 @@ delegate_to: localhost when: database.target.type == 'rolling' -- name: Register target database name. +- name: Register target rolling database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}" when: database.target.type == 'rolling' -- name: Register target database name. +- name: Register target static database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}" when: not database.target.type == 'rolling' @@ -117,15 +136,25 @@ - name: Fetch dump file. ansible.builtin.fetch: src: "{{ mysql_sync_source_dump_path }}" - dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" + dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" flat: true delegate_to: "{{ database.source.host }}" - name: Copy dump file to destination. ansible.builtin.copy: - src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" + src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" dest: "{{ mysql_sync_target_dump_path }}" +- name: Unpack dump file. + ansible.builtin.shell: "{{ archival_command }} -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" + args: + executable: /bin/bash + +- name: Delete temporary dump file on target. + ansible.builtin.file: + path: "{{ mysql_sync_target_dump_path }}" + state: absent + - name: Drop target database. ansible.builtin.command: cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" @@ -135,14 +164,31 @@ cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - name: Repopulate database from dump. - ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}" args: executable: /bin/bash -- name: Remove tmp dump file. +- name: Delete temporary unpacked dump file on target. ansible.builtin.file: - path: "{{ mysql_sync_target_dump_path }}" + path: "{{ mysql_sync_target_dump_unpacked_path }}" + state: absent + +- name: Delete temporary dump file on source. + ansible.builtin.file: + path: "{{ mysql_sync_source_dump_path }}" state: absent + delegate_to: "{{ database.source.host }}" + +- name: Delete temporary dump file on deploy server. + ansible.builtin.file: + path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}" + state: absent + delegate_to: localhost + when: + - mysql_sync.cleanup + with_items: + - ".bz2" + - ".gz" - name: Enable all autoscale processes on source ASG. ansible.builtin.command: > diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index 96d9b65f..d13b0129 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -17,6 +17,12 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output # This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run. - name: Import configuration. @@ -31,6 +37,12 @@ when: - site.config_import_command - site.sync_config_import is not defined or site.sync_config_import + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output - name: Clear the cache. ansible.builtin.include_role: diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml index 5b65b1fb..4e222b40 100644 --- a/roles/sync/files_sync/defaults/main.yml +++ b/roles/sync/files_sync/defaults/main.yml @@ -1,5 +1,8 @@ --- files_sync: + unique_workspace: false # set to true to grab a complete full set of files every sync + # Generally speaking you will *not* want to clean up after file syncs, as leaving the files there makes the next rsync far quicker. + cleanup: false # set to true to delete the synced files after a sync directories: - source: # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index e2160008..516059c2 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -1,7 +1,16 @@ --- +- name: Register file sync location. + ansible.builtin.set_fact: + file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + +- name: Register unique file sync location. + ansible.builtin.set_fact: + file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}_{{ build_number }}" + when: files_sync.unique_workspace + - name: Create a temporary directory for source files on localhost. ansible.builtin.file: - path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + path: "{{ file_sync_path }}" state: directory owner: "{{ deploy_user }}" group: "{{ deploy_user }}" @@ -10,12 +19,20 @@ - name: Copy the source files onto the deploy server. ansible.builtin.command: - cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" - delegate_to: "localhost" + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ file_sync_path }}/" + delegate_to: localhost run_once: true - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.command: - cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" - delegate_to: "localhost" + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ file_sync_path }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + delegate_to: localhost run_once: true + +- name: Delete synced files on deploy server. + ansible.builtin.file: + path: "{{ file_sync_path }}" + state: absent + delegate_to: localhost + when: + - files_sync.cleanup diff --git a/scripts/_common.sh b/scripts/_common.sh index 3715c97a..74ba9a60 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -9,6 +9,7 @@ export ANSIBLE_CONFIG="$OWN_DIR/ansible.cfg" TARGET_DEPLOY_REPO="" TARGET_DEPLOY_PLAYBOOK="" TARGET_DEPLOY_BRANCH="" +TARGET_DEPLOY_HOST="" PREVIOUS_BUILD_NUMBER="" CURRENT_BUILD_NUMBER="" ANSIBLE_EXTRA_VARS="" @@ -49,6 +50,10 @@ parse_options(){ shift TARGET_DEPLOY_PLAYBOOK="$1" ;; + "--host") + shift + TARGET_DEPLOY_HOST="$1" + ;; "--build-number") shift CURRENT_BUILD_NUMBER="$1" @@ -146,6 +151,25 @@ cleanup_build_tmp_dir(){ fi } +# Call Ansible playbook to ensure host exists. +ansible_host_check(){ + if [ -n "$TARGET_DEPLOY_HOST" ]; then + ANSIBLE_BIN=$(command -v ansible-playbook) + ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml" + if [ "$VERBOSE" = "yes" ]; then + ANSIBLE_CMD="$ANSIBLE_CMD -vvvv" + fi + if [ -n "$BOTO_PROFILE" ]; then + export AWS_PROFILE="$BOTO_PROFILE" + fi + $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS" + return $? + # No host to check provided, just return a clean exit code. + else + return 0 + fi +} + # Trigger actual Ansible job. # $1 (string) # Operation to perform. diff --git a/scripts/build.sh b/scripts/build.sh index 5f7fa21b..93f36887 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -19,6 +19,7 @@ usage(){ echo '--branch: The branch to deploy.' echo '' echo 'Available options:' + echo '--host: Valid Ansible hostname, if you want to run a host check. Can also be a group name.' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' @@ -94,6 +95,15 @@ fi # Get Ansible defaults. get_ansible_defaults_vars +# Optionally carry out a host check if --host is provided. +ansible_host_check +ANSIBLE_HOST_CHECK_RESULT=$? +# Exit early if host not found. +if [ -n "$ANSIBLE_HOST_CHECK_RESULT" ] && [ "$ANSIBLE_HOST_CHECK_RESULT" != 0 ]; then + echo "ce-deploy failed to find the host. Aborting." + exit 1 +fi + # From this point on, we want to trigger the "revert" if anything fails. ANSIBLE_BUILD_RESULT=1 # Trigger deploy. @@ -109,5 +119,5 @@ if [ -n "$ANSIBLE_BUILD_RESULT" ] && [ "$ANSIBLE_BUILD_RESULT" = 0 ]; then exit 0 fi # Failed somehow. Normally unreachable in strict mode. -echo "Something went wrong. Please fill a bug report against ce-deploy." +echo "Something went unexpectedly wrong with ce-deploy. Please file a bug report - https://github.com/codeenigma/ce-deploy/issues/new" exit 1 \ No newline at end of file diff --git a/scripts/host-check.yml b/scripts/host-check.yml new file mode 100644 index 00000000..980c63cb --- /dev/null +++ b/scripts/host-check.yml @@ -0,0 +1,15 @@ +- hosts: localhost + connection: local + become: false + tasks: + - name: Ensure the hostname check variable is empty. + ansible.builtin.set_fact: + _ce_deploy_ansible_host_check: "" + - name: Check to see if the Ansible host or hostgroup exists. + ansible.builtin.set_fact: + _ce_deploy_ansible_host_check: "{{ item }}" + with_inventory_hostnames: + - "{{ _deploy_host }}" + - ansible.builtin.fail: + msg: "Host does not exist!" + when: _ce_deploy_ansible_host_check | length == 0