diff --git a/.github/workflows/ce-deploy-lint.yml b/.github/workflows/ce-deploy-lint.yml new file mode 100644 index 00000000..283cd3b9 --- /dev/null +++ b/.github/workflows/ce-deploy-lint.yml @@ -0,0 +1,28 @@ +name: Linting + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + run-linting: + # Name the Job + name: Linting + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Linter checks. + - name: Run linters. + if: ${{ always() }} + run: | + sudo apt-get update + sudo apt-get install -y ansible-lint yamllint shellcheck + find ./roles -name "*.yml" | xargs ansible-lint + yamllint ./roles + cd scripts && shellcheck *.sh diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-publish-docs.yml similarity index 61% rename from .github/workflows/ce-deploy-test.yml rename to .github/workflows/ce-deploy-publish-docs.yml index b93a6d3e..45148600 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-publish-docs.yml @@ -1,14 +1,18 @@ -name: Run tests and build docs +name: Publish documentation # Run this workflow every time a new commit pushed to your repository -on: pull_request +on: + pull_request: + types: [closed] + branches: + - 1.x jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided - run-tests: + publish-docs: # Name the Job - name: Run tests against Ansible code base + name: Publish ce-deploy documentation # Set the type of machine to run on runs-on: ubuntu-20.04 @@ -17,15 +21,11 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - # Linter checks. - - name: Run linters. - if: ${{ always() }} + # Configures global Git variables for committing + - name: Configure Git run: | - sudo apt-get update - sudo apt-get install -y ansible-lint yamllint shellcheck - find ./roles -name "*.yml" | xargs ansible-lint - yamllint ./roles - cd scripts && shellcheck *.sh + git config --global user.email "sysadm@codeenigma.com" + git config --global user.name "Code Enigma CI" # Installs the ce-dev stack - name: Install ce-dev @@ -41,46 +41,6 @@ jobs: rm -Rf mkcert curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux - # Configures global Git variables for committing - - name: Configure Git - run: | - git config --global user.email "sysadm@codeenigma.com" - git config --global user.name "Code Enigma CI" - - # Uses the ce-dev stack to run a test Drupal deploy - - name: Run a test ce-dev deploy with Drupal 8 - run: | - ce-dev create -p test -t drupal8 -d ~/test - cd ~/test - ce-dev init - ce-dev start - ce-dev provision - ce-dev deploy - curl https://www.test.local - shell: bash - - - name: Run a test ce-dev deploy with Drupal 9 - run: | - ce-dev create -p testnine -t drupal9 -d ~/testnine - cd ~/testnine - ce-dev init - ce-dev start - ce-dev provision - ce-dev deploy - curl https://www.testnine.local - shell: bash - - - name: Run a test ce-dev deploy with Localgov - run: | - ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov - cd ~/testlocalgov - ce-dev init - ce-dev start - ce-dev provision - ce-dev deploy - curl https://www.testlocalgov.local - shell: bash - # Builds the table of contents for the docs - name: Documentation (build table of contents) if: ${{ github.event.pull_request.base.ref == '1.x' }} diff --git a/.github/workflows/ce-deploy-test-drupal8.yml b/.github/workflows/ce-deploy-test-drupal8.yml new file mode 100644 index 00000000..ec9909a9 --- /dev/null +++ b/.github/workflows/ce-deploy-test-drupal8.yml @@ -0,0 +1,44 @@ +name: Run test for Drupal 8 + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + run-tests: + # Name the Job + name: Run Drupal 8 test build + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + run: | + cd /tmp + wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test Drupal deploy + - name: Run a test ce-dev deploy with Drupal 8 + run: | + ce-dev create -p test -t drupal8 -d ~/test + cd ~/test + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.test.local + shell: bash diff --git a/.github/workflows/ce-deploy-test-drupal9.yml b/.github/workflows/ce-deploy-test-drupal9.yml new file mode 100644 index 00000000..afaaca7a --- /dev/null +++ b/.github/workflows/ce-deploy-test-drupal9.yml @@ -0,0 +1,44 @@ +name: Run test for Drupal 9 + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + run-tests: + # Name the Job + name: Run Drupal 9 test build + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + run: | + cd /tmp + wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test Drupal deploy + - name: Run a test ce-dev deploy with Drupal 9 + run: | + ce-dev create -p test -t drupal9 -d ~/test + cd ~/test + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.test.local + shell: bash diff --git a/ce-dev/ansible/common.yml b/ce-dev/ansible/common.yml index 98babe50..2cffc05e 100644 --- a/ce-dev/ansible/common.yml +++ b/ce-dev/ansible/common.yml @@ -1,5 +1,5 @@ # Common vars for local development. -is_local: yes +is_local: true ce_deploy: username: "ce-dev" own_repository: "https://github.com/codeenigma/ce-deploy.git" diff --git a/ce-dev/ansible/examples/drupal8/deploy.yml b/ce-dev/ansible/examples/drupal8/deploy.yml index 87c6efad..7c8def57 100644 --- a/ce-dev/ansible/examples/drupal8/deploy.yml +++ b/ce-dev/ansible/examples/drupal8/deploy.yml @@ -37,14 +37,35 @@ pre_tasks: # You can safely remove these steps once you have a working composer.json. - name: Download composer file. - get_url: - url: https://raw.githubusercontent.com/drupal/recommended-project/8.8.x/composer.json + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/8.9.x/composer.json dest: "{{ deploy_path }}/composer.json" - force: no + force: false + - name: Adding composer/installers plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.composer/installers true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-composer-scaffold plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-composer-scaffold true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-project-message plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-project-message true + working_dir: "{{ deploy_path }}" + - name: Adding dealerdirect/phpcodesniffer-composer-installer plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true + working_dir: "{{ deploy_path }}" - name: Install drush. - command: - cmd: composer require drush/drush - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: drush/drush:10.* + working_dir: "{{ deploy_path }}" roles: - _init # Sets some variables the deploy scripts rely on. - composer # Composer install step. diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml index deeef9ee..b7579ac4 100644 --- a/ce-dev/ansible/examples/drupal9/deploy.yml +++ b/ce-dev/ansible/examples/drupal9/deploy.yml @@ -37,14 +37,15 @@ pre_tasks: # You can safely remove these steps once you have a working composer.json. - name: Download composer file. - get_url: - url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.5.x/composer.json dest: "{{ deploy_path }}/composer.json" - force: no + force: false - name: Install drush. - command: - cmd: composer require drush/drush:11.* - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: drush/drush:11.* + working_dir: "{{ deploy_path }}" roles: - _init # Sets some variables the deploy scripts rely on. - composer # Composer install step. diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml index 9844206b..09db8ead 100644 --- a/ce-dev/ansible/examples/localgov/deploy.yml +++ b/ce-dev/ansible/examples/localgov/deploy.yml @@ -37,18 +37,45 @@ pre_tasks: # You can safely remove these steps once you have a working composer.json. - name: Download composer file. - get_url: - url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json - dest: "{{ deploy_path }}/composer.json" - force: false + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.1.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: false + - name: Adding composer/installers plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.composer/installers true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-composer-scaffold plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-composer-scaffold true + working_dir: "{{ deploy_path }}" + - name: Adding drupal/core-project-message plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.drupal/core-project-message true + working_dir: "{{ deploy_path }}" + - name: Adding dealerdirect/phpcodesniffer-composer-installer plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true + working_dir: "{{ deploy_path }}" + - name: Adding cweagans/composer-patches plugin to composer config. + community.general.composer: + command: config + arguments: allow-plugins.cweagans/composer-patches true + working_dir: "{{ deploy_path }}" - name: Install drush. - command: - cmd: composer require drush/drush:11.* - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: drush/drush:10.* + working_dir: "{{ deploy_path }}" - name: Install localgov. - command: - cmd: composer require localgovdrupal/localgov - chdir: "{{ deploy_path }}" + community.general.composer: + command: require + arguments: localgovdrupal/localgov + working_dir: "{{ deploy_path }}" roles: - _init # Sets some variables the deploy scripts rely on. - composer # Composer install step. @@ -56,4 +83,4 @@ - config_generate # Generates settings.php # - sync/database_sync # Grab database from a remote server. - database_apply # Run drush updb and config import. - - _exit # Some common housekeeping. \ No newline at end of file + - _exit # Some common housekeeping. diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 16139d8f..3d47469a 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -31,6 +31,7 @@ - [Data backups](/roles/database_backup) - [MySQL backups](/roles/database_backup/database_backup-mysql) - [Deploy](/roles/deploy_code) + - [Deploy container](/roles/deploy_container) - [Init](/roles/_init) - [LHCI run](/roles/lhci_run) - ["Meta"](/roles/_meta) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 9cb9e024..98179138 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -15,6 +15,9 @@ deploy_user: "deploy" # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: + drush_verbose_output: false + # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error. + truncate_cache_table: false sites: - folder: "default" public_files: "sites/default/files" @@ -33,12 +36,25 @@ drupal: cron: - minute: "*/{{ 10 | random(start=1) }}" job: cron + feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment + # For syncing database and files on a feature branch initial build - include all variables if used + mysql_sync: {} # see sync/database_sync for docs + # mysqldump_params: "{{ _mysqldump_params }}" + # cleanup: true + # archival_method: gzip + # databases: [] + files_sync: {} # see sync/files_sync for docs + # unique_workspace: false + # cleanup: true + # directories: [] mautic: image_path: "media/images" force_install: false +# Used for custom build time tools like cachetool bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md index 465307e2..267485f4 100644 --- a/docs/roles/cache_clear/cache_clear-opcache.md +++ b/docs/roles/cache_clear/cache_clear-opcache.md @@ -14,11 +14,11 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. + # adapter: "--fcgi=127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false - clear_stat: false + clear_stat: true # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined ``` diff --git a/docs/roles/composer.md b/docs/roles/composer.md index 17144004..74765170 100644 --- a/docs/roles/composer.md +++ b/docs/roles/composer.md @@ -14,7 +14,7 @@ composer: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -24,6 +24,7 @@ composer: # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') templates: [] + ``` diff --git a/docs/roles/config_generate.md b/docs/roles/config_generate.md index 5428dd8b..fe7aecfe 100644 --- a/docs/roles/config_generate.md +++ b/docs/roles/config_generate.md @@ -11,7 +11,7 @@ config_generate: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' diff --git a/docs/roles/config_generate/config_generate-drupal8.md b/docs/roles/config_generate/config_generate-drupal8.md index 20e32b5b..75382886 100644 --- a/docs/roles/config_generate/config_generate-drupal8.md +++ b/docs/roles/config_generate/config_generate-drupal8.md @@ -4,4 +4,12 @@ Generates settings.php file for Drupal 8. +## Default variables +```yaml +config_generate_drupal: + # If you need to specify an exact file name for a template file to use for a build, you can do it here. + drupal_settings_file_name: settings.php + drush_settings_file_name: drush.yml +``` + diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md index b52413bf..2766a4f3 100644 --- a/docs/roles/database_backup/database_backup-mysql.md +++ b/docs/roles/database_backup/database_backup-mysql.md @@ -1,5 +1,19 @@ # MySQL backups Generate MySQL backups for each build. + +## Replicas +If you are using a read only replica in your application and you need to add it to `databases` in order to have access to the credentials for your app settings, be sure to set the database up in a similar way to this: + +```yaml +mysql_backup: + databases: + - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" + user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" + credentials_file: "/home/{{ deploy_user }}/.mysql.creds" + handling: none # prevents the replica from being backed up + is_replica: true # tells ce-deploy we are working with a replica, so it will implement a pause +``` + ## Default variables ```yaml @@ -8,6 +22,8 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage + mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}" # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: @@ -22,9 +38,11 @@ mysql_backup: # This is useful for locked-down setups where you do not have GRANT permissions. credentials_handling: rotate databases: - - database: "{{ project_name }}_{{ build_type }}" - user: "{{ project_name }}_{{ build_type }}" + - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names + user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit credentials_file: "/home/{{ deploy_user }}/.mysql.creds" + #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas + #is_replica: true # tell ce-deploy this database is a replica - can only be true, remove/comment out if not required ``` diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md index 377ca80c..5d4cc722 100644 --- a/docs/roles/deploy_code.md +++ b/docs/roles/deploy_code.md @@ -121,6 +121,8 @@ deploy_code: service_action: reload # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false + # Used to skip tasks to fix ownership and permissions, drupal needs this set to true by default + fix_cleanup_perms: true # Details of API call to trigger. See api_call role. api_call: type: gitlab diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md new file mode 100644 index 00000000..06847722 --- /dev/null +++ b/docs/roles/deploy_container.md @@ -0,0 +1,148 @@ +# Deploy container +Step that deploys the codebase in a Docker container image. Requires Docker and the `community.docker` collection for Ansible to be installed on your deploy server. You will also need to add a `docker` group and make sure your local deploy user is in that group, for example: + +``` +sudo groupadd docker +sudo usermod -aG docker deploy +``` + +This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles. + +If you set the `deploy_container.action` to `destroy` then the role will also take care of tidying up containers. If it is an AWS ECS deployment then it will also tidy up the ECS service for you. + +## AWS IAM requirements +AWS integration requires the AWS CLI user provided for `ce-deploy` to have certain managed AWS policies attached. + +If you enable AWS ECR registry integration by setting `deploy_container.aws_ecr.enabled` to `true` then you will need the `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers. + +Similarly, if you set `deploy_container.aws_ecs.acm.create_cert` to `true` then you will need the `AWSCertificateManagerFullAccess` policy attaching to create SSL certificates. + +If you enable full AWS ECS integration by setting `deploy_container.aws_ecs.enabled` to `true` then this requires the following policies to be attached to the AWS CLI user: +* `AmazonECS_FullAccess` - to create task definitions and services +* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups + +Finally, if you set `deploy_container.aws_ecs.route_53.zone` to another other than an empty string then you will also need `AmazonRoute53FullAccess` attaching to manipulate DNS entries in Route 53. + +The full list is: +* `EC2InstanceProfileForImageBuilderECRContainerBuilds` - to manipulate images in AWS ECR +* `AWSCertificateManagerFullAccess` - to manage SSL certificates +* `AmazonECS_FullAccess` - to create task definitions and services +* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups +* `AmazonRoute53FullAccess` - to manage DNS entries + +Naturally you can always create custom policies and roles to have tighter access control. This document simply gives you the broad strokes AWS managed policies you can use in conjunction with this Ansible role. + +# Peculiarities of AWS ECS +It is worth noting that even if you put your containers on private subnets and configure your apps to use internal addressing, traffic will pass via the public interface. Therefore any safelisting of IP addresses needs to include the IP addresses of the NAT gateways of your private subnets. [More on how this works here.](https://docs.aws.amazon.com/AmazonECS/latest/bestpracticesguide/networking-connecting-vpc.html) + + + + + +## Default variables +```yaml +--- +deploy_container: + action: create # can also be destroy + container_name: example-container + container_tag: latest # tag will take format container_name:container_tag + container_force_build: true # force Docker to build and tag a new image + docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name + docker_registry_user: example + docker_registry_pass: asdf1234 + docker_base_command: "docker image build" + docker_build_dir: "{{ _ce_deploy_build_dir }}" + dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name + environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template + # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached + aws_ecr: + enabled: false # set to true if using AWS ECR + region: eu-west-1 + aws_profile: example + # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached + # Note, you can if you wish make more restrictive roles and policies + aws_ecs: + enabled: false + region: eu-west-1 + aws_profile: example + tags: {} + domain_name: www.example.com + route_53: + zone: example.com + aws_profile: example2 # might not be the same account + vpc_name: example + #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use + security_groups: [] # list of security groups, accepts names or IDs + cluster_name: example-cluster + family_name: example-task-definition + task_definition_revision: "" # integer, but must be presented as a string for Jinja2 + task_definition_force_create: false # creates a task definition revision every time if set to true + task_execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable + #task_role_arn: "" # required if you set service_enable_ssm to true + task_count: 1 + task_minimum_count: 1 + task_maximum_count: 4 + # These subnets are usually the subnets created by ce-provision when you made your ECS cluster and must have a NAT gateway for ECR access. + service_subnets: # list of private subnet names + - example-cluster-dev-a + - example-cluster-dev-b + # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html + service_autoscale_metric_type: ECSServiceAverageCPUUtilization + service_autoscale_up_cooldown: 120 + service_autoscale_down_cooldown: 120 + service_autoscale_target_value: 70 # the value to trigger a scaling event at + service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973 + service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API + service_force_refresh: false # forces a refresh of all containers if set to true + containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers + - name: example-container + essential: true + image: index.docker.io/example:latest + portMappings: + - containerPort: 8080 # should match target_group_port + hostPort: 8080 + logConfiguration: + logDriver: awslogs + options: + awslogs-group: /ecs/example-cluster + awslogs-region: eu-west-1 + awslogs-stream-prefix: "ecs-example-task" + cpu: 512 # these values can be set globally or per container + memory: 1024 + launch_type: FARGATE + network_mode: awsvpc + #volumes: [] # list of additional volumes to attach + target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen + target_group_protocol: http + target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile + target_group_wait_timeout: 200 # how long to wait for target group events to complete + targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service + #- Id: 10.0.0.2 + # Port: 80 + # AvailabilityZone: all + health_check: + protocol: http + path: / + response_codes: "200" + # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached + acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm + create_cert: false + extra_domains: [] # list of Subject Alternative Name domains and zones + ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager + elb_security_groups: [] # default SG is used if none provided - module supports names or IDs + elb_subnets: # must be public subnets for public facing applications + - example-dev-a + - example-dev-b + elb_http_port: 80 + elb_https_port: 443 + elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + elb_listener_http_rules: [] + elb_listener_https_rules: [] + # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html + elb_listeners: [] + elb_idle_timeout: 60 + elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses). + +``` + + diff --git a/docs/roles/npm.md b/docs/roles/npm.md index 385e1c84..97f06cdc 100644 --- a/docs/roles/npm.md +++ b/docs/roles/npm.md @@ -18,7 +18,7 @@ npm: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -28,6 +28,7 @@ npm: # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') templates: [] + ``` diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md index cf7e5311..c0c2954b 100644 --- a/docs/roles/sync/database_sync/database_sync-mysql.md +++ b/docs/roles/sync/database_sync/database_sync-mysql.md @@ -5,7 +5,9 @@ Sync MySQL databases between environments. ```yaml --- mysql_sync: - mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here. + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes. + archival_method: "gzip" # oprions are "bzip2" or "gzip". databases: - source: # Name of the database to take a dump from. diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 00000000..58e31070 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,16 @@ +# Ansible Plugins +This directory contains extra plugins for Ansible. + +## Vars plugins +Here we include a plugin for handling SOPS decryption. + +## Callback plugins +Here we have a small custom override that fails builds if there are no matching hosts found, to avoid `ce-deploy` incrementing the track file when it didn't actually run because of a host issue. + +## Enabling plugins +To use these plugins you need to find the `# set plugin path directories here` section of `ansible.cfg` which should be kept in your `ce-deploy-config` repository. Add the paths to the plugin directories to enable the plugins, e.g. + +``` +vars_plugins = /home/deploy/ce-deploy/plugins/vars:/usr/share/ansible/plugins/vars +callback_plugins = /home/deploy/ce-deploy/plugins/callback:/usr/share/ansible/plugins/callback +``` diff --git a/plugins/callback/fail_on_no_hosts.py b/plugins/callback/fail_on_no_hosts.py new file mode 100644 index 00000000..e022c0a4 --- /dev/null +++ b/plugins/callback/fail_on_no_hosts.py @@ -0,0 +1,31 @@ +import sys + +from ansible import constants as C +from ansible.plugins.callback import CallbackBase + +DOCUMENTATION = ''' +name: fail_on_no_hosts +callback_type: aggregate +requirements: + - enable in configuration +short_description: Exits with code 1 if no play hosts are matched +version_added: "2.0" +description: + - This callback overrides the default 'v2_playbook_on_no_hosts_matched' method with one that exits instead of just notifying. +''' + +class CallbackModule(CallbackBase): + """ + This callback module exists non-zero if no hosts match + """ + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'fail_on_no_hosts' + CALLBACK_NEEDS_WHITELIST = False + + def __init__(self): + super(CallbackModule, self).__init__() + + def v2_playbook_on_no_hosts_matched(self): + self._display.display("failed: no hosts matched", color=C.COLOR_ERROR) + sys.exit(1) diff --git a/plugins/vars/sops_vars.py b/plugins/vars/sops_vars.py index eca3f3b9..9193d61c 100644 --- a/plugins/vars/sops_vars.py +++ b/plugins/vars/sops_vars.py @@ -160,4 +160,4 @@ def get_vars(self, loader, path, entities, cache=True): except Exception as e: raise AnsibleParserError(to_native(e)) - return data + return data \ No newline at end of file diff --git a/roles/_init/README.md b/roles/_init/README.md index 9cb9e024..98179138 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -15,6 +15,9 @@ deploy_user: "deploy" # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: + drush_verbose_output: false + # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error. + truncate_cache_table: false sites: - folder: "default" public_files: "sites/default/files" @@ -33,12 +36,25 @@ drupal: cron: - minute: "*/{{ 10 | random(start=1) }}" job: cron + feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment + # For syncing database and files on a feature branch initial build - include all variables if used + mysql_sync: {} # see sync/database_sync for docs + # mysqldump_params: "{{ _mysqldump_params }}" + # cleanup: true + # archival_method: gzip + # databases: [] + files_sync: {} # see sync/files_sync for docs + # unique_workspace: false + # cleanup: true + # directories: [] mautic: image_path: "media/images" force_install: false +# Used for custom build time tools like cachetool bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9a955f3e..2e4eb0a8 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -6,6 +6,9 @@ deploy_user: "deploy" # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: + drush_verbose_output: false + # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error. + truncate_cache_table: false sites: - folder: "default" public_files: "sites/default/files" @@ -24,9 +27,22 @@ drupal: cron: - minute: "*/{{ 10 | random(start=1) }}" job: cron + feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment + # For syncing database and files on a feature branch initial build - include all variables if used + mysql_sync: {} # see sync/database_sync for docs + # mysqldump_params: "{{ _mysqldump_params }}" + # cleanup: true + # archival_method: gzip + # databases: [] + files_sync: {} # see sync/files_sync for docs + # unique_workspace: false + # cleanup: true + # directories: [] mautic: image_path: "media/images" force_install: false +# Used for custom build time tools like cachetool bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml index 133b3d73..3053c5c4 100644 --- a/roles/_init/tasks/drupal7.yml +++ b/roles/_init/tasks/drupal7.yml @@ -1,8 +1,17 @@ --- -- name: Define Drush path. +- name: Define path to drush for this build. set_fact: drush_bin: "{{ drush_bin | default('{{ bin_directory }}/drush.phar') }}" -- name: Ensure we have Drush binary. +- name: Define path to drush for currently live build. # usually the same as drush_bin for Drupal 7 + set_fact: + drush_live_bin: "{{ drush_live_bin | default('{{ bin_directory }}/drush.phar') }}" + +- name: Ensure we have a drush binary. import_role: name: cli/drush + +- name: Ensure we have a cachetool binary. + ansible.builtin.import_role: + name: cli/cachetool + when: install_php_cachetool diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml index ebe63b82..7c64719e 100644 --- a/roles/_init/tasks/drupal8.yml +++ b/roles/_init/tasks/drupal8.yml @@ -1,4 +1,13 @@ --- -- name: Define Drush path. +- name: Define path to drush for this build. set_fact: drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}" + +- name: Define path to drush for currently live build. + set_fact: + drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}" + +- name: Ensure we have a cachetool binary. + ansible.builtin.import_role: + name: cli/cachetool + when: install_php_cachetool diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a0d90571..f57be10e 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -3,27 +3,35 @@ - name: Define deploy user. ansible.builtin.set_fact: deploy_user: "{{ deploy_user | default('deploy') }}" + - name: Define deploy base path. ansible.builtin.set_fact: deploy_base_path: "{{ deploy_base_path | default('/home/{{ deploy_user }}/deploy/{{ project_name }}_{{ build_type }}') }}" + - name: Define mounted directory for assets. ansible.builtin.set_fact: deploy_assets_base_path: "{{ deploy_assets_base_path | default('/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/assets') }}" + - name: Define webroot. ansible.builtin.set_fact: webroot: "{{ webroot | default('web') }}" + - name: Define build deploy path prefix. ansible.builtin.set_fact: deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_" + - name: Define build deploy path. ansible.builtin.set_fact: deploy_path: "{{ deploy_path | default('{{ deploy_path_prefix }}{{ build_number }}') }}" + - name: Define live_symlink dest. ansible.builtin.set_fact: live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" + - name: Define opcache cachetool path. ansible.builtin.set_fact: cachetool_bin: "{{ deploy_base_path }}/cachetool.phar" + - name: Set opcache cachetool path from variable if provided. ansible.builtin.set_fact: cachetool_bin: "{{ cache_clear_opcache.cachetool_bin }}" @@ -31,38 +39,37 @@ - cache_clear_opcache.cachetool_bin is defined - cache_clear_opcache.cachetool_bin | length > 0 -# Manipulate variables for SquashFS builds. -- name: Define image builds base path. - ansible.builtin.set_fact: - build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" - when: - - deploy_code.mount_type is defined - - deploy_code.mount_type == "squashfs" -- name: Define image builds build path prefix. - ansible.builtin.set_fact: - build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" - when: - - deploy_code.mount_type is defined - - deploy_code.mount_type == "squashfs" -- name: Define live_symlink dest for image builds. - ansible.builtin.set_fact: - live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" - when: - - deploy_code.mount_type is defined - - deploy_code.mount_type == "squashfs" -- name: Overwrite deploy and live_symlink paths if SquashFS deploy. - ansible.builtin.set_fact: - deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" - when: - - deploy_code.mount_type is defined - - deploy_code.mount_type == "squashfs" -- name: Overwrite cachetool path if SquashFS deploy and path not provided. - ansible.builtin.set_fact: - cachetool_bin: "{{ build_base_path }}/cachetool.phar" +- name: Manipulate variables for SquashFS builds. + block: + - name: Define image builds base path. + ansible.builtin.set_fact: + build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" + + - name: Define image builds build path prefix. + ansible.builtin.set_fact: + build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" + + - name: Define live_symlink dest for image builds. + ansible.builtin.set_fact: + live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" + + - name: Overwrite deploy and live_symlink paths if SquashFS deploy. + ansible.builtin.set_fact: + deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" + + - name: Overwrite cachetool path if SquashFS deploy and path not provided. + ansible.builtin.set_fact: + cachetool_bin: "{{ build_base_path }}/cachetool.phar" + when: + - cache_clear_opcache.cachetool_bin is not defined + + - name: Ensure build target directory exists. + ansible.builtin.file: + path: "{{ build_base_path }}" + state: directory when: - deploy_code.mount_type is defined - deploy_code.mount_type == "squashfs" - - cache_clear_opcache.cachetool_bin is not defined # Gather last known good build directly from symlink. # This can happen: @@ -99,15 +106,6 @@ path: "{{ deploy_base_path }}" state: directory -# Make sure the build target exists. -- name: Ensure build target directory exists. - ansible.builtin.file: - path: "{{ build_base_path }}" - state: directory - when: - - deploy_code.mount_type is defined - - deploy_code.mount_type == "squashfs" - # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. ansible.builtin.stat: @@ -120,7 +118,3 @@ ansible.builtin.include_tasks: "{{ project_type }}.yml" when: - _project_type_task_result.stat.exists - -- name: Ensure we have a cachetool binary. - ansible.builtin.import_role: - name: cli/cachetool diff --git a/roles/_init/tasks/matomo.yml b/roles/_init/tasks/matomo.yml index 6f34c92f..32f693a1 100644 --- a/roles/_init/tasks/matomo.yml +++ b/roles/_init/tasks/matomo.yml @@ -1,3 +1,5 @@ --- - -# Nothing to do here. \ No newline at end of file +- name: Ensure we have a cachetool binary. + ansible.builtin.import_role: + name: cli/cachetool + when: install_php_cachetool diff --git a/roles/_init/tasks/mautic.yml b/roles/_init/tasks/mautic.yml index 6f34c92f..32f693a1 100644 --- a/roles/_init/tasks/mautic.yml +++ b/roles/_init/tasks/mautic.yml @@ -1,3 +1,5 @@ --- - -# Nothing to do here. \ No newline at end of file +- name: Ensure we have a cachetool binary. + ansible.builtin.import_role: + name: cli/cachetool + when: install_php_cachetool diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml index 6ce31d4f..3e66875b 100644 --- a/roles/_meta/deploy-drupal8/tasks/main.yml +++ b/roles/_meta/deploy-drupal8/tasks/main.yml @@ -20,14 +20,14 @@ name: database_backup - import_role: name: config_generate +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: database_apply - import_role: name: sanitize/admin_creds - import_role: name: live_symlink -- import_role: - name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal8 - import_role: diff --git a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml index 6f72b3a5..b68e1979 100644 --- a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -1,11 +1,11 @@ --- - name: Clear Drupal 7 cache. command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" - chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site_drupal.folder }} -y cc all" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" with_items: "{{ drupal.sites }}" loop_control: - loop_var: site + loop_var: site_drupal run_once: true diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml index 5b780086..a0f960da 100644 --- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,11 +1,56 @@ --- +- name: Override paths for squashfs build cleanup step. + block: + - name: Override path to drush binary. + set_fact: + drush_bin: "{{ drush_live_bin }}" + + - name: Override deploy_path. + set_fact: + deploy_path: "{{ live_symlink_dest }}" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" + - deploy_operation == 'cleanup' + - name: Clear Drupal cache. - command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" - chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site_drupal.folder }} -y cr" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" with_items: "{{ drupal.sites }}" loop_control: - loop_var: site + loop_var: site_drupal run_once: true + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output + +- name: Truncate SQL table 'cache_container'. + block: + - name: Truncate table 'cache_container'. + ansible.builtin.command: + cmd: "{{ drush_bin }} sql-query 'TRUNCATE TABLE cache_container'" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site_drupal + run_once: true + register: _drush_output + + - name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: + - drupal.drush_verbose_output + when: + - drupal.truncate_cache_table + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" + - deploy_operation == 'cleanup' diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md index 465307e2..267485f4 100644 --- a/roles/cache_clear/cache_clear-opcache/README.md +++ b/roles/cache_clear/cache_clear-opcache/README.md @@ -14,11 +14,11 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. + # adapter: "--fcgi=127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false - clear_stat: false + clear_stat: true # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined ``` diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index 222325ce..d7f4af62 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -4,9 +4,9 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. + # adapter: "--fcgi=127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false - clear_stat: false + clear_stat: true # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined diff --git a/roles/composer/README.md b/roles/composer/README.md index 17144004..74765170 100644 --- a/roles/composer/README.md +++ b/roles/composer/README.md @@ -14,7 +14,7 @@ composer: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -24,6 +24,7 @@ composer: # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') templates: [] + ``` diff --git a/roles/composer/defaults/main.yml b/roles/composer/defaults/main.yml index 2e6eec4a..d0dfbd1d 100644 --- a/roles/composer/defaults/main.yml +++ b/roles/composer/defaults/main.yml @@ -7,7 +7,7 @@ composer: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -16,4 +16,4 @@ composer: # Specify any additional templates to generate, with src (template) and dest (file). # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') - templates: [] \ No newline at end of file + templates: [] diff --git a/roles/config_generate/README.md b/roles/config_generate/README.md index 5428dd8b..fe7aecfe 100644 --- a/roles/config_generate/README.md +++ b/roles/config_generate/README.md @@ -11,7 +11,7 @@ config_generate: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f8e3b55e..55d9dee7 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -19,6 +19,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ playbook_dir }}/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" diff --git a/roles/config_generate/config_generate-drupal8/README.md b/roles/config_generate/config_generate-drupal8/README.md index 20e32b5b..75382886 100644 --- a/roles/config_generate/config_generate-drupal8/README.md +++ b/roles/config_generate/config_generate-drupal8/README.md @@ -4,4 +4,12 @@ Generates settings.php file for Drupal 8. +## Default variables +```yaml +config_generate_drupal: + # If you need to specify an exact file name for a template file to use for a build, you can do it here. + drupal_settings_file_name: settings.php + drush_settings_file_name: drush.yml +``` + diff --git a/roles/config_generate/config_generate-drupal8/defaults/main.yml b/roles/config_generate/config_generate-drupal8/defaults/main.yml new file mode 100644 index 00000000..a6b7634d --- /dev/null +++ b/roles/config_generate/config_generate-drupal8/defaults/main.yml @@ -0,0 +1,4 @@ +config_generate_drupal: + # If you need to specify an exact file name for a template file to use for a build, you can do it here. + drupal_settings_file_name: settings.php + drush_settings_file_name: drush.yml \ No newline at end of file diff --git a/roles/config_generate/config_generate-drupal8/tasks/drush.yml b/roles/config_generate/config_generate-drupal8/tasks/drush.yml index f4b7ae7a..0bedd46d 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/drush.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/drush.yml @@ -4,6 +4,7 @@ src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml.j2" - - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml" + - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drush_settings_file_name }}.j2" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drush_settings_file_name }}" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ config_generate_drupal.drush_settings_file_name }}" - "drush.yml.j2" \ No newline at end of file diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index ebf182e8..2b07a09b 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -18,11 +18,30 @@ mode: 0775 become: "{{ false if www_user == deploy_user else true }}" +- name: Generate or retrieve hash salt value. + ansible.builtin.set_fact: + _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name + '_' + build_type + '/drupal/hash_salt length=74 chars=ascii_letters,digits') }}" + - name: Generates settings.php file. ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" - - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" + - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drupal_settings_file_name }}.j2" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drupal_settings_file_name }}" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ config_generate_drupal.drupal_settings_file_name }}" - "settings.php.j2" + +- name: Generates default.settings.php file in any additional multi-site directories. + ansible.builtin.template: + src: "{{ item }}" + dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/default.settings.php" + with_first_found: + - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/default.settings.php" + - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.default.settings.php" + - "{{ deploy_path }}/{{ webroot }}/sites/default/default.settings.php" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.default.{{ config_generate_drupal.drupal_settings_file_name }}" + - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/default.{{ config_generate_drupal.drupal_settings_file_name }}" + - "default.settings.php.j2" + when: + - site.folder != "default" diff --git a/roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2 b/roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2 new file mode 100644 index 00000000..63fb2df7 --- /dev/null +++ b/roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2 @@ -0,0 +1,883 @@ + 'databasename', + * 'username' => 'sql_username', + * 'password' => 'sql_password', + * 'host' => 'localhost', + * 'port' => '3306', + * 'driver' => 'mysql', + * 'prefix' => '', + * 'collation' => 'utf8mb4_general_ci', + * ]; + * @endcode + */ +$databases = []; + +/** + * Customizing database settings. + * + * Many of the values of the $databases array can be customized for your + * particular database system. Refer to the sample in the section above as a + * starting point. + * + * The "driver" property indicates what Drupal database driver the + * connection should use. This is usually the same as the name of the + * database type, such as mysql or sqlite, but not always. The other + * properties will vary depending on the driver. For SQLite, you must + * specify a database file name in a directory that is writable by the + * webserver. For most other drivers, you must specify a + * username, password, host, and database name. + * + * Drupal core implements drivers for mysql, pgsql, and sqlite. Other drivers + * can be provided by contributed or custom modules. To use a contributed or + * custom driver, the "namespace" property must be set to the namespace of the + * driver. The code in this namespace must be autoloadable prior to connecting + * to the database, and therefore, prior to when module root namespaces are + * added to the autoloader. To add the driver's namespace to the autoloader, + * set the "autoload" property to the PSR-4 base directory of the driver's + * namespace. This is optional for projects managed with Composer if the + * driver's namespace is in Composer's autoloader. + * + * For each database, you may optionally specify multiple "target" databases. + * A target database allows Drupal to try to send certain queries to a + * different database if it can but fall back to the default connection if not. + * That is useful for primary/replica replication, as Drupal may try to connect + * to a replica server when appropriate and if one is not available will simply + * fall back to the single primary server (The terms primary/replica are + * traditionally referred to as master/slave in database server documentation). + * + * The general format for the $databases array is as follows: + * @code + * $databases['default']['default'] = $info_array; + * $databases['default']['replica'][] = $info_array; + * $databases['default']['replica'][] = $info_array; + * $databases['extra']['default'] = $info_array; + * @endcode + * + * In the above example, $info_array is an array of settings described above. + * The first line sets a "default" database that has one primary database + * (the second level default). The second and third lines create an array + * of potential replica databases. Drupal will select one at random for a given + * request as needed. The fourth line creates a new database with a name of + * "extra". + * + * For MySQL, MariaDB or equivalent databases the 'isolation_level' option can + * be set. The recommended transaction isolation level for Drupal sites is + * 'READ COMMITTED'. The 'REPEATABLE READ' option is supported but can result + * in deadlocks, the other two options are 'READ UNCOMMITTED' and 'SERIALIZABLE'. + * They are available but not supported; use them at your own risk. For more + * info: + * https://dev.mysql.com/doc/refman/5.7/en/innodb-transaction-isolation-levels.html + * + * On your settings.php, change the isolation level: + * @code + * $databases['default']['default']['init_commands'] = [ + * 'isolation_level' => 'SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED', + * ]; + * @endcode + * + * You can optionally set a prefix for all database table names by using the + * 'prefix' setting. If a prefix is specified, the table name will be prepended + * with its value. Be sure to use valid database characters only, usually + * alphanumeric and underscore. If no prefix is desired, do not set the 'prefix' + * key or set its value to an empty string ''. + * + * For example, to have all database table prefixed with 'main_', set: + * @code + * 'prefix' => 'main_', + * @endcode + * + * Advanced users can add or override initial commands to execute when + * connecting to the database server, as well as PDO connection settings. For + * example, to enable MySQL SELECT queries to exceed the max_join_size system + * variable, and to reduce the database connection timeout to 5 seconds: + * @code + * $databases['default']['default'] = [ + * 'init_commands' => [ + * 'big_selects' => 'SET SQL_BIG_SELECTS=1', + * ], + * 'pdo' => [ + * PDO::ATTR_TIMEOUT => 5, + * ], + * ]; + * @endcode + * + * WARNING: The above defaults are designed for database portability. Changing + * them may cause unexpected behavior, including potential data loss. See + * https://www.drupal.org/developing/api/database/configuration for more + * information on these defaults and the potential issues. + * + * More details can be found in the constructor methods for each driver: + * - \Drupal\mysql\Driver\Database\mysql\Connection::__construct() + * - \Drupal\pgsql\Driver\Database\pgsql\Connection::__construct() + * - \Drupal\sqlite\Driver\Database\sqlite\Connection::__construct() + * + * Sample Database configuration format for PostgreSQL (pgsql): + * @code + * $databases['default']['default'] = [ + * 'driver' => 'pgsql', + * 'database' => 'databasename', + * 'username' => 'sql_username', + * 'password' => 'sql_password', + * 'host' => 'localhost', + * 'prefix' => '', + * ]; + * @endcode + * + * Sample Database configuration format for SQLite (sqlite): + * @code + * $databases['default']['default'] = [ + * 'driver' => 'sqlite', + * 'database' => '/path/to/database_filename', + * ]; + * @endcode + * + * Sample Database configuration format for a driver in a contributed module: + * @code + * $databases['default']['default'] = [ + * 'driver' => 'my_driver', + * 'namespace' => 'Drupal\my_module\Driver\Database\my_driver', + * 'autoload' => 'modules/my_module/src/Driver/Database/my_driver/', + * 'database' => 'databasename', + * 'username' => 'sql_username', + * 'password' => 'sql_password', + * 'host' => 'localhost', + * 'prefix' => '', + * ]; + * @endcode + * + * Sample Database configuration format for a driver that is extending another + * database driver. + * @code + * $databases['default']['default'] = [ + * 'driver' => 'my_driver', + * 'namespace' => 'Drupal\my_module\Driver\Database\my_driver', + * 'autoload' => 'modules/my_module/src/Driver/Database/my_driver/', + * 'database' => 'databasename', + * 'username' => 'sql_username', + * 'password' => 'sql_password', + * 'host' => 'localhost', + * 'prefix' => '', + * 'dependencies' => [ + * 'parent_module' => [ + * 'namespace' => 'Drupal\parent_module', + * 'autoload' => 'core/modules/parent_module/src/', + * ], + * ], + * ]; + * @endcode + */ + +/** + * Location of the site configuration files. + * + * The $settings['config_sync_directory'] specifies the location of file system + * directory used for syncing configuration data. On install, the directory is + * created. This is used for configuration imports. + * + * The default location for this directory is inside a randomly-named + * directory in the public files path. The setting below allows you to set + * its location. + */ +# $settings['config_sync_directory'] = '/directory/outside/webroot'; + +/** + * Settings: + * + * $settings contains environment-specific configuration, such as the files + * directory and reverse proxy address, and temporary configuration, such as + * security overrides. + * + * @see \Drupal\Core\Site\Settings::get() + */ + +/** + * Salt for one-time login links, cancel links, form tokens, etc. + * + * This variable will be set to a random value by the installer. All one-time + * login links will be invalidated if the value is changed. Note that if your + * site is deployed on a cluster of web servers, you must ensure that this + * variable has the same value on each server. + * + * For enhanced security, you may set this variable to the contents of a file + * outside your document root, and vary the value across environments (like + * production and development); you should also ensure that this file is not + * stored with backups of your database. + * + * Example: + * @code + * $settings['hash_salt'] = file_get_contents('/home/example/salt.txt'); + * @endcode + */ +$settings['hash_salt'] = ''; + +/** + * Deployment identifier. + * + * Drupal's dependency injection container will be automatically invalidated and + * rebuilt when the Drupal core version changes. When updating contributed or + * custom code that changes the container, changing this identifier will also + * allow the container to be invalidated as soon as code is deployed. + */ +# $settings['deployment_identifier'] = \Drupal::VERSION; + +/** + * Access control for update.php script. + * + * If you are updating your Drupal installation using the update.php script but + * are not logged in using either an account with the "Administer software + * updates" permission or the site maintenance account (the account that was + * created during installation), you will need to modify the access check + * statement below. Change the FALSE to a TRUE to disable the access check. + * After finishing the upgrade, be sure to open this file again and change the + * TRUE back to a FALSE! + */ +$settings['update_free_access'] = FALSE; + +/** + * Fallback to HTTP for Update Manager and for fetching security advisories. + * + * If your site fails to connect to updates.drupal.org over HTTPS (either when + * fetching data on available updates, or when fetching the feed of critical + * security announcements), you may uncomment this setting and set it to TRUE to + * allow an insecure fallback to HTTP. Note that doing so will open your site up + * to a potential man-in-the-middle attack. You should instead attempt to + * resolve the issues before enabling this option. + * @see https://www.drupal.org/docs/system-requirements/php-requirements#openssl + * @see https://en.wikipedia.org/wiki/Man-in-the-middle_attack + * @see \Drupal\update\UpdateFetcher + * @see \Drupal\system\SecurityAdvisories\SecurityAdvisoriesFetcher + */ +# $settings['update_fetch_with_http_fallback'] = TRUE; + +/** + * External access proxy settings: + * + * If your site must access the Internet via a web proxy then you can enter the + * proxy settings here. Set the full URL of the proxy, including the port, in + * variables: + * - $settings['http_client_config']['proxy']['http']: The proxy URL for HTTP + * requests. + * - $settings['http_client_config']['proxy']['https']: The proxy URL for HTTPS + * requests. + * You can pass in the user name and password for basic authentication in the + * URLs in these settings. + * + * You can also define an array of host names that can be accessed directly, + * bypassing the proxy, in $settings['http_client_config']['proxy']['no']. + */ +# $settings['http_client_config']['proxy']['http'] = 'http://proxy_user:proxy_pass@example.com:8080'; +# $settings['http_client_config']['proxy']['https'] = 'http://proxy_user:proxy_pass@example.com:8080'; +# $settings['http_client_config']['proxy']['no'] = ['127.0.0.1', 'localhost']; + +/** + * Reverse Proxy Configuration: + * + * Reverse proxy servers are often used to enhance the performance + * of heavily visited sites and may also provide other site caching, + * security, or encryption benefits. In an environment where Drupal + * is behind a reverse proxy, the real IP address of the client should + * be determined such that the correct client IP address is available + * to Drupal's logging, statistics, and access management systems. In + * the most simple scenario, the proxy server will add an + * X-Forwarded-For header to the request that contains the client IP + * address. However, HTTP headers are vulnerable to spoofing, where a + * malicious client could bypass restrictions by setting the + * X-Forwarded-For header directly. Therefore, Drupal's proxy + * configuration requires the IP addresses of all remote proxies to be + * specified in $settings['reverse_proxy_addresses'] to work correctly. + * + * Enable this setting to get Drupal to determine the client IP from the + * X-Forwarded-For header. If you are unsure about this setting, do not have a + * reverse proxy, or Drupal operates in a shared hosting environment, this + * setting should remain commented out. + * + * In order for this setting to be used you must specify every possible + * reverse proxy IP address in $settings['reverse_proxy_addresses']. + * If a complete list of reverse proxies is not available in your + * environment (for example, if you use a CDN) you may set the + * $_SERVER['REMOTE_ADDR'] variable directly in settings.php. + * Be aware, however, that it is likely that this would allow IP + * address spoofing unless more advanced precautions are taken. + */ +# $settings['reverse_proxy'] = TRUE; + +/** + * Reverse proxy addresses. + * + * Specify every reverse proxy IP address in your environment, as an array of + * IPv4/IPv6 addresses or subnets in CIDR notation. This setting is required if + * $settings['reverse_proxy'] is TRUE. + */ +# $settings['reverse_proxy_addresses'] = ['a.b.c.d', 'e.f.g.h/24', ...]; + +/** + * Reverse proxy trusted headers. + * + * Sets which headers to trust from your reverse proxy. + * + * Common values are: + * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR + * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST + * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT + * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO + * - \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED + * + * Note the default value of + * @code + * \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO | \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED + * @endcode + * is not secure by default. The value should be set to only the specific + * headers the reverse proxy uses. For example: + * @code + * \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO + * @endcode + * This would trust the following headers: + * - X_FORWARDED_FOR + * - X_FORWARDED_HOST + * - X_FORWARDED_PROTO + * - X_FORWARDED_PORT + * + * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR + * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST + * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT + * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO + * @see \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED + * @see \Symfony\Component\HttpFoundation\Request::setTrustedProxies + */ +# $settings['reverse_proxy_trusted_headers'] = \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO | \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED; + + +/** + * Page caching: + * + * By default, Drupal sends a "Vary: Cookie" HTTP header for anonymous page + * views. This tells a HTTP proxy that it may return a page from its local + * cache without contacting the web server, if the user sends the same Cookie + * header as the user who originally requested the cached page. Without "Vary: + * Cookie", authenticated users would also be served the anonymous page from + * the cache. If the site has mostly anonymous users except a few known + * editors/administrators, the Vary header can be omitted. This allows for + * better caching in HTTP proxies (including reverse proxies), i.e. even if + * clients send different cookies, they still get content served from the cache. + * However, authenticated users should access the site directly (i.e. not use an + * HTTP proxy, and bypass the reverse proxy if one is used) in order to avoid + * getting cached pages from the proxy. + */ +# $settings['omit_vary_cookie'] = TRUE; + + +/** + * Cache TTL for client error (4xx) responses. + * + * Items cached per-URL tend to result in a large number of cache items, and + * this can be problematic on 404 pages which by their nature are unbounded. A + * fixed TTL can be set for these items, defaulting to one hour, so that cache + * backends which do not support LRU can purge older entries. To disable caching + * of client error responses set the value to 0. Currently applies only to + * page_cache module. + */ +# $settings['cache_ttl_4xx'] = 3600; + +/** + * Expiration of cached forms. + * + * Drupal's Form API stores details of forms in a cache and these entries are + * kept for at least 6 hours by default. Expired entries are cleared by cron. + * + * @see \Drupal\Core\Form\FormCache::setCache() + */ +# $settings['form_cache_expiration'] = 21600; + +/** + * Class Loader. + * + * If the APCu extension is detected, the classloader will be optimized to use + * it. Set to FALSE to disable this. + * + * @see https://getcomposer.org/doc/articles/autoloader-optimization.md + */ +# $settings['class_loader_auto_detect'] = FALSE; + +/** + * Authorized file system operations: + * + * The Update Manager module included with Drupal provides a mechanism for + * site administrators to securely install missing updates for the site + * directly through the web user interface. On securely-configured servers, + * the Update manager will require the administrator to provide SSH or FTP + * credentials before allowing the installation to proceed; this allows the + * site to update the new files as the user who owns all the Drupal files, + * instead of as the user the webserver is running as. On servers where the + * webserver user is itself the owner of the Drupal files, the administrator + * will not be prompted for SSH or FTP credentials (note that these server + * setups are common on shared hosting, but are inherently insecure). + * + * Some sites might wish to disable the above functionality, and only update + * the code directly via SSH or FTP themselves. This setting completely + * disables all functionality related to these authorized file operations. + * + * @see https://www.drupal.org/node/244924 + * + * Remove the leading hash signs to disable. + */ +# $settings['allow_authorize_operations'] = FALSE; + +/** + * Default mode for directories and files written by Drupal. + * + * Value should be in PHP Octal Notation, with leading zero. + */ +# $settings['file_chmod_directory'] = 0775; +# $settings['file_chmod_file'] = 0664; + +/** + * Optimized assets path: + * + * A local file system path where optimized assets will be stored. This directory + * must exist and be writable by Drupal. This directory must be relative to + * the Drupal installation directory and be accessible over the web. + */ +# $settings['file_assets_path'] = 'sites/default/files'; + +/** + * Public file base URL: + * + * An alternative base URL to be used for serving public files. This must + * include any leading directory path. + * + * A different value from the domain used by Drupal to be used for accessing + * public files. This can be used for a simple CDN integration, or to improve + * security by serving user-uploaded files from a different domain or subdomain + * pointing to the same server. Do not include a trailing slash. + */ +# $settings['file_public_base_url'] = 'http://downloads.example.com/files'; + +/** + * Public file path: + * + * A local file system path where public files will be stored. This directory + * must exist and be writable by Drupal. This directory must be relative to + * the Drupal installation directory and be accessible over the web. + */ +# $settings['file_public_path'] = 'sites/default/files'; + +/** + * Additional public file schemes: + * + * Public schemes are URI schemes that allow download access to all users for + * all files within that scheme. + * + * The "public" scheme is always public, and the "private" scheme is always + * private, but other schemes, such as "https", "s3", "example", or others, + * can be either public or private depending on the site. By default, they're + * private, and access to individual files is controlled via + * hook_file_download(). + * + * Typically, if a scheme should be public, a module makes it public by + * implementing hook_file_download(), and granting access to all users for all + * files. This could be either the same module that provides the stream wrapper + * for the scheme, or a different module that decides to make the scheme + * public. However, in cases where a site needs to make a scheme public, but + * is unable to add code in a module to do so, the scheme may be added to this + * variable, the result of which is that system_file_download() grants public + * access to all files within that scheme. + */ +# $settings['file_additional_public_schemes'] = ['example']; + +/** + * File schemes whose paths should not be normalized: + * + * Normally, Drupal normalizes '/./' and '/../' segments in file URIs in order + * to prevent unintended file access. For example, 'private://css/../image.png' + * is normalized to 'private://image.png' before checking access to the file. + * + * On Windows, Drupal also replaces '\' with '/' in URIs for the local + * filesystem. + * + * If file URIs with one or more scheme should not be normalized like this, then + * list the schemes here. For example, if 'porcelain://china/./plate.png' should + * not be normalized to 'porcelain://china/plate.png', then add 'porcelain' to + * this array. In this case, make sure that the module providing the 'porcelain' + * scheme does not allow unintended file access when using '/../' to move up the + * directory tree. + */ +# $settings['file_sa_core_2023_005_schemes'] = ['porcelain']; + +/** + * Configuration for phpinfo() admin status report. + * + * Drupal's admin UI includes a report at admin/reports/status/php which shows + * the output of phpinfo(). The full output can contain sensitive information + * so by default Drupal removes some sections. + * + * This behavior can be configured by setting this variable to a different + * value corresponding to the flags parameter of phpinfo(). + * + * If you need to expose more information in the report - for example to debug a + * problem - consider doing so temporarily. + * + * @see https://www.php.net/manual/function.phpinfo.php + */ +# $settings['sa_core_2023_004_phpinfo_flags'] = ~ (INFO_VARIABLES | INFO_ENVIRONMENT); + +/** + * Private file path: + * + * A local file system path where private files will be stored. This directory + * must be absolute, outside of the Drupal installation directory and not + * accessible over the web. + * + * Note: Caches need to be cleared when this value is changed to make the + * private:// stream wrapper available to the system. + * + * See https://www.drupal.org/documentation/modules/file for more information + * about securing private files. + */ +# $settings['file_private_path'] = ''; + +/** + * Temporary file path: + * + * A local file system path where temporary files will be stored. This directory + * must be absolute, outside of the Drupal installation directory and not + * accessible over the web. + * + * If this is not set, the default for the operating system will be used. + * + * @see \Drupal\Component\FileSystem\FileSystem::getOsTemporaryDirectory() + */ +# $settings['file_temp_path'] = '/tmp'; + +/** + * Session write interval: + * + * Set the minimum interval between each session write to database. + * For performance reasons it defaults to 180. + */ +# $settings['session_write_interval'] = 180; + +/** + * String overrides: + * + * To override specific strings on your site with or without enabling the Locale + * module, add an entry to this list. This functionality allows you to change + * a small number of your site's default English language interface strings. + * + * Remove the leading hash signs to enable. + * + * The "en" part of the variable name, is dynamic and can be any langcode of + * any added language. (eg locale_custom_strings_de for german). + */ +# $settings['locale_custom_strings_en'][''] = [ +# 'Home' => 'Front page', +# '@count min' => '@count minutes', +# ]; + +/** + * A custom theme for the offline page: + * + * This applies when the site is explicitly set to maintenance mode through the + * administration page or when the database is inactive due to an error. + * The template file should also be copied into the theme. It is located inside + * 'core/modules/system/templates/maintenance-page.html.twig'. + * + * Note: This setting does not apply to installation and update pages. + */ +# $settings['maintenance_theme'] = 'claro'; + +/** + * PHP settings: + * + * To see what PHP settings are possible, including whether they can be set at + * runtime (by using ini_set()), read the PHP documentation: + * http://php.net/manual/ini.list.php + * See \Drupal\Core\DrupalKernel::bootEnvironment() for required runtime + * settings and the .htaccess file for non-runtime settings. + * Settings defined there should not be duplicated here so as to avoid conflict + * issues. + */ + +/** + * If you encounter a situation where users post a large amount of text, and + * the result is stripped out upon viewing but can still be edited, Drupal's + * output filter may not have sufficient memory to process it. If you + * experience this issue, you may wish to uncomment the following two lines + * and increase the limits of these variables. For more information, see + * http://php.net/manual/pcre.configuration.php. + */ +# ini_set('pcre.backtrack_limit', 200000); +# ini_set('pcre.recursion_limit', 200000); + +/** + * Configuration overrides. + * + * To globally override specific configuration values for this site, + * set them here. You usually don't need to use this feature. This is + * useful in a configuration file for a vhost or directory, rather than + * the default settings.php. + * + * Note that any values you provide in these variable overrides will not be + * viewable from the Drupal administration interface. The administration + * interface displays the values stored in configuration so that you can stage + * changes to other environments that don't have the overrides. + * + * There are particular configuration values that are risky to override. For + * example, overriding the list of installed modules in 'core.extension' is not + * supported as module install or uninstall has not occurred. Other examples + * include field storage configuration, because it has effects on database + * structure, and 'core.menu.static_menu_link_overrides' since this is cached in + * a way that is not config override aware. Also, note that changing + * configuration values in settings.php will not fire any of the configuration + * change events. + */ +# $config['system.site']['name'] = 'My Drupal site'; +# $config['user.settings']['anonymous'] = 'Visitor'; + +/** + * Load services definition file. + */ +$settings['container_yamls'][] = $app_root . '/' . $site_path . '/services.yml'; + +/** + * Override the default service container class. + * + * This is useful for example to trace the service container for performance + * tracking purposes, for testing a service container with an error condition or + * to test a service container that throws an exception. + */ +# $settings['container_base_class'] = '\Drupal\Core\DependencyInjection\Container'; + +/** + * Override the default yaml parser class. + * + * Provide a fully qualified class name here if you would like to provide an + * alternate implementation YAML parser. The class must implement the + * \Drupal\Component\Serialization\SerializationInterface interface. + */ +# $settings['yaml_parser_class'] = NULL; + +/** + * Trusted host configuration. + * + * Drupal core can use the Symfony trusted host mechanism to prevent HTTP Host + * header spoofing. + * + * To enable the trusted host mechanism, you enable your allowable hosts + * in $settings['trusted_host_patterns']. This should be an array of regular + * expression patterns, without delimiters, representing the hosts you would + * like to allow. + * + * For example: + * @code + * $settings['trusted_host_patterns'] = [ + * '^www\.example\.com$', + * ]; + * @endcode + * will allow the site to only run from www.example.com. + * + * If you are running multisite, or if you are running your site from + * different domain names (eg, you don't redirect http://www.example.com to + * http://example.com), you should specify all of the host patterns that are + * allowed by your site. + * + * For example: + * @code + * $settings['trusted_host_patterns'] = [ + * '^example\.com$', + * '^.+\.example\.com$', + * '^example\.org$', + * '^.+\.example\.org$', + * ]; + * @endcode + * will allow the site to run off of all variants of example.com and + * example.org, with all subdomains included. + * + * @see https://www.drupal.org/docs/installing-drupal/trusted-host-settings + */ +# $settings['trusted_host_patterns'] = []; + +/** + * The default list of directories that will be ignored by Drupal's file API. + * + * By default ignore node_modules and bower_components folders to avoid issues + * with common frontend tools and recursive scanning of directories looking for + * extensions. + * + * @see \Drupal\Core\File\FileSystemInterface::scanDirectory() + * @see \Drupal\Core\Extension\ExtensionDiscovery::scanDirectory() + */ +$settings['file_scan_ignore_directories'] = [ + 'node_modules', + 'bower_components', +]; + +/** + * The default number of entities to update in a batch process. + * + * This is used by update and post-update functions that need to go through and + * change all the entities on a site, so it is useful to increase this number + * if your hosting configuration (i.e. RAM allocation, CPU speed) allows for a + * larger number of entities to be processed in a single batch run. + */ +$settings['entity_update_batch_size'] = 50; + +/** + * Entity update backup. + * + * This is used to inform the entity storage handler that the backup tables as + * well as the original entity type and field storage definitions should be + * retained after a successful entity update process. + */ +$settings['entity_update_backup'] = TRUE; + +/** + * Node migration type. + * + * This is used to force the migration system to use the classic node migrations + * instead of the default complete node migrations. The migration system will + * use the classic node migration only if there are existing migrate_map tables + * for the classic node migrations and they contain data. These tables may not + * exist if you are developing custom migrations and do not want to use the + * complete node migrations. Set this to TRUE to force the use of the classic + * node migrations. + */ +$settings['migrate_node_migrate_type_classic'] = FALSE; + +/** + * The default settings for migration sources. + * + * These settings are used as the default settings on the Credential form at + * /upgrade/credentials. + * + * - migrate_source_version - The version of the source database. This can be + * '6' or '7'. Defaults to '7'. + * - migrate_source_connection - The key in the $databases array for the source + * site. + * - migrate_file_public_path - The location of the source Drupal 6 or Drupal 7 + * public files. This can be a local file directory containing the source + * Drupal 6 or Drupal 7 site (e.g /var/www/docroot), or the site address + * (e.g http://example.com). + * - migrate_file_private_path - The location of the source Drupal 7 private + * files. This can be a local file directory containing the source Drupal 7 + * site (e.g /var/www/docroot), or empty to use the same value as Public + * files directory. + * + * Sample configuration for a drupal 6 source site with the source files in a + * local directory. + * + * @code + * $settings['migrate_source_version'] = '6'; + * $settings['migrate_source_connection'] = 'migrate'; + * $settings['migrate_file_public_path'] = '/var/www/drupal6'; + * @endcode + * + * Sample configuration for a drupal 7 source site with public source files on + * the source site and the private files in a local directory. + * + * @code + * $settings['migrate_source_version'] = '7'; + * $settings['migrate_source_connection'] = 'migrate'; + * $settings['migrate_file_public_path'] = 'https://drupal7.com'; + * $settings['migrate_file_private_path'] = '/var/www/drupal7'; + * @endcode + */ +# $settings['migrate_source_connection'] = ''; +# $settings['migrate_source_version'] = ''; +# $settings['migrate_file_public_path'] = ''; +# $settings['migrate_file_private_path'] = ''; + +/** + * Load local development override configuration, if available. + * + * Create a settings.local.php file to override variables on secondary (staging, + * development, etc.) installations of this site. + * + * Typical uses of settings.local.php include: + * - Disabling caching. + * - Disabling JavaScript/CSS compression. + * - Rerouting outgoing emails. + * + * Keep this code block at the end of this file to take full effect. + */ +# +# if (file_exists($app_root . '/' . $site_path . '/settings.local.php')) { +# include $app_root . '/' . $site_path . '/settings.local.php'; +# } diff --git a/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 b/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 index e5d43204..895f654f 100644 --- a/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 +++ b/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 @@ -23,3 +23,5 @@ $settings['file_public_path'] = '{{ build_public_file_path }}'; $config_directories['sync'] = '{{ build_config_sync_directory }}'; // Drupal 8.8 $settings['config_sync_directory'] = '{{ build_config_sync_directory }}'; + +$settings['hash_salt'] = '{{ _drupal_hash_salt }}'; diff --git a/roles/config_generate/defaults/main.yml b/roles/config_generate/defaults/main.yml index 5258730e..8f1987eb 100644 --- a/roles/config_generate/defaults/main.yml +++ b/roles/config_generate/defaults/main.yml @@ -3,7 +3,7 @@ config_generate: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml index ffd14157..e87cad12 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml @@ -4,7 +4,7 @@ # add a keep mechanism for backup scripts, like for the dumps themselves. # Nice to have more than anything. - name: Setup PATH in crontab. - cron: + ansible.builtin.cron: name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" @@ -13,7 +13,7 @@ - drupal.defer is defined - drupal.defer -- include_tasks: setup.yml +- ansible.builtin.include_tasks: setup.yml vars: database: database with_items: "{{ build_databases }}" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index d09fbbac..f12301da 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -1,39 +1,40 @@ # Probably not worth adding more granularity (weeks, months) for backups, should be at least daily? -- set_fact: +- ansible.builtin.set_fact: _cron_mysql_backup_random_minute: "{{ 59 | random }}" -- set_fact: +- ansible.builtin.set_fact: _cron_mysql_backup_minute: "{{ database.original.backup.minute | default(_cron_mysql_backup_random_minute) }}" -- set_fact: +- ansible.builtin.set_fact: _cron_mysql_backup_hour: "{{ database.original.backup.hour | default(0) }}" -- set_fact: +- ansible.builtin.set_fact: _cron_mysql_backup_keep: "{{ database.original.backup.keep | default(10) }}" - name: Ensure backup directory exists. - file: + ansible.builtin.file: path: "{{ cron_mysql_backup.dumps_directory }}/{{ database.host }}" state: directory mode: 0700 recurse: true - name: Ensure cron directory exists. - file: + ansible.builtin.file: path: "/home/{{ deploy_user }}/cron/{{ project_name }}_{{ build_type }}" state: directory mode: 0700 recurse: true - name: Create backup script. - template: + ansible.builtin.template: src: "regular-backups.sh.j2" dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" mode: 0700 + when: database.is_replica is not defined - name: Define backup cron job command. - set_fact: + ansible.builtin.set_fact: _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" - name: Define backup cron job command if deferred (ASG). - set_fact: + ansible.builtin.set_fact: _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\"" when: - drupal.defer is defined @@ -42,9 +43,10 @@ - drupal.defer_target | length > 0 - name: Setup regular backup for MySQL. - cron: + ansible.builtin.cron: name: "cron_mysql_{{ database.host }}_{{ database.original.database }}" minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" job: "{{ _backup_cron_job_command }}" delegate_to: "{{ 'localhost' if drupal.defer is defined and drupal.defer else inventory_hostname }}" + when: database.is_replica is not defined diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index 33c27a5a..691163f5 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -6,20 +6,20 @@ DBHOST='{{ database.host }}' DB_NAME='{{ database.original.database }}' CURRENT_DBNAME='{{ database.name }}' TARGET_DIR="{{ cron_mysql_backup.dumps_directory }}/$DBHOST" -TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" +TARBALL="$DB_NAME-$(date -Iseconds).sql.gz" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ mysqldump {{ cron_mysql_backup.mysqldump_params }} \ - -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" + -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | gzip > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } cleanup(){ - if [ "$(find "$TARGET_DIR" -name '*sql.bz2' | wc -l)" -lt "$KEEP" ]; then + if [ "$(find "$TARGET_DIR" -name '*.sql.*' | wc -l)" -lt "$KEEP" ]; then return 0 fi - rm "$(find $TARGET_DIR -name '*.sql.bz2' | sort | head -n 1)" + rm "$(find $TARGET_DIR -name '*.sql.*' | sort | head -n 1)" cleanup } diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 4f476253..3012032f 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -26,11 +26,6 @@ name: "cache_clear/cache_clear-{{ project_type }}" when: previous_build_number > 0 -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 4dc66a3a..1935ac79 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -13,6 +13,21 @@ - www_user != deploy_user - previous_build_number == 0 +- name: Fix file permissions for config directory. + ansible.builtin.file: + state: directory + path: "{{ deploy_path }}/{{ site.config_sync_directory }}" + owner: "{{ www_user }}" + group: "{{ www_user }}" + mode: '0775' # in this case often our deploy user is in the web user group and will need to be able to manipulate config + become: true + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - www_user != deploy_user + - previous_build_number == 0 + - name: Install Drupal. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" @@ -23,6 +38,38 @@ loop_control: loop_var: site when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) + register: _drush_output + +- name: Sync database. + ansible.builtin.include_role: + name: sync/database_sync + vars: + mysql_sync: "{{ site.mysql_sync }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - (previous_build_number == 0) or (site.force_install is defined and site.force_install) + - site.feature_branch | default(false) + - site.mysql_sync | length > 0 + +- name: Sync files. + ansible.builtin.include_role: + name: sync/files_sync + vars: + files_sync: "{{ site.files_sync }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - (previous_build_number == 0) or (site.force_install is defined and site.force_install) + - site.feature_branch | default(false) + - site.files_sync | length > 0 + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output - name: Fix permissions on Drupal directory. ansible.builtin.file: @@ -44,11 +91,6 @@ - previous_build_number > 0 - site.config_import_command != 'deploy' -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" @@ -59,6 +101,12 @@ loop_control: loop_var: site when: site.config_import_command != 'deploy' + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output - name: Import configuration. ansible.builtin.command: @@ -72,3 +120,9 @@ when: - previous_build_number > 0 - site.config_import_command + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md index b52413bf..2766a4f3 100644 --- a/roles/database_backup/database_backup-mysql/README.md +++ b/roles/database_backup/database_backup-mysql/README.md @@ -1,5 +1,19 @@ # MySQL backups Generate MySQL backups for each build. + +## Replicas +If you are using a read only replica in your application and you need to add it to `databases` in order to have access to the credentials for your app settings, be sure to set the database up in a similar way to this: + +```yaml +mysql_backup: + databases: + - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" + user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" + credentials_file: "/home/{{ deploy_user }}/.mysql.creds" + handling: none # prevents the replica from being backed up + is_replica: true # tells ce-deploy we are working with a replica, so it will implement a pause +``` + ## Default variables ```yaml @@ -8,6 +22,8 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage + mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}" # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: @@ -22,9 +38,11 @@ mysql_backup: # This is useful for locked-down setups where you do not have GRANT permissions. credentials_handling: rotate databases: - - database: "{{ project_name }}_{{ build_type }}" - user: "{{ project_name }}_{{ build_type }}" + - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names + user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit credentials_file: "/home/{{ deploy_user }}/.mysql.creds" + #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas + #is_replica: true # tell ce-deploy this database is a replica - can only be true, remove/comment out if not required ``` diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index f7e070e9..e7351d89 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -3,6 +3,8 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage + mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}" # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: @@ -17,6 +19,8 @@ mysql_backup: # This is useful for locked-down setups where you do not have GRANT permissions. credentials_handling: rotate databases: - - database: "{{ project_name }}_{{ build_type }}" - user: "{{ project_name }}_{{ build_type }}" + - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names + user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit credentials_file: "/home/{{ deploy_user }}/.mysql.creds" + #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas + #is_replica: true # tell ce-deploy this database is a replica - can only be true, remove/comment out if not required diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index 2609cabe..734aad4d 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -2,7 +2,7 @@ # We assume it's safe to only go back 50 build back. - name: Delete mysql dumps. ansible.builtin.file: - path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" + path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.*" state: absent with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index 89cbf171..23654247 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -14,6 +14,7 @@ _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" - ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" + when: database.is_replica is not defined - name: Delete mysql users. community.mysql.mysql_user: @@ -24,5 +25,7 @@ login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} - when: mysql_backup.credentials_handling == 'rotate' + when: + - mysql_backup.credentials_handling == 'rotate' + - database.is_replica is not defined run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 9f6f6959..49bda27e 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -4,6 +4,15 @@ - ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" +- name: Check if the database exists already. + ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}" + register: _build_db_status + failed_when: _build_db_status.rc > 1 # only exit on abnormal errors + run_once: true + args: + executable: /bin/bash + +# Create database if this is an initial build or it doesn't exist yet - name: Create initial database. community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" @@ -11,7 +20,7 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - when: previous_build_number == 0 + when: previous_build_number == 0 or _build_db_status.rc == 1 run_once: true - name: Ensure the dump directory exists. @@ -23,7 +32,7 @@ run_once: true - name: Take a database dump. - ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | gzip > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz" args: executable: /bin/bash when: previous_build_number > 0 diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml index dea8fe9f..4c4b1624 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml @@ -4,6 +4,15 @@ - ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" +- name: Check if the database exists already. + ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}" + register: _build_db_status + failed_when: _build_db_status.rc > 1 # only exit on abnormal errors + run_once: true + args: + executable: /bin/bash + +# Create database if this is an initial build or it doesn't exist yet - name: Create initial database. community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" @@ -11,5 +20,5 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - when: previous_build_number == 0 + when: previous_build_number == 0 or _build_db_status.rc == 1 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index aec45596..30fbaa62 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -4,26 +4,46 @@ _mysql_build_database_name: "{{ database.database }}_{{ build_number }}" - ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}_{{ previous_build_number }}" - # Note: we don't use the mysql_db Ansible module on purpose. - # If database already exists, we want to fail and not override it - # with previous build. - # @TODO fix this so we check if the database exists and exit with - # the proper plugin instead of using command. -- name: Create new database. - ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE `{{ _mysql_build_database_name }}`;" + +- name: Check if the new database exists already. + ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}" + register: _build_db_status + failed_when: _build_db_status.rc == 0 # we want the build to fail if the database exists + run_once: true + args: + executable: /bin/bash + +- name: Create a new database. + community.mysql.mysql_db: + name: "{{ _mysql_build_database_name }}" + state: present + config_file: "{{ database.credentials_file }}" + config_overrides_defaults: true run_once: true -#- name: Create a new database. -# community.mysql.mysql_db: -# name: "{{ _mysql_build_database_name }}" -# state: present -# config_file: "{{ database.credentials_file }}" -# config_overrides_defaults: true -# run_once: true +- name: Check if the previous database exists. + ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_previous_build_database_name }}" + register: _previous_db_status + failed_when: _previous_db_status.rc > 1 # only fail if we get an unexpected exitcode + run_once: true + args: + executable: /bin/bash +# Importing with shell for speed, we cannot import with mysql_db unless we create a dump file first - name: Populate new database. ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" args: executable: /bin/bash - when: previous_build_number > 0 + when: + - previous_build_number > 0 + - _previous_db_status.rc == 0 # only run if we have a database + run_once: true + +# Making it clear if we skipped the population of the new database +- name: Populating database skipped. + ansible.builtin.debug: + msg: "### Attention - the new database was NOT populated!" + when: + - previous_build_number > 0 + - _previous_db_status.rc != 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index 4160806c..f8ac6f62 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -71,6 +71,11 @@ - _legacy_static_creds.stat.exists ### End of legacy handling. +- name: Ensure the temporary password directory exists. + ansible.builtin.file: + state: directory + path: "{{ mysql_backup.mysql_password_path }}" + - ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' @@ -79,10 +84,23 @@ _mysql_build_user_name: "{{ database.user }}_{{ build_number }}" when: mysql_backup.credentials_handling == 'rotate' - ansible.builtin.set_fact: - _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}" + _mysql_build_password: "{{ lookup('password', '{{ mysql_backup.mysql_password_path }}') }}" when: mysql_backup.credentials_handling == 'rotate' -- ansible.builtin.include_tasks: "deploy-{{ mysql_backup.handling }}.yml" +- name: "Set database handling type to {{ mysql_backup.handling }}." + ansible.builtin.set_fact: + _mysql_handling: "{{ mysql_backup.handling }}" +# If we have a specific instruction for handling this database differently, use it. +- name: Override database handling type for this database, if specified. + ansible.builtin.set_fact: + _mysql_handling: "{{ database.handling }}" + when: + - database.handling is defined + - database.handling | length > 0 + +- name: Execute backup tasks. + ansible.builtin.include_tasks: "deploy-{{ _mysql_handling }}.yml" + when: database.is_replica is not defined # We append privileges instead of replacing, # to allow this role to be looped over, @@ -91,12 +109,16 @@ # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user for TCP connections. ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'%';" - when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + when: + - ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + - database.is_replica is not defined run_once: true - name: Create/update mysql user for unix socket connections. ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'localhost';" - when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + when: + - ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + - database.is_replica is not defined run_once: true - ansible.builtin.set_fact: diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 80898e7d..6040c73a 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,7 +1,29 @@ --- +- name: Move dump file to local disk. + ansible.builtin.copy: + remote_src: true + src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz" + dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql.gz" + run_once: true + when: previous_build_number > 0 + +# unarchive module does not support archives not in tar.gz format +- name: Unpack dump file. + ansible.builtin.command: + cmd: "gunzip /tmp/{{ database.database }}-{{ previous_build_number }}.sql.gz" + run_once: true + when: previous_build_number > 0 + - name: Revert database from dump. - ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql" args: executable: /bin/bash + run_once: true when: previous_build_number > 0 + +- name: Delete unpacked dump file. + ansible.builtin.file: + path: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + state: absent run_once: true + when: previous_build_number > 0 diff --git a/roles/database_backup/database_backup-mysql/tasks/revert.yml b/roles/database_backup/database_backup-mysql/tasks/revert.yml index f6076262..9e698c91 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert.yml @@ -17,3 +17,4 @@ when: - previous_build_number > 0 - database_backup.revert + - database.is_replica is not defined diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md index 377ca80c..5d4cc722 100644 --- a/roles/deploy_code/README.md +++ b/roles/deploy_code/README.md @@ -121,6 +121,8 @@ deploy_code: service_action: reload # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false + # Used to skip tasks to fix ownership and permissions, drupal needs this set to true by default + fix_cleanup_perms: true # Details of API call to trigger. See api_call role. api_call: type: gitlab diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index f9b84739..7b68c3a7 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -35,6 +35,8 @@ deploy_code: service_action: reload # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false + # Used to skip tasks to fix ownership and permissions, drupal needs this set to true by default + fix_cleanup_perms: true # Details of API call to trigger. See api_call role. api_call: type: gitlab diff --git a/roles/deploy_code/deploy_code-custom/tasks/main.yml b/roles/deploy_code/deploy_code-custom/tasks/main.yml new file mode 100644 index 00000000..6f34c92f --- /dev/null +++ b/roles/deploy_code/deploy_code-custom/tasks/main.yml @@ -0,0 +1,3 @@ +--- + +# Nothing to do here. \ No newline at end of file diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index db2bb9bb..449c1103 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -4,7 +4,9 @@ cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true - when: "www_user != deploy_user" + when: + - "www_user != deploy_user" + - deploy_code.fix_cleanup_perms - name: Ensure permissions are set on deploy directory. ansible.builtin.shell: @@ -15,15 +17,6 @@ - deploy_code.perms_fix_path | length > 1 - deploy_code.mount_type != "squashfs" -- name: Ensure permissions are set on builds directory. - ansible.builtin.shell: - cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} - when: - - deploy_code.perms_fix_path is defined - - deploy_code.perms_fix_path | length > 1 - - deploy_code.mount_type == "squashfs" - - name: Delete codebases from deploy directory. ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" @@ -32,14 +25,6 @@ when: - deploy_code.mount_type != "squashfs" -- name: Delete codebases from builds directory. - ansible.builtin.file: - name: "{{ build_path_prefix }}{{ item }}" - state: absent - with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} - when: - - deploy_code.mount_type == "squashfs" - - name: Create a tarball of the deployed codebases. ansible.builtin.command: cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar --owner=0 --group=0 {{ deploy_base_path }}" @@ -49,15 +34,6 @@ - deploy_code.mount_type == "tarball" run_once: true -- name: Create a SquashFS image of the deployed codebases. - ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh" - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - run_once: true - - name: Create destination folder. ansible.builtin.file: path: "{{ deploy_code.mount_sync }}" @@ -77,132 +53,153 @@ - deploy_code.mount_type == "tarball" run_once: true -- name: Move SquashFS image to final destination. - ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - run_once: true - -- name: Check if we have a SquashFS image already there. - ansible.builtin.stat: - path: "{{ build_base_path }}/deploy.sqsh" - register: _deploy_code_mount_image - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Copy previous SquashFS image to shared mount point in case of rollback. - ansible.builtin.copy: - remote_src: true - force: true - src: "{{ build_base_path }}/deploy.sqsh" - dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh" - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_image.stat.islnk is defined - run_once: true - -- name: Ensure mounted SquashFS image is deleted. - ansible.builtin.file: - path: "{{ build_base_path }}/deploy.sqsh" - state: absent - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Copy SquashFS image to local server. - ansible.builtin.command: - cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Check if we have a mount already. - ansible.builtin.shell: - cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" - args: - executable: /bin/bash - failed_when: false - register: _deploy_code_mount_check - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Reload any services that might be keeping the loop device busy. - ansible.builtin.service: - name: "{{ www_service }}" - state: reloaded - with_items: "{{ deploy_code.services }}" - loop_control: - loop_var: www_service - become: true - when: - - deploy_code.service_action == "reload" - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded - - deploy_code.services | length > 0 - -- name: Stop any services that might be keeping the loop device busy. - ansible.builtin.service: - name: "{{ www_service }}" - state: stopped - with_items: "{{ deploy_code.services }}" - loop_control: - loop_var: www_service - become: true - when: - - deploy_code.service_action == "stop" - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded - - deploy_code.services | length > 0 - -- name: Unmount existing SquashFS image. - ansible.builtin.command: - cmd: "umount {{ deploy_base_path }}" - become: true - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded - -- name: Mount new SquashFS image. - ansible.builtin.command: - cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" - become: true - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Start any services we stopped. - ansible.builtin.service: - name: "{{ www_service }}" - state: started - with_items: "{{ deploy_code.services }}" - loop_control: - loop_var: www_service - become: true +# Beginning of the squashFS block. +- name: Execute tasks for squashFS mount type. + block: + - name: Ensure permissions are set on builds directory. + ansible.builtin.shell: + cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.perms_fix_path is defined + - deploy_code.perms_fix_path | length > 1 + + - name: Delete codebases from builds directory. + ansible.builtin.file: + name: "{{ build_path_prefix }}{{ item }}" + state: absent + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + + - name: Create a SquashFS image of the deployed codebases. + ansible.builtin.command: + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh" + run_once: true + + - name: Move SquashFS image to final destination. + ansible.builtin.command: + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + run_once: true + + - name: Check if we have a SquashFS image already there. + ansible.builtin.stat: + path: "{{ build_base_path }}/deploy.sqsh" + register: _deploy_code_mount_image + + - name: Copy previous SquashFS image to shared mount point in case of rollback. + ansible.builtin.copy: + remote_src: true + force: true + src: "{{ build_base_path }}/deploy.sqsh" + dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh" + when: + - _deploy_code_mount_image.stat.islnk is defined + run_once: true + + - name: Ensure mounted SquashFS image is deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + + - name: Copy SquashFS image to local server. + ansible.builtin.command: + cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" + + - name: Check if we have a mount already. + ansible.builtin.shell: + cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" + args: + executable: /bin/bash + failed_when: false + register: _deploy_code_mount_check + + - name: Get the current pts session. + ansible.builtin.shell: + cmd: "tty | sed 's#/dev/##'" + register: deploy_pts + + - name: "Check for active sessions in {{ deploy_base_path }}." + ansible.builtin.shell: + cmd: "ps -eo pid,tty | awk '{print $1}' | xargs -n 1 pwdx 2>&1 | grep -v 'No such process' | grep {{ deploy_base_path }} | cut -d: -f1 | xargs -n 1 ps -o tty= -p | sort | uniq" + register: sessions_in_deploy_path + become: true + + - name: Display active sessions. + ansible.builtin.debug: + msg: > + Deploy session: {{ deploy_pts.stdout | default('Unknown') }}. + Active sessions in {{ deploy_base_path }}: {{ sessions_in_deploy_path.stdout_lines | default([]) | join(', ') | default('None') }}. + + - name: Kill sessions except the current one. + ansible.builtin.command: + cmd: "pkill -9 -t {{ item }}" + loop: "{{ sessions_in_deploy_path.stdout_lines }}" + when: + - "item != deploy_pts.stdout" + - "item is match('^pts/\\d+$')" + failed_when: false + register: kill_sessions_result + become: true + + - name: Display killed sessions. + ansible.builtin.debug: + msg: > + Sessions terminated: {{ kill_sessions_result.results | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | map(attribute='item') | list | join(', ') | default('None') }}. + + - name: Reload any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: reloaded + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - _deploy_code_mount_check.rc == 0 + - deploy_code.service_action == "reload" + - deploy_code.services | length > 0 + + - name: Stop any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: stopped + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - _deploy_code_mount_check.rc == 0 + - deploy_code.service_action == "stop" + - deploy_code.services | length > 0 + + - name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount {{ deploy_base_path }}" + become: true + when: + - _deploy_code_mount_check.rc == 0 + + - name: Mount new SquashFS image. + ansible.builtin.command: + cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" + become: true + + - name: Start any services we stopped. + ansible.builtin.service: + name: "{{ www_service }}" + state: started + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - _deploy_code_mount_check.rc == 0 + - deploy_code.service_action == "stop" + - deploy_code.services | length > 0 when: - - deploy_code.service_action == "stop" - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded - - deploy_code.services | length > 0 +# End of the squashFS block. - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md new file mode 100644 index 00000000..06847722 --- /dev/null +++ b/roles/deploy_container/README.md @@ -0,0 +1,148 @@ +# Deploy container +Step that deploys the codebase in a Docker container image. Requires Docker and the `community.docker` collection for Ansible to be installed on your deploy server. You will also need to add a `docker` group and make sure your local deploy user is in that group, for example: + +``` +sudo groupadd docker +sudo usermod -aG docker deploy +``` + +This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles. + +If you set the `deploy_container.action` to `destroy` then the role will also take care of tidying up containers. If it is an AWS ECS deployment then it will also tidy up the ECS service for you. + +## AWS IAM requirements +AWS integration requires the AWS CLI user provided for `ce-deploy` to have certain managed AWS policies attached. + +If you enable AWS ECR registry integration by setting `deploy_container.aws_ecr.enabled` to `true` then you will need the `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers. + +Similarly, if you set `deploy_container.aws_ecs.acm.create_cert` to `true` then you will need the `AWSCertificateManagerFullAccess` policy attaching to create SSL certificates. + +If you enable full AWS ECS integration by setting `deploy_container.aws_ecs.enabled` to `true` then this requires the following policies to be attached to the AWS CLI user: +* `AmazonECS_FullAccess` - to create task definitions and services +* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups + +Finally, if you set `deploy_container.aws_ecs.route_53.zone` to another other than an empty string then you will also need `AmazonRoute53FullAccess` attaching to manipulate DNS entries in Route 53. + +The full list is: +* `EC2InstanceProfileForImageBuilderECRContainerBuilds` - to manipulate images in AWS ECR +* `AWSCertificateManagerFullAccess` - to manage SSL certificates +* `AmazonECS_FullAccess` - to create task definitions and services +* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups +* `AmazonRoute53FullAccess` - to manage DNS entries + +Naturally you can always create custom policies and roles to have tighter access control. This document simply gives you the broad strokes AWS managed policies you can use in conjunction with this Ansible role. + +# Peculiarities of AWS ECS +It is worth noting that even if you put your containers on private subnets and configure your apps to use internal addressing, traffic will pass via the public interface. Therefore any safelisting of IP addresses needs to include the IP addresses of the NAT gateways of your private subnets. [More on how this works here.](https://docs.aws.amazon.com/AmazonECS/latest/bestpracticesguide/networking-connecting-vpc.html) + + + + + +## Default variables +```yaml +--- +deploy_container: + action: create # can also be destroy + container_name: example-container + container_tag: latest # tag will take format container_name:container_tag + container_force_build: true # force Docker to build and tag a new image + docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name + docker_registry_user: example + docker_registry_pass: asdf1234 + docker_base_command: "docker image build" + docker_build_dir: "{{ _ce_deploy_build_dir }}" + dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name + environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template + # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached + aws_ecr: + enabled: false # set to true if using AWS ECR + region: eu-west-1 + aws_profile: example + # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached + # Note, you can if you wish make more restrictive roles and policies + aws_ecs: + enabled: false + region: eu-west-1 + aws_profile: example + tags: {} + domain_name: www.example.com + route_53: + zone: example.com + aws_profile: example2 # might not be the same account + vpc_name: example + #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use + security_groups: [] # list of security groups, accepts names or IDs + cluster_name: example-cluster + family_name: example-task-definition + task_definition_revision: "" # integer, but must be presented as a string for Jinja2 + task_definition_force_create: false # creates a task definition revision every time if set to true + task_execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable + #task_role_arn: "" # required if you set service_enable_ssm to true + task_count: 1 + task_minimum_count: 1 + task_maximum_count: 4 + # These subnets are usually the subnets created by ce-provision when you made your ECS cluster and must have a NAT gateway for ECR access. + service_subnets: # list of private subnet names + - example-cluster-dev-a + - example-cluster-dev-b + # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html + service_autoscale_metric_type: ECSServiceAverageCPUUtilization + service_autoscale_up_cooldown: 120 + service_autoscale_down_cooldown: 120 + service_autoscale_target_value: 70 # the value to trigger a scaling event at + service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973 + service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API + service_force_refresh: false # forces a refresh of all containers if set to true + containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers + - name: example-container + essential: true + image: index.docker.io/example:latest + portMappings: + - containerPort: 8080 # should match target_group_port + hostPort: 8080 + logConfiguration: + logDriver: awslogs + options: + awslogs-group: /ecs/example-cluster + awslogs-region: eu-west-1 + awslogs-stream-prefix: "ecs-example-task" + cpu: 512 # these values can be set globally or per container + memory: 1024 + launch_type: FARGATE + network_mode: awsvpc + #volumes: [] # list of additional volumes to attach + target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen + target_group_protocol: http + target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile + target_group_wait_timeout: 200 # how long to wait for target group events to complete + targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service + #- Id: 10.0.0.2 + # Port: 80 + # AvailabilityZone: all + health_check: + protocol: http + path: / + response_codes: "200" + # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached + acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm + create_cert: false + extra_domains: [] # list of Subject Alternative Name domains and zones + ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager + elb_security_groups: [] # default SG is used if none provided - module supports names or IDs + elb_subnets: # must be public subnets for public facing applications + - example-dev-a + - example-dev-b + elb_http_port: 80 + elb_https_port: 443 + elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + elb_listener_http_rules: [] + elb_listener_https_rules: [] + # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html + elb_listeners: [] + elb_idle_timeout: 60 + elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses). + +``` + + diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml new file mode 100644 index 00000000..ff99d2b9 --- /dev/null +++ b/roles/deploy_container/defaults/main.yml @@ -0,0 +1,101 @@ +--- +deploy_container: + action: create # can also be destroy + container_name: example-container + container_tag: latest # tag will take format container_name:container_tag + container_force_build: true # force Docker to build and tag a new image + docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name + docker_registry_user: example + docker_registry_pass: asdf1234 + docker_base_command: "docker image build" + docker_build_dir: "{{ _ce_deploy_build_dir }}" + dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name + environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template + # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached + aws_ecr: + enabled: false # set to true if using AWS ECR + region: eu-west-1 + aws_profile: example + # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached + # Note, you can if you wish make more restrictive roles and policies + aws_ecs: + enabled: false + region: eu-west-1 + aws_profile: example + tags: {} + domain_name: www.example.com + route_53: + zone: example.com + aws_profile: example2 # might not be the same account + vpc_name: example + #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use + security_groups: [] # list of security groups, accepts names or IDs + cluster_name: example-cluster + family_name: example-task-definition + task_definition_revision: "" # integer, but must be presented as a string for Jinja2 + task_definition_force_create: false # creates a task definition revision every time if set to true + task_execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable + #task_role_arn: "" # required if you set service_enable_ssm to true + task_count: 1 + task_minimum_count: 1 + task_maximum_count: 4 + # These subnets are usually the subnets created by ce-provision when you made your ECS cluster and must have a NAT gateway for ECR access. + service_subnets: # list of private subnet names + - example-cluster-dev-a + - example-cluster-dev-b + # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html + service_autoscale_metric_type: ECSServiceAverageCPUUtilization + service_autoscale_up_cooldown: 120 + service_autoscale_down_cooldown: 120 + service_autoscale_target_value: 70 # the value to trigger a scaling event at + service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973 + service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API + service_force_refresh: false # forces a refresh of all containers if set to true + containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers + - name: example-container + essential: true + image: index.docker.io/example:latest + portMappings: + - containerPort: 8080 # should match target_group_port + hostPort: 8080 + logConfiguration: + logDriver: awslogs + options: + awslogs-group: /ecs/example-cluster + awslogs-region: eu-west-1 + awslogs-stream-prefix: "ecs-example-task" + cpu: 512 # these values can be set globally or per container + memory: 1024 + launch_type: FARGATE + network_mode: awsvpc + #volumes: [] # list of additional volumes to attach + target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen + target_group_protocol: http + target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile + target_group_wait_timeout: 200 # how long to wait for target group events to complete + targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service + #- Id: 10.0.0.2 + # Port: 80 + # AvailabilityZone: all + health_check: + protocol: http + path: / + response_codes: "200" + # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached + acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm + create_cert: false + extra_domains: [] # list of Subject Alternative Name domains and zones + ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager + elb_security_groups: [] # default SG is used if none provided - module supports names or IDs + elb_subnets: # must be public subnets for public facing applications + - example-dev-a + - example-dev-b + elb_http_port: 80 + elb_https_port: 443 + elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + elb_listener_http_rules: [] + elb_listener_https_rules: [] + # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html + elb_listeners: [] + elb_idle_timeout: 60 + elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses). diff --git a/roles/deploy_container/tasks/action-cleanup.yml b/roles/deploy_container/tasks/action-cleanup.yml new file mode 100644 index 00000000..57312dbe --- /dev/null +++ b/roles/deploy_container/tasks/action-cleanup.yml @@ -0,0 +1,3 @@ +--- +# @TODO we need container image cleanup here and potentially ECS Task Definition cleanup too. +# Possibly useful link: https://stackoverflow.com/a/40949364 diff --git a/roles/deploy_container/tasks/action-create.yml b/roles/deploy_container/tasks/action-create.yml new file mode 100644 index 00000000..cc06163d --- /dev/null +++ b/roles/deploy_container/tasks/action-create.yml @@ -0,0 +1,363 @@ +--- +# Build and ship a container image +- name: Create Dockerfile from template. + ansible.builtin.template: + src: "{{ deploy_container.dockerfile_template }}" + dest: "{{ deploy_container.docker_build_dir }}/Dockerfile" + delegate_to: localhost + +- name: Set Docker registry username and password. + ansible.builtin.set_fact: + _docker_registry_username: "{{ deploy_container.docker_registry_user }}" + _docker_registry_password: "{{ deploy_container.docker_registry_pass }}" + delegate_to: localhost + +- name: Fetch AWS ECR registry login token. # token valid for 12 hours + ansible.builtin.command: + cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + register: _docker_registry_ecr_token + +- name: Set AWS ECR registry password. + ansible.builtin.set_fact: + _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + +- name: Set AWS ECR registry username. + ansible.builtin.set_fact: + _docker_registry_username: "AWS" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + +- name: Remove Docker credentials file. + ansible.builtin.file: + state: absent + path: "/home/{{ deploy_user }}/.docker/config.json" + delegate_to: localhost + +- name: Log into Docker registry. + community.docker.docker_login: + registry_url: "{{ deploy_container.docker_registry_url }}" + username: "{{ _docker_registry_username }}" + password: "{{ _docker_registry_password }}" + reauthorize: true + delegate_to: localhost + +- name: Build and push container image. + community.docker.docker_image: + build: + path: "{{ deploy_container.docker_build_dir }}" + name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}" + tag: "{{ deploy_container.container_tag | default('latest') }}" + push: true + source: build + force_source: "{{ deploy_container.container_force_build }}" + force_tag: "{{ deploy_container.container_force_build }}" + delegate_to: localhost + +# Fetch the ACM role from ce-provision +- name: Ensure the aws_acm directory exists. + ansible.builtin.file: + path: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}" + state: directory + mode: '0755' + delegate_to: localhost + with_items: + - tasks + - defaults + +- name: Fetch the aws_acm files. + ansible.builtin.get_url: + url: "https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/{{ item }}/main.yml" + dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}/main.yml" + delegate_to: localhost + with_items: + - tasks + - defaults + +- name: Fetch the aws_acm tasks. + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/tasks/main.yml + dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/tasks/main.yml" + delegate_to: localhost + +# Gather all network information +- name: Gather VPC information. + amazon.aws.ec2_vpc_net_info: + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + region: "{{ deploy_container.aws_ecs.region }}" + filters: + "tag:Name": "{{ deploy_container.aws_ecs.vpc_name }}" + register: _aws_ecs_cluster_vpc + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.vpc_name is defined + - deploy_container.aws_ecs.vpc_name | length > 0 + +- name: Set the VPC id from name. + ansible.builtin.set_fact: + _aws_ecs_cluster_vpc_id: "{{ _aws_ecs_cluster_vpc.vpcs[0].vpc_id }}" + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.vpc_name is defined + - deploy_container.aws_ecs.vpc_name | length > 0 + +- name: Use provided VPC id. + ansible.builtin.set_fact: + _aws_ecs_cluster_vpc_id: "{{ deploy_container.aws_ecs.vpc_id }}" + when: + - deploy_container.aws_ecs.enabled + - (deploy_container.aws_ecs.vpc_name is not defined or deploy_container.aws_ecs.vpc_name | length < 0) + +- name: Reset subnets lists. + ansible.builtin.set_fact: + _aws_ecs_cluster_public_subnets_ids: [] + _aws_ecs_cluster_private_subnets_ids: [] + when: deploy_container.aws_ecs.enabled + +- name: Construct list of public subnet IDs. + ansible.builtin.include_tasks: subnet-public.yml + with_items: "{{ deploy_container.aws_ecs.elb_subnets }}" + loop_control: + loop_var: subnet + when: deploy_container.aws_ecs.enabled + +- name: Construct list of private subnet IDs. + ansible.builtin.include_tasks: subnet-private.yml + with_items: "{{ deploy_container.aws_ecs.service_subnets }}" + loop_control: + loop_var: subnet + when: deploy_container.aws_ecs.enabled + +# Construct AWS supporting assets +- name: Create task definition. + community.aws.ecs_taskdefinition: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + family: "{{ deploy_container.aws_ecs.family_name }}" + execution_role_arn: "{{ deploy_container.aws_ecs.task_execution_role_arn }}" + task_role_arn: "{{ deploy_container.aws_ecs.task_role_arn | default(omit) }}" + containers: "{{ deploy_container.aws_ecs.containers }}" + launch_type: "{{ deploy_container.aws_ecs.launch_type }}" + cpu: "{{ deploy_container.aws_ecs.cpu | default(omit) }}" + memory: "{{ deploy_container.aws_ecs.memory | default(omit) }}" + state: present + network_mode: "{{ deploy_container.aws_ecs.network_mode }}" + volumes: "{{ deploy_container.aws_ecs.volumes | default(omit) }}" + force_create: "{{ deploy_container.aws_ecs.task_definition_force_create }}" + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Create a target group with IP address targets. + community.aws.elb_target_group: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + protocol: "{{ deploy_container.aws_ecs.target_group_protocol }}" + port: "{{ deploy_container.aws_ecs.target_group_port }}" + vpc_id: "{{ _aws_ecs_cluster_vpc_id }}" + health_check_protocol: "{{ deploy_container.aws_ecs.health_check.protocol }}" + health_check_path: "{{ deploy_container.aws_ecs.health_check.path }}" + successful_response_codes: "{{ deploy_container.aws_ecs.health_check.response_codes }}" + target_type: ip + targets: "{{ deploy_container.aws_ecs.targets }}" + state: present + wait_timeout: "{{ deploy_container.aws_ecs.target_group_wait_timeout }}" + wait: true + register: _aws_ecs_target_group + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Create SSL certificate for load balancer. + ansible.builtin.include_role: + name: aws_acm + vars: + aws_acm: + region: "{{ deploy_container.aws_ecs.region }}" + aws_profile: "{{ deploy_container.aws_ecs.aws_profile }}" + tags: "{{ deploy_container.aws_ecs.tags }}" + export: false + domain_name: "{{ deploy_container.aws_ecs.domain_name }}" + extra_domains: "{{ deploy_container.aws_ecs.acm.extra_domains }}" + route_53: + aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}" + zone: "{{ deploy_container.aws_ecs.route_53.zone }}" + when: + - deploy_container.aws_ecs.acm.create_cert + - deploy_container.aws_ecs.enabled + +- name: Default to provided SSL certificate ARN. + ansible.builtin.set_fact: + _ssl_certificate_ARN: "{{ deploy_container.aws_ecs.ssl_certificate_ARN }}" + when: deploy_container.aws_ecs.enabled + +- name: If provided, override SSL certificate ARN with the one received from ACM. + ansible.builtin.set_fact: + _ssl_certificate_ARN: "{{ aws_acm_certificate_arn }}" + when: + - deploy_container.aws_ecs.acm.create_cert + - deploy_container.aws_ecs.enabled + +- name: Define default ALB listeners. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners_http: + Protocol: HTTP + Port: "{{ deploy_container.aws_ecs.elb_http_port }}" + DefaultActions: + - Type: forward + TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" + Rules: "{{ deploy_container.aws_ecs.elb_listener_http_rules }}" + _aws_ecs_cluster_listeners_redirect: + Protocol: HTTP + Port: "{{ deploy_container.aws_ecs.elb_http_port }}" + DefaultActions: + - Type: redirect + RedirectConfig: + Protocol: HTTPS + Host: "#{host}" + Query: "#{query}" + Path: "/#{path}" + Port: "{{ deploy_container.aws_ecs.elb_https_port }}" + StatusCode: HTTP_301 + _aws_ecs_cluster_listeners_https: + Protocol: HTTPS + Port: "{{ deploy_container.aws_ecs.elb_https_port }}" + SslPolicy: "{{ deploy_container.aws_ecs.elb_ssl_policy }}" + Certificates: + - CertificateArn: "{{ _ssl_certificate_ARN }}" + DefaultActions: + - Type: forward + TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" + Rules: "{{ deploy_container.aws_ecs.elb_listener_https_rules }}" + when: deploy_container.aws_ecs.enabled + +- name: Add HTTP listeners. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_http ] }}" + when: + - _ssl_certificate_ARN | length < 1 + - deploy_container.aws_ecs.enabled + +- name: Add HTTPS Listener. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_redirect, _aws_ecs_cluster_listeners_https ] }}" + when: + - _ssl_certificate_ARN | length > 1 + - deploy_container.aws_ecs.enabled + +- name: Add custom Listeners. + ansible.builtin.set_fact: + _aws_ecs_cluster_listeners: "{{ _aws_ecs_cluster_listeners + deploy_container.aws_ecs.elb_listeners }}" + when: + - deploy_container.aws_ecs.elb_listeners is defined + - deploy_container.aws_ecs.elb_listeners | length + - deploy_container.aws_ecs.enabled + +- name: Create an ALB. + amazon.aws.elb_application_lb: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + state: present + tags: "{{ deploy_container.aws_ecs.tags }}" + subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}" + security_groups: "{{ deploy_container.aws_ecs.elb_security_groups }}" + listeners: "{{ _aws_ecs_cluster_listeners }}" + idle_timeout: "{{ deploy_container.aws_ecs.elb_idle_timeout }}" + ip_address_type: "{{ deploy_container.aws_ecs.elb_ip_address_type }}" + register: _aws_ecs_cluster_alb + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Set task definition name. + ansible.builtin.set_fact: + _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}" + when: deploy_container.aws_ecs.enabled + +- name: Set task definition revision if applicable. + ansible.builtin.set_fact: + _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}:{{ deploy_container.aws_ecs.task_definition_revision }}" + when: + - deploy_container.aws_ecs.task_definition_revision | length > 0 + - deploy_container.aws_ecs.enabled + +- name: Create ECS service. + community.aws.ecs_service: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: present + name: "{{ deploy_container.aws_ecs.family_name }}" + cluster: "{{ deploy_container.aws_ecs.cluster_name }}" + task_definition: "{{ _aws_ecs_service_task_definition }}" + desired_count: "{{ deploy_container.aws_ecs.task_count }}" + launch_type: "{{ deploy_container.aws_ecs.launch_type }}" + platform_version: LATEST + load_balancers: # see https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_LoadBalancer.html + - containerName: "{{ deploy_container.container_name }}" + containerPort: "{{ deploy_container.aws_ecs.target_group_port }}" + targetGroupArn: "{{ _aws_ecs_target_group.target_group_arn }}" + network_configuration: + subnets: "{{ _aws_ecs_cluster_private_subnets_ids }}" # internal private subnet + security_groups: "{{ deploy_container.aws_ecs.security_groups }}" + assign_public_ip: "{{ deploy_container.aws_ecs.service_public_container_ip }}" + tags: "{{ deploy_container.aws_ecs.tags }}" + enable_execute_command: "{{ deploy_container.aws_ecs.service_enable_ssm }}" + force_new_deployment: "{{ deploy_container.aws_ecs.service_force_refresh }}" + wait: true + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Create target tracking scaling policy for ECS service. + community.aws.application_autoscaling_policy: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: present + policy_name: "{{ deploy_container.aws_ecs.family_name }}" + service_namespace: ecs + resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}" + scalable_dimension: ecs:service:DesiredCount + minimum_tasks: "{{ deploy_container.aws_ecs.task_minimum_count }}" + maximum_tasks: "{{ deploy_container.aws_ecs.task_maximum_count }}" + policy_type: TargetTrackingScaling + target_tracking_scaling_policy_configuration: + PredefinedMetricSpecification: + PredefinedMetricType: "{{ deploy_container.aws_ecs.service_autoscale_metric_type }}" + ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_up_cooldown }}" + ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_down_cooldown }}" + DisableScaleIn: false + TargetValue: "{{ deploy_container.aws_ecs.service_autoscale_target_value }}" + delegate_to: localhost + when: deploy_container.aws_ecs.enabled + +- name: Initialise the domains loop var with main domain entry DNS settings. + ansible.builtin.set_fact: + _aws_ecs_cluster_dns_all_domains: + - domain: "{{ deploy_container.aws_ecs.domain_name }}" + zone: "{{ deploy_container.aws_ecs.route_53.zone }}" + aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}" + when: deploy_container.aws_ecs.enabled + +- name: Add extra_domains so we can loop through DNS records. + ansible.builtin.set_fact: + _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}" + loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}" + when: + - deploy_container.aws_ecs.acm.extra_domains | length > 0 + - deploy_container.aws_ecs.enabled + +- name: Add DNS records in Route 53. + amazon.aws.route53: + state: present + profile: "{{ item.aws_profile }}" + zone: "{{ item.zone }}" + record: "{{ item.domain }}" + type: CNAME + value: "{{ _aws_ecs_cluster_alb.dns_name }}" + overwrite: true + loop: "{{ _aws_ecs_cluster_dns_all_domains }}" + when: + - deploy_container.aws_ecs.route_53.zone | length > 0 + - deploy_container.aws_ecs.enabled diff --git a/roles/deploy_container/tasks/action-destroy.yml b/roles/deploy_container/tasks/action-destroy.yml new file mode 100644 index 00000000..b0b6bd99 --- /dev/null +++ b/roles/deploy_container/tasks/action-destroy.yml @@ -0,0 +1,254 @@ +--- +# Be sure to include your deploy_container variables in your playbook +- name: Set Docker registry username and password. + ansible.builtin.set_fact: + _docker_registry_username: "{{ deploy_container.docker_registry_user }}" + _docker_registry_password: "{{ deploy_container.docker_registry_pass }}" + delegate_to: localhost + +- name: Fetch AWS ECR registry login token. # token valid for 12 hours + ansible.builtin.command: + cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + register: _docker_registry_ecr_token + +- name: Set AWS ECR registry password. + ansible.builtin.set_fact: + _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + +- name: Set AWS ECR registry username. + ansible.builtin.set_fact: + _docker_registry_username: "AWS" + when: deploy_container.aws_ecr.enabled + delegate_to: localhost + +- name: Remove Docker credentials file. + ansible.builtin.file: + state: absent + path: "/home/{{ deploy_user }}/.docker/config.json" + delegate_to: localhost + +- name: Log into Docker registry. + community.docker.docker_login: + registry_url: "{{ deploy_container.docker_registry_url }}" + username: "{{ _docker_registry_username }}" + password: "{{ _docker_registry_password }}" + reauthorize: true + delegate_to: localhost + +- name: Destroy matching container images. + community.docker.docker_image: + name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}" + tag: "{{ deploy_container.container_tag | default('latest') }}" + force_absent: true + state: absent + delegate_to: localhost + +# Destroy AWS services +- name: Get minimal ALB information before we destroy it. + amazon.aws.elb_application_lb_info: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + names: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + #include_attributes: false # @TODO - these attributes added in amazon.aws 7.0.0 + #include_listeners: false + #include_listener_rules: false + register: _aws_ecs_cluster_alb + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +- name: Destroy ALB. + amazon.aws.elb_application_lb: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + state: absent + wait: true + when: + - deploy_container.aws_ecs.enabled + +- name: Destroy target group. + community.aws.elb_target_group: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit + state: absent + wait: true + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +- name: Reduce task count to zero on ECS service. + community.aws.ecs_service: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: present + name: "{{ deploy_container.aws_ecs.family_name }}" + cluster: "{{ deploy_container.aws_ecs.cluster_name }}" + desired_count: 0 + force_new_deployment: true + wait: true + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +- name: Destroy ECS service. + community.aws.ecs_service: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: absent + name: "{{ deploy_container.aws_ecs.family_name }}" + cluster: "{{ deploy_container.aws_ecs.cluster_name }}" + wait: true + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +# @TODO: We cannot currently use the module for this +# See feature request: https://github.com/ansible-collections/community.aws/issues/2023 +#- name: Get task definition details. +# community.aws.ecs_taskdefinition_info: +# region: "{{ deploy_container.aws_ecs.region }}" +# profile: "{{ deploy_container.aws_ecs.aws_profile }}" +# task_definition: "{{ deploy_container.aws_ecs.family_name }}" +# register: _task_definition_info +# delegate_to: localhost + +- name: Ensure the task definitions ARN list variable is empty. + ansible.builtin.set_fact: + _task_definition_arns_list: [] + when: + - deploy_container.aws_ecs.enabled + +- name: Get active task definition details. + ansible.builtin.command: + cmd: "aws ecs list-task-definitions --status ACTIVE --family-prefix {{ deploy_container.aws_ecs.family_name }} --region {{ deploy_container.aws_ecs.region }} --profile {{ deploy_container.aws_ecs.aws_profile }}" + register: _task_definition_arns_raw + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +- name: Convert CLI output of active ARNs to a YAML variable. + ansible.builtin.set_fact: + _task_definition_arns: "{{ _task_definition_arns_raw.stdout | from_json }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Create a clean list of task definition ARNs. + ansible.builtin.set_fact: + _task_definition_arns_list: "{{ _task_definition_arns.taskDefinitionArns }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Get inactive task definition details. + ansible.builtin.command: + cmd: "aws ecs list-task-definitions --status INACTIVE --family-prefix {{ deploy_container.aws_ecs.family_name }} --region {{ deploy_container.aws_ecs.region }} --profile {{ deploy_container.aws_ecs.aws_profile }}" + register: _task_definition_arns_raw + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +- name: Convert CLI output of inactive ARNs to a YAML variable. + ansible.builtin.set_fact: + _task_definition_arns: "{{ _task_definition_arns_raw.stdout | from_json }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Add inactive task definition ARNs to the YAML list. + ansible.builtin.set_fact: + _task_definition_arns_list: "{{ _task_definition_arns_list + _task_definition_arns.taskDefinitionArns }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Deregister task definitions. + community.aws.ecs_taskdefinition: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + arn: "{{ item }}" + containers: "{{ deploy_container.aws_ecs.containers }}" + state: absent + delegate_to: localhost + with_items: "{{ _task_definition_arns_list }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Delete task definitions. + ansible.builtin.command: + cmd: "aws ecs delete-task-definitions --task-definitions {{ item }} --region {{ deploy_container.aws_ecs.region }} --profile {{ deploy_container.aws_ecs.aws_profile }}" + delegate_to: localhost + with_items: "{{ _task_definition_arns_list }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Destroy scaling policy for ECS service. + community.aws.application_autoscaling_policy: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + state: absent + policy_name: "{{ deploy_container.aws_ecs.family_name }}" + service_namespace: ecs + policy_type: TargetTrackingScaling + resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}" + scalable_dimension: ecs:service:DesiredCount + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + +# Clean up SSL certificates +- name: Delete the main ACM certificate. + community.aws.acm_certificate: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + domain_name: "{{ deploy_container.aws_ecs.domain_name }}" + state: absent + delegate_to: localhost + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.acm.create_cert + +- name: Delete any extra ACM certificates. + community.aws.acm_certificate: + region: "{{ deploy_container.aws_ecs.region }}" + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + domain_name: "{{ item.domain }}" + state: absent + with_items: "{{ deploy_container.aws_ecs.acm.extra_domains }}" + when: + - deploy_container.aws_ecs.acm.extra_domains | length > 0 + - deploy_container.aws_ecs.enabled + delegate_to: localhost + +# Clean up DNS +- name: Initialise the domains loop var with main domain entry DNS settings. + ansible.builtin.set_fact: + _aws_ecs_cluster_dns_all_domains: + - domain: "{{ deploy_container.aws_ecs.domain_name }}" + zone: "{{ deploy_container.aws_ecs.route_53.zone }}" + aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}" + when: + - deploy_container.aws_ecs.enabled + +- name: Add extra_domains so we can loop through DNS records. + ansible.builtin.set_fact: + _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}" + loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}" + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.acm.extra_domains | length > 0 + +- name: Remove DNS records in Route 53. + amazon.aws.route53: + state: absent + profile: "{{ item.aws_profile }}" + zone: "{{ item.zone }}" + record: "{{ item.domain }}" + type: CNAME + value: "{{ _aws_ecs_cluster_alb.load_balancers[0].dns_name }}" + loop: "{{ _aws_ecs_cluster_dns_all_domains }}" + when: + - deploy_container.aws_ecs.enabled + - deploy_container.aws_ecs.route_53.zone | length > 0 diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml new file mode 100644 index 00000000..ce3e618f --- /dev/null +++ b/roles/deploy_container/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Execute a container deployment. + ansible.builtin.include_tasks: + file: "action-{{ deploy_container.action }}.yml" + when: deploy_operation == 'deploy' + +- name: Cleanup dangling containers. + ansible.builtin.include_tasks: + file: action-cleanup.yml + when: deploy_operation == 'cleanup' diff --git a/roles/deploy_container/tasks/subnet-private.yml b/roles/deploy_container/tasks/subnet-private.yml new file mode 100644 index 00000000..2335cb89 --- /dev/null +++ b/roles/deploy_container/tasks/subnet-private.yml @@ -0,0 +1,13 @@ +- name: Gather private subnet information. + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + region: "{{ deploy_container.aws_ecs.region }}" + filters: + vpc-id: "{{ _aws_ecs_cluster_vpc_id }}" + tag:Name: "{{ subnet }}" + register: _aws_ecs_cluster_private_subnet + delegate_to: localhost + +- name: Add private subnet to the list. + ansible.builtin.set_fact: + _aws_ecs_cluster_private_subnets_ids: "{{ _aws_ecs_cluster_private_subnets_ids + [ _aws_ecs_cluster_private_subnet.subnets[0].subnet_id ] }}" diff --git a/roles/deploy_container/tasks/subnet-public.yml b/roles/deploy_container/tasks/subnet-public.yml new file mode 100644 index 00000000..a0701a36 --- /dev/null +++ b/roles/deploy_container/tasks/subnet-public.yml @@ -0,0 +1,13 @@ +- name: Gather public subnet information. + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ deploy_container.aws_ecs.aws_profile }}" + region: "{{ deploy_container.aws_ecs.region }}" + filters: + vpc-id: "{{ _aws_ecs_cluster_vpc_id }}" + tag:Name: "{{ subnet }}" + register: _aws_ecs_cluster_public_subnet + delegate_to: localhost + +- name: Add public subnet to the list. + ansible.builtin.set_fact: + _aws_ecs_cluster_public_subnets_ids: "{{ _aws_ecs_cluster_public_subnets_ids + [ _aws_ecs_cluster_public_subnet.subnets[0].subnet_id ] }}" diff --git a/roles/deploy_container/templates/example.j2 b/roles/deploy_container/templates/example.j2 new file mode 100644 index 00000000..5a155197 --- /dev/null +++ b/roles/deploy_container/templates/example.j2 @@ -0,0 +1,7 @@ +# Basic Dockerfile example +FROM debian:bullseye-slim +MAINTAINER sysadm@codeenigma.com + +RUN apt-get update +RUN apt-get install -y nginx +CMD ["echo","Image created"] \ No newline at end of file diff --git a/roles/live_symlink/defaults/main.yml b/roles/live_symlink/defaults/main.yml index 4b9fb449..eca23ad6 100644 --- a/roles/live_symlink/defaults/main.yml +++ b/roles/live_symlink/defaults/main.yml @@ -6,7 +6,7 @@ live_symlink: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -15,4 +15,4 @@ live_symlink: # Specify any additional templates to generate, with src (template) and dest (file). # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') - templates: [] \ No newline at end of file + templates: [] diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 2d6a9310..78314024 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -1,7 +1,7 @@ --- - name: Enable maintenance mode. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -13,7 +13,7 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Enable maintenance mode D7. ansible.builtin.shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 1" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/npm/README.md b/roles/npm/README.md index 385e1c84..97f06cdc 100644 --- a/roles/npm/README.md +++ b/roles/npm/README.md @@ -18,7 +18,7 @@ npm: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -28,6 +28,7 @@ npm: # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') templates: [] + ``` diff --git a/roles/npm/defaults/main.yml b/roles/npm/defaults/main.yml index be6bdc3f..a29fcac7 100644 --- a/roles/npm/defaults/main.yml +++ b/roles/npm/defaults/main.yml @@ -11,7 +11,7 @@ npm: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -20,4 +20,4 @@ npm: # Specify any additional templates to generate, with src (template) and dest (file). # src: name of a template, in the "templates" dir relative to your playbook. # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php') - templates: [] \ No newline at end of file + templates: [] diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md index cf7e5311..c0c2954b 100644 --- a/roles/sync/database_sync/database_sync-mysql/README.md +++ b/roles/sync/database_sync/database_sync-mysql/README.md @@ -5,7 +5,9 @@ Sync MySQL databases between environments. ```yaml --- mysql_sync: - mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here. + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes. + archival_method: "gzip" # oprions are "bzip2" or "gzip". databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 0ab99e98..98ac19bf 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,6 +1,8 @@ --- mysql_sync: - mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here. + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes. + archival_method: "gzip" # oprions are "bzip2" or "gzip". databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index c695982c..3f94e47d 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -35,9 +35,24 @@ - database.target.asg is defined - database.target.asg | length > 0 + +- name: Register bzip2 archive type vars. + ansible.builtin.set_fact: + archive_file_type: "bz2" + archival_command: "bzip2" + when: + - mysql_sync.archival_method == 'bzip2' + +- name: Register gunzip archive type vars. + ansible.builtin.set_fact: + archive_file_type: "gz" + archival_command: "gzip" + when: + - mysql_sync.archival_method == 'gzip' + - name: Register remote dump name (from database). ansible.builtin.set_fact: - mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" + mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.{{ archive_file_type }}" - name: Get source last known good build number. ansible.builtin.command: @@ -61,7 +76,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}" args: executable: /bin/bash delegate_to: "{{ database.source.host }}" @@ -91,7 +106,11 @@ - name: Register tmp target dump name. ansible.builtin.set_fact: - mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" + mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.{{ archive_file_type }}" + +- name: Register tmp unpacked target dump name. + ansible.builtin.set_fact: + mysql_sync_target_dump_unpacked_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql" - name: Get target last known good build number. ansible.builtin.command: @@ -104,12 +123,12 @@ delegate_to: localhost when: database.target.type == 'rolling' -- name: Register target database name. +- name: Register target rolling database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}" when: database.target.type == 'rolling' -- name: Register target database name. +- name: Register target static database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}" when: not database.target.type == 'rolling' @@ -117,32 +136,59 @@ - name: Fetch dump file. ansible.builtin.fetch: src: "{{ mysql_sync_source_dump_path }}" - dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" + dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" flat: true delegate_to: "{{ database.source.host }}" - name: Copy dump file to destination. ansible.builtin.copy: - src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" + src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}" dest: "{{ mysql_sync_target_dump_path }}" +- name: Unpack dump file. + ansible.builtin.shell: "{{ archival_command }} -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" + args: + executable: /bin/bash + +- name: Delete temporary dump file on target. + ansible.builtin.file: + path: "{{ mysql_sync_target_dump_path }}" + state: absent + - name: Drop target database. ansible.builtin.command: - cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" + cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists `{{ mysql_sync_target_database }}`;'" - name: Recreate target database. ansible.builtin.command: - cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" + cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database `{{ mysql_sync_target_database }}`;'" - name: Repopulate database from dump. - ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}" args: executable: /bin/bash -- name: Remove tmp dump file. +- name: Delete temporary unpacked dump file on target. ansible.builtin.file: - path: "{{ mysql_sync_target_dump_path }}" + path: "{{ mysql_sync_target_dump_unpacked_path }}" + state: absent + +- name: Delete temporary dump file on source. + ansible.builtin.file: + path: "{{ mysql_sync_source_dump_path }}" state: absent + delegate_to: "{{ database.source.host }}" + +- name: Delete temporary dump file on deploy server. + ansible.builtin.file: + path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}" + state: absent + delegate_to: localhost + when: + - mysql_sync.cleanup + with_items: + - ".bz2" + - ".gz" - name: Enable all autoscale processes on source ASG. ansible.builtin.command: > diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml index 431366cf..0f116adc 100644 --- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,16 +1,8 @@ --- -# Drupal 8 ships drush with the website code so we need the previous build in the path. -- name: Stash the drush_bin variable. - ansible.builtin.set_fact: - _drush_bin_deploy: "{{ drush_bin }}" - -- name: Update location of drush for the sync cache clear command. - ansible.builtin.set_fact: - drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush" - +# Drupal 8 ships drush with the website code so to use the drush_live_bin variable - name: Clear Drupal cache. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y cr" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -18,7 +10,3 @@ loop_control: loop_var: site run_once: true - -- name: Restore the drush_bin variable. - ansible.builtin.set_fact: - drush_bin: "{{ _drush_bin_deploy }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index 96d9b65f..297e476b 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -1,27 +1,25 @@ --- -# Drupal 8 ships drush with the website code so we need the previous build in the path. -- name: Stash the drush_bin variable. - ansible.builtin.set_fact: - _drush_bin_deploy: "{{ drush_bin }}" - -- name: Update location of drush for the update/config import commands. - ansible.builtin.set_fact: - drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush" - +# Drupal 8 ships drush with the website code so to use the drush_live_bin variable - name: Apply Drupal database updates. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y updb" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" with_items: "{{ drupal.sites }}" loop_control: loop_var: site + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output # This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run. - name: Import configuration. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" + cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -31,11 +29,13 @@ when: - site.config_import_command - site.sync_config_import is not defined or site.sync_config_import + register: _drush_output + +- name: Show drush output. + ansible.builtin.debug: + msg: "{{ _drush_output }}" + when: drupal.drush_verbose_output - name: Clear the cache. ansible.builtin.include_role: name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" - -- name: Restore the drush_bin variable. - ansible.builtin.set_fact: - drush_bin: "{{ _drush_bin_deploy }}" diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml index 5b65b1fb..4e222b40 100644 --- a/roles/sync/files_sync/defaults/main.yml +++ b/roles/sync/files_sync/defaults/main.yml @@ -1,5 +1,8 @@ --- files_sync: + unique_workspace: false # set to true to grab a complete full set of files every sync + # Generally speaking you will *not* want to clean up after file syncs, as leaving the files there makes the next rsync far quicker. + cleanup: false # set to true to delete the synced files after a sync directories: - source: # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index e2160008..516059c2 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -1,7 +1,16 @@ --- +- name: Register file sync location. + ansible.builtin.set_fact: + file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + +- name: Register unique file sync location. + ansible.builtin.set_fact: + file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}_{{ build_number }}" + when: files_sync.unique_workspace + - name: Create a temporary directory for source files on localhost. ansible.builtin.file: - path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + path: "{{ file_sync_path }}" state: directory owner: "{{ deploy_user }}" group: "{{ deploy_user }}" @@ -10,12 +19,20 @@ - name: Copy the source files onto the deploy server. ansible.builtin.command: - cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" - delegate_to: "localhost" + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ file_sync_path }}/" + delegate_to: localhost run_once: true - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.command: - cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" - delegate_to: "localhost" + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ file_sync_path }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + delegate_to: localhost run_once: true + +- name: Delete synced files on deploy server. + ansible.builtin.file: + path: "{{ file_sync_path }}" + state: absent + delegate_to: localhost + when: + - files_sync.cleanup diff --git a/scripts/_common.sh b/scripts/_common.sh index 3715c97a..07aa7368 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -9,10 +9,13 @@ export ANSIBLE_CONFIG="$OWN_DIR/ansible.cfg" TARGET_DEPLOY_REPO="" TARGET_DEPLOY_PLAYBOOK="" TARGET_DEPLOY_BRANCH="" +TARGET_DEPLOY_HOST="" PREVIOUS_BUILD_NUMBER="" CURRENT_BUILD_NUMBER="" ANSIBLE_EXTRA_VARS="" ANSIBLE_DEFAULT_EXTRA_VARS="" +ANSIBLE_PATH="" +PYTHON_INTERPRETER="" BUILD_WORKSPACE="" BUILD_TRACK_FILE="" BUILD_ID="" @@ -20,19 +23,34 @@ BUILD_WORKSPACE_BASE="$OWN_DIR/build" DRY_RUN="no" VERBOSE="no" BOTO_PROFILE="" +# Ensure build workspace exists. if [ ! -d "$BUILD_WORKSPACE_BASE" ]; then mkdir "$BUILD_WORKSPACE_BASE" fi BUILD_TMP_DIR=$(mktemp -d -p "$BUILD_WORKSPACE_BASE") +# Ensure ce-deploy data directory exists. ANSIBLE_DATA_DIR="$OWN_DIR/data" if [ ! -d "$ANSIBLE_DATA_DIR" ]; then mkdir "$ANSIBLE_DATA_DIR" fi +# Ensure directory for build track files exists. BUILD_TRACK_DIR="$OWN_DIR/track" if [ ! -d "$BUILD_TRACK_DIR" ]; then mkdir "$BUILD_TRACK_DIR" fi +# Load the contents of profile.d in case we added items to $PATH there. +if [ -n "$(ls -A /etc/profile.d)" ]; then + for f in /etc/profile.d/*; do + # shellcheck source=/dev/null + . "$f" + done +fi ANSIBLE_LOCATION=$(command -v ansible) +# Load the contents of profile.d in case we added items to $PATH there. +for f in /etc/profile.d/*; do +# shellcheck source=/dev/null + . "$f" +done # Parse options arguments. parse_options(){ while [ "${1:-}" ]; do @@ -49,6 +67,10 @@ parse_options(){ shift TARGET_DEPLOY_PLAYBOOK="$1" ;; + "--host") + shift + TARGET_DEPLOY_HOST="$1" + ;; "--build-number") shift CURRENT_BUILD_NUMBER="$1" @@ -83,6 +105,14 @@ parse_options(){ shift BOTO_PROFILE="$1" ;; + "--ansible-path") + shift + ANSIBLE_PATH="$1" + ;; + "--python-interpreter") + shift + PYTHON_INTERPRETER="$1" + ;; "--build-id") shift BUILD_ID="$1" @@ -108,7 +138,11 @@ get_build_workspace(){ # Common extra-vars to pass to Ansible. get_ansible_defaults_vars(){ - ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}" + if [ -n "$PYTHON_INTERPRETER" ]; then + ANSIBLE_DEFAULT_EXTRA_VARS="{ansible_python_interpreter: $PYTHON_INTERPRETER, _ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}" + else + ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}" + fi } # Fetch previous build number from track file. @@ -146,6 +180,30 @@ cleanup_build_tmp_dir(){ fi } +# Call Ansible playbook to ensure host exists. +ansible_host_check(){ + if [ -n "$TARGET_DEPLOY_HOST" ]; then + if [ -z "$ANSIBLE_PATH" ]; then + ANSIBLE_BIN=$(command -v ansible-playbook) + else + ANSIBLE_BIN="$ANSIBLE_PATH/ansible-playbook" + fi + ANSIBLE_BIN=$(command -v ansible-playbook) + ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml" + if [ "$VERBOSE" = "yes" ]; then + ANSIBLE_CMD="$ANSIBLE_CMD -vvvv" + fi + if [ -n "$BOTO_PROFILE" ]; then + export AWS_PROFILE="$BOTO_PROFILE" + fi + $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS" + return $? + # No host to check provided, just return a clean exit code. + else + return 0 + fi +} + # Trigger actual Ansible job. # $1 (string) # Operation to perform. diff --git a/scripts/build.sh b/scripts/build.sh index 5f7fa21b..a29ec43b 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -19,7 +19,10 @@ usage(){ echo '--branch: The branch to deploy.' echo '' echo 'Available options:' + echo '--host: Valid Ansible hostname, if you want to run a host check. Can also be a group name.' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output' @@ -94,6 +97,15 @@ fi # Get Ansible defaults. get_ansible_defaults_vars +# Optionally carry out a host check if --host is provided. +ansible_host_check +ANSIBLE_HOST_CHECK_RESULT=$? +# Exit early if host not found. +if [ -n "$ANSIBLE_HOST_CHECK_RESULT" ] && [ "$ANSIBLE_HOST_CHECK_RESULT" != 0 ]; then + echo "ce-deploy failed to find the host. Aborting." + exit 1 +fi + # From this point on, we want to trigger the "revert" if anything fails. ANSIBLE_BUILD_RESULT=1 # Trigger deploy. @@ -109,5 +121,5 @@ if [ -n "$ANSIBLE_BUILD_RESULT" ] && [ "$ANSIBLE_BUILD_RESULT" = 0 ]; then exit 0 fi # Failed somehow. Normally unreachable in strict mode. -echo "Something went wrong. Please fill a bug report against ce-deploy." +echo "Something went unexpectedly wrong with ce-deploy. Please file a bug report - https://github.com/codeenigma/ce-deploy/issues/new" exit 1 \ No newline at end of file diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 9f0e50cf..abe24675 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -20,6 +20,8 @@ usage(){ echo '' echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output' diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 341af1ce..66ab7fe3 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -20,6 +20,8 @@ usage(){ echo '' echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output' diff --git a/scripts/host-check.yml b/scripts/host-check.yml new file mode 100644 index 00000000..980c63cb --- /dev/null +++ b/scripts/host-check.yml @@ -0,0 +1,15 @@ +- hosts: localhost + connection: local + become: false + tasks: + - name: Ensure the hostname check variable is empty. + ansible.builtin.set_fact: + _ce_deploy_ansible_host_check: "" + - name: Check to see if the Ansible host or hostgroup exists. + ansible.builtin.set_fact: + _ce_deploy_ansible_host_check: "{{ item }}" + with_inventory_hostnames: + - "{{ _deploy_host }}" + - ansible.builtin.fail: + msg: "Host does not exist!" + when: _ce_deploy_ansible_host_check | length == 0 diff --git a/scripts/revert.sh b/scripts/revert.sh index a10655a3..7b36b42b 100755 --- a/scripts/revert.sh +++ b/scripts/revert.sh @@ -20,6 +20,8 @@ usage(){ echo '' echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' + echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output'