From 80986fbbd18b9ef504421b1a9b99550716d25425 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Tue, 7 Jun 2022 11:06:37 +0100 Subject: [PATCH 1/5] Add a files_sync role. --- roles/sync/files_sync/defaults/main.yml | 14 ++++++++++++++ roles/sync/files_sync/tasks/main.yml | 7 +++++++ roles/sync/files_sync/tasks/sync.yml | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 roles/sync/files_sync/defaults/main.yml create mode 100644 roles/sync/files_sync/tasks/main.yml create mode 100644 roles/sync/files_sync/tasks/sync.yml diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml new file mode 100644 index 00000000..06f6a818 --- /dev/null +++ b/roles/sync/files_sync/defaults/main.yml @@ -0,0 +1,14 @@ +--- +files_sync: + directories: + - source: + # Location of the files to sync from. + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_prod/assets/{{ project_name }}_prod_default_public_files" + # Host that can connect to the database. + host: "localhost" + # For "rolling builds", so we can compute the database name. + build_id: mybuildprod + target: + # Location of the files to sync to. + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_dev/assets/{{ project_name }}_dev_default_public_files" + build_id: mybuilddev diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml new file mode 100644 index 00000000..47c3667a --- /dev/null +++ b/roles/sync/files_sync/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Sync files. + include_tasks: "sync.yml" + with_items: "{{ files_sync.directories }}" + loop_control: + loop_var: files + run_once: true diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml new file mode 100644 index 00000000..cb34430b --- /dev/null +++ b/roles/sync/files_sync/tasks/sync.yml @@ -0,0 +1,22 @@ +--- +- name: Create a temporary directory for source files on localhost. + ansible.builtin.file: + path: "/tmp/{{ files.source.build_id }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + delegate_to: localhost + run_once: true + +- name: Copy the source files onto the deploy server. + ansible.posix.synchronize: + mode: pull + src: "{{ files.source.files_dir }}" + dest: "/tmp/{{ files.source.build_id }}" + +- name: Copy the source files from the deploy server onto the destination server. + ansible.builtin.copy: + src: "/tmp/{{ files.source.build_id }}" + dest: "{{ files.target.files_dir }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" From e59e6bdfb79a0bdb02e9c0b807062f2b0e820ee0 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Tue, 7 Jun 2022 12:33:55 +0100 Subject: [PATCH 2/5] Clear up some comments, add temp_dir variable and use rsync with command module instead of synchronize module, in files_sync role. --- roles/sync/files_sync/defaults/main.yml | 10 ++++++---- roles/sync/files_sync/tasks/sync.yml | 11 +++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml index 06f6a818..5b65b1fb 100644 --- a/roles/sync/files_sync/defaults/main.yml +++ b/roles/sync/files_sync/defaults/main.yml @@ -2,13 +2,15 @@ files_sync: directories: - source: - # Location of the files to sync from. + # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_prod/assets/{{ project_name }}_prod_default_public_files" - # Host that can connect to the database. + # Host that contains source files. host: "localhost" - # For "rolling builds", so we can compute the database name. + # Location on deploy server where source files get copied to first. NO TRAILING SLASH. + temp_dir: "/tmp" + # Used to create directory in /tmp. build_id: mybuildprod target: - # Location of the files to sync to. + # Location of the files to sync to. DO NOT INCLUDE TRAILING SLASH! files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_dev/assets/{{ project_name }}_dev_default_public_files" build_id: mybuilddev diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index cb34430b..d830f11f 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -1,7 +1,7 @@ --- - name: Create a temporary directory for source files on localhost. ansible.builtin.file: - path: "/tmp/{{ files.source.build_id }}" + path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" state: directory owner: "{{ deploy_user }}" group: "{{ deploy_user }}" @@ -9,14 +9,13 @@ run_once: true - name: Copy the source files onto the deploy server. - ansible.posix.synchronize: - mode: pull - src: "{{ files.source.files_dir }}" - dest: "/tmp/{{ files.source.build_id }}" + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" + delegate_to: "localhost" - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.copy: - src: "/tmp/{{ files.source.build_id }}" + src: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" dest: "{{ files.target.files_dir }}" owner: "{{ deploy_user }}" group: "{{ deploy_user }}" From e849548db7228a83367c8b5ab0e3d9f741a5852d Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Tue, 7 Jun 2022 13:00:48 +0100 Subject: [PATCH 3/5] Need trailing slash on src when syncing files to destination server so the contents get synced and not the parent directory itself. --- roles/sync/files_sync/tasks/sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index d830f11f..986e9e32 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -15,7 +15,7 @@ - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.copy: - src: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + src: "{{ files.source.temp_dir }}/{{ files.source.build_id }}/" dest: "{{ files.target.files_dir }}" owner: "{{ deploy_user }}" group: "{{ deploy_user }}" From d6995261de490668ee3d6039157708e7ceaa86ec Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Tue, 7 Jun 2022 15:12:22 +0100 Subject: [PATCH 4/5] Use rsync instead of copy when syncing files to target server as copy is so slow. --- roles/sync/files_sync/tasks/sync.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index 986e9e32..44e72743 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -14,8 +14,6 @@ delegate_to: "localhost" - name: Copy the source files from the deploy server onto the destination server. - ansible.builtin.copy: - src: "{{ files.source.temp_dir }}/{{ files.source.build_id }}/" - dest: "{{ files.target.files_dir }}" - owner: "{{ deploy_user }}" - group: "{{ deploy_user }}" + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + delegate_to: "localhost" From a5934b0f661009d62bf1b2451e9f1e23cb7f64a4 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Tue, 7 Jun 2022 15:14:52 +0100 Subject: [PATCH 5/5] Ensure file sync tasks are run only once. --- roles/sync/files_sync/tasks/sync.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index 44e72743..e2160008 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -12,8 +12,10 @@ ansible.builtin.command: cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" delegate_to: "localhost" + run_once: true - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.command: cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" delegate_to: "localhost" + run_once: true