From 1530f4d03082223498f402108fc6c65eaa54abff Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Dec 2023 13:33:25 -0500
Subject: [PATCH 01/10] #328 rewrite localstack/minio, keep docker, initial
 commit

---
 tasks/localstack.yml         | 34 ++++++++++++++++++++++++++++------
 tasks/minio.yml              | 10 ++++++----
 tests/group_vars/jenkins.yml |  7 ++++++-
 3 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/tasks/localstack.yml b/tasks/localstack.yml
index 01e81180..3309ccf4 100644
--- a/tasks/localstack.yml
+++ b/tasks/localstack.yml
@@ -4,7 +4,30 @@
 
 - ansible.builtin.import_tasks: aws_cli.yml
 
-- ansible.builtin.import_tasks: aws_config.yml
+- name: create localstack user
+  ansible.builtin.user:
+    name: '{{ localstack.user }}'
+
+- name: ensure dot dir exists
+  file:
+    path: '~{{ localstack.user }}/.aws'
+    state: directory
+    owner: '{{ localstack.user }}'
+    mode: '0700'
+
+- name: place localstack aws credentials
+  template:
+    src: localstack_aws_credentials.j2
+    dest: '~{{ localstack.user }}/.aws/credentials'
+    owner: '{{ localstack.user }}'
+    mode: '0600'
+
+- name: place localstack aws config
+  template:
+    src: localstack_aws_config.j2
+    dest: '~{{ localstack.user }}/.aws/config'
+    owner: '{{ localstack.user }}'
+    mode: '0600'
 
 # so we can run localstack task stand-alone
 - ansible.builtin.import_tasks: dataverse-fqdn.yml
@@ -17,19 +40,18 @@
 - name: place docker-compose.yml template
   ansible.builtin.template:
     src: localstack-compose.yml.j2
-    dest: '~{{ dataverse.payara.user }}/localstack-compose.yml'
+    dest: '~{{ dataverse.localstack.user }}/localstack-compose.yml'
 
 - name: ensure DATA_DIR exists
   ansible.builtin.file:
     path: '{{ localstack.data_dir }}'
     state: directory
-    owner: '{{ dataverse.payara.user }}'
-    group: '{{ dataverse.payara.group }}'
+    owner: '{{ dataverse.localstack.user }}'
     mode: 0755
 
 - name: start s3-test container
   ansible.builtin.shell: 'docker compose -f localstack-compose.yml up -d'
   become: true
-  become_user: '{{ dataverse.payara.user }}'
+  become_user: '{{ dataverse.localstack.user }}'
   args:
-    chdir: '~{{ dataverse.payara.user }}'
+    chdir: '~{{ dataverse.localstack.user }}'
diff --git a/tasks/minio.yml b/tasks/minio.yml
index 1c1b99a9..aa68ef2c 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -2,6 +2,10 @@
 
 - ansible.builtin.import_tasks: docker.yml
 
+- name: ensure minio service account exists
+  ansible.builtin.user:
+    name: '{{ minio.user }}'
+
 - name: STORAGE | Import assert.yml
   ansible.builtin.import_tasks: minio_assert.yml
   run_once: true
@@ -33,8 +37,7 @@
       ansible.builtin.file:
         path: "{{ minio.docker.project_location }}/data"
         state: directory
-        owner: '{{ dataverse.payara.user }}'
-        group: '{{ dataverse.payara.group }}'
+        owner: '{{ dataverse.minio.user }}'
         mode: "0755"
         recurse: true
       when:
@@ -45,8 +48,7 @@
       ansible.builtin.template:
         src: minio_compose.j2
         dest: "{{ minio.docker.project_location }}/minio_compose.yml"
-        owner: '{{ dataverse.payara.user }}'
-        group: '{{ dataverse.payara.group }}'
+        owner: '{{ dataverse.minio.user }}'
         mode: "0644"
         lstrip_blocks: true
         force: true
diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml
index 22215e05..ae2677b8 100644
--- a/tests/group_vars/jenkins.yml
+++ b/tests/group_vars/jenkins.yml
@@ -304,11 +304,15 @@ java:
 localstack:
   enabled: true
   container_name: dev_localstack
+  access_key: 4cc355_k3y
+  secret_access_key: s3cr3t_4cc355_k3y
+  region: us-east-2
   # set to /tmp/localstack/data to enable persistence
   data_dir: /tmp/localstack/data
   debug: true
   hostname_external: localstack
   port: 4566
+  user: localstack
   web_ui: 8888
   buckets:
     - label: LocalStack
@@ -336,13 +340,14 @@ maven:
 
 minio:
   enabled: true
+  user: minio
   docker:
     version: "latest"
     service_name: "minio"
     network:
       name: "minio"
       external: false
-    project_location: "/home/dataverse/minio"
+    project_location: "/home/minio/minio"
     timezone: "America/New_York"
     user: dataverse
     group: dataverse

From 098e726fa26adfa0312d735bec34f0800f1c9b08 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Dec 2023 13:54:49 -0500
Subject: [PATCH 02/10] #328 add localstack aws templates

---
 templates/localstack_aws_config.j2      | 2 ++
 templates/localstack_aws_credentials.j2 | 3 +++
 2 files changed, 5 insertions(+)
 create mode 100644 templates/localstack_aws_config.j2
 create mode 100644 templates/localstack_aws_credentials.j2

diff --git a/templates/localstack_aws_config.j2 b/templates/localstack_aws_config.j2
new file mode 100644
index 00000000..8a42ed75
--- /dev/null
+++ b/templates/localstack_aws_config.j2
@@ -0,0 +1,2 @@
+[default]
+region = {{ localstack.region }}
diff --git a/templates/localstack_aws_credentials.j2 b/templates/localstack_aws_credentials.j2
new file mode 100644
index 00000000..ab0151cf
--- /dev/null
+++ b/templates/localstack_aws_credentials.j2
@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = {{ localstack.access_key }}
+aws_secret_access_key = {{ localstack.secret_access_key }}

From 18cf73ee16a29a18fd9e2d32ee859758c23b39fd Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Dec 2023 14:14:34 -0500
Subject: [PATCH 03/10] #328 correct service account variables

---
 tasks/localstack.yml | 8 ++++----
 tasks/minio.yml      | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tasks/localstack.yml b/tasks/localstack.yml
index 3309ccf4..95babd24 100644
--- a/tasks/localstack.yml
+++ b/tasks/localstack.yml
@@ -40,18 +40,18 @@
 - name: place docker-compose.yml template
   ansible.builtin.template:
     src: localstack-compose.yml.j2
-    dest: '~{{ dataverse.localstack.user }}/localstack-compose.yml'
+    dest: '~{{ localstack.user }}/localstack-compose.yml'
 
 - name: ensure DATA_DIR exists
   ansible.builtin.file:
     path: '{{ localstack.data_dir }}'
     state: directory
-    owner: '{{ dataverse.localstack.user }}'
+    owner: '{{ localstack.user }}'
     mode: 0755
 
 - name: start s3-test container
   ansible.builtin.shell: 'docker compose -f localstack-compose.yml up -d'
   become: true
-  become_user: '{{ dataverse.localstack.user }}'
+  become_user: '{{ localstack.user }}'
   args:
-    chdir: '~{{ dataverse.localstack.user }}'
+    chdir: '~{{ localstack.user }}'
diff --git a/tasks/minio.yml b/tasks/minio.yml
index aa68ef2c..22f22fca 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -37,7 +37,7 @@
       ansible.builtin.file:
         path: "{{ minio.docker.project_location }}/data"
         state: directory
-        owner: '{{ dataverse.minio.user }}'
+        owner: '{{ minio.user }}'
         mode: "0755"
         recurse: true
       when:
@@ -48,7 +48,7 @@
       ansible.builtin.template:
         src: minio_compose.j2
         dest: "{{ minio.docker.project_location }}/minio_compose.yml"
-        owner: '{{ dataverse.minio.user }}'
+        owner: '{{ minio.user }}'
         mode: "0644"
         lstrip_blocks: true
         force: true

From 724512d0453a9c657dba40f53068966788a4fe3b Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Dec 2023 14:32:12 -0500
Subject: [PATCH 04/10] #328 add service accounts to docker group

---
 tasks/localstack.yml | 1 +
 tasks/minio.yml      | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tasks/localstack.yml b/tasks/localstack.yml
index 95babd24..49927821 100644
--- a/tasks/localstack.yml
+++ b/tasks/localstack.yml
@@ -7,6 +7,7 @@
 - name: create localstack user
   ansible.builtin.user:
     name: '{{ localstack.user }}'
+    group: docker
 
 - name: ensure dot dir exists
   file:
diff --git a/tasks/minio.yml b/tasks/minio.yml
index 22f22fca..df34423a 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -5,6 +5,7 @@
 - name: ensure minio service account exists
   ansible.builtin.user:
     name: '{{ minio.user }}'
+    group: docker
 
 - name: STORAGE | Import assert.yml
   ansible.builtin.import_tasks: minio_assert.yml

From b5a1a24621f6a5d5daf4b68822ef5fab49f2342f Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Dec 2023 14:49:14 -0500
Subject: [PATCH 05/10] #328 correct localstack bucket creation by localstack
 service account

---
 tasks/localstack_create_bucket.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tasks/localstack_create_bucket.yml b/tasks/localstack_create_bucket.yml
index b2ed39ed..7e98bcc6 100644
--- a/tasks/localstack_create_bucket.yml
+++ b/tasks/localstack_create_bucket.yml
@@ -7,7 +7,7 @@
 - name: create localstack bucket for testing
   ansible.builtin.shell: "aws --endpoint={{ custom_endpoint_escaped }} s3 mb s3://{{ bucket_options.bucket_name }}"
   become: yes
-  become_user: "{{ dataverse.payara.user }}"
+  become_user: "{{ localstack.user }}"
   when: bucket_options.custom_endpoint_url | length > 0
 
 - name: place CORS.json
@@ -21,7 +21,7 @@
 - name: set CORS on bucket when upload_redirect is true
   ansible.builtin.shell: 'aws --endpoint={{ custom_endpoint_escaped }} s3api put-bucket-cors --bucket {{ bucket_options.bucket_name }} --cors-configuration file:///tmp/cors.json'
   become: yes
-  become_user: "{{ dataverse.payara.user }}"
+  become_user: "{{ localstack.user }}"
   when: bucket_options.upload_redirect == true
 
 - name: debug

From ee267cab646c65ef50eb60178786eba492df069f Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Wed, 20 Dec 2023 09:38:42 -0500
Subject: [PATCH 06/10] #328 let's not use the Ansible docker-compose module
 due to RHEL9 conflicts

---
 tasks/docker.yml                   | 49 +++++++++---------------------
 tasks/localstack_create_bucket.yml |  1 -
 tasks/minio.yml                    | 17 ++++++-----
 3 files changed, 24 insertions(+), 43 deletions(-)

diff --git a/tasks/docker.yml b/tasks/docker.yml
index cbe4c796..93d7d403 100644
--- a/tasks/docker.yml
+++ b/tasks/docker.yml
@@ -11,20 +11,21 @@
     name: ['docker-ce', 'docker-compose-plugin']
     state: latest
 
-- name: Ansible docker module requires python-docker
-  ansible.builtin.package:
-    name: 'python3-docker'
-    state: latest
-
-- name: pip must update itself
-  ansible.builtin.pip:
-    name: pip
-    executable: pip3
-    state: latest
-
-- name: and docker-compose python package
-  ansible.builtin.pip:
-    name: docker-compose
+# this breaks on RHEL9. let's use 'docker compose' instead
+#- name: Ansible docker module requires python-docker
+#  ansible.builtin.package:
+#    name: 'python3-docker'
+#    state: latest
+#
+#- name: pip must update itself
+#  ansible.builtin.pip:
+#    name: pip
+#    executable: pip3
+#    state: latest
+#
+#- name: and docker-compose python package
+#  ansible.builtin.pip:
+#    name: docker-compose
 
 - name: ensure /etc/docker exists
   ansible.builtin.file:
@@ -43,26 +44,6 @@
     mode: 0644
   when: docker.cidr is undefined
 
-- name: infer become_user
-  ansible.builtin.command: whoami
-  register: whoami_output
-
-- name: register docker_user
-  ansible.builtin.set_fact:
-    docker_user: "{{ whoami_output.stdout }}"
-
-- name: add ansible_ssh_user to docker group
-  ansible.builtin.user:
-    name: '{{ docker_user }}'
-    groups: docker
-    append: yes
-
-- name: add dataverse user to docker group
-  ansible.builtin.user:
-    name: '{{ dataverse.payara.user }}'
-    groups: docker
-    append: yes
-
 - name: reload systemd, enable docker, start
   ansible.builtin.systemd:
     name: docker
diff --git a/tasks/localstack_create_bucket.yml b/tasks/localstack_create_bucket.yml
index 7e98bcc6..743d7d5a 100644
--- a/tasks/localstack_create_bucket.yml
+++ b/tasks/localstack_create_bucket.yml
@@ -8,7 +8,6 @@
   ansible.builtin.shell: "aws --endpoint={{ custom_endpoint_escaped }} s3 mb s3://{{ bucket_options.bucket_name }}"
   become: yes
   become_user: "{{ localstack.user }}"
-  when: bucket_options.custom_endpoint_url | length > 0
 
 - name: place CORS.json
   ansible.builtin.copy:
diff --git a/tasks/minio.yml b/tasks/minio.yml
index df34423a..53d33a73 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -69,21 +69,22 @@
       register: compose_file
 
     - name: STORAGE | Stop `docker-compose down` MinIO
-      community.docker.docker_compose:
-        project_src: "{{ minio.docker.project_location }}"
-        state: absent
-        remove_orphans: true
+      ansible.builtin.shell: 'docker compose -f minio_compose.yml down'
       become: true
+      become_user: '{{ minio.user }}'
+      args:
+        chdir: '~{{ minio.user }}'
       register: continer_stop
       when:
         - minio_container.exists
         - copy_compose.changed
 
     - name: STORAGE | Run `docker-compose up` MinIO
-      community.docker.docker_compose:
-        project_src: "{{ minio.docker.project_location }}"
-        build: true
-        files: minio_compose.yml
+      ansible.builtin.shell: 'docker compose -f minio_compose.yml up -d'
+      become: true
+      become_user: '{{ minio.user }}'
+      args:
+        chdir: '~{{ minio.user }}'
       when: (not minio_container.exists and minio_dir.stat.isdir and compose_file.stat.exists) or continer_stop.changed
 
 - ansible.builtin.import_tasks: minio_jvm_options.yml

From e6c205500bd56b093fb19f7b57d3a011923a7afb Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Wed, 20 Dec 2023 10:21:37 -0500
Subject: [PATCH 07/10] #328 correct minio_compose.yml location

---
 tasks/minio.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tasks/minio.yml b/tasks/minio.yml
index 53d33a73..7e6cd776 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -69,7 +69,7 @@
       register: compose_file
 
     - name: STORAGE | Stop `docker-compose down` MinIO
-      ansible.builtin.shell: 'docker compose -f minio_compose.yml down'
+      ansible.builtin.shell: 'docker compose -f {{ minio.docker.project_location }}/minio_compose.yml down'
       become: true
       become_user: '{{ minio.user }}'
       args:
@@ -80,7 +80,7 @@
         - copy_compose.changed
 
     - name: STORAGE | Run `docker-compose up` MinIO
-      ansible.builtin.shell: 'docker compose -f minio_compose.yml up -d'
+      ansible.builtin.shell: 'docker compose -f {{ minio.docker.project_location }}/minio_compose.yml up -d'
       become: true
       become_user: '{{ minio.user }}'
       args:

From 019206fe7add4c965c87a15e34ac58bc1aa14b65 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Wed, 20 Dec 2023 12:02:26 -0500
Subject: [PATCH 08/10] #328 MinIO S3Access requires hard-coded credentials

---
 tasks/minio.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tasks/minio.yml b/tasks/minio.yml
index 7e6cd776..0b64f8eb 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -7,6 +7,34 @@
     name: '{{ minio.user }}'
     group: docker
 
+# S3AccessIT tests will fail without hard-coded AWS credentials
+
+- name: ensure dot dir exists
+  file:
+    path: '~{{ dataverse.payara.user }}/.aws'
+    state: directory
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+    mode: '0700'
+
+- name: place aws credentials
+  template:
+    src: aws_credentials.j2
+    dest: '~{{ dataverse.payara.user }}/.aws/credentials'
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+    mode: '0600'
+
+- name: place aws config
+  template:
+    src: aws_config.j2
+    dest: '~{{ dataverse.payara.user }}/.aws/config'
+    owner: '{{ dataverse.payara.user }}'
+    group: '{{ dataverse.payara.group }}'
+    mode: '0600'
+
+# remove these once #### has been merged.
+
 - name: STORAGE | Import assert.yml
   ansible.builtin.import_tasks: minio_assert.yml
   run_once: true

From 8caca9bf8b7145f48c9d2e38cd4a1c1a751d8749 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Thu, 21 Dec 2023 15:40:27 -0500
Subject: [PATCH 09/10] #328 correct secret-access-key to secret-key per Jim

---
 tasks/localstack_create_bucket.yml | 2 +-
 tasks/minio_jvm_options.yml        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tasks/localstack_create_bucket.yml b/tasks/localstack_create_bucket.yml
index 743d7d5a..59f5068b 100644
--- a/tasks/localstack_create_bucket.yml
+++ b/tasks/localstack_create_bucket.yml
@@ -43,7 +43,7 @@
     - { 'key' : 'upload-redirect' , 'value' : '{{ bucket_options.upload_redirect }}' }
     - { 'key' : 'download-redirect' , 'value' : '{{ bucket_options.download_redirect }}' }
     - { 'key' : 'access-key' , 'value' : '{{ bucket_options.access_key }}' }
-    - { 'key' : 'secret-access-key' , 'value' : '{{ bucket_options.secret_access_key }}' }
+    - { 'key' : 'secret-key' , 'value' : '{{ bucket_options.secret_access_key }}' }
 
 - name: debug
   ansible.builtin.debug:
diff --git a/tasks/minio_jvm_options.yml b/tasks/minio_jvm_options.yml
index 90f45ba8..10289556 100644
--- a/tasks/minio_jvm_options.yml
+++ b/tasks/minio_jvm_options.yml
@@ -19,7 +19,7 @@
     - { 'key' : 'upload-redirect' , 'value' : 'false' }
     - { 'key' : 'download-redirect' , 'value' : 'false' }
     - { 'key' : 'access-key' , 'value' : '{{ minio.docker.access_key }}' }
-    - { 'key' : 'secret-access-key' , 'value' : '{{ minio.docker.secret_key }}' }
+    - { 'key' : 'secret-key' , 'value' : '{{ minio.docker.secret_key }}' }
 
 - name: debug
   ansible.builtin.debug:

From 0e02f42fadfc56713ef56980b9cd24093d04033d Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Fri, 22 Dec 2023 10:50:00 -0500
Subject: [PATCH 10/10] #328 final clean-up

---
 tasks/aws_cli.yml |  9 +--------
 tasks/minio.yml   | 28 ----------------------------
 2 files changed, 1 insertion(+), 36 deletions(-)

diff --git a/tasks/aws_cli.yml b/tasks/aws_cli.yml
index b5d840fe..c3ec4168 100644
--- a/tasks/aws_cli.yml
+++ b/tasks/aws_cli.yml
@@ -1,13 +1,6 @@
 ---
 
-#- name: we need awscli on 8
-#  pip:
-#    name: awscli
-#    executable: pip3
-#  when: (ansible_os_family == "RedHat") and
-#        (ansible_distribution_major_version) == "8"
-
-- name: just use OS-included binary for now
+- name: AWS CLI OS-included binary for now
   ansible.builtin.package:
     name: awscli
     state: latest
diff --git a/tasks/minio.yml b/tasks/minio.yml
index 0b64f8eb..7e6cd776 100644
--- a/tasks/minio.yml
+++ b/tasks/minio.yml
@@ -7,34 +7,6 @@
     name: '{{ minio.user }}'
     group: docker
 
-# S3AccessIT tests will fail without hard-coded AWS credentials
-
-- name: ensure dot dir exists
-  file:
-    path: '~{{ dataverse.payara.user }}/.aws'
-    state: directory
-    owner: '{{ dataverse.payara.user }}'
-    group: '{{ dataverse.payara.group }}'
-    mode: '0700'
-
-- name: place aws credentials
-  template:
-    src: aws_credentials.j2
-    dest: '~{{ dataverse.payara.user }}/.aws/credentials'
-    owner: '{{ dataverse.payara.user }}'
-    group: '{{ dataverse.payara.group }}'
-    mode: '0600'
-
-- name: place aws config
-  template:
-    src: aws_config.j2
-    dest: '~{{ dataverse.payara.user }}/.aws/config'
-    owner: '{{ dataverse.payara.user }}'
-    group: '{{ dataverse.payara.group }}'
-    mode: '0600'
-
-# remove these once #### has been merged.
-
 - name: STORAGE | Import assert.yml
   ansible.builtin.import_tasks: minio_assert.yml
   run_once: true