diff --git a/.gitignore b/.gitignore index 6bb015b2..7a7c9d43 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,37 @@ venv .vagrant/ /.idea /*.iml +inventory_ec2 +.DS_Store + + +### ucla local + +# Conda environment artifacts +.env/ +.venv/ +conda-meta/ +*.conda +*.egg-info/ + +# Python bytecode +__pycache__/ +*.py[cod] +*.pyo + +# Pip-tools generated lockfile (optional to ignore) +# If you want reproducibility, keep this file checked in. +# If you want to force all devs to recompile, uncomment below: +# requirements.txt + +# pip-sync temporary install log +pip-log.txt + +# Molecule test artifacts +.molecule/ +*.retry +*.log + +# VSCode & Editor configs +.vscode/ +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 67a35092..405fd69e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The role installs Apache, PostgreSQL, GlassFish/Payara and other prerequisites, Running the following commands as root should install the latest released version of Dataverse. - $ git clone https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible.git dataverse + $ git clone https://github.com/ucla-data-science-center/dataverse-ansible.git dataverse $ ansible-playbook --connection=local -v -i dataverse/inventory dataverse/dataverse.pb -e "@dataverse/defaults/main.yml" Recent, specific versions of Dataverse (namely, 4.20 and 5.0) may be installed using branches tagged with that version. @@ -89,8 +89,8 @@ It is possible to run certain portions of the playbook to avoid running the enti **Note:** While Ansible in general strives to achieve role idempotence, the dataverse-ansible role is merely a wrapper for the Dataverse installer, which itself is not idempotent. If you strongly desire that the role be idempotent and would like achieve this via semaphores, pull requests are welcome! ### To test using Vagrant: - $ git clone https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible - $ cd dataverse-ansible + $ git clone https://github.com/ucla-data-science-center/dataverse-ansible.git ucla-dataverse + $ cd ucla-dataverse $ vagrant up On successful completion of the Vagrant run, you should be able to log in to your test Dataverse as dataverseAdmin using the dataverse_adminpass from tests/group_vars/vagrant.yml using the address: diff --git a/Vagrantfile b/Vagrantfile index e7ea6530..3ecf9879 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,8 +4,9 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "bento/rockylinux-9" - + #config.vm.box = "bento/rockylinux-9" + config.vm.box = "bento/rockylinux-9-arm64" + config.vm.synced_folder ".", "/vagrant" config.vm.synced_folder ".", "/etc/ansible/roles/dataverse" @@ -20,7 +21,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 9090, host: 9090, auto_correct: true # Prometheus config.vm.provision :ansible_local do |ansible| - ansible.playbook = "tests/site.yml" + ansible.playbook = "site.yml" ansible.groups = { "dataverse" => %(default), "db" => %(default), @@ -34,8 +35,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ansible.verbose = true end - config.vm.provider "virtualbox" do |vbox| - vbox.cpus = 4 - vbox.memory = 8192 + config.vm.provider "vmware_desktop" do |vmware| + vmware.vmx["tools.upgrade.policy"] = "manual" + vmware.gui = false + vmware.ssh_info_public = true + vmware.allowlist_verified = true + vmware.linked_clone = false + vmware.vmx["ethernet0.virtualdev"] = "vmxnet3" + vmware.vmx["ethernet1.virtualdev"] = "vmxnet3" + vmware.vmx["memsize"] = "8192" + vmware.vmx["numvcpus"] = "4" end -end +end \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index 0989718b..da09072e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -8,7 +8,7 @@ dataverse_repo: https://github.com/IQSS/dataverse.git # dataverse_installer_url: https://example.com/path/to/dvinstall.zip # set this to true for troubleshooting -any_errors_fatal: false +#any_errors_fatal: true apache: enabled: true @@ -51,9 +51,9 @@ dataverse: adminpass: admin1 allow_signups: true api: - allow_lookup: false - blocked_endpoints: "admin,builtin-users,test" - blocked_policy: "localhost-only" + allow_lookup: true + blocked_endpoints: "builtin-users,test" + blocked_policy: "" location: "http://localhost:8080/api" test_suite: false # possible test values from https://github.com/IQSS/dataverse/blob/develop/conf/docker-aio/run-test-suite.sh#L11 @@ -61,7 +61,7 @@ dataverse: #tests: "DataversesIT,DatasetsIT,AdminIT" tests: default branding: - enabled: false + enabled: true directory: "{{ playbook_dir }}/files/branding" favicons_directory: "{{ playbook_dir }}/files/favicons" fileSettings: @@ -70,23 +70,23 @@ dataverse: - setting: StyleCustomizationFile file: custom-stylesheet.css - setting: LogoCustomizationFile - file: topbanner001w425_darkbg.png' + file: dataverseUCLA_logo.png otherSettings: - setting: FooterCopyright - value: Your institute name here + value: " UC Regents" language: - enabled: false # setting this to true allows the language task to run + enabled: true # setting this to true allows the language task to run languages: - locale: en_US title: English - - locale: de_DE - title: Deutsch + - locale: de_ES + title: Spanish language_packs: source: https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs.git version: develop lang_directory: "{{ dataverse_misc_files_dir }}/lang" licenses: - enabled: false + enabled: true user: dataverseAdmin licenses: - name: CC0 1.0 @@ -131,9 +131,9 @@ dataverse: iconUrl: https://licensebuttons.net/l/by-sa/4.0/88x31.png active: true sortOrder: 7 - copyright: "Your Institution" + copyright: "UC Regents" counter: - enabled: false + enabled: true #geoipdir: maxmind_geoip #geoipfile: GeoLite2-Country.mmdb hub_api_token: set_me_in_secrets @@ -149,7 +149,7 @@ dataverse: user: counter year_month: "2018-05" custom_metadata_blocks: - enabled: false + enabled: true urls: - https://github.com/IQSS/dataverse/files/3744336/codemeta.tsv.txt default: @@ -247,7 +247,7 @@ dataverse: custom_sampledataverses: "{{ playbook_dir }}/custom_sampledata/dataverses" custom_sampleusers: "{{ playbook_dir }}/custom_sampledata/users" custom_samplefiles: "{{ playbook_dir }}/custom_sampledata/files" - service_email: noreply@dataverse.yourinstitution.edu + service_email: noreply@dataverse.ucla.edu smtp: localhost # or the FQDN of your organization's SMTP relay solr: download_url: https://archive.apache.org/dist/solr/solr/9.8.0/solr-9.8.0.tgz @@ -327,7 +327,7 @@ localstack: web_ui: 8888 buckets: - label: LocalStack - id: localstack1 + id: loalstack1 bucket_name: mybucket enabled: false access_key: 4cc355_k3y @@ -416,7 +416,8 @@ s3: label: s3-test # for localstack this must be true path_style_access: true - region: us-east-1 + region: us-west-2 + location_constraint: us-west-2 storage_driver_id: s3 url_expiration_minutes: 60 payload_signing: false diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..29f6bd68 --- /dev/null +++ b/environment.yml @@ -0,0 +1,7 @@ +name: dataverse-ansible +channels: + - conda-forge +dependencies: + - python=3.11 + - pip + - pip-tools \ No newline at end of file diff --git a/files/branding/custom-header.html b/files/branding/custom-header.html new file mode 100644 index 00000000..c4bdd3c3 --- /dev/null +++ b/files/branding/custom-header.html @@ -0,0 +1,35 @@ + +
+
+

+ SAMPLE IMAGE + SAMPLE HEADER LINK +

+
+
diff --git a/files/branding/dataverseUCLA_logo.png b/files/branding/dataverseUCLA_logo.png new file mode 100644 index 00000000..a24c8e3f Binary files /dev/null and b/files/branding/dataverseUCLA_logo.png differ diff --git a/files/branding/dataverseUCLA_logo2.png b/files/branding/dataverseUCLA_logo2.png new file mode 100644 index 00000000..30ca777d Binary files /dev/null and b/files/branding/dataverseUCLA_logo2.png differ diff --git a/group_vars/all.yml b/group_vars/all.yml new file mode 100644 index 00000000..19144333 --- /dev/null +++ b/group_vars/all.yml @@ -0,0 +1,2 @@ +dataverse_system_email: "admin@example.org" + diff --git a/inventory b/inventory index 05b10603..6141eb8c 100644 --- a/inventory +++ b/inventory @@ -1,2 +1,2 @@ [dataverse] -localhost +rocky9 ansible_connection=docker \ No newline at end of file diff --git a/meta/main.yml b/meta/main.yml index 3bc82dd7..a809d53b 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -11,3 +11,5 @@ galaxy_info: - 7 galaxy_tags: - dataverse + role_name: dataverse + namespace: ucla-data-science-center diff --git a/molecule/rocky9/Dockerfile.j2 b/molecule/rocky9/Dockerfile.j2 new file mode 100644 index 00000000..b9120bbe --- /dev/null +++ b/molecule/rocky9/Dockerfile.j2 @@ -0,0 +1,3 @@ +FROM rockylinux:9 +RUN yum install -y sudo systemd systemd-sysv postfix +CMD ["/usr/sbin/init"] \ No newline at end of file diff --git a/molecule/rocky9/converge.yml b/molecule/rocky9/converge.yml new file mode 100644 index 00000000..e6007562 --- /dev/null +++ b/molecule/rocky9/converge.yml @@ -0,0 +1,8 @@ +--- +- name: Converge + hosts: all + gather_facts: false + tasks: + - name: Replace this task with one that validates your content + ansible.builtin.debug: + msg: "This is the effective test" diff --git a/molecule/rocky9/molecule.yml b/molecule/rocky9/molecule.yml new file mode 100644 index 00000000..2b3f7c9b --- /dev/null +++ b/molecule/rocky9/molecule.yml @@ -0,0 +1,34 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: rocky9 + image: eniocarboni/docker-rockylinux-systemd:latest + pre_build_image: true + privileged: true + command: /usr/sbin/init + groups: + - dataverse + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + cgroupns_mode: host + published_ports: + - "8080:8080" +# gpt lies! +# ports: +# - "8080:8080" +provisioner: + name: ansible + log: true # Enable logging + config_options: + defaults: + log_path: ./ansible-output.log # Path where the log will be saved + roles_path: + - .. + playbooks: + converge: ../../site.yml +verifier: + name: testinfra diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..d17ba25d --- /dev/null +++ b/requirements.in @@ -0,0 +1,4 @@ +ansible-core +molecule +molecule-docker +docker diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..b992eb06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,107 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile requirements.in +# +ansible-compat==25.6.0 + # via molecule +ansible-core==2.19.0 + # via + # -r requirements.in + # ansible-compat + # molecule +attrs==25.3.0 + # via + # jsonschema + # referencing +bracex==2.6 + # via wcmatch +certifi==2025.7.14 + # via requests +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.2 + # via requests +click==8.2.2 + # via + # click-help-colors + # molecule +click-help-colors==0.9.4 + # via molecule +cryptography==45.0.5 + # via ansible-core +docker==7.1.0 + # via + # -r requirements.in + # molecule-docker +enrich==1.2.7 + # via molecule +idna==3.10 + # via requests +jinja2==3.1.6 + # via + # ansible-core + # molecule +jsonschema==4.25.0 + # via + # ansible-compat + # molecule +jsonschema-specifications==2025.4.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +markupsafe==3.0.2 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +molecule==25.7.0 + # via + # -r requirements.in + # molecule-docker +molecule-docker==2.1.0 + # via -r requirements.in +packaging==25.0 + # via + # ansible-compat + # ansible-core + # molecule +pluggy==1.6.0 + # via molecule +pycparser==2.22 + # via cffi +pygments==2.19.2 + # via rich +pyyaml==6.0.2 + # via + # ansible-compat + # ansible-core + # molecule +referencing==0.36.2 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.4 + # via + # docker + # molecule-docker +resolvelib==1.2.0 + # via ansible-core +rich==14.1.0 + # via + # enrich + # molecule +rpds-py==0.26.0 + # via + # jsonschema + # referencing +subprocess-tee==0.4.2 + # via ansible-compat +typing-extensions==4.14.1 + # via referencing +urllib3==2.5.0 + # via + # docker + # requests +wcmatch==10.1 + # via molecule diff --git a/site.yml b/site.yml index 54bdcc06..a2ee780a 100644 --- a/site.yml +++ b/site.yml @@ -1,8 +1,20 @@ ---- -# dataverse.pb - - name: Install Dataverse - hosts: dataverse - become: true - roles: - - role: dataverse + hosts: all # or use "dataverse" if it's correctly defined + #become: true + tasks: + - name: Install sudo + package: + name: sudo + state: present + + - name: Install which and pip3 + package: + name: "{{ item }}" + state: present + loop: + - which + - python3-pip + + - name: Install Dataverse + include_role: + name: ../ # Your Dataverse role path diff --git a/tasks/bak_dataverse-prereqs.yml b/tasks/bak_dataverse-prereqs.yml new file mode 100644 index 00000000..14ff1973 --- /dev/null +++ b/tasks/bak_dataverse-prereqs.yml @@ -0,0 +1,104 @@ +--- +# dataverse/tasks/dataverse-prereqs.yml + +- name: install prerequisite packages + debug: + msg: '##### INSTALL PREREQUISITE PACKAGES #####' + +- name: yum clean all + shell: 'yum clean all' + when: ansible_os_family == "RedHat" + +- name: let's use the closest mirror + file: + path: /var/cache/yum/x86_64/7/timedhosts.txt + state: absent + when: ansible_os_family == "RedHat" and + ansible_distribution_major_version == "7" + +- name: let's use the fastest mirror + lineinfile: + path: /etc/dnf/dnf.conf + line: 'fastestmirror=1' + insertafter: '^gpgcheck' + when: ansible_os_family == "RedHat" and + ansible_distribution_major_version == "8" + +- name: makecache on RedHat + yum: + update_cache: yes + when: ansible_os_family == "RedHat" and + ansible_distribution_major_version == "7" + +- name: makecache + apt: + update_cache: yes + when: ansible_os_family == "Debian" + +- name: ensure EPEL repository for RedHat/Rocky + yum: + name: epel-release + state: latest + when: ansible_os_family == "RedHat" + +- name: install some necessary packages + ansible.builtin.package: + name: ['bash-completion', 'git', 'jq', 'mlocate', 'net-tools', 'sudo', 'unzip', 'python3-psycopg2', 'zip', 'tar'] + state: latest + +- name: "RHEL/Rocky 8.6-packaged Ansible wants Python-3.8" + ansible.builtin.package: + name: ['python38-psycopg2'] + state: latest + when: ansible_os_family == "RedHat" and + ansible_distribution_major_version == "8" + +- name: "RHEL/Rocky 9 provides Python-3.9" + ansible.builtin.package: + name: python3-psycopg2 + state: latest + when: ansible_os_family == "RedHat" and + ansible_distribution_major_version == "9" + +- name: install java-nnn-openjdk and other packages for RedHat/Rocky + yum: + name: ['java-{{ java.version }}-openjdk-devel', 'tzdata-java', 'vim-enhanced'] + state: latest + when: ansible_os_family == "RedHat" + +- name: install java-nnn-openjdk and other packages for Debian/Ubuntu. + package: + name: ['acl', 'openjdk-{{ java.version }}-jdk-headless', 'python3', 'vim'] + when: ansible_os_family == "Debian" + +# it is strongly recommended to check for open CVEs before enabling this. +- name: install GraphicsMagic on RHEL/Rocky for thumbnail generation + dnf: + name: GraphicsMagick + when: + - ansible_os_family == "RedHat" + - ansible_distribution_major_version == "8" or ansible_distribution_major_version == "9" + - dataverse.thumbnails + +- name: install GraphicsMagic on Debian/Ubuntu for thumbnail generation + package: + name: graphicsmagick + when: + - ansible_os_family == "Debian" + - dataverse.thumbnails + +- name: install curl on Debian/Ubuntu + package: + name: curl + when: + - ansible_os_family == "Debian" + +- name: Payara service account must exist + import_tasks: payara_service_account.yml + +- name: create dataverse misc files directory for language and handle and other similar auxilliary files + file: + path: "{{ dataverse_misc_files_dir }}" + state: directory + owner: "{{ dataverse.payara.user }}" + group: "{{ dataverse.payara.group }}" diff --git a/tasks/dataverse-counter.yml b/tasks/dataverse-counter.yml index 8c112fbd..cd8c725f 100644 --- a/tasks/dataverse-counter.yml +++ b/tasks/dataverse-counter.yml @@ -9,16 +9,16 @@ name: python38-pip state: latest when: - - ansible_os_family == "RedHat" - - ansible_distribution_major_version == "8" + - ansible_os_family == "RedHat" + - ansible_distribution_major_version == "8" - name: ensure python39-pip on RHEL/Rocky 9 ansible.builtin.package: name: python3-pip state: latest when: - - ansible_os_family == "RedHat" - - ansible_distribution_major_version == "9" + - ansible_os_family == "RedHat" + - ansible_distribution_major_version == "9" - name: ensure counter user exists user: @@ -31,8 +31,14 @@ remote_src: yes owner: "{{ dataverse.counter.user }}" +# Find the correct path to python3 +- name: Find path to python3 + command: "which python3" + register: python_path + +# Use the dynamically found python path to install the requirements - name: pip install requirements - shell: "/usr/bin/python3.8 -m pip install -r /usr/local/counter-processor-{{ dataverse.counter.version }}/requirements.txt" + shell: "{{ python_path.stdout }} -m pip install -r /usr/local/counter-processor-{{ dataverse.counter.version }}/requirements.txt" become: yes - name: create sample log dir @@ -56,8 +62,8 @@ - name: launch counter shell: - cmd: 'cd /usr/local/counter-processor-{{ dataverse.counter.version }} || python3.9 main.py' + cmd: 'cd /usr/local/counter-processor-{{ dataverse.counter.version }} || {{ python_path.stdout }} main.py' vars: CONFIG_FILE: "/usr/local/counter-processor-{{ dataverse.counter.version }}/counter-processor-config.yml" become: yes - become_user: "{{ dataverse.counter.user }}" + become_user: "{{ dataverse.counter.user }}" \ No newline at end of file diff --git a/tasks/dataverse-gui.yml b/tasks/dataverse-gui.yml index 16309bda..889604cd 100644 --- a/tasks/dataverse-gui.yml +++ b/tasks/dataverse-gui.yml @@ -1,9 +1,8 @@ --- - # dataverse/tasks/dataverse-gui.yml # install gui modifications (branding and favicons), if defined -- name: calculate destination directories +- name: Calculate destination directories set_fact: gui_file_path: '{{ payara_dir }}/glassfish/domains/{{ dataverse.payara.domain }}/applications/dataverse' favicon_file_path: '{{ payara_dir }}/glassfish/domains/{{ dataverse.payara.domain }}/applications/dataverse/resources/images/fav/' @@ -21,9 +20,9 @@ group: '{{ dataverse.payara.group }}' with_fileglob: '{{ dataverse.branding.favicons_directory }}/*.ico' -- name: copy branding files +- name: Copy branding files copy: - src: branding + src: '{{ dataverse.branding.directory }}' dest: '{{ gui_file_path }}' owner: '{{ dataverse.payara.user }}' group: '{{ dataverse.payara.group }}' @@ -33,7 +32,7 @@ uri: url: http://localhost:8080/api/admin/settings/:{{ item.setting }} method: PUT - body: '{{ gui_file_path }}/branding/{{ item.file }}' + body: '/branding/{{ item.file }}' status_code: 200 with_items: '{{ dataverse.branding.fileSettings }}' @@ -45,4 +44,3 @@ body: "{{ item.value }}" status_code: 200 with_items: '{{ dataverse.branding.otherSettings }}' - diff --git a/tasks/dataverse-languages.yml b/tasks/dataverse-languages.yml index 336bc4d9..0ee60413 100644 --- a/tasks/dataverse-languages.yml +++ b/tasks/dataverse-languages.yml @@ -1,38 +1,40 @@ --- -- name: install and configure dataverse languages +- name: Install and configure Dataverse languages debug: msg: '##### DATAVERSE LANGUAGES #####' - set_fact: lang_git_dir: /usr/local/src/dataverse_language_packs -- name: get dataverse language file path +- name: Get Dataverse language file path shell: "{{ payara_dir }}/bin/asadmin list-jvm-options | grep dataverse.lang.directory | sed 's/.*=//'" register: dataverse_lang_directory changed_when: false -- name: create dataverse language file path if not set +- name: Create Dataverse language file path if not set file: - path: "{{ dataverse.language.lang_directory }}" + path: "{{ dataverse.language.lang_directory | default('/opt/dv/lang') }}" state: directory owner: '{{ dataverse.payara.user }}' when: (dataverse_lang_directory.stdout | trim) == '' -- name: copy default bundle to the language directory if it was just created +- name: Copy default bundle to the language directory if it was just created copy: src: "{{ lang_git_dir }}/en_US/Bundle.properties" dest: "{{ dataverse.language.lang_directory }}" owner: '{{ dataverse.payara.user }}' remote_src: yes - when: (dataverse_lang_directory.stdout | trim) == '' + when: lookup('file', "{{ lang_git_dir }}/en_US/Bundle.properties", errors='ignore') is not none -- name: set dataverse language file path if not set +- name: Set Dataverse language file path if not set shell: "{{ payara_dir }}/bin/asadmin create-jvm-options -Ddataverse.lang.directory={{ dataverse.language.lang_directory }}" when: (dataverse_lang_directory.stdout | trim) == '' -- name: restart payara after setting language directory - service: name=payara state=restarted +- name: Restart Payara after setting language directory + service: + name: payara + state: restarted when: (dataverse_lang_directory.stdout | trim) == '' - ansible.builtin.import_tasks: check_index_status.yml @@ -44,32 +46,63 @@ version: "{{ dataverse.language.language_packs.version }}" run_once: true -- name: prepare language file temporary directory - shell: cd {{ lang_git_dir }} ; rm -rf tmp.bak ; [ -d tmp ] && mv tmp tmp.bak && rm tmp.bak/*.zip ; mkdir tmp +# Updated Section: Prepare language file temporary directory +- name: Ensure language directory exists + file: + path: "{{ lang_git_dir }}" + state: directory + +- name: Remove old tmp.bak directory if it exists + file: + path: "{{ lang_git_dir }}/tmp.bak" + state: absent + +- name: Backup existing tmp directory if it exists + command: mv tmp tmp.bak + args: + chdir: "{{ lang_git_dir }}" + when: lookup('file', "{{ lang_git_dir }}/tmp", errors='ignore') is not none + +- name: Remove old zip files in tmp.bak if they exist + shell: "rm -f {{ lang_git_dir }}/tmp.bak/*.zip" + ignore_errors: yes changed_when: false -- name: copy language files to temporary directory - shell: cd {{ lang_git_dir }} ; cp -R {{ item.locale }}*/*.properties tmp/ +- name: Create new tmp directory + file: + path: "{{ lang_git_dir }}/tmp" + state: directory + +# New Check for Each Locale Directory Using a Fact +- name: Check if language directory exists for each locale + set_fact: + locale_dir_exists: "{{ lookup('file', lang_git_dir + '/' + item.locale, errors='ignore') is not none }}" + with_items: "{{ dataverse.language.languages }}" + loop_control: + label: "{{ item.locale }}" + +- name: Copy language files to temporary directory + shell: "cp -R {{ lang_git_dir }}/{{ item.locale }}*/*.properties {{ lang_git_dir }}/tmp/" changed_when: false with_items: "{{ dataverse.language.languages }}" + when: locale_dir_exists -- name: check if there was a change in the temporary directory - shell: cd {{ lang_git_dir }} ; diff -r tmp tmp.bak +- name: Check if there was a change in the temporary directory + shell: "cd {{ lang_git_dir }} && diff -r tmp tmp.bak || true" register: diff changed_when: diff.rc != 0 failed_when: diff.rc > 2 -- name: create language pack - shell: cd {{ lang_git_dir }}/tmp ; zip languages.zip *.properties +- name: Create language pack + shell: "cd {{ lang_git_dir }}/tmp && zip languages.zip *.properties" when: diff.changed -- name: upload language pack +- name: Upload language pack uri: url: "{{ dataverse.api.location }}/admin/datasetfield/loadpropertyfiles" method: POST headers: Content-type: "application/zip" -# Accept: application/json src: "{{ lang_git_dir }}/tmp/languages.zip" remote_src: yes status_code: 200 @@ -77,7 +110,7 @@ when: diff.changed notify: enable and restart payara -- name: configure available languages +- name: Configure available languages uri: url: "{{ dataverse.api.location }}/admin/settings/:Languages" method: PUT @@ -86,12 +119,11 @@ status_code: 200 when: diff.changed -- name: configure available languages +- name: Configure metadata languages uri: url: "{{ dataverse.api.location }}/admin/settings/:MetadataLanguages" method: PUT body: '{{ dataverse.language.languages }}' body_format: json status_code: 200 - when: diff.changed - + when: diff.changed \ No newline at end of file diff --git a/tasks/dataverse-licenses.yml b/tasks/dataverse-licenses.yml index 2499dc07..69247d7f 100644 --- a/tasks/dataverse-licenses.yml +++ b/tasks/dataverse-licenses.yml @@ -1,11 +1,11 @@ --- -- name: install and configure dataverse languages +- name: Install and configure Dataverse licenses debug: msg: '##### DATAVERSE LICENSES #####' -#### you need community.postgresql from ansible galaxy for this -- name: get api key for dataverseAdmin from the database +#### Requires community.postgresql from Ansible Galaxy +- name: Get API key for dataverseAdmin from the database community.postgresql.postgresql_query: db: "{{ db.postgres.name }}" login_user: "{{ db.postgres.user }}" @@ -16,21 +16,28 @@ register: token_result failed_when: token_result.rowcount != 1 -- name: calculate api_token +- name: Calculate API token set_fact: api_token: '{{ token_result.query_result[0].tokenstring }}' -#- name: get installed licenses -- this is needed for true idempotence, but not used currently -# uri: -# url: "{{ dataverse.api.location }}/licenses" -# method: GET -# status_code: 200 -# register: current_licenses +- name: Get existing licenses from Dataverse + uri: + url: "{{ dataverse.api.location }}/licenses" + method: GET + headers: + X-Dataverse-key: '{{ api_token }}' + Content-Type: 'application/json' + status_code: 200 + register: current_licenses -#- debug: -# msg: '{{ current_licenses }}' -- name: configure available licenses +# Filter desired licenses to add only if not already present +- name: Filter licenses to add + set_fact: + licenses_to_add: "{{ dataverse.licenses.licenses | rejectattr('name', 'in', current_licenses.json.data | map(attribute='name')) | list }}" + +# Add licenses conditionally +- name: Configure available licenses uri: url: "{{ dataverse.api.location }}/licenses" method: POST @@ -41,5 +48,6 @@ status_code: 201,409 register: license_update changed_when: license_update.status != 409 - with_items: '{{ dataverse.licenses.licenses }}' - + loop: "{{ licenses_to_add }}" + loop_control: + label: "{{ item.name }}" diff --git a/tasks/dataverse-optional-settings.yml b/tasks/dataverse-optional-settings.yml index 627af429..58160e72 100644 --- a/tasks/dataverse-optional-settings.yml +++ b/tasks/dataverse-optional-settings.yml @@ -16,9 +16,35 @@ shell: 'curl -X PUT -d {{ dataverse.options.provcollectionenabled }} {{ dataverse.api.location }}/admin/settings/:ProvCollectionEnabled' when: dataverse.options.provcollectionenabled -- name: set SystemEmail as a jvm-option now - shell: '{{ payara_dir}}/bin/asadmin create-jvm-options "-Ddataverse.mail.system-email={{ dataverse.service_email }}"' - when: dataverse.service_email is defined +- name: Get JVM options + command: /usr/local/payara6/bin/asadmin list-jvm-options + register: jvm_options + changed_when: false + +- name: Check if system email JVM option exists + set_fact: + has_system_email: "{{ '-Ddataverse.mail.system-email' in jvm_options.stdout }}" + +- name: Set SystemEmail as a JVM option if it doesn't already exist + command: > + /usr/local/payara6/bin/asadmin create-jvm-options "-Ddataverse.mail.system-email={{ dataverse_system_email }}" + when: not has_system_email + + + +#- name: Check if system email JVM option already exists +# shell: "{{ payara_dir}}/bin/asadmin list-jvm-options | grep -q \"-Ddataverse.mail.system-email\"" +# ignore_errors: yes +# register: check_jvm_option + +# name: Set SystemEmail as a JVM option if it doesn't already exist +# shell: "{{ payara_dir}}/bin/asadmin create-jvm-options \"-Ddataverse.mail.system-email=noreply@dataverse.yourinstitution.edu\"" +# when: check_jvm_option.rc != 0 +# become: yes + +#- name: set SystemEmail as a jvm-option now +# shell: '{{ payara_dir}}/bin/asadmin create-jvm-options "-Ddataverse.mail.system-email={{ dataverse.service_email }}"' +# when: dataverse.service_email is defined - name: set TabularIngestSizeLimit when provided shell: 'curl -X PUT -d {{ dataverse.options.tabularingestsizelimit }} {{ dataverse.api.location }}/admin/settings/:TabularIngestSizeLimit' diff --git a/tasks/dataverse-prereqs.yml b/tasks/dataverse-prereqs.yml index f9a45dc8..3acb74d2 100644 --- a/tasks/dataverse-prereqs.yml +++ b/tasks/dataverse-prereqs.yml @@ -1,42 +1,33 @@ --- # dataverse/tasks/dataverse-prereqs.yml -- name: install prerequisite packages +- name: Install prerequisite packages debug: - msg: '##### INSTALL PREREQUISITE PACKAGES #####' + msg: '##### INSTALLING PREREQUISITE PACKAGES #####' -- name: yum clean all - shell: 'yum clean all' +# Ensure the package manager cache is cleaned +- name: Clean all yum/dnf caches + shell: 'dnf clean all' when: ansible_os_family == "RedHat" -- name: let's use the closest mirror - file: - path: /var/cache/yum/x86_64/7/timedhosts.txt - state: absent - when: ansible_os_family == "RedHat" and - ansible_distribution_major_version == "7" - -- name: let's use the fastest mirror +# Ensure the fastest mirror is used for Rocky 8 and Rocky 9 +- name: Use the fastest mirror on Rocky lineinfile: path: /etc/dnf/dnf.conf line: 'fastestmirror=1' insertafter: '^gpgcheck' when: ansible_os_family == "RedHat" and - ansible_distribution_major_version == "8" - -- name: makecache on RedHat - yum: - update_cache: yes - when: ansible_os_family == "RedHat" and - ansible_distribution_major_version == "7" + ansible_distribution_major_version in ["8", "9"] -- name: makecache - apt: +# Make sure the package manager cache is updated +- name: Refresh dnf cache on RedHat/Rocky + dnf: update_cache: yes - when: ansible_os_family == "Debian" + when: ansible_os_family == "RedHat" -- name: ensure EPEL repository for RedHat/Rocky - yum: +# Ensure EPEL repository is installed for RedHat/Rocky +- name: Ensure EPEL repository for RedHat/Rocky + dnf: name: epel-release state: latest when: ansible_os_family == "RedHat" @@ -45,60 +36,60 @@ ansible.builtin.package: name: ['bash-completion', 'bc', 'ed', 'git', 'jq', 'mlocate', 'net-tools', 'sudo', 'unzip', 'python3-psycopg2', 'zip', 'tar', 'wget'] state: latest + when: ansible_os_family == "RedHat" -- name: "RHEL/Rocky 8.6-packaged Ansible wants Python-3.8" - ansible.builtin.package: - name: ['python38-psycopg2'] - state: latest +# Ensure pip is installed for Python 3.9 +- name: Ensure pip is installed for Python 3.9 + dnf: + name: python3-pip + state: present when: ansible_os_family == "RedHat" and - ansible_distribution_major_version == "8" + ansible_distribution_major_version == "9" -- name: "RHEL/Rocky 9 provides Python-3.9" - ansible.builtin.package: - name: python3-psycopg2 +# Find the path to pip3 for Rocky 9 +- name: Find path to pip3 + command: "which pip3" + register: pip_path + +# Install psycopg2 via pip for Rocky 9 if not available via package manager +- name: Install psycopg2 via pip + pip: + name: psycopg2-binary state: latest + executable: "{{ pip_path.stdout }}" when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "9" -- name: install java-nnn-openjdk and other packages for RedHat/Rocky - yum: +# Install OpenJDK and other Java packages +- name: Install Java (OpenJDK) and other packages for Rocky + dnf: name: ['java-{{ java.version }}-openjdk-devel', 'tzdata-java', 'vim-enhanced'] state: latest when: ansible_os_family == "RedHat" -- name: install java-nnn-openjdk and other packages for Debian/Ubuntu. - package: - name: ['acl', 'openjdk-{{ java.version }}-jdk-headless', 'python3', 'vim'] - when: ansible_os_family == "Debian" - -# it is strongly recommended to check for open CVEs before enabling this. -- name: install GraphicsMagic on RHEL/Rocky for thumbnail generation +# Install GraphicsMagick for thumbnail generation on Rocky +- name: Install GraphicsMagick for thumbnail generation dnf: name: GraphicsMagick + state: latest when: - ansible_os_family == "RedHat" - - ansible_distribution_major_version == "8" or ansible_distribution_major_version == "9" - - dataverse.thumbnails - -- name: install GraphicsMagic on Debian/Ubuntu for thumbnail generation - package: - name: graphicsmagick - when: - - ansible_os_family == "Debian" + - ansible_distribution_major_version in ["8", "9"] - dataverse.thumbnails -- name: install curl on Debian/Ubuntu - package: - name: curl - when: - - ansible_os_family == "Debian" - +# Ensure the Payara service account exists - name: Payara service account must exist import_tasks: payara_service_account.yml -- name: create dataverse misc files directory for language and handle and other similar auxilliary files +# Create directory for miscellaneous Dataverse files +- name: Create Dataverse misc files directory file: path: "{{ dataverse_misc_files_dir }}" state: directory owner: "{{ dataverse.payara.user }}" group: "{{ dataverse.payara.group }}" + +- name: Install required system utilities + package: + name: procps + state: present \ No newline at end of file diff --git a/tasks/minio.yml b/tasks/minio.yml index 1c1b99a9..f8a25f98 100644 --- a/tasks/minio.yml +++ b/tasks/minio.yml @@ -66,7 +66,7 @@ register: compose_file - name: STORAGE | Stop `docker-compose down` MinIO - community.docker.docker_compose: + community.docker.docker_compose_v2: project_src: "{{ minio.docker.project_location }}" state: absent remove_orphans: true @@ -77,7 +77,7 @@ - copy_compose.changed - name: STORAGE | Run `docker-compose up` MinIO - community.docker.docker_compose: + community.docker.docker_compose_v2: project_src: "{{ minio.docker.project_location }}" build: true files: minio_compose.yml diff --git a/tasks/payara.yml b/tasks/payara.yml index 00081e52..54dd62db 100644 --- a/tasks/payara.yml +++ b/tasks/payara.yml @@ -26,6 +26,7 @@ - name: download payara zip get_url: url: '{{ dataverse.payara.zipurl }}' + #url: 'https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip' checksum: '{{ dataverse.payara.zipchecksum }}' dest: /tmp/payara.zip register: payara_zip_download diff --git a/tasks/postfix.yml b/tasks/postfix.yml index 35991dc3..ec5fd1ef 100644 --- a/tasks/postfix.yml +++ b/tasks/postfix.yml @@ -16,6 +16,9 @@ - ansible_os_family == "RedHat" - ansible_distribution_major_version == "8" +#- name: start postfix without systemctl +# command: /usr/sbin/postfix start + - name: enable and start postfix systemd: name: postfix diff --git a/tasks/postgres_redhat.yml b/tasks/postgres_redhat.yml index 8a4a0ec3..1d353125 100644 --- a/tasks/postgres_redhat.yml +++ b/tasks/postgres_redhat.yml @@ -1,17 +1,25 @@ --- -- name: import RPM-GPG-KEY-PGDG - rpm_key: +- name: import PGDG-RPM-GPG-KEY-RHEL + ansible.builtin.rpm_key: state: present key: https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL + become: yes - name: install postgres repo RPM - ansible.builtin.package: - name: 'https://download.postgresql.org/pub/repos/yum/reporpms/EL-{{ ansible_distribution_major_version }}-x86_64/pgdg-redhat-repo-latest.noarch.rpm' + ansible.builtin.yum: + name: 'https://download.postgresql.org/pub/repos/yum/reporpms/EL-{{ ansible_distribution_major_version }}-{{ ansible_architecture }}/pgdg-redhat-repo-latest.noarch.rpm' state: present + disable_gpg_check: true + become: yes - name: "RHEL/Rocky: disable PostgreSQL proper in the OS" - shell: 'dnf -qy module disable postgresql' + shell: | + dnf -qy module disable postgresql + args: + executable: /bin/bash + ignore_errors: yes + become: yes - name: get postgres config directory set_fact: diff --git a/tasks/s3.yml b/tasks/s3.yml index 99d3cb82..496e50f8 100644 --- a/tasks/s3.yml +++ b/tasks/s3.yml @@ -27,12 +27,12 @@ mode: '0600' - name: set s3 settings in dataverse + environment: + PATH: "{{ lookup('env', 'PATH') }}:/usr/local/bin" shell: 'asadmin-create-or-replace-option.sh "{{ item.key }}" "{{ item.value }}"' register: output changed_when: "'Command create-jvm-options executed successfully.' in output.stdout" - when: - - item.value is defined - - item.value != '' + with_items: - key: dataverse.files.storage-driver-id value: "{{ s3.storage_driver_id }}" @@ -52,9 +52,8 @@ value: "{{ s3.payload_signing }}" - key: dataverse.files.s3.chunked-encoding value: "{{ s3.chunked_encoding }}" - - key: dataverse.files.s3.custom-endpoint-region - value: "{{ s3.custom_endpoint_region }}" - + - key: dataverse.files.s3.path-style-access + value: "{{ s3.path_style_access }}" # optional s3 settings - name: expose custom_endpoint_url as variable @@ -66,7 +65,7 @@ - name: create S3 bucket shell: - 'aws s3api create-bucket --bucket {{ s3.bucket_name }}' + 'aws s3api create-bucket --bucket {{ s3.bucket_name }} --create-bucket-configuration LocationConstraint={{ s3.location_constraint }} --region {{ s3.region }}' args: executable: /bin/bash become_user: '{{ dataverse.payara.user }}' @@ -98,6 +97,8 @@ and custom_endpoint_url | length == 0 and s3.cors_already_set == false - name: set s3 direct download + environment: + PATH: "{{ lookup('env', 'PATH') }}:/usr/local/bin" shell: 'asadmin-create-or-replace-option.sh "dataverse.files.s3.download-redirect" "{{ s3.download_redirect }}"' register: output changed_when: "'Command create-jvm-options executed successfully.' in output.stdout" diff --git a/tasks/selinux.yml b/tasks/selinux.yml index 95ae828b..5e03fd19 100644 --- a/tasks/selinux.yml +++ b/tasks/selinux.yml @@ -8,9 +8,8 @@ - setools - setools-console - policycoreutils - when: ansible_os_family == 'RedHat' + when: ansible_os_family == 'RedHat' and ansible_selinux.status == "enabled" -# Ansible seboolean works on Rocky 9, 8.6 handled below. - name: set httpd_can_network_connect on and keep it persistent across reboots seboolean: name: httpd_can_network_connect @@ -19,15 +18,15 @@ when: - ansible_os_family == 'RedHat' - ansible_distribution_major_version == "9" + - ansible_selinux.status == "enabled" -# Ansible seboolean module is broken on RHEL/Rocky 8.6, use shell cmd instead. -- name: allow apache to make outbound connections +- name: allow apache to make outbound connections (for Rocky 8.6) shell: '/usr/sbin/setsebool -P httpd_can_network_connect 1' when: - ansible_os_family == "RedHat" - ansible_distribution_major_version == "8" + - ansible_selinux.status == "enabled" -# Ansible seboolean works on Rocky 9, 8.6 handled below. - name: allow apache to read user content by default seboolean: name: httpd_read_user_content @@ -36,20 +35,23 @@ when: - ansible_os_family == "RedHat" - ansible_distribution_major_version == "9" + - ansible_selinux.status == "enabled" -# Ansible seboolean module is broken on RHEL/Rocky 8.6. dls 20220602 -- name: allow apache to read user content by default +- name: allow apache to read user content by default (for Rocky 8.6) shell: 'setsebool -P httpd_read_user_content 1' when: - ansible_os_family == "RedHat" - ansible_distribution_major_version == "8" + - ansible_selinux.status == "enabled" -- name: "both redhat and ubuntu default to /var/www/html" +- name: "both RedHat and Ubuntu default to /var/www/html" shell: 'semanage fcontext -a -t httpd_sys_content_t "/var/www/html(/.*)?" || semanage fcontext -m -t httpd_sys_content_t "/var/www/html(/.*)?"' when: - ansible_os_family == "RedHat" + - ansible_selinux.status == "enabled" - name: "allow apache read-only access to /var/www/html" shell: 'restorecon -R -v /var/www/html' when: - ansible_os_family == "RedHat" + - ansible_selinux.status == "enabled" \ No newline at end of file diff --git a/tasks/solr.yml b/tasks/solr.yml index b4eae0cd..64cfe762 100644 --- a/tasks/solr.yml +++ b/tasks/solr.yml @@ -39,6 +39,7 @@ url: "{{ solr_download_url }}" checksum: "{{ dataverse.solr.checksum }}" dest: /tmp/solr-{{ dataverse.solr.version }}.tgz + timeout: 600 register: solr_installer_download - name: untar solr diff --git a/ucla_readme.md b/ucla_readme.md new file mode 100644 index 00000000..7dc60bf8 --- /dev/null +++ b/ucla_readme.md @@ -0,0 +1,131 @@ +## Local Setup (Recommended) + +These instructions assume: + +- You have already cloned this repository locally: + + ```bash + git clone https://github.com/ucla-data-science-center/dataverse-ansible.git + cd dataverse-ansible + ``` + +- You have [Conda](https://docs.conda.io/en/latest/miniconda.html) installed (e.g. via [Miniforge](https://github.com/conda-forge/miniforge)). +- Docker is installed and running on your system. + +--- + +### 1. Create the Conda Environment + +To create a consistent development environment using `pip-tools`: + + ```bash + conda env create -f environment.yml + conda activate dataverse-ansible + ``` + +This will install: + +- Python 3.11 +- `pip-tools` (to manage Python packages via lockfiles) + +--- + +### 2. Compile and Install Python Dependencies + +This project uses [`pip-tools`](https://pip-tools.readthedocs.io/) for dependency management. After activating the environment: + + ```bash + pip-compile requirements.in + pip-sync + ``` + +This will install: + +- `ansible-core` +- `molecule` +- `molecule-docker` +- `docker` (Python SDK) + +> You only need to run `pip-compile` again if `requirements.in` changes. Use `pip-sync` to reinstall the locked dependencies. + +--- + +### Optional: Manual Environment Creation + +If you prefer not to use `environment.yml` or `pip-tools`, you can manually create and install dependencies: + + ```bash + conda create -n dataverse-ansible python=3.11 -y + conda activate dataverse-ansible + pip install ansible-core molecule molecule-docker docker + ``` + +--- + +## Running with Molecule and Docker + +The `rocky9` Molecule scenario uses Docker as a provisioner. It relies on a custom image with `systemd` support, allowing `sudo` commands to run inside the container. This avoids modifying the Ansible role's privilege escalation behavior. + +From the root of the cloned repository, run: + + ```bash + molecule converge --scenario-name rocky9 + ``` + +This will build a Docker container, install Dataverse, and configure services. + +Once complete, you should be able to access Dataverse at: + + http://localhost:8080 + +**Default admin login:** + +- **Username**: `dataverseAdmin` +- **Password**: defined in `tests/group_vars/vagrant.yml` (see `dataverse_adminpass`) + +To verify the server is responding: + + ```bash + curl -I http://localhost:8080 + ``` + +--- + +## Teardown and Rebuild + +Because the Dataverse installer is not idempotent, it’s recommended to fully reset the container between changes. + +To stop and delete the container: + + ```bash + molecule reset --scenario-name rocky9 + ``` + +Then rebuild with: + + ```bash + molecule converge --scenario-name rocky9 + ``` + +To open a shell inside the running container: + + ```bash + molecule login --scenario-name rocky9 + ``` + +To see additional Molecule commands: + + ```bash + molecule --help + ``` + +More documentation: [https://ansible.readthedocs.io/projects/molecule/](https://ansible.readthedocs.io/projects/molecule/) + +--- + +## Notes + +- If port `8080` is already in use on your machine, update the port mapping in `molecule/rocky9/molecule.yml`. +- Ensure Docker Desktop (macOS) or the Docker daemon (Linux/WSL2) is running before launching `molecule converge`. + +---