Modify Deployment for multi data source searchengine

ansible/backup_searchengine_data.yml

@@ -0,0 +1,18 @@
+- name: Backup searchengine data
+  hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
+
+  tasks:
+  # Backup the elastic indexed data
+  - name: backup elastic search indexed data
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: restore_elasticsearch_data
+      cleanup: True
+      #auto_remove: yes
+      networks:
+        - name: searchengine-net
+      command: "backup_elasticsearch_data"
+      state: started
+      volumes:
+        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"

ansible/decommission/archive-logs.yml

@@ -10,7 +10,6 @@
     set_fact:
       decommission_archive_prefix: "{{ ansible_hostname }}-{{ ansible_date_time.date | replace('-', '') }}"
 
-
 - hosts: "{{ idr_environment | default('idr') }}-proxy-hosts"
   vars:
     logs:
@@ -42,7 +41,6 @@
         from {{ ansible_hostname }}:{{ item.expanded_paths | join(',') }}
     with_items: "{{_decommission_archive_proxy.results}}"
 
-
 - hosts: "{{ idr_environment | default('idr') }}-management-hosts"
   vars:
     logs:
@@ -74,11 +72,11 @@
         from {{ ansible_hostname }}:{{ item.expanded_paths | join(',') }}
     with_items: "{{_decommission_archive_management.results}}"
 
-
 - hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
   vars:
     logs:
       - { name: 'searchengine', path: '/data/searchengine/searchengine/logs' }
+      - {"name": "searchengine_backup_data", path: "/searchengine_backup"}
   tasks:
 
   - name: Archive searchengine logs

ansible/group_vars/searchengine-hosts.yml

@@ -2,19 +2,22 @@ apps_folder: /data
 #database_server_url: "{{ omero_db_host_ansible }}"
 database_port: 5432
 database_name: idr
+default_datasource: idr
 database_username: omeroreadonly
 database_user_password: "{{ idr_secret_postgresql_password_ro | default('omero') }}"
 searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
 search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
-searchengine_docker_image: openmicroscopy/omero-searchengine:0.6.0
-#ansible_python_interpreter: path/to/bin/python
+searchengine_docker_image: openmicroscopy/omero-searchengine:0.7
+# ansible_python_interpreter: path/to/bin/python
 searchengine_index: searchengine_index
+automatic_refresh: true
 cache_rows: 100000
 # I think that the following two variables should be in secret
 searchengine_secret_key: "{{ idr_searchengine_secret_key | default('secret_key') }}"
 searchengineurlprefix: "searchengine"
 IDR_TEST_FILE_URL: "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv"
 elasticsearch_no_nodes: 3
+# This should be a separate volume, cloned as part of the routine deployment process.
 elasticsearch_backup_folder: "/searchengine_backup"
 ca_password: "{{ idr_secret_elastic_ca_password | default('ca_password') }}"
 keystore_password: "{{ idr_secret_elastic_keystore_password | default('keystore_password') }}"

ansible/idr-02-services.yml

@@ -9,3 +9,6 @@
 ## Search services
 - import_playbook: idr-elasticsearch.yml
 - import_playbook: idr-searchengine.yml
+
+## Restore searchengine data if existence
+- import_playbook: restore_searchengine_data.yml

ansible/idr-elasticsearch.yml

@@ -73,18 +73,6 @@
       mode: 0755
     with_sequence: start=1 count={{ elasticsearch_no_nodes }}
 
-  - name: Create ELasticsearch backup folder
-    become: yes
-    file:
-      path: "{{ elasticsearch_backup_folder }}"
-      recurse: yes
-      state: directory
-      owner: 1000
-      group: root
-      mode: 0755
-
-
-
   - name: Add elastic nodes to instances_nodes
     set_fact:
       instances_nodes: "{{instances_nodes + [( {'name' : 'searchengine_elasticsearch_node'+item, 'dns': ['searchengine_elasticsearch_node'+item,'localhost'],'ip': '127.0.0.1'})] }}"
@@ -130,7 +118,7 @@
           #fi;
       state: started
       volumes:
-          - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
+          - "{{ apps_folder }}/searchengine/elasticsearch/certs:/certs"
           - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
 
   - name: Wait for CA file
@@ -163,7 +151,7 @@
 
       state: started
       volumes:
-          - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
+          - "{{ apps_folder }}/searchengine/elasticsearch/certs:/certs"
           - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
           - /tmp/instances.yaml:/tmp/instances.yaml
 

ansible/idr-searchengine.yml

@@ -68,7 +68,6 @@
       volumes:
         - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
 
-
   - name: configure database for docker searchengine
     become: yes
     docker_container:
@@ -77,16 +76,28 @@
       cleanup: True
       #auto_remove: yes
       command: >
-        set_database_configuration -u {{ database_server_url }}
-        -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}
-      #networks:
+        set_database_configuration -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }} -b {{ apps_folder }} -w {{ default_datasource }} -u {{ database_server_url }}
+      #networks::
       #- name: searchengine-net
       #published_ports:
       #- "5577:5577"
       state: started
       volumes:
       - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
 
+  - name: set default data source
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine_database
+      cleanup: True
+      #auto_remove: yes
+      command: >
+        set_default_datasource -d  {{ default_datasource }}
+      state: started
+      volumes:
+        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
   - name: configure IDR_TEST_FILE_URL item
     become: yes
     docker_container:
@@ -98,7 +109,6 @@
       volumes:
       - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
 
-
   - name: configure cache folder  for docker searchengine
     become: yes
     docker_container:
@@ -151,6 +161,20 @@
       volumes:
         - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
 
+# automatic refreshing  when configuration change is detected
+
+  - name: Set automatic refreshing the configuration
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: elastics_password
+      cleanup: True
+      #auto_remove: yes
+      command: "set_automatic_refresh -a {{ automatic_refresh }}"
+      state: started
+      volumes:
+        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
   - name: Run docker searchengine
     become: yes
     docker_container:

ansible/openstack-create-publicidr.yml

@@ -169,3 +169,11 @@
       openstack_volume_name: "{{ idr_environment_idr }}-searchengine-data"
       openstack_volume_device: /dev/vdb
       openstack_volume_type: "{{ idr_volume_searchengine_data_type | default(omit) }}"
+
+    #searchengine searchengine_backup volume
+    - role: ome.openstack_volume_storage
+      openstack_volume_size: 25
+      openstack_volume_vmname: "{{ idr_environment_idr }}-searchengine"
+      openstack_volume_name: "{{ idr_environment_idr }}-searchengine-searchengine_backup"
+      openstack_volume_device: /dev/vdb
+      openstack_volume_type: "{{ idr_volume_searchengine_data_type | default(omit) }}"

ansible/restore_searchengine_data.yml

@@ -0,0 +1,24 @@
+- name: Restore searchengine data
+  hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
+
+  tasks:
+  - name: Verify whether the snapshot exists
+    stat:
+      path: "{{ elasticsearch_backup_folder }}/backup_data"
+    register: snapshot
+
+  # restore the elastic indexed data from the backup
+  - name: Restore the backup elastic search indexed data
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: restore_elasticsearch_data
+      cleanup: True
+      #auto_remove: yes
+      networks:
+        - name: searchengine-net
+      command: "restore_elasticsearch_data"
+      state: started
+      volumes:
+        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+    when: snapshot.stat.exists == True

scripts/os-idr-snapshot.sh

@@ -18,6 +18,7 @@ for vol in \
         database-db \
         omeroreadwrite-data \
         proxy-nginxcache \
+        searchengine_backup \
         ; do
     volume="$vm_prefix-$vol"
     echo "Snapshotting volume $volume"