From 9cb5163e625a8b30f3578be8aa7da3074e0c43c2 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Tue, 25 Jan 2022 19:29:37 +0000
Subject: [PATCH 01/22] Adding searchengine, searchengineclinet, and
 Elasticsearch deployment using  ansible playbooks

---
 ansible/check_caching_service.yml             |  22 ++
 ansible/check_indexing_service.yml            |  22 ++
 ansible/management-searchengine.yml           | 237 ++++++++++++++++++
 .../run_searchengine_index_cache_services.yml |  40 +++
 ansible/searchengine_vars.yml                 |  13 +
 docs/searchengine_deployemnt.md               |  22 ++
 6 files changed, 356 insertions(+)
 create mode 100644 ansible/check_caching_service.yml
 create mode 100644 ansible/check_indexing_service.yml
 create mode 100644 ansible/management-searchengine.yml
 create mode 100644 ansible/run_searchengine_index_cache_services.yml
 create mode 100644 ansible/searchengine_vars.yml
 create mode 100644 docs/searchengine_deployemnt.md

diff --git a/ansible/check_caching_service.yml b/ansible/check_caching_service.yml
new file mode 100644
index 00000000..c6990d59
--- /dev/null
+++ b/ansible/check_caching_service.yml
@@ -0,0 +1,22 @@
+#Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
+- name: Check caching
+  connection: local
+  hosts: local 
+  vars_files:
+    searchengine_vars.yml
+  tasks:  
+  - name: Get infos on container
+    docker_container_info:
+      name: "{{ searchengine_cache }}"
+    register: result
+  
+  - name: does caching data container exist?
+    debug:
+      msg: "The caching container {{ 'exists' if result.exists else 'does not exist' }}"
+  
+  - name: did Indexing finish?
+    debug: 
+      msg: "The caching container status is {{ result.container['State']['Status'] }}" 
+    when: result.exists
+
+  
diff --git a/ansible/check_indexing_service.yml b/ansible/check_indexing_service.yml
new file mode 100644
index 00000000..b14beb87
--- /dev/null
+++ b/ansible/check_indexing_service.yml
@@ -0,0 +1,22 @@
+#Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
+- name: Check  indexing
+  connection: local
+  hosts: local 
+  vars_files:
+    searchengine_vars.yml
+  tasks:
+  - name: Get infos on container
+    docker_container_info:
+      name: "{{ searchengine_index }}"
+    register: result
+  
+  - name: does Indexing exisit?
+    debug:
+      msg: "The indexing container {{ 'exists' if result.exists else 'does not exist' }}"
+      
+  - name: did Indexing finish?
+    debug: 
+      msg: "The indexing container status is {{ result.container['State']['Status'] }}"  
+    when: result.exists
+
+  
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
new file mode 100644
index 00000000..ac44d7b2
--- /dev/null
+++ b/ansible/management-searchengine.yml
@@ -0,0 +1,237 @@
+# Search Engine + Search Engine Client + Elasticsearch
+
+#- hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+
+
+- name: Deploying search engine
+  connection: local
+  hosts: local
+  vars_files:
+    searchengine_vars.yml
+
+  pre_tasks:
+    - raw: sudo yum update -y
+  tasks:
+  - name: Create app top level directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine"
+      recurse: yes
+      state: directory
+      owner: root
+      group: root
+
+  - name: Create searchengine folder directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/searchengine"
+      recurse: yes
+      state: directory
+      owner: root
+      group: root
+      
+  - name: Create searchengine logs directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/searchengine/logs"
+      state: directory
+
+  - name: Create searchengine cached directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/searchengine/cacheddata"
+      state: directory
+
+  - name: Create client  directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/client"
+      recurse: yes
+      state: directory
+      owner: root
+      group: root
+
+  - name: Create client logs directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/client/logs"
+      state: directory
+
+  - name: Create elasticsearch directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/elasticsearch"
+      state: directory
+      # User id in elasticsearch Docker image
+      owner: 1000
+      group: root
+      
+  - name: Create elasticsearch logs directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/elasticsearch/logs"
+      state: directory
+      # User id in elasticsearch Docker image
+      owner: 1000
+      group: root
+
+  - name: Create elasticsearch data directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/elasticsearch{{ apps_folder }}"
+      state: directory
+      # User id in elasticsearch Docker image
+      owner: 1000
+      group: root
+
+  - name: Create docker network
+    become: yes
+    docker_network:
+      name: searchengine-net
+      ipam_config:
+        - subnet=10.11.0.0/16
+      #networks_cli_compatible: yes
+      #state: present
+
+  #- name: Pull search engine docker image
+  #  docker_image:
+  #      name: "{{ search_engineicsearch_docker_image }}"
+  #      tag: latest
+  #      source: pull
+
+  - name: Run docker elasticsearch
+    become: yes
+    docker_container:
+      image: "{{ search_engineelasticsearch_docker_image }}"
+      name: searchengineelasticsearch
+      cleanup: True
+      env:
+         discovery.type: single-node
+         path.data: /var/lib/elasticsearch
+         path.logs: /var/log/elasticsearch
+         network.host:  0.0.0.0
+         cluster.name: docker-cluster
+         http.host: 0.0.0.0
+         #http.port: 9200
+         discovery.type: single-node
+         ES_JAVA_OPTS: "-Xmx4096m"
+      networks:
+      - name: searchengine-net
+      published_ports:
+      - "9201:9200"
+      - "9301:9300"
+      state: started
+      restart_policy: always
+      volumes:
+      - "{{ apps_folder }}/searchengine/elasticsearch{{ apps_folder }}:/var/lib/elasticsearch"
+      - "{{ apps_folder }}/searchengine/elasticsearch/logs:/var/log/elasticsearch"
+      - 
+  - name: configure elasticsearch  for docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine
+      cleanup: True
+      command: "set_elasticsearch_configuration -e searchengineelasticsearch"
+      #networks:
+      #- name: searchengine-net
+      #published_ports:
+      #- "5577:5577"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+ 
+  - name: configure database for docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine
+      cleanup: True
+      command: "set_database_configuration -u {{ database_server_url }}  -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}"
+      #networks:
+      #- name: searchengine-net
+      #published_ports:
+      #- "5577:5577"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+ 
+  - name: configure cache folder  for docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine
+      cleanup: True
+      command: "set_cache_folder -c /etc/searchengine/cachedata"
+      #networks:
+      #- name: searchengine-net
+      #published_ports:
+      #- "5577:5577"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+      
+  - name: create elasticsearch all indcies for docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine
+      cleanup: True
+      command: create_index
+      networks:
+      - name: searchengine-net
+      #published_ports:
+      #- "5577:5577"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+  - name: configure search engine url for search client
+    become: yes
+    docker_container:
+      image: "{{ searchengineclient_docker_image }}"
+      name: searchengineclient
+      cleanup: True
+      command: "set_searchengine_url -u http://searchengine:5577/"# http://127.0.0.1:5556/"
+      #networks:
+      #- name: searchengine-net
+      #published_ports:
+      #- "5567:5567"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"    
+      
+  - name: Run docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image}}"
+      name: searchengine
+      cleanup: True
+      networks:
+      - name: searchengine-net
+      published_ports:
+      - "5577:5577"
+      #restart: "{{ searchengine_conf_status | changed }}"
+      state: started
+      restart_policy: always
+      volumes:
+        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+        - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
+
+
+  - name: Run docker searchengineclient
+    become: yes
+    docker_container:
+      image: "{{ searchengineclient_docker_image }}"
+      name: searchengineclient
+      cleanup: True
+      networks:
+      - name: searchengine-net
+      published_ports:
+      - "5567:5567"
+      #restart: "{{ searchengineclient_conf_status | changed }}"
+      state: started
+      restart_policy: always
+      volumes:
+        - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
diff --git a/ansible/run_searchengine_index_cache_services.yml b/ansible/run_searchengine_index_cache_services.yml
new file mode 100644
index 00000000..c5a93d3e
--- /dev/null
+++ b/ansible/run_searchengine_index_cache_services.yml
@@ -0,0 +1,40 @@
+#Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
+- name: Deploying search engine cache and indexing
+  connection: local
+  hosts: local 
+  vars_files:
+    searchengine_vars.yml
+  tasks:
+   - name: Set cache data from postgres database using docker searchengine
+     become: yes
+     docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine_cache
+      cleanup: True
+      auto_remove: yes
+      command: "set_resource_cached_name_value"
+      networks:
+        - name: searchengine-net
+          ipv4_address: 10.11.0.10
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"      
+      - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
+
+   - name: Get data from postgres database and insert  them to Elasticsearch index using docker searchengine
+       become: yes
+       docker_container:
+         image: "{{ searchengine_docker_image }}"
+         name: searchengine_index
+         cleanup: True
+         auto_remove: yes
+         command: "get_index_data_from_database"
+         networks:
+           - name: searchengine-net
+             ipv4_address: 10.11.0.11
+         published_ports:
+           - "5571:5577"
+         state: started
+         volumes:
+           - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"      
+      
\ No newline at end of file
diff --git a/ansible/searchengine_vars.yml b/ansible/searchengine_vars.yml
new file mode 100644
index 00000000..7ffb50e3
--- /dev/null
+++ b/ansible/searchengine_vars.yml
@@ -0,0 +1,13 @@
+apps_folder: host_data_folder
+database_server_url: "{{ omero_db_host_ansible }}"
+database_port: 5432
+database_name: idr
+database_username: omeroreadonly
+database_user_password:  "{{ idr_secret_postgresql_password_ro | default('omero') }}"
+searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
+search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
+searchengine_docker_image: khaledk2/searchengine:0.1
+searchengineclient_docker_image: khaledk2/searchengineclient:0.1
+ansible_python_interpreter: path/to/bin/python
+searchengine_cache: searchengine_cache
+searchengine_index: searchengine_index
\ No newline at end of file
diff --git a/docs/searchengine_deployemnt.md b/docs/searchengine_deployemnt.md
new file mode 100644
index 00000000..d6ff2799
--- /dev/null
+++ b/docs/searchengine_deployemnt.md
@@ -0,0 +1,22 @@
+Searchengine installation and configuration using Ansible:
+==========================================================
+
+There is an ansible playbook (management-searchengine.yml) that has been written to deploy the apps:
+* It will config and run searchengine, Elasticsearch and searchengine client
+* It will configure and create the required folders
+* It will configure the three apps and run them
+* There is a variables file (searchengine_vars.yml) that the user needs to edit before running the playbook
+    * The variable names are self-explained
+* To check that the apps have been installed and run, the user can use wget or curl to call:
+  * for searchengine, http://127.0.0.1:5556/api/v2/resources/
+  * for searchengine client, http://127.0.0.1:5556
+  * for Elasticsearch, http://127.0.0.1:9201  
+* After deploying the apps using the playbook, it is needed to run another two playbooks:
+    * Caching playbook (run_searchengine_cache_service.yml)
+    * indexing playbook (run_searchengine_index_service.yml)
+    * If the Postgresql database server is located at the same machine which hosts the searchengine, it is needed to:
+        * Edit pg_hba.conf file (one of the postgresql configuration files) and add two client ips (i.e. 10.11.0.10 and 10.11.0.11)
+        * Reload the configuration; so the PostgreSQL accepts the connection from indexing and caching services.
+    * As the caching and indexing processes take a long time, there are another two playbooks that enable the user to check if they have finished or not:
+        * check_indexing_service.yml
+        * check_caching_service.yml
\ No newline at end of file

From f8531150d1931870bb4f33b652defc5fe6e6d3ac Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Tue, 25 Jan 2022 19:36:33 +0000
Subject: [PATCH 02/22] update

---
 ansible/management-searchengine.yml | 34 +++++++++++++++++++++--------
 docs/searchengine_deployemnt.md     |  5 ++---
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index ac44d7b2..37d005fd 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -1,7 +1,18 @@
-# Search Engine + Search Engine Client + Elasticsearch
+# Search ngine + Search Engine Client + Elasticsearch
 
 #- hosts: "{{ idr_environment | default('idr') }}-management-hosts"
 
+ 
+#hosts: localhost
+#connection: local
+
+#- hosts: all
+#  remote_user: root
+#  vars_files:
+#    - /vars/external_vars.yml
+#- hosts: all
+#  vars_files:
+#    file: searchengine_vars.yml
 
 - name: Deploying search engine
   connection: local
@@ -104,7 +115,7 @@
     docker_container:
       image: "{{ search_engineelasticsearch_docker_image }}"
       name: searchengineelasticsearch
-      cleanup: True
+      cleanup: True      
       env:
          discovery.type: single-node
          path.data: /var/lib/elasticsearch
@@ -130,8 +141,9 @@
     become: yes
     docker_container:
       image: "{{ searchengine_docker_image }}"
-      name: searchengine
+      name: searchengine_elasticsearch
       cleanup: True
+      auto_remove: yes
       command: "set_elasticsearch_configuration -e searchengineelasticsearch"
       #networks:
       #- name: searchengine-net
@@ -146,8 +158,9 @@
     become: yes
     docker_container:
       image: "{{ searchengine_docker_image }}"
-      name: searchengine
+      name: searchengine_database
       cleanup: True
+      auto_remove: yes
       command: "set_database_configuration -u {{ database_server_url }}  -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}"
       #networks:
       #- name: searchengine-net
@@ -161,8 +174,9 @@
     become: yes
     docker_container:
       image: "{{ searchengine_docker_image }}"
-      name: searchengine
+      name: searchengine_cache
       cleanup: True
+      auto_remove: yes
       command: "set_cache_folder -c /etc/searchengine/cachedata"
       #networks:
       #- name: searchengine-net
@@ -176,8 +190,9 @@
     become: yes
     docker_container:
       image: "{{ searchengine_docker_image }}"
-      name: searchengine
+      name: searchengine_create_index
       cleanup: True
+      auto_remove: yes
       command: create_index
       networks:
       - name: searchengine-net
@@ -191,8 +206,9 @@
     become: yes
     docker_container:
       image: "{{ searchengineclient_docker_image }}"
-      name: searchengineclient
+      name: searchengineclient_search_uri
       cleanup: True
+      auto_remove: yes
       command: "set_searchengine_url -u http://searchengine:5577/"# http://127.0.0.1:5556/"
       #networks:
       #- name: searchengine-net
@@ -207,7 +223,7 @@
     docker_container:
       image: "{{ searchengine_docker_image}}"
       name: searchengine
-      cleanup: True
+      cleanup: True      
       networks:
       - name: searchengine-net
       published_ports:
@@ -225,7 +241,7 @@
     docker_container:
       image: "{{ searchengineclient_docker_image }}"
       name: searchengineclient
-      cleanup: True
+      cleanup: True       
       networks:
       - name: searchengine-net
       published_ports:
diff --git a/docs/searchengine_deployemnt.md b/docs/searchengine_deployemnt.md
index d6ff2799..0ded71d8 100644
--- a/docs/searchengine_deployemnt.md
+++ b/docs/searchengine_deployemnt.md
@@ -11,9 +11,8 @@ There is an ansible playbook (management-searchengine.yml) that has been written
   * for searchengine, http://127.0.0.1:5556/api/v2/resources/
   * for searchengine client, http://127.0.0.1:5556
   * for Elasticsearch, http://127.0.0.1:9201  
-* After deploying the apps using the playbook, it is needed to run another two playbooks:
-    * Caching playbook (run_searchengine_cache_service.yml)
-    * indexing playbook (run_searchengine_index_service.yml)
+* After deploying the apps using the playbook, it is needed to run another playbook for caching and indexing:
+    * run_searchengine_index_cache_services.yml    
     * If the Postgresql database server is located at the same machine which hosts the searchengine, it is needed to:
         * Edit pg_hba.conf file (one of the postgresql configuration files) and add two client ips (i.e. 10.11.0.10 and 10.11.0.11)
         * Reload the configuration; so the PostgreSQL accepts the connection from indexing and caching services.

From 7c58b65b5c5cd88daf27fce57e2b7a3149355ad3 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Tue, 25 Jan 2022 20:10:55 +0000
Subject: [PATCH 03/22] Fix syntex issue

---
 .../run_searchengine_index_cache_services.yml | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/ansible/run_searchengine_index_cache_services.yml b/ansible/run_searchengine_index_cache_services.yml
index c5a93d3e..d6b837eb 100644
--- a/ansible/run_searchengine_index_cache_services.yml
+++ b/ansible/run_searchengine_index_cache_services.yml
@@ -1,7 +1,7 @@
-#Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
+#Issue setup ip address inside the hpa config file for postgres to accept the connection from it
 - name: Deploying search engine cache and indexing
   connection: local
-  hosts: local 
+  hosts: local
   vars_files:
     searchengine_vars.yml
   tasks:
@@ -18,23 +18,23 @@
           ipv4_address: 10.11.0.10
       state: started
       volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"      
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
       - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
 
    - name: Get data from postgres database and insert  them to Elasticsearch index using docker searchengine
-       become: yes
-       docker_container:
-         image: "{{ searchengine_docker_image }}"
-         name: searchengine_index
-         cleanup: True
-         auto_remove: yes
-         command: "get_index_data_from_database"
-         networks:
-           - name: searchengine-net
-             ipv4_address: 10.11.0.11
-         published_ports:
-           - "5571:5577"
-         state: started
-         volumes:
-           - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"      
+     become: yes
+     docker_container:
+       image: "{{ searchengine_docker_image }}"
+       name: searchengine_index
+       cleanup: True
+       auto_remove: yes
+       command: "get_index_data_from_database"
+       networks:
+         - name: searchengine-net
+           ipv4_address: 10.11.0.11
+       published_ports:
+         - "5571:5577"
+       state: started
+       volumes:
+         - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
       
\ No newline at end of file

From 35fd7d5822c5561f209c701a197ae312788e8400 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 27 Jan 2022 12:00:51 +0000
Subject: [PATCH 04/22] Update searchengine playbook

---
 .../{ => group_vars}/searchengine_vars.yml    |  0
 ansible/management-searchengine.yml           | 19 +------------------
 2 files changed, 1 insertion(+), 18 deletions(-)
 rename ansible/{ => group_vars}/searchengine_vars.yml (100%)

diff --git a/ansible/searchengine_vars.yml b/ansible/group_vars/searchengine_vars.yml
similarity index 100%
rename from ansible/searchengine_vars.yml
rename to ansible/group_vars/searchengine_vars.yml
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 37d005fd..346b4acc 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -1,27 +1,10 @@
 # Search ngine + Search Engine Client + Elasticsearch
 
-#- hosts: "{{ idr_environment | default('idr') }}-management-hosts"
-
- 
-#hosts: localhost
-#connection: local
-
-#- hosts: all
-#  remote_user: root
-#  vars_files:
-#    - /vars/external_vars.yml
-#- hosts: all
-#  vars_files:
-#    file: searchengine_vars.yml
-
 - name: Deploying search engine
-  connection: local
-  hosts: local
+  hosts:  "{{ idr_environment | default('idr') }}-management-hosts"
   vars_files:
     searchengine_vars.yml
 
-  pre_tasks:
-    - raw: sudo yum update -y
   tasks:
   - name: Create app top level directory
     become: yes

From 5486209a25d83d83ad5c3d81dcc1bcac2833bd45 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 27 Jan 2022 12:09:32 +0000
Subject: [PATCH 05/22] Update the searchengine playbooks

---
 ansible/check_caching_service.yml                 | 6 ++----
 ansible/check_indexing_service.yml                | 6 ++----
 ansible/management-searchengine.yml               | 4 +---
 ansible/run_searchengine_index_cache_services.yml | 6 ++----
 4 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/ansible/check_caching_service.yml b/ansible/check_caching_service.yml
index c6990d59..59ef2e29 100644
--- a/ansible/check_caching_service.yml
+++ b/ansible/check_caching_service.yml
@@ -1,9 +1,7 @@
 #Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
 - name: Check caching
-  connection: local
-  hosts: local 
-  vars_files:
-    searchengine_vars.yml
+  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+
   tasks:  
   - name: Get infos on container
     docker_container_info:
diff --git a/ansible/check_indexing_service.yml b/ansible/check_indexing_service.yml
index b14beb87..5a3f5e3d 100644
--- a/ansible/check_indexing_service.yml
+++ b/ansible/check_indexing_service.yml
@@ -1,9 +1,7 @@
 #Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
 - name: Check  indexing
-  connection: local
-  hosts: local 
-  vars_files:
-    searchengine_vars.yml
+  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+
   tasks:
   - name: Get infos on container
     docker_container_info:
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 346b4acc..76067574 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -1,9 +1,7 @@
 # Search ngine + Search Engine Client + Elasticsearch
 
 - name: Deploying search engine
-  hosts:  "{{ idr_environment | default('idr') }}-management-hosts"
-  vars_files:
-    searchengine_vars.yml
+  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
 
   tasks:
   - name: Create app top level directory
diff --git a/ansible/run_searchengine_index_cache_services.yml b/ansible/run_searchengine_index_cache_services.yml
index d6b837eb..409c7546 100644
--- a/ansible/run_searchengine_index_cache_services.yml
+++ b/ansible/run_searchengine_index_cache_services.yml
@@ -1,9 +1,7 @@
 #Issue setup ip address inside the hpa config file for postgres to accept the connection from it
 - name: Deploying search engine cache and indexing
-  connection: local
-  hosts: local
-  vars_files:
-    searchengine_vars.yml
+  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+
   tasks:
    - name: Set cache data from postgres database using docker searchengine
      become: yes

From 96b4ee4a7690bfe0e827900e82bd154e9b2e1da7 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 27 Jan 2022 12:17:06 +0000
Subject: [PATCH 06/22] set /data as main searchengine main folder

---
 ansible/group_vars/searchengine_vars.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/group_vars/searchengine_vars.yml b/ansible/group_vars/searchengine_vars.yml
index 7ffb50e3..66374dae 100644
--- a/ansible/group_vars/searchengine_vars.yml
+++ b/ansible/group_vars/searchengine_vars.yml
@@ -1,4 +1,4 @@
-apps_folder: host_data_folder
+apps_folder: /data
 database_server_url: "{{ omero_db_host_ansible }}"
 database_port: 5432
 database_name: idr

From b2505870166d649262df6ae0682a7a20053ed9b0 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 27 Jan 2022 16:37:28 +0000
Subject: [PATCH 07/22] comment auto_remove out

---
 ansible/management-searchengine.yml               | 10 +++++-----
 ansible/run_searchengine_index_cache_services.yml |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 76067574..0c19d7f8 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -124,7 +124,7 @@
       image: "{{ searchengine_docker_image }}"
       name: searchengine_elasticsearch
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: "set_elasticsearch_configuration -e searchengineelasticsearch"
       #networks:
       #- name: searchengine-net
@@ -141,7 +141,7 @@
       image: "{{ searchengine_docker_image }}"
       name: searchengine_database
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: "set_database_configuration -u {{ database_server_url }}  -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}"
       #networks:
       #- name: searchengine-net
@@ -157,7 +157,7 @@
       image: "{{ searchengine_docker_image }}"
       name: searchengine_cache
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: "set_cache_folder -c /etc/searchengine/cachedata"
       #networks:
       #- name: searchengine-net
@@ -173,7 +173,7 @@
       image: "{{ searchengine_docker_image }}"
       name: searchengine_create_index
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: create_index
       networks:
       - name: searchengine-net
@@ -189,7 +189,7 @@
       image: "{{ searchengineclient_docker_image }}"
       name: searchengineclient_search_uri
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: "set_searchengine_url -u http://searchengine:5577/"# http://127.0.0.1:5556/"
       #networks:
       #- name: searchengine-net
diff --git a/ansible/run_searchengine_index_cache_services.yml b/ansible/run_searchengine_index_cache_services.yml
index 409c7546..00bad09f 100644
--- a/ansible/run_searchengine_index_cache_services.yml
+++ b/ansible/run_searchengine_index_cache_services.yml
@@ -9,7 +9,7 @@
       image: "{{ searchengine_docker_image }}"
       name: searchengine_cache
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: "set_resource_cached_name_value"
       networks:
         - name: searchengine-net
@@ -25,7 +25,7 @@
        image: "{{ searchengine_docker_image }}"
        name: searchengine_index
        cleanup: True
-       auto_remove: yes
+       #auto_remove: yes
        command: "get_index_data_from_database"
        networks:
          - name: searchengine-net

From 48c099c624cba308a166b4e8dd68f766a7057732 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Fri, 28 Jan 2022 18:43:11 +0000
Subject: [PATCH 08/22] add set_fact to get the database host

---
 ...rchengine_vars.yml => management-searchengine-hosts.yml} | 4 ++--
 ansible/management-searchengine.yml                         | 6 ++++++
 docs/searchengine_deployemnt.md                             | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)
 rename ansible/group_vars/{searchengine_vars.yml => management-searchengine-hosts.yml} (84%)

diff --git a/ansible/group_vars/searchengine_vars.yml b/ansible/group_vars/management-searchengine-hosts.yml
similarity index 84%
rename from ansible/group_vars/searchengine_vars.yml
rename to ansible/group_vars/management-searchengine-hosts.yml
index 66374dae..5b447a59 100644
--- a/ansible/group_vars/searchengine_vars.yml
+++ b/ansible/group_vars/management-searchengine-hosts.yml
@@ -1,5 +1,5 @@
 apps_folder: /data
-database_server_url: "{{ omero_db_host_ansible }}"
+#database_server_url: "{{ omero_db_host_ansible }}"
 database_port: 5432
 database_name: idr
 database_username: omeroreadonly
@@ -8,6 +8,6 @@ searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
 search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
 searchengine_docker_image: khaledk2/searchengine:0.1
 searchengineclient_docker_image: khaledk2/searchengineclient:0.1
-ansible_python_interpreter: path/to/bin/python
+#ansible_python_interpreter: path/to/bin/python
 searchengine_cache: searchengine_cache
 searchengine_index: searchengine_index
\ No newline at end of file
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 0c19d7f8..0b5c1c05 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -4,6 +4,12 @@
   hosts: "{{ idr_environment | default('idr') }}-management-hosts"
 
   tasks:
+  - name: Get database host
+      set_fact:
+        database_server_url: >-
+          {{ hostvars[groups[idr_environment | default('idr') +
+          '-database-hosts'][0]][
+          'ansible_' + (idr_net_iface | default('eth0'))]['ipv4']['address'] }}
   - name: Create app top level directory
     become: yes
     file:
diff --git a/docs/searchengine_deployemnt.md b/docs/searchengine_deployemnt.md
index 0ded71d8..8b5c792a 100644
--- a/docs/searchengine_deployemnt.md
+++ b/docs/searchengine_deployemnt.md
@@ -5,7 +5,7 @@ There is an ansible playbook (management-searchengine.yml) that has been written
 * It will config and run searchengine, Elasticsearch and searchengine client
 * It will configure and create the required folders
 * It will configure the three apps and run them
-* There is a variables file (searchengine_vars.yml) that the user needs to edit before running the playbook
+* There is a variables file (groups_vars/management-searchengine-hosts.yml) that the user needs to edit before running the playbook
     * The variable names are self-explained
 * To check that the apps have been installed and run, the user can use wget or curl to call:
   * for searchengine, http://127.0.0.1:5556/api/v2/resources/

From a4b7855248cc3d33ec46e268cb339afd8205f647 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 10 Feb 2022 17:43:14 +0000
Subject: [PATCH 09/22] modify searchengine deployment playbook to allow set no
 of rows for caching and set secret key for search engine and client

---
 .../management-searchengine-hosts.yml         |  6 ++-
 ansible/management-searchengine.yml           | 40 ++++++++++++++++++-
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/ansible/group_vars/management-searchengine-hosts.yml b/ansible/group_vars/management-searchengine-hosts.yml
index 5b447a59..270aa2ed 100644
--- a/ansible/group_vars/management-searchengine-hosts.yml
+++ b/ansible/group_vars/management-searchengine-hosts.yml
@@ -10,4 +10,8 @@ searchengine_docker_image: khaledk2/searchengine:0.1
 searchengineclient_docker_image: khaledk2/searchengineclient:0.1
 #ansible_python_interpreter: path/to/bin/python
 searchengine_cache: searchengine_cache
-searchengine_index: searchengine_index
\ No newline at end of file
+searchengine_index: searchengine_index
+cache_rows: 10000
+# I think that the following two variables should be in secret
+searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
+searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
\ No newline at end of file
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 0b5c1c05..449ea45e 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -172,7 +172,31 @@
       state: started
       volumes:
       - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-      
+
+  - name: configure number of cache rows for docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine_cache
+      cleanup: True
+      #auto_remove: yes
+      command: "set_cache_rows_number -n {{ cache_rows }}"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+  - name: configure secret key for docker searchengine
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: searchengine_cache
+      cleanup: True
+      #auto_remove: yes
+      command: "set_searchengine_secret_key -s {{ searchengine_secret_key }}"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
   - name: create elasticsearch all indcies for docker searchengine
     become: yes
     docker_container:
@@ -203,7 +227,19 @@
       #- "5567:5567"
       state: started
       volumes:
-      - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"    
+      - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
+
+  - name: configure secret key for search client
+    become: yes
+    docker_container:
+      image: "{{ searchengineclient_docker_image }}"
+      name: searchengineclient_search_uri
+      cleanup: True
+      #auto_remove: yes
+      command: "set_client_secret_key -s {{ searchengineclient_secret_key }}"
+      state: started
+      volumes:
+      - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
       
   - name: Run docker searchengine
     become: yes

From 9a7e40f4c51aed7db3c2df090d0e53ffd8d37ee1 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Fri, 18 Feb 2022 17:48:20 +0000
Subject: [PATCH 10/22] change searchengine searchengieclient  names

---
 ansible/group_vars/management-searchengine-hosts.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/group_vars/management-searchengine-hosts.yml b/ansible/group_vars/management-searchengine-hosts.yml
index 270aa2ed..68384b14 100644
--- a/ansible/group_vars/management-searchengine-hosts.yml
+++ b/ansible/group_vars/management-searchengine-hosts.yml
@@ -6,8 +6,8 @@ database_username: omeroreadonly
 database_user_password:  "{{ idr_secret_postgresql_password_ro | default('omero') }}"
 searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
 search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
-searchengine_docker_image: khaledk2/searchengine:0.1
-searchengineclient_docker_image: khaledk2/searchengineclient:0.1
+searchengine_docker_image: openmicroscopy/omero-searchengine:0.1
+searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:0.1
 #ansible_python_interpreter: path/to/bin/python
 searchengine_cache: searchengine_cache
 searchengine_index: searchengine_index

From 622358154660e17243bfc533b7ddb3afbedb3de6 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Fri, 18 Feb 2022 20:52:16 +0000
Subject: [PATCH 11/22] change the image tag to be latest

---
 ansible/group_vars/management-searchengine-hosts.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/group_vars/management-searchengine-hosts.yml b/ansible/group_vars/management-searchengine-hosts.yml
index 68384b14..65548ebe 100644
--- a/ansible/group_vars/management-searchengine-hosts.yml
+++ b/ansible/group_vars/management-searchengine-hosts.yml
@@ -6,8 +6,8 @@ database_username: omeroreadonly
 database_user_password:  "{{ idr_secret_postgresql_password_ro | default('omero') }}"
 searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
 search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
-searchengine_docker_image: openmicroscopy/omero-searchengine:0.1
-searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:0.1
+searchengine_docker_image: openmicroscopy/omero-searchengine:latest
+searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:latest
 #ansible_python_interpreter: path/to/bin/python
 searchengine_cache: searchengine_cache
 searchengine_index: searchengine_index

From d745f5c052cc21938418bf94d9f573921cd68bf0 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Tue, 22 Feb 2022 21:12:41 +0000
Subject: [PATCH 12/22] create and configure app data folder for the client

---
 ansible/management-searchengine.yml | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 449ea45e..f5bce683 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -49,6 +49,15 @@
       owner: root
       group: root
 
+  - name: Create client data directory
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/client/app_data"
+      recurse: yes
+      state: directory
+      owner: root
+      group: root
+
   - name: Create client logs directory
     become: yes
     file:
@@ -240,7 +249,20 @@
       state: started
       volumes:
       - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
-      
+
+  - name: configure app data folder for search client
+    become: yes
+    docker_container:
+      image: "{{ searchengineclient_docker_image }}"
+      name: searchengineclient_search_uri
+      cleanup: True
+      auto_remove: yes
+      command: "set_app_data_folder -a /etc/searchengineclient/app_data"
+      state: started
+      volumes:
+        - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
+
+
   - name: Run docker searchengine
     become: yes
     docker_container:
@@ -274,3 +296,4 @@
       restart_policy: always
       volumes:
         - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
+        - "{{ apps_folder }}/searchengine/client/app_data:/etc/searchengineclient/app_data"

From c44b87f782631965eb856de6c1ff76409d4fa790 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 24 Feb 2022 11:18:49 +0000
Subject: [PATCH 13/22] requested changes

---
 ...agement-searchengine-hosts.yml => management-hosts.yml} | 0
 ansible/management-searchengine.yml                        | 7 ++-----
 docs/searchengine_deployemnt.md                            | 2 +-
 3 files changed, 3 insertions(+), 6 deletions(-)
 rename ansible/group_vars/{management-searchengine-hosts.yml => management-hosts.yml} (100%)

diff --git a/ansible/group_vars/management-searchengine-hosts.yml b/ansible/group_vars/management-hosts.yml
similarity index 100%
rename from ansible/group_vars/management-searchengine-hosts.yml
rename to ansible/group_vars/management-hosts.yml
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index f5bce683..c246143d 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -5,11 +5,8 @@
 
   tasks:
   - name: Get database host
-      set_fact:
-        database_server_url: >-
-          {{ hostvars[groups[idr_environment | default('idr') +
-          '-database-hosts'][0]][
-          'ansible_' + (idr_net_iface | default('eth0'))]['ipv4']['address'] }}
+    set_fact:
+      database_server_url: "{{ hostvars[groups[idr_environment | default('idr') + '-database-hosts'][0]]['ansible_' + (idr_net_iface | default('eth0'))]['ipv4']['address']}}"
   - name: Create app top level directory
     become: yes
     file:
diff --git a/docs/searchengine_deployemnt.md b/docs/searchengine_deployemnt.md
index 8b5c792a..597de83d 100644
--- a/docs/searchengine_deployemnt.md
+++ b/docs/searchengine_deployemnt.md
@@ -5,7 +5,7 @@ There is an ansible playbook (management-searchengine.yml) that has been written
 * It will config and run searchengine, Elasticsearch and searchengine client
 * It will configure and create the required folders
 * It will configure the three apps and run them
-* There is a variables file (groups_vars/management-searchengine-hosts.yml) that the user needs to edit before running the playbook
+* There is a variables file (groups_vars/management-hosts.yml) that the user needs to edit before running the playbook
     * The variable names are self-explained
 * To check that the apps have been installed and run, the user can use wget or curl to call:
   * for searchengine, http://127.0.0.1:5556/api/v2/resources/

From 6a5fc7a965509504ac35c35c04e44ba4a9c2f971 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 24 Feb 2022 11:45:51 +0000
Subject: [PATCH 14/22] Suggested change

---
 ansible/check_caching_service.yml   | 1 +
 ansible/check_indexing_service.yml  | 1 +
 ansible/management-searchengine.yml | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/ansible/check_caching_service.yml b/ansible/check_caching_service.yml
index 59ef2e29..b278b7e1 100644
--- a/ansible/check_caching_service.yml
+++ b/ansible/check_caching_service.yml
@@ -4,6 +4,7 @@
 
   tasks:  
   - name: Get infos on container
+    become: yes
     docker_container_info:
       name: "{{ searchengine_cache }}"
     register: result
diff --git a/ansible/check_indexing_service.yml b/ansible/check_indexing_service.yml
index 5a3f5e3d..ec1e5f21 100644
--- a/ansible/check_indexing_service.yml
+++ b/ansible/check_indexing_service.yml
@@ -4,6 +4,7 @@
 
   tasks:
   - name: Get infos on container
+    become: yes
     docker_container_info:
       name: "{{ searchengine_index }}"
     register: result
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index c246143d..69f3f482 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -1,5 +1,7 @@
 # Search ngine + Search Engine Client + Elasticsearch
 
+- hosts: "{{ idr_environment | default('idr') }}-database-hosts"
+
 - name: Deploying search engine
   hosts: "{{ idr_environment | default('idr') }}-management-hosts"
 

From a2dd386b9670e63db617069b739a8e3f4b00da7c Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 24 Feb 2022 12:11:42 +0000
Subject: [PATCH 15/22] comment auto_remove

---
 ansible/management-searchengine.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 69f3f482..140a23d1 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -255,7 +255,7 @@
       image: "{{ searchengineclient_docker_image }}"
       name: searchengineclient_search_uri
       cleanup: True
-      auto_remove: yes
+      #auto_remove: yes
       command: "set_app_data_folder -a /etc/searchengineclient/app_data"
       state: started
       volumes:

From 3761f3c373d1c6631766b48545ada97ea5e853dc Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Tue, 19 Apr 2022 22:56:36 +0100
Subject: [PATCH 16/22] Changes to run on the searchengine-hosts group, remove
 hdf5 caching service as all the cached data now is saved in  Elasticsearch.

---
 ansible/check_caching_service.yml             | 21 -------------------
 ansible/check_indexing_service.yml            |  2 +-
 ansible/group_vars/management-hosts.yml       |  1 -
 ansible/management-searchengine.yml           |  2 +-
 ...yml => run_searchengine_index_service.yml} | 17 +--------------
 docs/searchengine_deployemnt.md               |  5 ++---
 6 files changed, 5 insertions(+), 43 deletions(-)
 delete mode 100644 ansible/check_caching_service.yml
 rename ansible/{run_searchengine_index_cache_services.yml => run_searchengine_index_service.yml} (53%)

diff --git a/ansible/check_caching_service.yml b/ansible/check_caching_service.yml
deleted file mode 100644
index b278b7e1..00000000
--- a/ansible/check_caching_service.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
-- name: Check caching
-  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
-
-  tasks:  
-  - name: Get infos on container
-    become: yes
-    docker_container_info:
-      name: "{{ searchengine_cache }}"
-    register: result
-  
-  - name: does caching data container exist?
-    debug:
-      msg: "The caching container {{ 'exists' if result.exists else 'does not exist' }}"
-  
-  - name: did Indexing finish?
-    debug: 
-      msg: "The caching container status is {{ result.container['State']['Status'] }}" 
-    when: result.exists
-
-  
diff --git a/ansible/check_indexing_service.yml b/ansible/check_indexing_service.yml
index ec1e5f21..6b1b259c 100644
--- a/ansible/check_indexing_service.yml
+++ b/ansible/check_indexing_service.yml
@@ -1,6 +1,6 @@
 #Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
 - name: Check  indexing
-  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+  hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
 
   tasks:
   - name: Get infos on container
diff --git a/ansible/group_vars/management-hosts.yml b/ansible/group_vars/management-hosts.yml
index 65548ebe..e27a58fa 100644
--- a/ansible/group_vars/management-hosts.yml
+++ b/ansible/group_vars/management-hosts.yml
@@ -9,7 +9,6 @@ search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elastic
 searchengine_docker_image: openmicroscopy/omero-searchengine:latest
 searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:latest
 #ansible_python_interpreter: path/to/bin/python
-searchengine_cache: searchengine_cache
 searchengine_index: searchengine_index
 cache_rows: 10000
 # I think that the following two variables should be in secret
diff --git a/ansible/management-searchengine.yml b/ansible/management-searchengine.yml
index 140a23d1..fa019aa3 100644
--- a/ansible/management-searchengine.yml
+++ b/ansible/management-searchengine.yml
@@ -3,7 +3,7 @@
 - hosts: "{{ idr_environment | default('idr') }}-database-hosts"
 
 - name: Deploying search engine
-  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+  hosts: "{{ idr_parent_environment | default('idr') }}-searchengine-hosts"
 
   tasks:
   - name: Get database host
diff --git a/ansible/run_searchengine_index_cache_services.yml b/ansible/run_searchengine_index_service.yml
similarity index 53%
rename from ansible/run_searchengine_index_cache_services.yml
rename to ansible/run_searchengine_index_service.yml
index 00bad09f..d4c849a9 100644
--- a/ansible/run_searchengine_index_cache_services.yml
+++ b/ansible/run_searchengine_index_service.yml
@@ -1,23 +1,8 @@
 #Issue setup ip address inside the hpa config file for postgres to accept the connection from it
 - name: Deploying search engine cache and indexing
-  hosts: "{{ idr_environment | default('idr') }}-management-hosts"
+  hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
 
   tasks:
-   - name: Set cache data from postgres database using docker searchengine
-     become: yes
-     docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_cache
-      cleanup: True
-      #auto_remove: yes
-      command: "set_resource_cached_name_value"
-      networks:
-        - name: searchengine-net
-          ipv4_address: 10.11.0.10
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-      - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
 
    - name: Get data from postgres database and insert  them to Elasticsearch index using docker searchengine
      become: yes
diff --git a/docs/searchengine_deployemnt.md b/docs/searchengine_deployemnt.md
index 597de83d..a4750bc5 100644
--- a/docs/searchengine_deployemnt.md
+++ b/docs/searchengine_deployemnt.md
@@ -16,6 +16,5 @@ There is an ansible playbook (management-searchengine.yml) that has been written
     * If the Postgresql database server is located at the same machine which hosts the searchengine, it is needed to:
         * Edit pg_hba.conf file (one of the postgresql configuration files) and add two client ips (i.e. 10.11.0.10 and 10.11.0.11)
         * Reload the configuration; so the PostgreSQL accepts the connection from indexing and caching services.
-    * As the caching and indexing processes take a long time, there are another two playbooks that enable the user to check if they have finished or not:
-        * check_indexing_service.yml
-        * check_caching_service.yml
\ No newline at end of file
+    * As the indexing processe takes a long time, there is a playbooks that enable the user to check if it have finished or not:
+        * check_indexing_service.yml        
\ No newline at end of file

From 4781f0f0715b91a984226c94a79bc03ee817c252 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Wed, 20 Apr 2022 17:46:18 +0100
Subject: [PATCH 17/22] rename files, and incrase cache_rows

---
 ansible/group_vars/management-hosts.yml                         | 2 +-
 .../{dockermanager-hosts.yml => searchengine-hosts.yml}         | 0
 ansible/{management-searchengine.yml => idr-searchengine.yml}   | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename ansible/group_vars/{dockermanager-hosts.yml => searchengine-hosts.yml} (100%)
 rename ansible/{management-searchengine.yml => idr-searchengine.yml} (100%)

diff --git a/ansible/group_vars/management-hosts.yml b/ansible/group_vars/management-hosts.yml
index e27a58fa..27ca8298 100644
--- a/ansible/group_vars/management-hosts.yml
+++ b/ansible/group_vars/management-hosts.yml
@@ -10,7 +10,7 @@ searchengine_docker_image: openmicroscopy/omero-searchengine:latest
 searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:latest
 #ansible_python_interpreter: path/to/bin/python
 searchengine_index: searchengine_index
-cache_rows: 10000
+cache_rows: 50000
 # I think that the following two variables should be in secret
 searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
 searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
\ No newline at end of file
diff --git a/ansible/group_vars/dockermanager-hosts.yml b/ansible/group_vars/searchengine-hosts.yml
similarity index 100%
rename from ansible/group_vars/dockermanager-hosts.yml
rename to ansible/group_vars/searchengine-hosts.yml
diff --git a/ansible/management-searchengine.yml b/ansible/idr-searchengine.yml
similarity index 100%
rename from ansible/management-searchengine.yml
rename to ansible/idr-searchengine.yml

From fbc7e09d6170ad6892d27620ed8cb9b090578b76 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 21 Apr 2022 16:16:16 +0100
Subject: [PATCH 18/22] Revert "rename files, and incrase cache_rows"

This reverts commit 4781f0f0715b91a984226c94a79bc03ee817c252.
---
 .../{searchengine-hosts.yml => dockermanager-hosts.yml}         | 0
 ansible/group_vars/management-hosts.yml                         | 2 +-
 ansible/{idr-searchengine.yml => management-searchengine.yml}   | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename ansible/group_vars/{searchengine-hosts.yml => dockermanager-hosts.yml} (100%)
 rename ansible/{idr-searchengine.yml => management-searchengine.yml} (100%)

diff --git a/ansible/group_vars/searchengine-hosts.yml b/ansible/group_vars/dockermanager-hosts.yml
similarity index 100%
rename from ansible/group_vars/searchengine-hosts.yml
rename to ansible/group_vars/dockermanager-hosts.yml
diff --git a/ansible/group_vars/management-hosts.yml b/ansible/group_vars/management-hosts.yml
index 27ca8298..e27a58fa 100644
--- a/ansible/group_vars/management-hosts.yml
+++ b/ansible/group_vars/management-hosts.yml
@@ -10,7 +10,7 @@ searchengine_docker_image: openmicroscopy/omero-searchengine:latest
 searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:latest
 #ansible_python_interpreter: path/to/bin/python
 searchengine_index: searchengine_index
-cache_rows: 50000
+cache_rows: 10000
 # I think that the following two variables should be in secret
 searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
 searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
\ No newline at end of file
diff --git a/ansible/idr-searchengine.yml b/ansible/management-searchengine.yml
similarity index 100%
rename from ansible/idr-searchengine.yml
rename to ansible/management-searchengine.yml

From 23c76cb396ca2d658a55273ee6982155ca169a92 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 21 Apr 2022 16:33:59 +0100
Subject: [PATCH 19/22] rename files

---
 .../group_vars/{management-hosts.yml => searchengine-hosts.yml} | 2 +-
 ansible/{management-searchengine.yml => idr-searchengine.yml}   | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename ansible/group_vars/{management-hosts.yml => searchengine-hosts.yml} (97%)
 rename ansible/{management-searchengine.yml => idr-searchengine.yml} (100%)

diff --git a/ansible/group_vars/management-hosts.yml b/ansible/group_vars/searchengine-hosts.yml
similarity index 97%
rename from ansible/group_vars/management-hosts.yml
rename to ansible/group_vars/searchengine-hosts.yml
index e27a58fa..27ca8298 100644
--- a/ansible/group_vars/management-hosts.yml
+++ b/ansible/group_vars/searchengine-hosts.yml
@@ -10,7 +10,7 @@ searchengine_docker_image: openmicroscopy/omero-searchengine:latest
 searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:latest
 #ansible_python_interpreter: path/to/bin/python
 searchengine_index: searchengine_index
-cache_rows: 10000
+cache_rows: 50000
 # I think that the following two variables should be in secret
 searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
 searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
\ No newline at end of file
diff --git a/ansible/management-searchengine.yml b/ansible/idr-searchengine.yml
similarity index 100%
rename from ansible/management-searchengine.yml
rename to ansible/idr-searchengine.yml

From 2249cf2574a49f5d795728e5f93de171fa8da28c Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Tue, 26 Apr 2022 19:23:20 +0100
Subject: [PATCH 20/22] change  hosts

---
 ansible/idr-searchengine.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/idr-searchengine.yml b/ansible/idr-searchengine.yml
index fa019aa3..5fe110e8 100644
--- a/ansible/idr-searchengine.yml
+++ b/ansible/idr-searchengine.yml
@@ -3,7 +3,7 @@
 - hosts: "{{ idr_environment | default('idr') }}-database-hosts"
 
 - name: Deploying search engine
-  hosts: "{{ idr_parent_environment | default('idr') }}-searchengine-hosts"
+  hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
 
   tasks:
   - name: Get database host

From f71306f89abbe1f5f4aeb0d8a4b332c738823669 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Wed, 27 Apr 2022 15:23:31 +0100
Subject: [PATCH 21/22] increase cache_rows to 100000

---
 ansible/group_vars/searchengine-hosts.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/group_vars/searchengine-hosts.yml b/ansible/group_vars/searchengine-hosts.yml
index 27ca8298..0b7f64ec 100644
--- a/ansible/group_vars/searchengine-hosts.yml
+++ b/ansible/group_vars/searchengine-hosts.yml
@@ -10,7 +10,7 @@ searchengine_docker_image: openmicroscopy/omero-searchengine:latest
 searchengineclient_docker_image: openmicroscopy/omero-searchengineclient:latest
 #ansible_python_interpreter: path/to/bin/python
 searchengine_index: searchengine_index
-cache_rows: 50000
+cache_rows: 100000
 # I think that the following two variables should be in secret
 searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
 searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
\ No newline at end of file

From 8614035daf44ebbe95574b10f2f87291515676ee Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Fri, 29 Apr 2022 00:40:19 +0100
Subject: [PATCH 22/22] adding searchengineurlprefix varaible to set up
 script_name

---
 ansible/group_vars/searchengine-hosts.yml | 3 ++-
 ansible/idr-searchengine.yml              | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ansible/group_vars/searchengine-hosts.yml b/ansible/group_vars/searchengine-hosts.yml
index 0b7f64ec..6b3a976b 100644
--- a/ansible/group_vars/searchengine-hosts.yml
+++ b/ansible/group_vars/searchengine-hosts.yml
@@ -13,4 +13,5 @@ searchengine_index: searchengine_index
 cache_rows: 100000
 # I think that the following two variables should be in secret
 searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
-searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
\ No newline at end of file
+searchengineclient_secret_key: "gfdgfdggregb3tyttnmnymytmasfd"
+searchengineurlprefix: "searchengineapi"
\ No newline at end of file
diff --git a/ansible/idr-searchengine.yml b/ansible/idr-searchengine.yml
index 5fe110e8..22fbe1d6 100644
--- a/ansible/idr-searchengine.yml
+++ b/ansible/idr-searchengine.yml
@@ -267,7 +267,8 @@
     docker_container:
       image: "{{ searchengine_docker_image}}"
       name: searchengine
-      cleanup: True      
+      cleanup: True
+      command: "run_app {{ searchengineurlprefix }}"
       networks:
       - name: searchengine-net
       published_ports: