[Not for merging] test90

ansible/grafana-dashboards/idr-per-server.json

@@ -78,7 +78,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "(1 - node_filesystem_free{fstype!~\"(nfs|nfs4|overlay|rootfs|rpc_pipefs|tmpfs)\", instance=\"$hostname\"} / node_filesystem_size{fstype!~\"(nfs|nfs4|overlay|rootfs|rpc_pipefs|tmpfs)\", instance=\"$hostname\"}) * 100",
+              "expr": "(1 - node_filesystem_free_bytes{fstype!~\"(nfs|nfs4|overlay|rootfs|rpc_pipefs|tmpfs)\", instance=\"$hostname\"} / node_filesystem_size_bytes{fstype!~\"(nfs|nfs4|overlay|rootfs|rpc_pipefs|tmpfs)\", instance=\"$hostname\"}) * 100",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "{{mountpoint}} ({{device}})",

ansible/grafana-dashboards/idr-sessions.json

@@ -309,7 +309,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(django_http_responses_total_by_status[$quantileint])) without (instance)",
+              "expr": "sum(rate(django_http_responses_total_by_status_total[$quantileint])) without (instance)",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "{{status}}",

ansible/grafana-dashboards/idr-vertical.json

@@ -80,7 +80,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "(100 - (avg by (instance) (irate(node_cpu{mode=\"idle\", instance=~\".*$servergroup.*\"}[5m])) * 100)) * on(instance)  (count(node_cpu{mode=\"idle\", instance=~\".*$servergroup.*\"}) without (cpu, mode))",
+              "expr": "(100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\", instance=~\".*$servergroup.*\"}[5m])) * 100)) * on(instance)  (count(node_cpu_seconds_total{mode=\"idle\", instance=~\".*$servergroup.*\"}) without (cpu, mode))",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "{{instance}}",
@@ -173,7 +173,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "node_memory_Active{instance=~\".*$servergroup.*\"} / 1024 / 1024 / 2014",
+              "expr": "node_memory_Active_bytes{instance=~\".*$servergroup.*\"} / 1024 / 1024 / 2014",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 2,
@@ -182,7 +182,7 @@
               "step": 600
             },
             {
-              "expr": "node_memory_Active{instance=~\".*$servergroup.*\"} / node_memory_MemTotal{instance=~\".*$servergroup.*\"} * 100",
+              "expr": "node_memory_Active_bytes{instance=~\".*$servergroup.*\"} / node_memory_MemTotal_bytes{instance=~\".*$servergroup.*\"} * 100",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 2,
@@ -355,7 +355,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum (irate(node_network_receive_bytes{instance=~\".*$servergroup.*\"}[1m])) by (instance) / 1024 / 1024",
+              "expr": "sum (irate(node_network_receive_bytes_total{instance=~\".*$servergroup.*\"}[1m])) by (instance) / 1024 / 1024",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 2,
@@ -365,7 +365,7 @@
               "step": 600
             },
             {
-              "expr": "- sum (irate(node_network_transmit_bytes{instance=~\".*$servergroup.*\"}[1m])) by (instance) / 1024 / 1024",
+              "expr": "- sum (irate(node_network_transmit_bytes_total{instance=~\".*$servergroup.*\"}[1m])) by (instance) / 1024 / 1024",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 2,
@@ -428,12 +428,10 @@
       {
         "allValue": null,
         "current": {
-          "tags": [],
-          "text": "omero + database + docker",
+          "text": "omero + database",
           "value": [
             "omero",
-            "database",
-            "docker"
+            "database"
           ]
         },
         "hide": 0,
@@ -451,14 +449,9 @@
             "selected": true,
             "text": "database",
             "value": "database"
-          },
-          {
-            "selected": true,
-            "text": "docker",
-            "value": "docker"
           }
         ],
-        "query": "omero, database, docker",
+        "query": "omero, database",
         "type": "custom"
       }
     ]

ansible/group_vars/omero-hosts.yml

@@ -25,7 +25,7 @@ idr_bf_components:
   - formats-api
   - formats-bsd
   - formats-gpl
-idr_bf_release: "0.6.4"
+idr_bf_release: "0.6.5"
 idr_bf_baseurl: "https://artifacts.openmicroscopy.org/artifactory/maven/idr"
 
 ice_install_devel: false
@@ -37,6 +37,11 @@ ice_install_python: false
 postgresql_version: "11"
 
 
+######################################################################
+# redis (for OMERO.web and microservices)
+redis_listen: 0.0.0.0
+
+
 ######################################################################
 # openmicroscopy.fluentd vars
 fluentd_source_configs:

ansible/group_vars/proxy-hosts.yml

@@ -30,6 +30,22 @@ nginx_proxy_log_format: main_timed_cache_upstream
 nginx_proxy_websockets_enable: True
 
 nginx_proxy_upstream_servers:
+- name: omeromsthumbnailreadonly
+  balance: ip_hash
+  servers: "{{ omeroreadonly_omeromsthumbnail_hosts | sort }}"
+- name: omeromsthumbnailreadwrite
+  servers: "{{ omeroreadwrite_omeromsthumbnail_hosts }}"
+- name: omeromsimageregionreadonly
+  balance: ip_hash
+  servers: "{{ omeroreadonly_omeromsimageregion_hosts | sort }}"
+- name: omeromsimageregionreadwrite
+  servers: "{{ omeroreadwrite_omeromsimageregion_hosts }}"
+- name: omeromszarrreadonly
+  balance: ip_hash
+  servers: "{{ omeroreadonly_omeromszarr_hosts | sort }}"
+- name: omeromszarrreadwrite
+  servers: "{{ omeroreadwrite_omeromszarr_hosts }}"
+
 - name: omeroreadonly
   balance: ip_hash
   servers: "{{ omero_omeroreadonly_hosts_reserved | sort }}"
@@ -56,7 +72,48 @@ _nginx_proxy_omero_locations:
 # Gallery is hosted at /
 - /gallery-api/*
 
+_nginx_proxy_omeromsthumbnail_locations:
+- /webgateway/render_thumbnail/*
+- /webclient/render_thumbnail/*
+- /webgateway/render_birds_eye_view/*
+- /webclient/render_birds_eye_view/*
+- /webgateway/get_thumbnails/*
+- /webclient/get_thumbnails/*
+
+_nginx_proxy_omeromsimageregion_locations:
+- /webgateway/render_image_region/*
+- /webgateway/render_image/*
+- /webclient/render_image_region/*
+- /webclient/render_image/*
+- /webgateway/render_shape_mask/*
+
+_nginx_proxy_omeromszarr_locations:
+- /idr/zarr/*
+
+
+# Order is important since we want the longer omeroms*cached
+# locations to have precedence over the shorter omerocached ones
+# https://www.digitalocean.com/community/tutorials/understanding-nginx-server-and-location-block-selection-algorithms
 _nginx_proxy_backends_omero:
+- name: omeromsthumbnailscached
+  location: ~ {{ _nginx_proxy_omeromsthumbnail_locations | join('|') }}
+  server: http://omeromsthumbnailreadonly
+  cache_validity: 1d
+  maintenance_flag: "{{ omero_maintenance_flag }}"
+  maintenance_uri: "{{ omero_maintenance_uri }}"
+- name: omeromsimagregioncached
+  location: ~ {{ _nginx_proxy_omeromsimageregion_locations | join('|') }}
+  server: http://omeromsimageregionreadonly
+  cache_validity: 1d
+  maintenance_flag: "{{ omero_maintenance_flag }}"
+  maintenance_uri: "{{ omero_maintenance_uri }}"
+- name: omeromszarr
+  location: ~ {{ _nginx_proxy_omeromszarr_locations | join('|') }}
+  server: http://omeromszarrreadonly
+  # Don't cache, files are too large
+  # cache_validity: 1d
+  maintenance_flag: "{{ omero_maintenance_flag }}"
+  maintenance_uri: "{{ omero_maintenance_uri }}"
 - name: omerocached
   location: ~ {{ _nginx_proxy_omero_locations | join('|') }}
   server: http://omeroreadonly
@@ -292,9 +349,11 @@ _nginx_proxy_sites:
   # This enables the default site (configured using the global
   # nginx_proxy_* variables):
   - nginx_proxy_is_default: True
-    # Only add this CORS header to the default public site
+
     nginx_proxy_additional_directives:
-    - "add_header Access-Control-Allow-Origin $allow_origin"
+    # Only add this CORS header to the default public site
+    # Always set a header including on 404 responses
+    - "add_header Access-Control-Allow-Origin $allow_origin always"
     # Study redirects
     - "if ($request_uri ~ /search/\\?query=Name:(?<prefix>idr00)?68) {
        return 302 /about/download.html;}"
@@ -310,6 +369,19 @@ _nginx_proxy_sites:
     nginx_proxy_ssl: False
     nginx_proxy_cachebuster_enabled: True
     nginx_proxy_backends:
+    - name: omeromsthumbnailscached
+      location: ~ {{ _nginx_proxy_omeromsthumbnail_locations | join('|') }}
+      server: http://omeromsthumbnailreadwrite
+      cache_validity: 1d
+    - name: omeromsimageregioncached
+      location: ~ {{ _nginx_proxy_omeromsimageregion_locations | join('|') }}
+      server: http://omeromsimageregionreadwrite
+      cache_validity: 1d
+    - name: omeromszarr
+      location: ~ {{ _nginx_proxy_omeromszarr_locations | join('|') }}
+      server: http://omeromszarrreadwrite
+      # Don't cache, files are too large
+      # cache_validity: 1d
     - name: omerocached
       location: ~ {{ _nginx_proxy_omero_locations | join('|') }}|/mapr/*
       server: http://omeroreadwrite

ansible/idr-02-services.yml

@@ -3,7 +3,7 @@
 ### TODO: Remove these in the next version
 - include: idr-downloads.yml
 
-- include: idr-mineotaur.yml
+- include: idr-microservices.yml
 
 - include: idr-kubernetes.yml
 # TODO: Not idempotent

ansible/idr-docker.yml

@@ -16,24 +16,3 @@
     yum:
       name: docker-python
       state: present
-
-
-- hosts: "{{ idr_environment | default('idr') }}-dockermanager-hosts"
-
-  pre_tasks:
-  - name: create NFS export directories
-    become: yes
-    file:
-      path: /data/{{ item }}
-      state: directory
-    with_items:
-    - mineotaur
-    - volumes
-
-  roles:
-  - role: ome.versioncontrol_utils
-  - role: ome.nfs_share
-    nfs_shares:
-      /data/mineotaur:
-      - host: "*"
-        options: 'rw'

ansible/idr-downloads.yml

@@ -19,9 +19,6 @@
     - name: sql
       path: /srv/omero-sql
       comment: PostgreSQL 9.4 database dump of the IDR
-    - name: mineotaur
-      path: /data/mineotaur
-      comment: Neo4j databases for various IDR studies
 
   tasks:
 

ansible/idr-firewall.yml

@@ -12,7 +12,7 @@
       become: yes
       iptables_raw_25:
         name: default_accept
-        keep_unmanaged: no
+        keep_unmanaged: yes
         rules: |
           -A INPUT -j ACCEPT
           -A FORWARD -j ACCEPT

ansible/idr-ftp-monitoring.yml

@@ -4,9 +4,9 @@
 - hosts: "{{ idr_environment | default('idr') }}-ftp-hosts"
 
   roles:
-  - role: ome.prometheus_node_0_3_0
+  - role: ome.prometheus_node
 
-  - role: ome.prometheus_0_4_0
+  - role: ome.prometheus
     prometheus_docker_network: monitoring
 
     prometheus_alertmanager_slack_webhook: "{{ idr_secret_management_slack_webhook }}"