From d1d20a218ff2de578c3c620be380af0402922294 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 5 May 2020 15:36:35 +0200 Subject: [PATCH 01/76] wip --- docker-compose.yml | 21 +++++++++++++++++++++ server/workers/services/src/models.py | 6 ++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d5da493d7..706ca77dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,25 @@ version: '3.7' services: + pgsql: + image: 'postgres:12.2-alpine' + restart: always + environment: + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + ports: + - '127.0.0.1:54322:5432' + volumes: + - db-data:/var/lib/postgresql/data + + pgadmin: + image: 'dpage/pgadmin4' + ports: + - '127.0.0.1:54323:80' + env_file: + - .env + volumes: + - /home/chris/data/OKMaps:/pgadmin4/data + api: build: context: server @@ -65,3 +84,5 @@ services: volumes: redis: + db-data: + driver: local diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index d7f9e8dd0..2f3a1b727 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -3,13 +3,11 @@ class Revisions(db.Model): rev_id = db.Column(db.Integer, - db.ForeignKey('visualizations.vis_latest'), nullable=False, primary_key=True) rev_vis = db.Column(db.Text, nullable=False, primary_key=True) - vis_query = db.Column(db.Text, - db.ForeignKey('visualizations.vis_clean_query')) + vis_query = db.Column(db.Text) rev_user = db.Column(db.Text) rev_timestamp = db.Column(db.DateTime) rev_comment = db.Column(db.Text) @@ -17,7 +15,7 @@ class Revisions(db.Model): class Visualizations(db.Model): - vis_id = db.Column(db.Text, nullable=False, unique=True, + vis_id = db.Column(db.Text, nullable=False, unique=False, primary_key=True) vis_query = db.Column(db.Text) vis_clean_query = db.Column(db.Text) From abc95c73b96c35d14b1cda50845f318ced261b55 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 29 Sep 2020 19:04:13 +0200 Subject: [PATCH 02/76] persistence wip --- .../workers/services/src/apis/persistence.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 server/workers/services/src/apis/persistence.py diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py new file mode 100644 index 000000000..882afe7ae --- /dev/null +++ b/server/workers/services/src/apis/persistence.py @@ -0,0 +1,60 @@ +from datetime import datetime +from models import Revisions, Visualizations + +from flask import Blueprint, request, make_response, jsonify, abort +from flask_restx import Namespace, Resource, fields + +persistence_ns = Namespace("persistence", description="OKMAps persistence operations") + + +def write_revision(vis_id, data, rev_id=None): + + vis = Visualizations.query.filter_by(vis_id=vis_id).first() + + rev = rev_id + if rev is None: + r_id = vis.vis_latest + rev = r_id + 1 + + query = vis.vis_clean_query + + new_rev = { + "rev_id": rev, + "rev_vis": vis_id, + "rev_user": "System", + "rev_timestamp": datetime.utcnow(), + "rev_comment": "Visualization created", + "rev_data": data, + "vis_query": query + } + Revisions.create(**new_rev) + + +def create_visualization(vis_id, vis_title, data, + vis_clean_query=None, vis_query=None, + params=None): + pass + + +def exists_visualization(vis_id): + map = Visualizations.query.filter_by(vis_id=vis_id).first() + exists = True if map else False + return exists + + +@persistence_ns.route('/createVisualization') +class createVisualization(Resource): + # (vis_id, vis_title, data) + pass + + +@persistence_ns.route('/getRevision') +class getRevision(Resource): + # (vis_id, rev_id) + pass + + +@persistence_ns.route('/writeRevision') +class writeRevision(Resource): + # (vis_id, data) + pass From aa3e488eee757f0cf0099eb3c43747fbbe950114 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 29 Sep 2020 23:22:53 +0200 Subject: [PATCH 03/76] new map id route for persistence API --- server/services/search.php | 16 +++++++++- .../workers/services/src/apis/persistence.py | 30 +++++++++++++++++-- server/workers/tests/test_persistence.py | 13 ++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 server/workers/tests/test_persistence.py diff --git a/server/services/search.php b/server/services/search.php index f213321ad..ce3dd89e1 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -69,7 +69,21 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $params_for_id_creation = ($params_for_id === null)?($params_json):(packParamsJSON($params_for_id, $post_params)); - $unique_id = ($precomputed_id === null)?($persistence->createID(array($query, $params_for_id_creation))):($precomputed_id); + if ($backend === "api") { + $url = $ini_array["general"]["api_url"] . $repository . "/createID"; + $payload = json_encode(array("params" => $post_params, + "param_types" => $param_types)); + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $output_json = curl_exec($ch); + $unique_id = $output_json["unique_id"]; + } else { + $unique_id = ($precomputed_id === null)?($persistence->createID(array($query, $params_for_id_creation))):($precomputed_id); + } if($retrieve_cached_map) { $last_version = $persistence->getLastVersion($unique_id, false); diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 882afe7ae..05bc57d06 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -1,9 +1,13 @@ +from hashlib import md5 from datetime import datetime -from models import Revisions, Visualizations - +import json +from collections import OrderedDict from flask import Blueprint, request, make_response, jsonify, abort from flask_restx import Namespace, Resource, fields +from models import Revisions, Visualizations + + persistence_ns = Namespace("persistence", description="OKMAps persistence operations") @@ -58,3 +62,25 @@ class getRevision(Resource): class writeRevision(Resource): # (vis_id, data) pass + + +@persistence_ns.route('/createID') +class createID(Resource): + + @persistence_ns.produces(["application/json"]) + def post(self): + payload = request.get_json() + params = payload.get("params") + param_types = payload.get("param_types") + # create map id + ordered_params = OrderedDict() + for k in param_types: + ordered_params[k] = params[k] + string_to_hash = json.dumps(ordered_params, separators=(',', ':')) + string_to_hash = " ".join([params["q"], string_to_hash]) + mapid = md5(string_to_hash.encode('utf-8')).hexdigest() + # create response + headers = {} + result = {"unique_id": mapid} + headers["Content-Type"] = "application/json" + return make_response(result, 200, headers) diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py new file mode 100644 index 000000000..36876397d --- /dev/null +++ b/server/workers/tests/test_persistence.py @@ -0,0 +1,13 @@ +import json +import pytest +import pandas as pd +import numpy as np + + +@pytest.mark.persistence +def test_add_map_to_database(): + pass + + +def test_add_revision_to_database(): + pass From 2e8283c9a3fabd037bc9445c3113f09304d69ecb Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 29 Sep 2020 23:34:57 +0200 Subject: [PATCH 04/76] map id route test --- server/workers/services/src/app.py | 2 + server/workers/tests/test_persistence.py | 416 ++++++++++++++++++++++- 2 files changed, 417 insertions(+), 1 deletion(-) diff --git a/server/workers/services/src/app.py b/server/workers/services/src/app.py index 61ba1ae6e..b49d3da1c 100644 --- a/server/workers/services/src/app.py +++ b/server/workers/services/src/app.py @@ -10,6 +10,7 @@ from apis.base import base_ns from apis.pubmed import pubmed_ns from apis.openaire import openaire_ns +from apis.persistence import persistence_ns from database import db from config import settings @@ -50,6 +51,7 @@ def api_patches(app, settings): api.add_namespace(base_ns, path='/base') api.add_namespace(pubmed_ns, path='/pubmed') api.add_namespace(openaire_ns, path='/openaire') +api.add_namespace(persistence_ns, path='/persistence') app.logger.debug(app.config) app.logger.debug(app.url_map) diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index 36876397d..953b34a09 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -1,7 +1,420 @@ import json import pytest -import pandas as pd import numpy as np +import pandas as pd +import requests + + +@pytest.mark.persistence +def test_map_id_creation(): + testcases = [ + # {"params": {"q": "air quality management", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, + # "param_types": ["from", "to", "document_types", "sorting"], + # "expected result": "2074f8b9eee26f53936abd16f6187ed8"}, + # {"params": {"q": "trump", "from": "1665-01-01", "to": "2017-10-27", "sorting": "most-relevant", "document_types": [121]}, + # "param_types": ["from", "to", "document_types", "sorting"], + # "expected result": "d9c930deef17b3f22e3030a9bd020f9f"}, + # {"params": {"q": "solar eclipse", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, + # "param_types": ["from", "to", "document_types", "sorting"], + # "expected result": "f9f0d52aaf3a91d0040c2acdacf00620"}, + # {"params": {"q": "pop music", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, + # "param_types": ["from", "to", "document_types", "sorting"], + # "expected result": "32df883cd04d85a710b6c4057f01dfd8"}, + # {"params": {"q": "fear-of-missing-out", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, + # "param_types": ["from", "to", "document_types", "sorting"], + # "expected result": "81baaafd6f0ddba4a7ce4e7237210fe7"}, + {"params": {"q": "sustainable development goals", 'from': '1809-01-01', + 'to': '2017-09-08', + 'sorting': 'most-relevant', + 'article_types': ['adaptive clinical trial', + 'address', + 'autobiography', + 'bibliography', + 'biography', + 'book illustrations', + 'case reports', + 'classical article', + 'clinical conference', + 'clinical study', + 'clinical trial', + 'clinical trial protocol', + 'clinical trial, phase i', + 'clinical trial, phase ii', + 'clinical trial, phase iii', + 'clinical trial, phase iv', + 'clinical trial, veterinary', + 'collected work', + 'collected works', + 'comment', + 'comparative study', + 'congress', + 'consensus development conference', + 'consensus development conference, nih', + 'controlled clinical trial', + 'corrected and republished article', + 'dataset', + 'dictionary', + 'directory', + 'duplicate publication', + 'editorial', + 'electronic supplementary materials', + 'english abstract', + 'ephemera', + 'equivalence trial', + 'evaluation studies', + 'evaluation study', + 'expression of concern', + 'festschrift', + 'government publication', + 'guideline', + 'historical article', + 'interactive tutorial', + 'interview', + 'introductory journal article', + 'journal article', + 'lecture', + 'legal case', + 'legislation', + 'letter', + 'manuscript', + 'meta analysis', + 'multicenter study', + 'news', + 'newspaper article', + 'observational study', + 'observational study, veterinary', + 'overall', + 'patient education handout', + 'periodical index', + 'personal narrative', + 'pictorial work', + 'popular work', + 'portrait', + 'practice guideline', + 'pragmatic clinical trial', + 'preprint', + 'publication components', + 'publication formats', + 'publication type category', + 'published erratum', + 'randomized controlled trial', + 'randomized controlled trial, veterinary', + 'research support, american recovery and reinvestment act', + 'research support, n i h, extramural', + 'research support, n i h, intramural', + "research support, non u s gov't", + "research support, u s gov't, non p h s", + "research support, u s gov't, p h s", + 'research support, u s government', + 'retraction of publication', + 'review', + 'scientific integrity review', + 'study characteristics', + 'support of research', + 'systematic review', + 'technical report', + 'twin study', + 'validation study', + 'video audio media', + 'webcasts']}, + "param_types": ["from", "to", "sorting", "article_types"], + "expected result": "60d2dd3caeade4c0eed6ed486d737fd3"}, + {"params": {"q": "athens", + 'from': '1809-01-01', + 'to': '2017-09-08', + 'sorting': 'most-relevant', + 'article_types': ['adaptive clinical trial', + 'address', + 'autobiography', + 'bibliography', + 'biography', + 'book illustrations', + 'case reports', + 'classical article', + 'clinical conference', + 'clinical study', + 'clinical trial', + 'clinical trial protocol', + 'clinical trial, phase i', + 'clinical trial, phase ii', + 'clinical trial, phase iii', + 'clinical trial, phase iv', + 'clinical trial, veterinary', + 'collected work', + 'collected works', + 'comment', + 'comparative study', + 'congress', + 'consensus development conference', + 'consensus development conference, nih', + 'controlled clinical trial', + 'corrected and republished article', + 'dataset', + 'dictionary', + 'directory', + 'duplicate publication', + 'editorial', + 'electronic supplementary materials', + 'english abstract', + 'ephemera', + 'equivalence trial', + 'evaluation studies', + 'evaluation study', + 'expression of concern', + 'festschrift', + 'government publication', + 'guideline', + 'historical article', + 'interactive tutorial', + 'interview', + 'introductory journal article', + 'journal article', + 'lecture', + 'legal case', + 'legislation', + 'letter', + 'manuscript', + 'meta analysis', + 'multicenter study', + 'news', + 'newspaper article', + 'observational study', + 'observational study, veterinary', + 'overall', + 'patient education handout', + 'periodical index', + 'personal narrative', + 'pictorial work', + 'popular work', + 'portrait', + 'practice guideline', + 'pragmatic clinical trial', + 'preprint', + 'publication components', + 'publication formats', + 'publication type category', + 'published erratum', + 'randomized controlled trial', + 'randomized controlled trial, veterinary', + 'research support, american recovery and reinvestment act', + 'research support, n i h, extramural', + 'research support, n i h, intramural', + "research support, non u s gov't", + "research support, u s gov't, non p h s", + "research support, u s gov't, p h s", + 'research support, u s government', + 'retraction of publication', + 'review', + 'scientific integrity review', + 'study characteristics', + 'support of research', + 'systematic review', + 'technical report', + 'twin study', + 'validation study', + 'video audio media', + 'webcasts']}, + "param_types": ["from", "to", "sorting", "article_types"], + "expected result": "fc3240ce14abf183f7a089ad8757f6a1"}, + {"params": {"q": "hannover", + 'from': '1809-01-01', + 'to': '2018-02-16', + 'sorting': 'most-relevant', + 'article_types': ['adaptive clinical trial', + 'address', + 'autobiography', + 'bibliography', + 'biography', + 'book illustrations', + 'case reports', + 'classical article', + 'clinical conference', + 'clinical study', + 'clinical trial', + 'clinical trial protocol', + 'clinical trial, phase i', + 'clinical trial, phase ii', + 'clinical trial, phase iii', + 'clinical trial, phase iv', + 'clinical trial, veterinary', + 'collected work', + 'collected works', + 'comment', + 'comparative study', + 'congress', + 'consensus development conference', + 'consensus development conference, nih', + 'controlled clinical trial', + 'corrected and republished article', + 'dataset', + 'dictionary', + 'directory', + 'duplicate publication', + 'editorial', + 'electronic supplementary materials', + 'english abstract', + 'ephemera', + 'equivalence trial', + 'evaluation studies', + 'evaluation study', + 'expression of concern', + 'festschrift', + 'government publication', + 'guideline', + 'historical article', + 'interactive tutorial', + 'interview', + 'introductory journal article', + 'journal article', + 'lecture', + 'legal case', + 'legislation', + 'letter', + 'manuscript', + 'meta analysis', + 'multicenter study', + 'news', + 'newspaper article', + 'observational study', + 'observational study, veterinary', + 'overall', + 'patient education handout', + 'periodical index', + 'personal narrative', + 'pictorial work', + 'popular work', + 'portrait', + 'practice guideline', + 'pragmatic clinical trial', + 'preprint', + 'publication components', + 'publication formats', + 'publication type category', + 'published erratum', + 'randomized controlled trial', + 'randomized controlled trial, veterinary', + 'research support, american recovery and reinvestment act', + 'research support, n i h, extramural', + 'research support, n i h, intramural', + "research support, non u s gov't", + "research support, u s gov't, non p h s", + "research support, u s gov't, p h s", + 'research support, u s government', + 'retraction of publication', + 'review', + 'scientific integrity review', + 'study characteristics', + 'support of research', + 'systematic review', + 'technical report', + 'twin study', + 'validation study', + 'video audio media', + 'webcasts']}, + "param_types": ["from", "to", "sorting", "article_types"], + "expected result": "3b39a6afad01a572d02122d15d3bf9bb"}, + {"params": {"q": "hangover", + 'from': '1809-01-01', + 'to': '2018-02-16', + 'sorting': 'most-relevant', + 'article_types': ['adaptive clinical trial', + 'address', + 'autobiography', + 'bibliography', + 'biography', + 'book illustrations', + 'case reports', + 'classical article', + 'clinical conference', + 'clinical study', + 'clinical trial', + 'clinical trial protocol', + 'clinical trial, phase i', + 'clinical trial, phase ii', + 'clinical trial, phase iii', + 'clinical trial, phase iv', + 'clinical trial, veterinary', + 'collected work', + 'collected works', + 'comment', + 'comparative study', + 'congress', + 'consensus development conference', + 'consensus development conference, nih', + 'controlled clinical trial', + 'corrected and republished article', + 'dataset', + 'dictionary', + 'directory', + 'duplicate publication', + 'editorial', + 'electronic supplementary materials', + 'english abstract', + 'ephemera', + 'equivalence trial', + 'evaluation studies', + 'evaluation study', + 'expression of concern', + 'festschrift', + 'government publication', + 'guideline', + 'historical article', + 'interactive tutorial', + 'interview', + 'introductory journal article', + 'journal article', + 'lecture', + 'legal case', + 'legislation', + 'letter', + 'manuscript', + 'meta analysis', + 'multicenter study', + 'news', + 'newspaper article', + 'observational study', + 'observational study, veterinary', + 'overall', + 'patient education handout', + 'periodical index', + 'personal narrative', + 'pictorial work', + 'popular work', + 'portrait', + 'practice guideline', + 'pragmatic clinical trial', + 'preprint', + 'publication components', + 'publication formats', + 'publication type category', + 'published erratum', + 'randomized controlled trial', + 'randomized controlled trial, veterinary', + 'research support, american recovery and reinvestment act', + 'research support, n i h, extramural', + 'research support, n i h, intramural', + "research support, non u s gov't", + "research support, u s gov't, non p h s", + "research support, u s gov't, p h s", + 'research support, u s government', + 'retraction of publication', + 'review', + 'scientific integrity review', + 'study characteristics', + 'support of research', + 'systematic review', + 'technical report', + 'twin study', + 'validation study', + 'video audio media', + 'webcasts']}, + "param_types": ["from", "to", "sorting", "article_types"], + "expected result": "3d7c033bf1dac0ca0895f9004d18db01"}, + ] + for tc in testcases: + res = requests.post("http://localhost/api/persistence/createID", json=tc) + result = res.json() + assert result["unique_id"] == tc["expected result"] @pytest.mark.persistence @@ -9,5 +422,6 @@ def test_add_map_to_database(): pass +@pytest.mark.persistence def test_add_revision_to_database(): pass From c56afa7d933ee000c0b9e69654af01e889a82d07 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 1 Oct 2020 15:18:59 +0200 Subject: [PATCH 05/76] wip --- .../workers/services/src/apis/persistence.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 05bc57d06..3b37dc34a 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -48,14 +48,31 @@ def exists_visualization(vis_id): @persistence_ns.route('/createVisualization') class createVisualization(Resource): - # (vis_id, vis_title, data) + + def post(self): + payload = request.get_json() + vis_id = payload.get('vis_id') + data = payload.get('data') + rev_id = payload.get('rev_id') + + + +@persistence_ns.route('/getLastVersion') +class getLastVersion(Resource): pass @persistence_ns.route('/getRevision') class getRevision(Resource): - # (vis_id, rev_id) - pass + + @persistence_ns.produces(["application/json"]) + def post(self): + + # create response + headers = {} + result = {} + headers["Content-Type"] = "application/json" + return make_response(result, 200, headers) @persistence_ns.route('/writeRevision') From 73c64d46ae110388e856ad10809827a7a7e0aa41 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 2 Oct 2020 13:00:52 +0200 Subject: [PATCH 06/76] map exists route --- server/services/search.php | 19 +++++++- .../workers/services/src/apis/persistence.py | 48 +++++++++++++++---- server/workers/tests/test_persistence.py | 13 ++++- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index 8ad71e2a2..f12b2f2a5 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -145,7 +145,24 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $vis_title = $repository; - $exists = $persistence->existsVisualization($unique_id); + if ($backend === "api") { + $url = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization"; + $payload = json_encode(array("vis_id" => $unique_id)); + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $output_json = curl_exec($ch); + $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpcode != 200) { + $output_json = NULL; + } + $exists = $output_json["exists"]; + } else { + $exists = $persistence->existsVisualization($unique_id); + } if (!$exists) { $persistence->createVisualization($unique_id, $vis_title, $input_json, $query, $dirty_query, $params_json); diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 3b37dc34a..327beb2fb 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -11,6 +11,17 @@ persistence_ns = Namespace("persistence", description="OKMAps persistence operations") +def create_map_id(params, param_types): + # create map id + ordered_params = OrderedDict() + for k in param_types: + ordered_params[k] = params[k] + string_to_hash = json.dumps(ordered_params, separators=(',', ':')) + string_to_hash = " ".join([params["q"], string_to_hash]) + mapid = md5(string_to_hash.encode('utf-8')).hexdigest() + return mapid + + def write_revision(vis_id, data, rev_id=None): vis = Visualizations.query.filter_by(vis_id=vis_id).first() @@ -46,6 +57,28 @@ def exists_visualization(vis_id): return exists +def get_last_version(vis_id, details=False, context=False): + return get_revision(vis_id, None, details=False, context=False) + + +def get_revision(vis_id, rev_id, details=False, context=False): + pass + + +@persistence_ns.route('/existsVisualization') +class existsVisualization(Resource): + + def post(self): + payload = request.get_json() + vis_id = payload.get("vis_id") + exists = exists_visualization(vis_id) + # create response + headers = {} + result = {"exists": exists} + headers["Content-Type"] = "application/json" + return make_response(result, 200, headers) + + @persistence_ns.route('/createVisualization') class createVisualization(Resource): @@ -56,9 +89,14 @@ def post(self): rev_id = payload.get('rev_id') - @persistence_ns.route('/getLastVersion') class getLastVersion(Resource): + """ + Is actually a call to getRevision but taking the latest one + + params: vis_id, details(false), context(false) + + """ pass @@ -89,13 +127,7 @@ def post(self): payload = request.get_json() params = payload.get("params") param_types = payload.get("param_types") - # create map id - ordered_params = OrderedDict() - for k in param_types: - ordered_params[k] = params[k] - string_to_hash = json.dumps(ordered_params, separators=(',', ':')) - string_to_hash = " ".join([params["q"], string_to_hash]) - mapid = md5(string_to_hash.encode('utf-8')).hexdigest() + mapid = create_map_id(params, param_types) # create response headers = {} result = {"unique_id": mapid} diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index 953b34a09..0bdb4b965 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -4,6 +4,8 @@ import pandas as pd import requests +from .test_helpers import CASENAMES, CASEDATA, RESULTS, get_dataprocessing_result + @pytest.mark.persistence def test_map_id_creation(): @@ -418,7 +420,16 @@ def test_map_id_creation(): @pytest.mark.persistence -def test_add_map_to_database(): +def test_get_last_version(): + # vis_id, details, context + pass + + +@pytest.mark.persistence +@pytest.mark.parametrize("testcase", CASENAMES) +def test_add_map_to_database(testcase): + testcase = RESULTS[testcase] + map_data = {} pass From 3bc7901dd07c6a92ee473b29ed8ca8771f4c64e9 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 2 Oct 2020 15:31:54 +0200 Subject: [PATCH 07/76] write revision route --- .../workers/services/src/apis/persistence.py | 9 +++++-- server/workers/services/src/models.py | 24 ++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 327beb2fb..d4f444407 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -115,8 +115,13 @@ def post(self): @persistence_ns.route('/writeRevision') class writeRevision(Resource): - # (vis_id, data) - pass + + @persistence_ns.produces(["application/json"]) + def post(self): + payload = request.get_json() + mapid = payload.get("mapid") + data = payload.get("data") + write_revision(vis_id, data, None) @persistence_ns.route('/createID') diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index 2f3a1b727..36ec12866 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -1,24 +1,26 @@ from database import db +from sqlalchemy.orm import relationship + + +class Visualizations(db.Model): + vis_id = db.Column(db.Text, nullable=False, unique=False, + primary_key=True) + vis_query = db.Column(db.Text) + vis_clean_query = db.Column(db.Text) + vis_title = db.Column(db.Text) + vis_latest = db.Column(db.Integer) + vis_params = db.Column(db.Text) class Revisions(db.Model): rev_id = db.Column(db.Integer, nullable=False, primary_key=True) - rev_vis = db.Column(db.Text, nullable=False, + rev_vis = db.Column(db.Text, + nullable=False, primary_key=True) vis_query = db.Column(db.Text) rev_user = db.Column(db.Text) rev_timestamp = db.Column(db.DateTime) rev_comment = db.Column(db.Text) rev_data = db.Column(db.Text) - - -class Visualizations(db.Model): - vis_id = db.Column(db.Text, nullable=False, unique=False, - primary_key=True) - vis_query = db.Column(db.Text) - vis_clean_query = db.Column(db.Text) - vis_title = db.Column(db.Text) - vis_latest = db.Column(db.Integer) - vis_params = db.Column(db.Text) From f0709a980f70016df988a61979305978420194b5 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 2 Oct 2020 15:37:34 +0200 Subject: [PATCH 08/76] create visualization route --- docker-compose.yml | 1 + server/services.docker | 8 +- server/workers/services/requirements.txt | 1 + .../workers/services/src/apis/persistence.py | 109 ++++++++++++------ server/workers/tests/test_persistence.py | 32 ++++- 5 files changed, 107 insertions(+), 44 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a24180394..cb18cbf44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: image: 'postgres:12.2-alpine' restart: always environment: + POSTGRES_USER: "${POSTGRES_USER}" POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" ports: - '127.0.0.1:54322:5432' diff --git a/server/services.docker b/server/services.docker index 1e1c0e680..de630010a 100644 --- a/server/services.docker +++ b/server/services.docker @@ -1,14 +1,14 @@ -FROM python:3.6.10-alpine3.10 +FROM python:3.6.10-slim MAINTAINER Chris Kittel "christopher.kittel@openknowledgemaps.org" -RUN apk update -RUN apk add build-base gcc +RUN apt-get update +RUN apt-get install -y --no-install-recommends gcc +RUN apt-get install -y --no-install-recommends git WORKDIR /headstart COPY workers/services/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -RUN apk add git RUN pip install git+https://github.com/python-restx/flask-restx COPY workers/services/src/ ./ COPY workers/redis_config.json . diff --git a/server/workers/services/requirements.txt b/server/workers/services/requirements.txt index 918093091..744213065 100644 --- a/server/workers/services/requirements.txt +++ b/server/workers/services/requirements.txt @@ -11,3 +11,4 @@ aioredis pandas pyyaml flasgger +psycopg2-binary diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index d4f444407..6d4430759 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -6,6 +6,7 @@ from flask_restx import Namespace, Resource, fields from models import Revisions, Visualizations +from database import db persistence_ns = Namespace("persistence", description="OKMAps persistence operations") @@ -24,31 +25,41 @@ def create_map_id(params, param_types): def write_revision(vis_id, data, rev_id=None): - vis = Visualizations.query.filter_by(vis_id=vis_id).first() + vis = db.session.query(Visualizations).filter_by(vis_id=vis_id).first() - rev = rev_id - if rev is None: - r_id = vis.vis_latest - rev = r_id + 1 + if rev_id is None: + if vis.vis_latest is None: + rev_id = 1 + else: + rev_id = vis.vis_latest + 1 query = vis.vis_clean_query - new_rev = { - "rev_id": rev, - "rev_vis": vis_id, - "rev_user": "System", - "rev_timestamp": datetime.utcnow(), - "rev_comment": "Visualization created", - "rev_data": data, - "vis_query": query - } - Revisions.create(**new_rev) + new_rev = Revisions( + rev_id=rev_id, + rev_vis=vis_id, + rev_user="System", + rev_timestamp=datetime.utcnow(), + rev_comment="Visualization created", + rev_data=data, + vis_query=query) + db.session.add(new_rev) + vis.vis_latest = rev_id + db.session.commit() def create_visualization(vis_id, vis_title, data, vis_clean_query=None, vis_query=None, - params=None): - pass + vis_params=None): + new_vis = Visualizations( + vis_id=vis_id, + vis_clean_query=vis_clean_query, + vis_query=vis_query, + vis_title=vis_title, + vis_params=vis_params) + db.session.add(new_vis) + db.session.commit() + write_revision(vis_id, data, 1) def exists_visualization(vis_id): @@ -83,10 +94,27 @@ def post(self): class createVisualization(Resource): def post(self): - payload = request.get_json() - vis_id = payload.get('vis_id') - data = payload.get('data') - rev_id = payload.get('rev_id') + try: + payload = request.get_json() + vis_id = payload.get('vis_id') + vis_title = payload.get('vis_title') + data = payload.get('data') + vis_clean_query = payload.get('vis_clean_query') + vis_query = payload.get('vis_query') + vis_params = payload.get('vis_params') + create_visualization(vis_id, vis_title, data, + vis_clean_query, vis_query, vis_params) + result = {'success': True} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), + 200, + headers) + except Exception as e: + result = {'success': False, 'reason': e} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), + 500, + headers) @persistence_ns.route('/getLastVersion') @@ -118,10 +146,18 @@ class writeRevision(Resource): @persistence_ns.produces(["application/json"]) def post(self): - payload = request.get_json() - mapid = payload.get("mapid") - data = payload.get("data") - write_revision(vis_id, data, None) + try: + payload = request.get_json() + vis_id = payload.get("vis_id") + data = payload.get("data") + write_revision(vis_id, data, None) + result = {'success': True} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), 200, headers) + except Exception as e: + result = {'success': False, 'reason': e} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), 500, headers) @persistence_ns.route('/createID') @@ -129,12 +165,17 @@ class createID(Resource): @persistence_ns.produces(["application/json"]) def post(self): - payload = request.get_json() - params = payload.get("params") - param_types = payload.get("param_types") - mapid = create_map_id(params, param_types) - # create response - headers = {} - result = {"unique_id": mapid} - headers["Content-Type"] = "application/json" - return make_response(result, 200, headers) + try: + payload = request.get_json() + params = payload.get("params") + param_types = payload.get("param_types") + mapid = create_map_id(params, param_types) + # create response + headers = {} + result = {"unique_id": mapid} + headers["Content-Type"] = "application/json" + return make_response(jsonify(result), 200, headers) + except Exception as e: + result = {'success': False, 'reason': e} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), 500, headers) diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index 0bdb4b965..5fc5c7c0a 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -4,7 +4,7 @@ import pandas as pd import requests -from .test_helpers import CASENAMES, CASEDATA, RESULTS, get_dataprocessing_result +from .test_helpers import KNOWNCASES, CASENAMES, CASEDATA, RESULTS, get_dataprocessing_result @pytest.mark.persistence @@ -426,11 +426,31 @@ def test_get_last_version(): @pytest.mark.persistence -@pytest.mark.parametrize("testcase", CASENAMES) -def test_add_map_to_database(testcase): - testcase = RESULTS[testcase] - map_data = {} - pass +@pytest.mark.parametrize("testcase", KNOWNCASES) +def test_create_visualization(testcase): + caseid = testcase["caseid"] + payload = {} + payload["vis_id"] = caseid + payload["vis_title"] = caseid + payload["data"] = RESULTS[caseid].to_json(orient='records') + payload["vis_clean_query"] = caseid + payload["vis_query"] = caseid + payload["vis_params"] = json.dumps(testcase["casedata"]["params"]) + res = requests.post("http://localhost/api/persistence/createVisualization", + json=payload) + assert res.status_code == 200, res.json().get('reason') + + +@pytest.mark.persistence +@pytest.mark.parametrize("testcase", KNOWNCASES) +def test_write_revision(testcase): + caseid = testcase["caseid"] + payload = {} + payload["vis_id"] = caseid + payload["data"] = RESULTS[caseid].to_json(orient='records') + res = requests.post("http://localhost/api/persistence/writeRevision", + json=payload) + assert res.status_code == 200, res.json().get('reason') @pytest.mark.persistence From 55568d40ee33f08eccff4dcdac57cc965459aeb0 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 2 Oct 2020 19:10:28 +0200 Subject: [PATCH 09/76] persistence error handling --- server/services/search.php | 27 +++++++++++++++++++++--- server/workers/tests/test_persistence.py | 5 ----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index f12b2f2a5..b9c028580 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -154,18 +154,39 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $output_json = curl_exec($ch); + $res = curl_exec($ch); $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($httpcode != 200) { - $output_json = NULL; + return res; } - $exists = $output_json["exists"]; + $exists = $res["exists"]; } else { $exists = $persistence->existsVisualization($unique_id); } if (!$exists) { + if ($backend === "api"){ + $url = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; + $payload = json_encode(array("vis_id" => $unique_id, + "vis_title" => $vis_title, + "data" => $input_json, + "vis_clean_query" => $query, + "vis_query" => $dirty_query, + "vis_params" => $params_json)); + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $res = curl_exec($ch); + $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpcode != 200) { + return res; + } + } else { $persistence->createVisualization($unique_id, $vis_title, $input_json, $query, $dirty_query, $params_json); + } } else { $persistence->writeRevision($unique_id, $input_json); } diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index 5fc5c7c0a..271080957 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -451,8 +451,3 @@ def test_write_revision(testcase): res = requests.post("http://localhost/api/persistence/writeRevision", json=payload) assert res.status_code == 200, res.json().get('reason') - - -@pytest.mark.persistence -def test_add_revision_to_database(): - pass From a147ade99caa8bad6e7e3fe52c3e860a0abdd925 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 2 Oct 2020 19:27:16 +0200 Subject: [PATCH 10/76] search re-routing --- server/services/search.php | 90 +++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index b9c028580..e5c6c11aa 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -51,6 +51,18 @@ function cleanQuery($dirty_query, $transform_query_tolowercase) { return $query; } +function call_api($route, $payload) { + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $route); + curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $result = curl_exec($ch); + $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + return array("result" => $result, "httpcode" => $httpcode); +} + function search($repository, $dirty_query, $post_params, $param_types, $keyword_separator, $taxonomy_separator, $transform_query_tolowercase = true , $retrieve_cached_map = true, $params_for_id = null, $num_labels = 3, $id = "area_uri", $subjects = "subject", $precomputed_id = null, $do_clean_query = true , $backend = "legacy") { @@ -70,17 +82,15 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $params_for_id_creation = ($params_for_id === null)?($params_json):(packParamsJSON($params_for_id, $post_params)); if ($backend === "api") { - $url = $ini_array["general"]["api_url"] . $repository . "/createID"; + $route = $ini_array["general"]["api_url"] . $repository . "/createID"; $payload = json_encode(array("params" => $post_params, "param_types" => $param_types)); - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $output_json = curl_exec($ch); - $unique_id = $output_json["unique_id"]; + $res = call_api($route, $payload); + if ($res["httpcode"] != 200) { + return $res; + } else { + $unique_id = $res["result"]["unique_id"]; + } } else { $unique_id = ($precomputed_id === null)?($persistence->createID(array($query, $params_for_id_creation))):($precomputed_id); } @@ -104,18 +114,13 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $WORKING_DIR = $ini_array["general"]["preprocessing_dir"] . $ini_array["output"]["output_dir"]; if ($backend === "api") { - $url = $ini_array["general"]["api_url"] . $repository . "/search"; + $route = $ini_array["general"]["api_url"] . $repository . "/search"; $payload = json_encode($post_params); - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $output_json = curl_exec($ch); - $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpcode != 200) { - $output_json = NULL; + $res = call_api($route, $payload); + if ($res["httpcode"] != 200) { + return $res; + } else { + $output_json = $res["result"]; } } else { $calculation = new \headstart\preprocessing\calculation\RCalculation($ini_array); @@ -146,49 +151,46 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $vis_title = $repository; if ($backend === "api") { - $url = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization"; $payload = json_encode(array("vis_id" => $unique_id)); - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $res = curl_exec($ch); - $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpcode != 200) { - return res; + $res = call_api($route, $payload); + if ($res["httpcode"] != 200) { + return $res; + } else { + $exists = $res["result"]["exists"]; } - $exists = $res["exists"]; } else { $exists = $persistence->existsVisualization($unique_id); } if (!$exists) { - if ($backend === "api"){ - $url = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; + if ($backend === "api") { + $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; $payload = json_encode(array("vis_id" => $unique_id, "vis_title" => $vis_title, "data" => $input_json, "vis_clean_query" => $query, "vis_query" => $dirty_query, "vis_params" => $params_json)); - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $res = curl_exec($ch); - $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpcode != 200) { - return res; + $res = call_api($route, $payload); + if ($res["httpcode"] != 200) { + return $res; } } else { $persistence->createVisualization($unique_id, $vis_title, $input_json, $query, $dirty_query, $params_json); } } else { + if ($backend === "api") { + $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; + $payload = json_encode(array("vis_id" => $unique_id, + "data" => $input_json)); + $res = call_api($route, $payload); + if ($res["httpcode"] != 200) { + return $res; + } + } else { $persistence->writeRevision($unique_id, $input_json); + } } $repo_mapping = array("plos" => "PLOS" From 107d5074a755e58f4e569cb891feb18ac583b3aa Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 2 Oct 2020 19:36:41 +0200 Subject: [PATCH 11/76] separate persistence backend flag --- server/services/search.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index e5c6c11aa..28c5b3168 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -65,7 +65,7 @@ function call_api($route, $payload) { function search($repository, $dirty_query, $post_params, $param_types, $keyword_separator, $taxonomy_separator, $transform_query_tolowercase = true , $retrieve_cached_map = true, $params_for_id = null, $num_labels = 3, $id = "area_uri", $subjects = "subject", $precomputed_id = null, $do_clean_query = true - , $backend = "legacy") { + , $backend = "legacy", $persistence_backend = "legacy") { $INI_DIR = dirname(__FILE__) . "/../preprocessing/conf/"; $ini_array = library\Toolkit::loadIni($INI_DIR); @@ -150,7 +150,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $vis_title = $repository; - if ($backend === "api") { + if ($persistence_backend === "api") { $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization"; $payload = json_encode(array("vis_id" => $unique_id)); $res = call_api($route, $payload); @@ -164,7 +164,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ } if (!$exists) { - if ($backend === "api") { + if ($persistence_backend === "api") { $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; $payload = json_encode(array("vis_id" => $unique_id, "vis_title" => $vis_title, @@ -180,7 +180,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $persistence->createVisualization($unique_id, $vis_title, $input_json, $query, $dirty_query, $params_json); } } else { - if ($backend === "api") { + if ($persistence_backend === "api") { $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; $payload = json_encode(array("vis_id" => $unique_id, "data" => $input_json)); From 1a748d1b9f2b3b2905e1f348b9755da7b2b7859f Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Sat, 3 Oct 2020 14:35:40 +0200 Subject: [PATCH 12/76] model updates --- .../workers/services/src/apis/persistence.py | 32 ++++++++++++------- server/workers/services/src/models.py | 5 +-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 6d4430759..7d94b7dec 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -51,20 +51,21 @@ def write_revision(vis_id, data, rev_id=None): def create_visualization(vis_id, vis_title, data, vis_clean_query=None, vis_query=None, vis_params=None): - new_vis = Visualizations( - vis_id=vis_id, - vis_clean_query=vis_clean_query, - vis_query=vis_query, - vis_title=vis_title, - vis_params=vis_params) - db.session.add(new_vis) - db.session.commit() - write_revision(vis_id, data, 1) + if not exists_visualization(vis_id): + new_vis = Visualizations( + vis_id=vis_id, + vis_clean_query=vis_clean_query, + vis_query=vis_query, + vis_title=vis_title, + vis_params=vis_params) + db.session.add(new_vis) + db.session.commit() + write_revision(vis_id, data, 1) def exists_visualization(vis_id): - map = Visualizations.query.filter_by(vis_id=vis_id).first() - exists = True if map else False + vis = db.session.query(Visualizations).filter_by(vis_id=vis_id).first() + exists = True if vis else False return exists @@ -73,7 +74,14 @@ def get_last_version(vis_id, details=False, context=False): def get_revision(vis_id, rev_id, details=False, context=False): - pass + result = (db.session.query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == Visualizations.vis_latest) + ).first() + res = {"id"} + return res @persistence_ns.route('/existsVisualization') diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index 36ec12866..f0fcfe7b0 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -1,9 +1,10 @@ from database import db +from sqlalchemy import ForeignKey, Table from sqlalchemy.orm import relationship class Visualizations(db.Model): - vis_id = db.Column(db.Text, nullable=False, unique=False, + vis_id = db.Column(db.Text, nullable=False, unique=True, primary_key=True) vis_query = db.Column(db.Text) vis_clean_query = db.Column(db.Text) @@ -17,7 +18,7 @@ class Revisions(db.Model): nullable=False, primary_key=True) rev_vis = db.Column(db.Text, - nullable=False, + nullable=False, unique=False, primary_key=True) vis_query = db.Column(db.Text) rev_user = db.Column(db.Text) From a08d05294f871760868f00abde408b9a78563ed0 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 6 Oct 2020 21:38:06 +0200 Subject: [PATCH 13/76] services integration --- server/services/getGSheetsMap.php | 4 ++-- server/services/getLatestRevision.php | 15 ++++++++++++--- .../workers/services/src/apis/persistence.py | 19 ++++++++++++------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index 42f035d71..7e3429fe8 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -13,11 +13,11 @@ $ini_array = library\Toolkit::loadIni($INI_DIR); $vis_id = library\CommUtils::getParameter($_GET, "vis_id"); -$backend = isset($_GET["backend"]) ? library\CommUtils::getParameter($_GET, "backend") : "legacy"; +$persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); -if ($backend == "api") { +if ($persistence_backend == "api") { $return_data = array("status" => "error", "reason" => "Not implemented."); $jsonData = json_encode($return_data); library\CommUtils::echoOrCallback($jsonData, $_GET); diff --git a/server/services/getLatestRevision.php b/server/services/getLatestRevision.php index b9c280f42..a9f7646f5 100644 --- a/server/services/getLatestRevision.php +++ b/server/services/getLatestRevision.php @@ -19,12 +19,17 @@ $streamgraph = filter_input(INPUT_GET, "streamgraph", FILTER_VALIDATE_BOOLEAN, array("flags" => FILTER_NULL_ON_FAILURE)); $backend = isset($_GET["backend"]) ? library\CommUtils::getParameter($_GET, "backend") : "legacy"; +$persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($backend == "api") { if ($context === true) { - $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; + if ($persistence_backend === "api") { + + } else { + $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; + } if ($streamgraph === true) { $packed_data = json_decode($data["rev_data"], true); $return_data = array("context" => array("id" => $data["rev_vis"], "query" => $data["vis_query"], "service" => $data["vis_title"] @@ -41,8 +46,12 @@ library\CommUtils::echoOrCallback($jsonData, $_GET); } } else { - $jsonData = $persistence->getLastVersion($vis_id); - library\CommUtils::echoOrCallback($jsonData[0], $_GET); + if ($persistence_backend === "api") { + + } else { + $jsonData = $persistence->getLastVersion($vis_id); + library\CommUtils::echoOrCallback($jsonData[0], $_GET); + } } } else { if ($context === true) { diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 7d94b7dec..43a8340ac 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -74,13 +74,18 @@ def get_last_version(vis_id, details=False, context=False): def get_revision(vis_id, rev_id, details=False, context=False): - result = (db.session.query(Visualizations, Revisions) - .select_from(Visualizations, Revisions) - .filter(Visualizations.vis_id == vis_id) - .filter(Revisions.rev_vis == vis_id) - .filter(Revisions.rev_id == Visualizations.vis_latest) - ).first() - res = {"id"} + vis, rev = (db.session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == Visualizations.vis_latest) + ).first() + res = {"rev_data": rev.rev_data} + if context is True: + res["context"] = { + "" + } return res From baf36b1a8e65060931a3524644ab64ff881e512f Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 6 Oct 2020 21:44:26 +0200 Subject: [PATCH 14/76] new commUtil function --- .../classes/headstart/library/CommUtils.php | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/server/classes/headstart/library/CommUtils.php b/server/classes/headstart/library/CommUtils.php index 60a2046cf..eea2c19be 100644 --- a/server/classes/headstart/library/CommUtils.php +++ b/server/classes/headstart/library/CommUtils.php @@ -18,25 +18,39 @@ private static function initialize() self::$initialized = true; } - + public static function echoOrCallback($data, $array) { self::initialize(); - + if(isset($array["jsoncallback"])) echo $array["jsoncallback"] . '(' . $data . ');'; - else + else echo $data; - + } - + public static function getParameter($array, $name) { self::initialize(); - + if(isset($array[$name])) { return $array[$name]; } else { throw new \Exception("The following parameter is not set: " . $name); } + + } + + public static function call_api($route, $payload) { + self::initialize(); + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $route); + curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $result = curl_exec($ch); + $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + return array("result" => $result, "httpcode" => $httpcode); } } From ea834e2d0bb18636fcf2bdf907151ba557542a6f Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 6 Oct 2020 23:32:35 +0200 Subject: [PATCH 15/76] getLatestRevision route --- server/services/getLatestRevision.php | 59 +++++++++++++++++-- server/services/search.php | 23 ++------ .../workers/services/src/apis/persistence.py | 36 +++++++++-- server/workers/services/src/models.py | 6 ++ 4 files changed, 98 insertions(+), 26 deletions(-) diff --git a/server/services/getLatestRevision.php b/server/services/getLatestRevision.php index a9f7646f5..b8b856905 100644 --- a/server/services/getLatestRevision.php +++ b/server/services/getLatestRevision.php @@ -24,13 +24,25 @@ $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($backend == "api") { + # case of streamgraph calculation in backend if ($context === true) { + # context data true start if ($persistence_backend === "api") { - + # get data + context from api + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = $res; + } } else { + # get data + context from legacy $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; } if ($streamgraph === true) { + # transform data formats if streamgraph requested $packed_data = json_decode($data["rev_data"], true); $return_data = array("context" => array("id" => $data["rev_vis"], "query" => $data["vis_query"], "service" => $data["vis_title"] , "timestamp" => $data["rev_timestamp"], "params" => $data["vis_params"]), @@ -45,21 +57,46 @@ $jsonData = json_encode($return_data); library\CommUtils::echoOrCallback($jsonData, $_GET); } + # context data true end } else { if ($persistence_backend === "api") { - + # return data without context from api + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = $res; + } } else { + # return data without context from legacy $jsonData = $persistence->getLastVersion($vis_id); library\CommUtils::echoOrCallback($jsonData[0], $_GET); } } + # end of streamgraph calculation in backend } else { + # case of streamgraph calculation in legacy way if ($context === true) { - $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; + if ($persistence_backend === "api") { + # get data + context from api + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = $res; + } + } else { + $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; + } $return_data = array("context" => array("id" => $data["rev_vis"], "query" => $data["vis_query"], "service" => $data["vis_title"] , "timestamp" => $data["rev_timestamp"], "params" => $data["vis_params"]), "data" => $data["rev_data"]); if ($streamgraph === true) { + # calculate streamgraph data live if requested $calculation = new headstart\preprocessing\calculation\RCalculation($ini_array); $working_dir = $ini_array["general"]["preprocessing_dir"] . $ini_array["output"]["output_dir"]; $sg_output = $calculation->performStreamgraphCalculation($working_dir, $return_data["context"]["service"], $return_data["data"]); @@ -75,7 +112,21 @@ $jsonData = json_encode($return_data); library\CommUtils::echoOrCallback($jsonData, $_GET); } else { - $jsonData = $persistence->getLastVersion($vis_id); + if ($persistence_backend === "api") { + # get data without context from api + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = $res; + } + } else { + # get data without context from legac + $jsonData = $persistence->getLastVersion($vis_id); + } library\CommUtils::echoOrCallback($jsonData[0], $_GET); } + # end of streamgraph calculation in legacy way } diff --git a/server/services/search.php b/server/services/search.php index 28c5b3168..c52eb3b88 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -51,17 +51,6 @@ function cleanQuery($dirty_query, $transform_query_tolowercase) { return $query; } -function call_api($route, $payload) { - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $route); - curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json')); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $result = curl_exec($ch); - $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - return array("result" => $result, "httpcode" => $httpcode); -} function search($repository, $dirty_query, $post_params, $param_types, $keyword_separator, $taxonomy_separator, $transform_query_tolowercase = true , $retrieve_cached_map = true, $params_for_id = null, $num_labels = 3, $id = "area_uri", $subjects = "subject", $precomputed_id = null, $do_clean_query = true @@ -81,11 +70,11 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $params_for_id_creation = ($params_for_id === null)?($params_json):(packParamsJSON($params_for_id, $post_params)); - if ($backend === "api") { + if ($persistence_backend === "api") { $route = $ini_array["general"]["api_url"] . $repository . "/createID"; $payload = json_encode(array("params" => $post_params, "param_types" => $param_types)); - $res = call_api($route, $payload); + $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { return $res; } else { @@ -116,7 +105,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ if ($backend === "api") { $route = $ini_array["general"]["api_url"] . $repository . "/search"; $payload = json_encode($post_params); - $res = call_api($route, $payload); + $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { return $res; } else { @@ -153,7 +142,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ if ($persistence_backend === "api") { $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization"; $payload = json_encode(array("vis_id" => $unique_id)); - $res = call_api($route, $payload); + $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { return $res; } else { @@ -172,7 +161,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ "vis_clean_query" => $query, "vis_query" => $dirty_query, "vis_params" => $params_json)); - $res = call_api($route, $payload); + $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { return $res; } @@ -184,7 +173,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; $payload = json_encode(array("vis_id" => $unique_id, "data" => $input_json)); - $res = call_api($route, $payload); + $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { return $res; } diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 43a8340ac..fc6b36615 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -81,12 +81,21 @@ def get_revision(vis_id, rev_id, details=False, context=False): .filter(Revisions.rev_vis == vis_id) .filter(Revisions.rev_id == Visualizations.vis_latest) ).first() - res = {"rev_data": rev.rev_data} if context is True: - res["context"] = { - "" + res = { + "rev_vis": rev.rev_vis, + "vis_query": rev.vis_query, + "vis_title": vis.vis_title, + "rev_timestamp": rev.rev_timestamp, + "vis_params": vis.vis_params, + "rev_data": rev.rev_data } - return res + return res + else: + if details is True: + return rev.as_dict() + else: + return rev.rev_data @persistence_ns.route('/existsVisualization') @@ -138,7 +147,24 @@ class getLastVersion(Resource): params: vis_id, details(false), context(false) """ - pass + + def post(self): + try: + payload = request.get_json() + vis_id = payload.get('vis_id') + details = payload.get('details') + context = payload.get('context') + result = get_last_version(vis_id, details, context) + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), + 200, + headers) + except Exception as e: + result = {'success': False, 'reason': e} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), + 500, + headers) @persistence_ns.route('/getRevision') diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index f0fcfe7b0..dc3d630eb 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -12,6 +12,9 @@ class Visualizations(db.Model): vis_latest = db.Column(db.Integer) vis_params = db.Column(db.Text) + def as_dict(self): + return {c.name: getattr(self, c.name) for c in self.__table__.columns} + class Revisions(db.Model): rev_id = db.Column(db.Integer, @@ -25,3 +28,6 @@ class Revisions(db.Model): rev_timestamp = db.Column(db.DateTime) rev_comment = db.Column(db.Text) rev_data = db.Column(db.Text) + + def as_dict(self): + return {c.name: getattr(self, c.name) for c in self.__table__.columns} From 2d4b8902bf30e8abae7efa67d7b16c9678e30d7d Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 6 Oct 2020 23:41:46 +0200 Subject: [PATCH 16/76] updated params in data integrations --- examples/search_repos/data-config_base.js | 7 ++++--- examples/search_repos/data-config_pubmed.js | 5 +++-- examples/triple/data-config_triple.js | 5 +++-- server/services/searchTRIPLE.php | 2 +- vis/js/headstart.js | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/examples/search_repos/data-config_base.js b/examples/search_repos/data-config_base.js index 5087be226..ea1dc9521 100644 --- a/examples/search_repos/data-config_base.js +++ b/examples/search_repos/data-config_base.js @@ -2,6 +2,7 @@ var data_config = { tag: "visualization", mode: "search_repos", backend: "legacy", + persistence_backend: "legacy", service: "base", @@ -18,7 +19,7 @@ var data_config = { show_list: true, content_based: true, url_prefix: "https://www.base-search.net/Record/", - + show_context: true, create_title_from_context: true, context_most_relevant_tooltip: true, @@ -27,10 +28,10 @@ var data_config = { filter_menu_dropdown: true, sort_menu_dropdown: true, filter_options: ["all", "open_access"], - + embed_modal: true, share_modal: false, - + show_keywords: true, highlight_query_terms: true, }; diff --git a/examples/search_repos/data-config_pubmed.js b/examples/search_repos/data-config_pubmed.js index 0fb754f34..4c944cffe 100644 --- a/examples/search_repos/data-config_pubmed.js +++ b/examples/search_repos/data-config_pubmed.js @@ -2,6 +2,7 @@ var data_config = { tag: "visualization", mode: "search_repos", backend: "legacy", + persistence_backend: "legacy", service: "pubmed", @@ -18,10 +19,10 @@ var data_config = { show_list: true, content_based: false, show_keywords: true, - + show_context: true, create_title_from_context: true, - + embed_modal: true, share_modal: true, diff --git a/examples/triple/data-config_triple.js b/examples/triple/data-config_triple.js index 8228cb75a..35de799c5 100644 --- a/examples/triple/data-config_triple.js +++ b/examples/triple/data-config_triple.js @@ -32,9 +32,10 @@ var data_config = { share_modal: false, backend: "api", - + persistence_backend: "api", + streamgraph_colors: ["#215A66", "#66214A", "#5D40FB", "#CB40FB", "#40C0FB", "#FB4068" , "#FBB240", "#40FBC8", "#fee4bc", "#bcfeec", "#c6bcfe", "#febcca"], - + highlight_query_terms: true, }; diff --git a/server/services/searchTRIPLE.php b/server/services/searchTRIPLE.php index 927a18732..b90eb3e9b 100644 --- a/server/services/searchTRIPLE.php +++ b/server/services/searchTRIPLE.php @@ -16,7 +16,7 @@ , $post_params, array("from", "to", "sorting", "vis_type", "language", "limit", "sg_method") , ";", null, true, true, null, 3 , "area_uri", "subject", $precomputed_id, false - , "api"); + , "api", "api"); echo $result diff --git a/vis/js/headstart.js b/vis/js/headstart.js index c2e339169..e1ea336a3 100644 --- a/vis/js/headstart.js +++ b/vis/js/headstart.js @@ -207,7 +207,7 @@ HeadstartFSM.prototype = { }, gsheets: function(that, setupVis) { - let url = config.server_url + "services/getGSheetsMap.php?vis_id=" + mediator.current_bubble.file + let url = config.server_url + "services/getGSheetsMap.php?vis_id=" + mediator.current_bubble.file + "&q=" +mediator.current_bubble.title + "&context=" + config.show_context + "&streamgraph=" + config.is_streamgraph; mediator.publish("get_data_from_files", url, 'json', setupVis); From 073ed9124286dfc1d45fffd0a60c0dfdd7cf1002 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 7 Oct 2020 00:13:41 +0200 Subject: [PATCH 17/76] parameter naming convention --- .../preprocessing/other-scripts/altmetrics.R | 2 +- server/preprocessing/other-scripts/base.R | 12 +++---- server/preprocessing/other-scripts/cluster.R | 12 +++---- server/preprocessing/other-scripts/openaire.R | 8 ++--- .../preprocessing/other-scripts/preprocess.R | 4 +-- server/preprocessing/other-scripts/pubmed.R | 8 ++--- server/preprocessing/other-scripts/run_base.R | 2 +- .../other-scripts/run_openaire.R | 2 +- .../preprocessing/other-scripts/run_pubmed.R | 2 +- .../other-scripts/run_vis_layout.R | 2 +- .../other-scripts/test/base-test.R | 2 +- .../other-scripts/test/params_base.json | 2 +- .../other-scripts/test/params_openaire.json | 2 +- .../other-scripts/test/params_pubmed.json | 2 +- .../other-scripts/test/pubmed-test.R | 2 +- .../other-scripts/test/test-openaire.R | 2 +- .../other-scripts/text_similarity.R | 2 +- .../preprocessing/other-scripts/vis_layout.R | 2 +- server/services/search.php | 4 +-- .../workers/services/src/apis/persistence.py | 4 +-- .../services/src/apis/request_validators.py | 1 + server/workers/tests/test_persistence.py | 2 +- vis/js/headstart.js | 34 +++++++++---------- 23 files changed, 58 insertions(+), 57 deletions(-) diff --git a/server/preprocessing/other-scripts/altmetrics.R b/server/preprocessing/other-scripts/altmetrics.R index d22132dfa..816031f93 100644 --- a/server/preprocessing/other-scripts/altmetrics.R +++ b/server/preprocessing/other-scripts/altmetrics.R @@ -38,7 +38,7 @@ enrich_output_json <- function(output_json){ end.time <- Sys.time() time.taken <- end.time - start.time - alog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Time taken:", time.taken, sep=" ")) + alog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Time taken:", time.taken, sep=" ")) return (output_json) } diff --git a/server/preprocessing/other-scripts/base.R b/server/preprocessing/other-scripts/base.R index 5fc5f2e05..2d3fafb84 100644 --- a/server/preprocessing/other-scripts/base.R +++ b/server/preprocessing/other-scripts/base.R @@ -41,7 +41,7 @@ get_papers <- function(query, params, limit=100, filter=NULL, retry_opts=rbace::bs_retry_options()) { - blog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Search:", query)) + blog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Search:", query)) start.time <- Sys.time() # remove pluses between terms @@ -58,7 +58,7 @@ get_papers <- function(query, params, limit=100, exact_query = gsub('([\"]+(.*?)[\"]+)|(?<=\\(\\b|\\+|-\\"\\b|\\s-\\b|^-\\b)|(?!or\\b|and\\b|[-]+[\\"\\(]*\\b)(? -1 && num_clusters > max_clusters) { num_clusters = MAX_CLUSTERS if(num_items >= 150) { - vclog$warn(paste("map_id:", .GlobalEnv$MAP_ID, "High content number, increasing max_k.")) + vclog$warn(paste("vis_id:", .GlobalEnv$VIS_ID, "High content number, increasing max_k.")) if(num_items >= 150 && num_items < 200) { num_clusters = 16 } else if (num_items >= 200 && num_items < 300) { @@ -63,7 +63,7 @@ create_clusters <- function(distance_matrix, max_clusters=-1, method="ward.D") { } if(num_items <= 30){ - vclog$warn(paste("map_id:", .GlobalEnv$MAP_ID, "Low content number, lowering max_k.")) + vclog$warn(paste("vis_id:", .GlobalEnv$VIS_ID, "Low content number, lowering max_k.")) num_clusters = round(sqrt(nrow(distance_matrix))) + 1 } @@ -88,7 +88,7 @@ create_clusters <- function(distance_matrix, max_clusters=-1, method="ward.D") { # dev.off() # } - vclog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Number of Clusters:", num_clusters, sep=" ")) + vclog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Number of Clusters:", num_clusters, sep=" ")) vclog$debug(paste("CutOff-Description:", attributes(cut_off)$description)) } clusters = list("labels"=labels, "cluster"=cluster, "groups"=groups, "num_clusters"=num_clusters) @@ -111,7 +111,7 @@ get_ndms <- function(distance_matrix, mindim=2, maxdim=2) { pc = TRUE, autotransform = FALSE, center = TRUE, halfchange = TRUE) - vclog$info(paste("map_id:", .GlobalEnv$MAP_ID, "NMDS-Stress:", min(ord$stress), sep=" ")) + vclog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "NMDS-Stress:", min(ord$stress), sep=" ")) points <- ord$points }, error=function(err){ points <- cbind(runif(nrow(distance_matrix), min=-1, max=0), @@ -129,7 +129,7 @@ get_ndms <- function(distance_matrix, mindim=2, maxdim=2) { pc = TRUE, autotransform = FALSE, center = TRUE, halfchange = TRUE) - vclog$info(paste("map_id:", .GlobalEnv$MAP_ID, "NMDS-Stress:", min(ord$stress), sep=" ")) + vclog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "NMDS-Stress:", min(ord$stress), sep=" ")) points <- ord$points }, error=function(err){ points <- cbind(runif(nrow(distance_matrix), min=-1, max=0), diff --git a/server/preprocessing/other-scripts/openaire.R b/server/preprocessing/other-scripts/openaire.R index 3ef62e180..1316fbade 100644 --- a/server/preprocessing/other-scripts/openaire.R +++ b/server/preprocessing/other-scripts/openaire.R @@ -35,7 +35,7 @@ olog <- getLogger('api.openaire') get_papers <- function(query, params, limit=NULL) { - olog$info(paste("map_id:", .GlobalEnv$MAP_ID, query)) + olog$info(paste("vis_id:", .GlobalEnv$VIS_ID, query)) start.time <- Sys.time() # parse params project_id <- params$project_id @@ -49,7 +49,7 @@ get_papers <- function(query, params, limit=NULL) { pubs_metadata <- fill_dois(pubs_metadata) }, error = function(err){ - olog$warn(paste0("map_id:", .GlobalEnv$MAP_ID, "publications: ", err)) + olog$warn(paste0("vis_id:", .GlobalEnv$VIS_ID, "publications: ", err)) pubs_metadata <- data.frame() return (data.frame()) }) @@ -61,7 +61,7 @@ get_papers <- function(query, params, limit=NULL) { datasets_metadata <- parse_response(response) }, error = function(err){ - olog$warn(paste0("map_id:", .GlobalEnv$MAP_ID, "datasets: ", err)) + olog$warn(paste0("vis_id:", .GlobalEnv$VIS_ID, "datasets: ", err)) datasets_metadata <- return (data.frame()) }) @@ -82,7 +82,7 @@ get_papers <- function(query, params, limit=NULL) { end.time <- Sys.time() time.taken <- end.time - start.time - olog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Time taken:", time.taken, sep=" ")) + olog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Time taken:", time.taken, sep=" ")) tryCatch({ get_return_values(all_artifacts) diff --git a/server/preprocessing/other-scripts/preprocess.R b/server/preprocessing/other-scripts/preprocess.R index 84baf5007..fc930eb5b 100644 --- a/server/preprocessing/other-scripts/preprocess.R +++ b/server/preprocessing/other-scripts/preprocess.R @@ -65,7 +65,7 @@ deduplicate_titles <- function(metadata, list_size) { output = head(output, max_replacements) } - vplog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Number of duplicate entries:", length(output))) + vplog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Number of duplicate entries:", length(output))) return(output) @@ -74,7 +74,7 @@ deduplicate_titles <- function(metadata, list_size) { replace_keywords_if_empty <- function(metadata, stops, service) { metadata$subject <- unlist(lapply(metadata$subject, function(x) {gsub(" +", " ", x)})) missing_subjects = which(lapply(metadata$subject, function(x) {nchar(x)}) <= 1) - vplog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Documents without subjects:", length(missing_subjects))) + vplog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Documents without subjects:", length(missing_subjects))) if (service == "linkedcat" || service == "linkedcat_authorview" || service == "linkedcat_browseview") { metadata$subject[missing_subjects] <- metadata$bkl_caption[missing_subjects] metadata$subject[is.na(metadata$subject)] <- "" diff --git a/server/preprocessing/other-scripts/pubmed.R b/server/preprocessing/other-scripts/pubmed.R index e80de6fae..9bd8a8796 100644 --- a/server/preprocessing/other-scripts/pubmed.R +++ b/server/preprocessing/other-scripts/pubmed.R @@ -46,7 +46,7 @@ get_papers <- function(query, params = NULL, limit = 100, retry_opts = rentrez:: # safe to add queries to the database query <- gsub("\\\\", "", query) - plog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Search:", query)) + plog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Search:", query)) start.time <- Sys.time() fields <- c('.//ArticleTitle', './/MedlineCitation/PMID', './/Title', './/Abstract') @@ -69,8 +69,8 @@ get_papers <- function(query, params = NULL, limit = 100, retry_opts = rentrez:: } else { query <- paste0(query, exclude_articles_with_abstract) } - plog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Query:", query)) - plog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Sort by:", sortby)) + plog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Query:", query)) + plog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Sort by:", sortby)) x <- rentrez::entrez_search(db = "pubmed", term = query, retmax = limit, mindate = from, maxdate = to, sort=sortby, use_history=TRUE, http_post = TRUE, retry = retry_opts) @@ -170,7 +170,7 @@ get_papers <- function(query, params = NULL, limit = 100, retry_opts = rentrez:: end.time <- Sys.time() time.taken <- end.time - start.time - plog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Time taken:", time.taken, sep=" ")) + plog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Time taken:", time.taken, sep=" ")) return(list(metadata = df, text = df[,c('id', 'content')])) } diff --git a/server/preprocessing/other-scripts/run_base.R b/server/preprocessing/other-scripts/run_base.R index 66935101e..cb03bd559 100644 --- a/server/preprocessing/other-scripts/run_base.R +++ b/server/preprocessing/other-scripts/run_base.R @@ -44,7 +44,7 @@ source('altmetrics.R') source('base.R') limit = 120 list_size = 100 -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id failed <- list(params=params) tryCatch({ diff --git a/server/preprocessing/other-scripts/run_openaire.R b/server/preprocessing/other-scripts/run_openaire.R index 829aa05cc..76b51af3d 100644 --- a/server/preprocessing/other-scripts/run_openaire.R +++ b/server/preprocessing/other-scripts/run_openaire.R @@ -33,7 +33,7 @@ f <- file("stdin") open(f) data = fromJSON(readLines(f)) params <- data$params -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id if (!is.null(params$lang_id)) { lang_id <- params$lang_id diff --git a/server/preprocessing/other-scripts/run_pubmed.R b/server/preprocessing/other-scripts/run_pubmed.R index bb57599f2..594c39acd 100644 --- a/server/preprocessing/other-scripts/run_pubmed.R +++ b/server/preprocessing/other-scripts/run_pubmed.R @@ -33,7 +33,7 @@ f <- file("stdin") open(f) data = fromJSON(readLines(f)) params <- data$params -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id if (!is.null(params$lang_id)) { lang_id <- params$lang_id diff --git a/server/preprocessing/other-scripts/run_vis_layout.R b/server/preprocessing/other-scripts/run_vis_layout.R index a8d3d7014..40d00e1e8 100644 --- a/server/preprocessing/other-scripts/run_vis_layout.R +++ b/server/preprocessing/other-scripts/run_vis_layout.R @@ -84,7 +84,7 @@ f <- file("stdin") open(f) data = fromJSON(readLines(f)) params <- data$params -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id if (!is.null(params$lang_id)) { lang_id <- params$lang_id diff --git a/server/preprocessing/other-scripts/test/base-test.R b/server/preprocessing/other-scripts/test/base-test.R index 88bbf0501..80e9ca8ec 100644 --- a/server/preprocessing/other-scripts/test/base-test.R +++ b/server/preprocessing/other-scripts/test/base-test.R @@ -40,7 +40,7 @@ if ('lang_id' %in% names(params)){ LANGUAGE <- get_service_lang(lang_id, valid_langs, service) ADDITIONAL_STOP_WORDS = LANGUAGE$name -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id #start.time <- Sys.time() failed <- list(params=params) diff --git a/server/preprocessing/other-scripts/test/params_base.json b/server/preprocessing/other-scripts/test/params_base.json index 55a6b2afc..9c5c8770a 100644 --- a/server/preprocessing/other-scripts/test/params_base.json +++ b/server/preprocessing/other-scripts/test/params_base.json @@ -4,6 +4,6 @@ "to":"2020-03-13", "sorting":"most-relevant", "lang_id":"eng", - "map_id": "TEST_ID", + "vis_id": "TEST_ID", "min_descsize": 300 } diff --git a/server/preprocessing/other-scripts/test/params_openaire.json b/server/preprocessing/other-scripts/test/params_openaire.json index 006cbda4c..fbcd00d59 100644 --- a/server/preprocessing/other-scripts/test/params_openaire.json +++ b/server/preprocessing/other-scripts/test/params_openaire.json @@ -1,3 +1,3 @@ {"funder":"EC", "project_id":"267439", - "map_id": "TEST_ID"} + "vis_id": "TEST_ID"} diff --git a/server/preprocessing/other-scripts/test/params_pubmed.json b/server/preprocessing/other-scripts/test/params_pubmed.json index 5e6cbb2d0..640b8fe04 100644 --- a/server/preprocessing/other-scripts/test/params_pubmed.json +++ b/server/preprocessing/other-scripts/test/params_pubmed.json @@ -95,4 +95,4 @@ "from":"1809-01-01", "to":"2020-09-04", "sorting":"most-relevant", - "map_id": "TEST_ID"} + "vis_id": "TEST_ID"} diff --git a/server/preprocessing/other-scripts/test/pubmed-test.R b/server/preprocessing/other-scripts/test/pubmed-test.R index a4a0e4ffb..8942e3d14 100644 --- a/server/preprocessing/other-scripts/test/pubmed-test.R +++ b/server/preprocessing/other-scripts/test/pubmed-test.R @@ -40,7 +40,7 @@ if ('lang_id' %in% names(params)){ LANGUAGE <- get_service_lang(lang_id, valid_langs, service) ADDITIONAL_STOP_WORDS = LANGUAGE$name -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id #start.time <- Sys.time() failed <- list(params=params) diff --git a/server/preprocessing/other-scripts/test/test-openaire.R b/server/preprocessing/other-scripts/test/test-openaire.R index dee0bea91..f7533f000 100644 --- a/server/preprocessing/other-scripts/test/test-openaire.R +++ b/server/preprocessing/other-scripts/test/test-openaire.R @@ -43,7 +43,7 @@ if ('lang_id' %in% names(params)){ LANGUAGE <- get_service_lang(lang_id, valid_langs, service) ADDITIONAL_STOP_WORDS = LANGUAGE$name -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id failed <- list(params=params) tryCatch({ diff --git a/server/preprocessing/other-scripts/text_similarity.R b/server/preprocessing/other-scripts/text_similarity.R index 53db947e0..7d53056b0 100644 --- a/server/preprocessing/other-scripts/text_similarity.R +++ b/server/preprocessing/other-scripts/text_similarity.R @@ -46,7 +46,7 @@ if (!is.null(params$vis_type)) { vis_type <- 'overview' } -.GlobalEnv$MAP_ID <- params$map_id +.GlobalEnv$VIS_ID <- params$vis_id taxonomy_separator = NULL limit = 100 list_size = -1 diff --git a/server/preprocessing/other-scripts/vis_layout.R b/server/preprocessing/other-scripts/vis_layout.R index 318b7dd76..0cde54e94 100644 --- a/server/preprocessing/other-scripts/vis_layout.R +++ b/server/preprocessing/other-scripts/vis_layout.R @@ -86,7 +86,7 @@ vis_layout <- function(text, metadata, service, end.time <- Sys.time() time.taken <- end.time - start.time - vlog$info(paste("map_id:", .GlobalEnv$MAP_ID, "Time taken:", time.taken, sep=" ")) + vlog$info(paste("vis_id:", .GlobalEnv$VIS_ID, "Time taken:", time.taken, sep=" ")) return(output) } diff --git a/server/services/search.php b/server/services/search.php index c52eb3b88..cd135e0e4 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -76,14 +76,14 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ "param_types" => $param_types)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { - return $res; + echo json_encode($res); } else { $unique_id = $res["result"]["unique_id"]; } } else { $unique_id = ($precomputed_id === null)?($persistence->createID(array($query, $params_for_id_creation))):($precomputed_id); } - $post_params["map_id"] = $unique_id; + $post_params["vis_id"] = $unique_id; $params_json = packParamsJSON($param_types, $post_params); if($retrieve_cached_map) { diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index fc6b36615..fbda556a6 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -12,7 +12,7 @@ persistence_ns = Namespace("persistence", description="OKMAps persistence operations") -def create_map_id(params, param_types): +def create_vis_id(params, param_types): # create map id ordered_params = OrderedDict() for k in param_types: @@ -208,7 +208,7 @@ def post(self): payload = request.get_json() params = payload.get("params") param_types = payload.get("param_types") - mapid = create_map_id(params, param_types) + mapid = create_vis_id(params, param_types) # create response headers = {} result = {"unique_id": mapid} diff --git a/server/workers/services/src/apis/request_validators.py b/server/workers/services/src/apis/request_validators.py index f2e66fa3b..b8f8a1993 100644 --- a/server/workers/services/src/apis/request_validators.py +++ b/server/workers/services/src/apis/request_validators.py @@ -21,6 +21,7 @@ class SearchParamSchema(Schema): unique_id = fields.Str() raw = fields.Boolean() sg_method = fields.Str() + vis_id = fields.Str() @pre_load def fix_years(self, in_data, **kwargs): diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index 271080957..d6dae7f20 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -8,7 +8,7 @@ @pytest.mark.persistence -def test_map_id_creation(): +def test_vis_id_creation(): testcases = [ # {"params": {"q": "air quality management", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, # "param_types": ["from", "to", "document_types", "sorting"], diff --git a/vis/js/headstart.js b/vis/js/headstart.js index e1ea336a3..770eedb8a 100644 --- a/vis/js/headstart.js +++ b/vis/js/headstart.js @@ -36,13 +36,13 @@ HeadstartFSM.prototype = { if(!config.is_evaluation) { return; } - + let services = config.evaluation_service; - + if(typeof services === "string") { services = [services]; } - + if (services.includes("log")) { this.recordActionLog(category, action, id, user, timestamp, additional_params, post_data); } @@ -53,7 +53,7 @@ HeadstartFSM.prototype = { this.recordActionGA(category, action, id, user, timestamp, additional_params, post_data); } }, - + recordActionLog: function(id, category, action, user, type, timestamp, additional_params, post_data) { timestamp = (typeof timestamp !== 'undefined') ? (escape(timestamp)) : (""); additional_params = (typeof additional_params !== 'undefined') ? ('&' + additional_params) : (""); @@ -81,13 +81,13 @@ HeadstartFSM.prototype = { } }); }, - + recordActionMatomo: function(category, action, id) { if(typeof _paq !== "undefined") { _paq.push(['trackEvent', category, action, id]); } }, - + recordActionGA: function(category, action, id) { //gtag.js if(typeof gtag === "function") { @@ -98,11 +98,11 @@ HeadstartFSM.prototype = { //analytics.js } else if (typeof ga === "function") { ga('send', 'event', category, action, id); - } + } }, - + markProjectChanged: function (id) { - + let php_script = config.server_url + "services/markProjectChanged.php"; $.ajax({ @@ -113,9 +113,9 @@ HeadstartFSM.prototype = { console.log(output); } }); - + }, - + dynamicForcePapers: function(num_items) { if (num_items >= 150 && num_items < 200) { config.papers_force_alpha = 0.2; @@ -127,13 +127,13 @@ HeadstartFSM.prototype = { config.papers_force_alpha = 0.6; } }, - + dynamicForceAreas: function(num_items) { if (num_items >= 200) { config.area_force_alpha = 0.02; } }, - + dynamicSizing: function(num_items) { if (num_items >= 150 && num_items < 200) { this.adjustSizes(0.9, 1.1); @@ -153,11 +153,11 @@ HeadstartFSM.prototype = { this.adjustSizes(0.6, 1.2); } }, - + adjustSizes: function(resize_paper_factor, resize_bubble_factor) { config.paper_min_scale *= resize_paper_factor; config.paper_max_scale *= resize_paper_factor; - + config.bubble_min_scale *= resize_bubble_factor; config.bubble_max_scale *= resize_bubble_factor; }, @@ -202,10 +202,10 @@ HeadstartFSM.prototype = { search_repos: function(that, setupVis) { let url = config.server_url + "services/getLatestRevision.php?vis_id=" + mediator.current_bubble.file + "&context=" + config.show_context + "&streamgraph=" + config.is_streamgraph - + "&backend=" + config.backend; + + "&backend=" + config.backend + "&persistence_backend=" + config.persistence_backend; mediator.publish("get_data_from_files", url, 'json', setupVis); }, - + gsheets: function(that, setupVis) { let url = config.server_url + "services/getGSheetsMap.php?vis_id=" + mediator.current_bubble.file + "&q=" +mediator.current_bubble.title From 54d2c85a2701e6e8b06f7a7c03cd9f106fa6ef66 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 7 Oct 2020 00:30:40 +0200 Subject: [PATCH 18/76] search api fix --- server/services/getLatestRevision.php | 2 +- server/services/search.php | 7 ++-- .../workers/services/src/apis/persistence.py | 33 ++++++++++++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/server/services/getLatestRevision.php b/server/services/getLatestRevision.php index b8b856905..4ff0ec2a2 100644 --- a/server/services/getLatestRevision.php +++ b/server/services/getLatestRevision.php @@ -35,7 +35,7 @@ if ($res["httpcode"] != 200) { library\CommUtils::echoOrCallback($res, $_GET); } else { - $data = $res; + $data = json_decode($res["result"], true); } } else { # get data + context from legacy diff --git a/server/services/search.php b/server/services/search.php index cd135e0e4..5dd0b95da 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -71,14 +71,15 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $params_for_id_creation = ($params_for_id === null)?($params_json):(packParamsJSON($params_for_id, $post_params)); if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . $repository . "/createID"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/createID"; $payload = json_encode(array("params" => $post_params, "param_types" => $param_types)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { echo json_encode($res); } else { - $unique_id = $res["result"]["unique_id"]; + $result = json_decode($res["result"], true); + $unique_id = $result["unique_id"]; } } else { $unique_id = ($precomputed_id === null)?($persistence->createID(array($query, $params_for_id_creation))):($precomputed_id); @@ -146,7 +147,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ if ($res["httpcode"] != 200) { return $res; } else { - $exists = $res["result"]["exists"]; + $exists = $result["exists"]; } } else { $exists = $persistence->existsVisualization($unique_id); diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index fbda556a6..06ee7658e 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -70,17 +70,26 @@ def exists_visualization(vis_id): def get_last_version(vis_id, details=False, context=False): - return get_revision(vis_id, None, details=False, context=False) + return get_revision(vis_id, None, details, context) def get_revision(vis_id, rev_id, details=False, context=False): - vis, rev = (db.session - .query(Visualizations, Revisions) - .select_from(Visualizations, Revisions) - .filter(Visualizations.vis_id == vis_id) - .filter(Revisions.rev_vis == vis_id) - .filter(Revisions.rev_id == Visualizations.vis_latest) - ).first() + if rev_id is None: + vis, rev = (db.session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == Visualizations.vis_latest) + ).first() + else: + vis, rev = (db.session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == rev_id) + ).first() if context is True: res = { "rev_vis": rev.rev_vis, @@ -118,12 +127,18 @@ class createVisualization(Resource): def post(self): try: payload = request.get_json() + persistence_ns.logger.debug(payload.keys()) vis_id = payload.get('vis_id') vis_title = payload.get('vis_title') data = payload.get('data') vis_clean_query = payload.get('vis_clean_query') vis_query = payload.get('vis_query') vis_params = payload.get('vis_params') + persistence_ns.logger.debug(vis_id) + persistence_ns.logger.debug(vis_title) + persistence_ns.logger.debug(vis_clean_query) + persistence_ns.logger.debug(vis_query) + persistence_ns.logger.debug(vis_params) create_visualization(vis_id, vis_title, data, vis_clean_query, vis_query, vis_params) result = {'success': True} @@ -151,6 +166,7 @@ class getLastVersion(Resource): def post(self): try: payload = request.get_json() + persistence_ns.logger.debug(payload) vis_id = payload.get('vis_id') details = payload.get('details') context = payload.get('context') @@ -189,6 +205,7 @@ def post(self): payload = request.get_json() vis_id = payload.get("vis_id") data = payload.get("data") + persistence_ns.logger.debug(data) write_revision(vis_id, data, None) result = {'success': True} headers = {'ContentType': 'application/json'} From b145aa384634df239904fc2b0566778253a39b90 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 8 Oct 2020 13:09:56 +0200 Subject: [PATCH 19/76] wip --- server/services/search.php | 1 + 1 file changed, 1 insertion(+) diff --git a/server/services/search.php b/server/services/search.php index 5dd0b95da..8194b6a76 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -147,6 +147,7 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ if ($res["httpcode"] != 200) { return $res; } else { + $result = json_decode($res, true); $exists = $result["exists"]; } } else { From 2c954770997b8f0617d5b9f67fccc08839285f24 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 8 Oct 2020 19:54:12 +0200 Subject: [PATCH 20/76] gsheets update --- server/services/getGSheetsMap.php | 25 ++++++++++++++++--- server/services/search.php | 3 ++- server/services/updateGSheetsMap.php | 2 +- .../workers/services/src/apis/persistence.py | 2 +- vis/js/headstart.js | 3 ++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index 7e3429fe8..7dd252716 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -17,10 +17,27 @@ $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); -if ($persistence_backend == "api") { - $return_data = array("status" => "error", "reason" => "Not implemented."); - $jsonData = json_encode($return_data); - library\CommUtils::echoOrCallback($jsonData, $_GET); +if ($persistence_backend === "api") { + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = json_decode($res["result"], true); + $rev_data = json_decode($data["rev_data"], true); + $return_data = array("context" => array("id" => $data["rev_vis"], + "query" => $data["vis_query"], + "service" => $data["vis_title"], + "timestamp" => $data["rev_timestamp"], + "params" => $data["vis_params"], + "sheet_id" => $rev_data["sheet_id"], + "last_update" => $rev_data["last_update"]), + "data" => $rev_data["data"], + "errors" => $rev_data["errors"]); + $jsonData = json_encode($return_data); + library\CommUtils::echoOrCallback($jsonData, $_GET); + } } else { $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; $rev_data = json_decode($data["rev_data"], true); diff --git a/server/services/search.php b/server/services/search.php index 8194b6a76..a92410bb2 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -82,8 +82,9 @@ function search($repository, $dirty_query, $post_params, $param_types, $keyword_ $unique_id = $result["unique_id"]; } } else { - $unique_id = ($precomputed_id === null)?($persistence->createID(array($query, $params_for_id_creation))):($precomputed_id); + $unique_id = $persistence->createID(array($query, $params_for_id_creation)); } + $unique_id = ($precomputed_id === null)?($unique_id):($precomputed_id); $post_params["vis_id"] = $unique_id; $params_json = packParamsJSON($param_types, $post_params); diff --git a/server/services/updateGSheetsMap.php b/server/services/updateGSheetsMap.php index 67edd273f..c6b79bdeb 100644 --- a/server/services/updateGSheetsMap.php +++ b/server/services/updateGSheetsMap.php @@ -37,7 +37,7 @@ , array("sheet_id") , ";", null, false, false, null, 3 , "area_uri", "subject", $sheet_id, false - , "api"); + , "api", "api"); echo $result; diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 06ee7658e..ed8a066b6 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -176,7 +176,7 @@ def post(self): 200, headers) except Exception as e: - result = {'success': False, 'reason': e} + result = {'success': False, 'reason': str(e)} headers = {'ContentType': 'application/json'} return make_response(jsonify(result), 500, diff --git a/vis/js/headstart.js b/vis/js/headstart.js index 770eedb8a..7f96127a6 100644 --- a/vis/js/headstart.js +++ b/vis/js/headstart.js @@ -209,7 +209,8 @@ HeadstartFSM.prototype = { gsheets: function(that, setupVis) { let url = config.server_url + "services/getGSheetsMap.php?vis_id=" + mediator.current_bubble.file + "&q=" +mediator.current_bubble.title - + "&context=" + config.show_context + "&streamgraph=" + config.is_streamgraph; + + "&context=" + config.show_context + "&streamgraph=" + config.is_streamgraph + + "&persistence_backend=" + config.persistence_backend; mediator.publish("get_data_from_files", url, 'json', setupVis); }, From c9565a282774421e6d1d8265ebcb6e65dbf35202 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 13 Oct 2020 23:12:42 +0200 Subject: [PATCH 21/76] added VIPER datatypes (unused for now) --- server/workers/services/src/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index dc3d630eb..58eacbd3f 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -11,6 +11,8 @@ class Visualizations(db.Model): vis_title = db.Column(db.Text) vis_latest = db.Column(db.Integer) vis_params = db.Column(db.Text) + vis_changed = db.Column(db.Boolean) + vis_changed_timestamp = db.Column(db.Time) def as_dict(self): return {c.name: getattr(self, c.name) for c in self.__table__.columns} From 5147d79325800b9787956bf0e974c161c826511a Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 13 Oct 2020 23:34:26 +0200 Subject: [PATCH 22/76] coding format --- server/services/search.php | 8 +++++-- server/services/searchBASE.php | 12 ++++++---- server/services/searchDOAJ.php | 12 ++++++---- server/services/searchOpenAire.php | 34 +++++++++++++++------------- server/services/searchPubmed.php | 8 ++++--- server/services/searchTRIPLE.php | 6 +++-- server/services/updateGSheetsMap.php | 10 ++++---- 7 files changed, 53 insertions(+), 37 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index a92410bb2..a96a093c6 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -52,8 +52,12 @@ function cleanQuery($dirty_query, $transform_query_tolowercase) { } -function search($repository, $dirty_query, $post_params, $param_types, $keyword_separator, $taxonomy_separator, $transform_query_tolowercase = true - , $retrieve_cached_map = true, $params_for_id = null, $num_labels = 3, $id = "area_uri", $subjects = "subject", $precomputed_id = null, $do_clean_query = true +function search($repository, $dirty_query + , $post_params, $param_types + , $keyword_separator, $taxonomy_separator, $transform_query_tolowercase = true + , $retrieve_cached_map = true, $params_for_id = null, $num_labels = 3 + , $id = "area_uri", $subjects = "subject" + , $precomputed_id = null, $do_clean_query = true , $backend = "legacy", $persistence_backend = "legacy") { $INI_DIR = dirname(__FILE__) . "/../preprocessing/conf/"; $ini_array = library\Toolkit::loadIni($INI_DIR); diff --git a/server/services/searchBASE.php b/server/services/searchBASE.php index eb8801a9d..df0a4310c 100644 --- a/server/services/searchBASE.php +++ b/server/services/searchBASE.php @@ -16,11 +16,13 @@ $post_params["min_descsize"] = 300; } -$result = search("base", $dirty_query, $post_params - , array("from", "to", "document_types", "sorting", "min_descsize") - , ";", null, true, true, null, 3 - , "area_uri", "subject", $precomputed_id, false - , "legacy"); +$result = search("base", $dirty_query + , $post_params, array("from", "to", "document_types", "sorting", "min_descsize") + , ";", null, true + , true, null, 3 + , "area_uri", "subject" + , $precomputed_id, false + , "legacy", "legacy"); echo $result diff --git a/server/services/searchDOAJ.php b/server/services/searchDOAJ.php index 3867df3fc..47e4cc7b0 100644 --- a/server/services/searchDOAJ.php +++ b/server/services/searchDOAJ.php @@ -12,11 +12,13 @@ $post_params = $_POST; -$result = search("doaj", $dirty_query, $post_params - , array("from", "to", "today", "sorting") - , ";", null, true, true, null, 3 - , "area_uri", "subject", $precomputed_id, false - , "legacy"); +$result = search("doaj", $dirty_query + , $post_params, array("from", "to", "today", "sorting") + , ";", null, true + , true, null, 3 + , "area_uri", "subject" + , $precomputed_id, false + , "legacy", "legacy"); echo $result diff --git a/server/services/searchOpenAire.php b/server/services/searchOpenAire.php index 73985395b..eb9fce663 100644 --- a/server/services/searchOpenAire.php +++ b/server/services/searchOpenAire.php @@ -11,23 +11,25 @@ $post_params = $_POST; -$result = search("openaire", $acronymtitle, $post_params, array("project_id", - "funder", - "title", - "funding_tree", - "call_id", - "start_date", - "end_date", - "oa_mandate", - "special_clause", - "organisations", - "openaire_link", - "obj_id", - "acronym"), - ";", null, false, true, array("project_id", "funder") - , 3, "area_uri", "subject" +$result = search("openaire", $acronymtitle + , $post_params, array("project_id", + "funder", + "title", + "funding_tree", + "call_id", + "start_date", + "end_date", + "oa_mandate", + "special_clause", + "organisations", + "openaire_link", + "obj_id", + "acronym") + , ";", null, false + , true, array("project_id", "funder"), 3 + , "area_uri", "subject" , null, true, - "legacy"); + "legacy", "legacy"); echo $result diff --git a/server/services/searchPubmed.php b/server/services/searchPubmed.php index af4cd5ff5..c3f535862 100644 --- a/server/services/searchPubmed.php +++ b/server/services/searchPubmed.php @@ -18,9 +18,11 @@ $result = search("pubmed", $dirty_query , $post_params, $query_params - , ";", null, true, true, null, 3 - , "area_uri", "subject", $precomputed_id, false - , "legacy"); + , ";", null, true + , true, null, 3 + , "area_uri", "subject" + , $precomputed_id, false + , "legacy", "legacy"); echo $result diff --git a/server/services/searchTRIPLE.php b/server/services/searchTRIPLE.php index b90eb3e9b..c3f36fbae 100644 --- a/server/services/searchTRIPLE.php +++ b/server/services/searchTRIPLE.php @@ -14,8 +14,10 @@ $result = search("triple", $dirty_query , $post_params, array("from", "to", "sorting", "vis_type", "language", "limit", "sg_method") - , ";", null, true, true, null, 3 - , "area_uri", "subject", $precomputed_id, false + , ";", null, true + , true, null, 3 + , "area_uri", "subject" + , $precomputed_id, false , "api", "api"); echo $result diff --git a/server/services/updateGSheetsMap.php b/server/services/updateGSheetsMap.php index c6b79bdeb..762031bc3 100644 --- a/server/services/updateGSheetsMap.php +++ b/server/services/updateGSheetsMap.php @@ -33,10 +33,12 @@ $params["last_update"] = $last_update; } -$result = search("gsheets", $dirty_query, $params - , array("sheet_id") - , ";", null, false, false, null, 3 - , "area_uri", "subject", $sheet_id, false +$result = search("gsheets", $dirty_query + , $params, array("sheet_id") + , ";", null, false + , false, null, 3 + , "area_uri", "subject" + , $sheet_id, false , "api", "api"); echo $result; From ad279b27076bb126af4a86b5c405cbb7614ff114 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 13 Oct 2020 23:49:04 +0200 Subject: [PATCH 23/76] fix API routing --- server/services/search.php | 2 +- .../workers/services/src/apis/persistence.py | 40 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index a96a093c6..84adfd559 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -177,7 +177,7 @@ function search($repository, $dirty_query } } else { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/writeRevision"; $payload = json_encode(array("vis_id" => $unique_id, "data" => $input_json)); $res = library\CommUtils::call_api($route, $payload); diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index ed8a066b6..8c8875762 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -154,6 +154,26 @@ def post(self): headers) +@persistence_ns.route('/writeRevision') +class writeRevision(Resource): + + @persistence_ns.produces(["application/json"]) + def post(self): + try: + payload = request.get_json() + vis_id = payload.get("vis_id") + data = payload.get("data") + persistence_ns.logger.debug(data) + write_revision(vis_id, data, None) + result = {'success': True} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), 200, headers) + except Exception as e: + result = {'success': False, 'reason': e} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), 500, headers) + + @persistence_ns.route('/getLastVersion') class getLastVersion(Resource): """ @@ -196,26 +216,6 @@ def post(self): return make_response(result, 200, headers) -@persistence_ns.route('/writeRevision') -class writeRevision(Resource): - - @persistence_ns.produces(["application/json"]) - def post(self): - try: - payload = request.get_json() - vis_id = payload.get("vis_id") - data = payload.get("data") - persistence_ns.logger.debug(data) - write_revision(vis_id, data, None) - result = {'success': True} - headers = {'ContentType': 'application/json'} - return make_response(jsonify(result), 200, headers) - except Exception as e: - result = {'success': False, 'reason': e} - headers = {'ContentType': 'application/json'} - return make_response(jsonify(result), 500, headers) - - @persistence_ns.route('/createID') class createID(Resource): From 4764193770a98dfda2815bc3cd08f5fbb6032fb6 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 14 Oct 2020 22:13:38 +0200 Subject: [PATCH 24/76] more logging --- server/workers/services/src/apis/persistence.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 8c8875762..5a5c5e76f 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -112,6 +112,7 @@ class existsVisualization(Resource): def post(self): payload = request.get_json() + persistence_ns.logger.debug("existsVisualization") vis_id = payload.get("vis_id") exists = exists_visualization(vis_id) # create response @@ -127,6 +128,7 @@ class createVisualization(Resource): def post(self): try: payload = request.get_json() + persistence_ns.logger.debug("createVisualization") persistence_ns.logger.debug(payload.keys()) vis_id = payload.get('vis_id') vis_title = payload.get('vis_title') @@ -161,6 +163,7 @@ class writeRevision(Resource): def post(self): try: payload = request.get_json() + persistence_ns.logger.debug("writeRevision") vis_id = payload.get("vis_id") data = payload.get("data") persistence_ns.logger.debug(data) @@ -186,6 +189,7 @@ class getLastVersion(Resource): def post(self): try: payload = request.get_json() + persistence_ns.logger.debug("getLastVersion") persistence_ns.logger.debug(payload) vis_id = payload.get('vis_id') details = payload.get('details') @@ -222,6 +226,7 @@ class createID(Resource): @persistence_ns.produces(["application/json"]) def post(self): try: + persistence_ns.logger.debug("createID") payload = request.get_json() params = payload.get("params") param_types = payload.get("param_types") From 1d83d2a5f377022b105741f5029d1485cb41ea90 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 15 Oct 2020 18:04:16 +0200 Subject: [PATCH 25/76] new exists check bugfix --- server/services/search.php | 2 +- server/workers/services/src/apis/persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/services/search.php b/server/services/search.php index 84adfd559..dc6de8219 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -152,7 +152,7 @@ function search($repository, $dirty_query if ($res["httpcode"] != 200) { return $res; } else { - $result = json_decode($res, true); + $result = json_decode($res["result"], true); $exists = $result["exists"]; } } else { diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 5a5c5e76f..f5ce93ef7 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -167,7 +167,7 @@ def post(self): vis_id = payload.get("vis_id") data = payload.get("data") persistence_ns.logger.debug(data) - write_revision(vis_id, data, None) + write_revision(vis_id, data) result = {'success': True} headers = {'ContentType': 'application/json'} return make_response(jsonify(result), 200, headers) From 7cca4fe642eaab313876e097111735672ed0d5e1 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 15 Oct 2020 18:54:43 +0200 Subject: [PATCH 26/76] persistence test updates --- server/workers/tests/test_persistence.py | 52 +++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index d6dae7f20..c8b8e3685 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -10,21 +10,18 @@ @pytest.mark.persistence def test_vis_id_creation(): testcases = [ - # {"params": {"q": "air quality management", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, - # "param_types": ["from", "to", "document_types", "sorting"], - # "expected result": "2074f8b9eee26f53936abd16f6187ed8"}, - # {"params": {"q": "trump", "from": "1665-01-01", "to": "2017-10-27", "sorting": "most-relevant", "document_types": [121]}, - # "param_types": ["from", "to", "document_types", "sorting"], - # "expected result": "d9c930deef17b3f22e3030a9bd020f9f"}, - # {"params": {"q": "solar eclipse", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, - # "param_types": ["from", "to", "document_types", "sorting"], - # "expected result": "f9f0d52aaf3a91d0040c2acdacf00620"}, - # {"params": {"q": "pop music", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, - # "param_types": ["from", "to", "document_types", "sorting"], - # "expected result": "32df883cd04d85a710b6c4057f01dfd8"}, - # {"params": {"q": "fear-of-missing-out", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121]}, - # "param_types": ["from", "to", "document_types", "sorting"], - # "expected result": "81baaafd6f0ddba4a7ce4e7237210fe7"}, + {"params": {"q": "air quality management", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121], "min_descsize": 300}, + "param_types": ["from", "to", "document_types", "sorting", "min_descsize"], + "expected result": "9810f6b4aefa7d0b8151e030c07c9514"}, + {"params": {"q": "solar eclipse", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121], "min_descsize": 300}, + "param_types": ["from", "to", "document_types", "sorting", "min_descsize"], + "expected result": "d76de62667fe956bd65862ebbf3c5448"}, + {"params": {"q": "lisbon treaty", "from": "1665-01-01", "to": "2018-10-04", "sorting": "most-relevant", "document_types": [121], "min_descsize": 300}, + "param_types": ["from", "to", "document_types", "sorting", "min_descsize"], + "expected result": "a2005cba90de92b6b8f13cdcb890dbfa "}, + {"params": {"q": "fear-of-missing-out", "from": "1665-01-01", "to": "2017-09-08", "sorting": "most-relevant", "document_types": [121], "min_descsize": 300}, + "param_types": ["from", "to", "document_types", "sorting", "min_descsize"], + "expected result": "f82bf0235217431d56a5afac8ef2c1d4 "}, {"params": {"q": "sustainable development goals", 'from': '1809-01-01', 'to': '2017-09-08', 'sorting': 'most-relevant', @@ -451,3 +448,28 @@ def test_write_revision(testcase): res = requests.post("http://localhost/api/persistence/writeRevision", json=payload) assert res.status_code == 200, res.json().get('reason') + + +@pytest.mark.persistence +@pytest.mark.parametrize("testcase", KNOWNCASES) +def test_map_exists(testcase): + caseid = testcase["caseid"] + payload = {} + payload["vis_id"] = caseid + res = requests.post("http://localhost/api/persistence/existsVisualization", + json=payload) + result = res.json() + assert result["exists"] is True + + +@pytest.mark.persistence +@pytest.mark.parametrize("testcase", KNOWNCASES) +def test_map_exists_not(testcase): + caseid = testcase["caseid"] + invalid_id = caseid[2:] + payload = {} + payload["vis_id"] = invalid_id + res = requests.post("http://localhost/api/persistence/existsVisualization", + json=payload) + result = res.json() + assert result["exists"] is False From f88fb6323daf1c0072affd8b6a4109aca585ab6c Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 15 Oct 2020 19:03:58 +0200 Subject: [PATCH 27/76] docs update --- examples/viper/js/data-config_openaire.js | 1 + server/workers/README.md | 8 ++++++++ vis/js/default-config.js | 4 +++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/viper/js/data-config_openaire.js b/examples/viper/js/data-config_openaire.js index 214b70ec9..96a671749 100644 --- a/examples/viper/js/data-config_openaire.js +++ b/examples/viper/js/data-config_openaire.js @@ -2,6 +2,7 @@ var data_config = { tag: "visualization", mode: "search_repos", backend: "legacy", + persistence_backend: "legacy", bubble_min_scale: 1, bubble_max_scale: 1, diff --git a/server/workers/README.md b/server/workers/README.md index a8e90ee10..90346abd6 100644 --- a/server/workers/README.md +++ b/server/workers/README.md @@ -84,6 +84,14 @@ Secure Redis: * In `server/workers` copy `example_redis_config.json` to `redis_config.json` and `example_redis.conf` to `redis.conf` and in both files replace "long_secure_password" with a long, secure password (Line 507 in redis.conf, parameter `requirepass`). +PostgreSQL service: +* In root folder create `.env` from the `example.env` and fill in the environment variables with the correct login data. +* Manual database creation: + +Enter container: `docker exec -it VARYINGNAME_pgsql_1 psql -U headstart` + +Execute command: `CREATE DATABASE databasename;` + ### Starting the backend services with docker-compose Following commands have to be executed from the root folder of the repository, where `docker-compose.yml` is located. diff --git a/vis/js/default-config.js b/vis/js/default-config.js index 120b6f9e4..d83395e48 100644 --- a/vis/js/default-config.js +++ b/vis/js/default-config.js @@ -98,8 +98,10 @@ var config = { /*** basic data- and third-party-service related settings ***/ //mode for retrieving data mode: "local_files", - //which backend to use for data retrieval (api or legacy) + //which backend to use for data processing (api or legacy) backend: "legacy", + //which backend to use for data persistence (api or legacy) + persistence_backend: "legacy", //language specification to use (see localization object) language: "eng", //language used for hyphenation From 71638337ea57d2413749af0ac20bd54017dd187d Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 22 Oct 2020 15:11:04 +0200 Subject: [PATCH 28/76] attempt correct escaping in query strings --- server/services/searchTRIPLE.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/services/searchTRIPLE.php b/server/services/searchTRIPLE.php index c3f36fbae..a04173cca 100644 --- a/server/services/searchTRIPLE.php +++ b/server/services/searchTRIPLE.php @@ -17,7 +17,7 @@ , ";", null, true , true, null, 3 , "area_uri", "subject" - , $precomputed_id, false + , $precomputed_id, true , "api", "api"); echo $result From 2667f7278da912b24b8722913854df2dc8000421 Mon Sep 17 00:00:00 2001 From: pkraker Date: Thu, 22 Oct 2020 23:09:30 +0200 Subject: [PATCH 29/76] First version of the GSheets example --- examples/gsheets/config.example.php | 13 ++ examples/gsheets/data-config.js | 55 ++++++++ examples/gsheets/index.php | 194 ++++++++++++++++++++++++++++ examples/gsheets/map.css | 89 +++++++++++++ vis/js/default-config.js | 63 +++++++++ 5 files changed, 414 insertions(+) create mode 100644 examples/gsheets/config.example.php create mode 100644 examples/gsheets/data-config.js create mode 100644 examples/gsheets/index.php create mode 100644 examples/gsheets/map.css diff --git a/examples/gsheets/config.example.php b/examples/gsheets/config.example.php new file mode 100644 index 000000000..3c12cd570 --- /dev/null +++ b/examples/gsheets/config.example.php @@ -0,0 +1,13 @@ + diff --git a/examples/gsheets/data-config.js b/examples/gsheets/data-config.js new file mode 100644 index 000000000..73a80548e --- /dev/null +++ b/examples/gsheets/data-config.js @@ -0,0 +1,55 @@ +var data_config = { + tag: "visualization", + mode: "gsheets", + + bubble_min_scale: 1.1, + bubble_max_scale: 1.1, + + paper_min_scale: 1, + + input_format: "json", + use_area_uri: true, + preview_type: "pdf", + use_hypothesis: true, + + show_multiples: false, + show_dropdown: false, + show_intro: false, + show_list:true, + is_force_papers: true, + is_title_clickable: false, + show_infolink: true, + show_infolink_areas: false, + + show_context: true, + create_title_from_context: false, + show_context_timestamp: true, + show_loading_screen: true, + + scale_toolbar: false, + + content_based: true, + is_evaluation: true, + evaluation_service: ["ga", "matomo"], + + is_force_areas: true, + area_force_alpha: 0.03, + papers_force_alpha: 0.2, + + language: "eng_gsheets", + + sort_options: ["year", "title", "area"], + filter_options: ["all", "open_access"], + + show_keywords: true, + hide_keywords_overview: false, + show_tags: true, + show_comments: true, + show_resulttype: true, + + sort_menu_dropdown: true, + filter_menu_dropdown: true, + + embed_modal: true, + share_modal: false, +}; diff --git a/examples/gsheets/index.php b/examples/gsheets/index.php new file mode 100644 index 000000000..b9a1d4e93 --- /dev/null +++ b/examples/gsheets/index.php @@ -0,0 +1,194 @@ + + + + + + + + + + + Knowledge Map of <?php echo $TOPIC ?> + + + +
+
+
An update is available
reload now or do it later
+ + + + + + + + + + + diff --git a/examples/gsheets/map.css b/examples/gsheets/map.css new file mode 100644 index 000000000..326b5c8a7 --- /dev/null +++ b/examples/gsheets/map.css @@ -0,0 +1,89 @@ +.errors-container { + margin: 20px 10px; + display: none; + padding-left: 50px; +} + +.show-errors { + display: block; +} + +.topheader { + padding-top: 59px; +} + +.error-row-top, .error-row-entry { + padding: 5px; +} + +.errors-info { + line-height: 110% !important; + z-index: 99; + font-family: 'Lato', sans-serif !important; + position: relative; +} + +.errors-info, .expand-icon { + cursor: pointer; + text-decoration: underline; +} + +.errors-table-hidden { + display: none; +} + +.reload-button { + color: white; + background-color: #cc3b6b; + position: fixed; + top: 192px; + left: 0%; + transform: translate(0%, 0); + z-index: 99; + cursor: pointer; + display: block; + padding: 12px 15px; + border-radius: 5px; + -moz-border-radius: 5px; + -webkit-border-radius: 5px; + max-width: 200px; + font-family: 'Lato', sans-serif; + display: none; + -moz-box-shadow: 0 2px 2px 0 rgba(0, 0, 0, 0.14), 0 3px 1px -2px rgba(0, 0, 0, 0.12), 0 1px 5px 0 rgba(0, 0, 0, 0.2); +box-shadow: 0 2px 2px 0 rgba(0, 0, 0, 0.14), 0 3px 1px -2px rgba(0, 0, 0, 0.12), 0 1px 5px 0 rgba(0, 0, 0, 0.2); +} + +.reload-button.mobile { + top: 298px; +} + +.dismiss-reload { + color: white; + text-decoration: underline; +} + +.dismiss-reload:hover { + color: white; + text-decoration: none; +} + +.show-reload-button { + display: block; +} + +.hide-reload-text { + display: none; +} + +@media screen and (max-width: 1150px) { + .reload-button { + padding: 10px 12px; + max-width: 180px; + } +} + +@media screen and (max-width: 640px) { + .errors-container { + padding-left: 0px; + } +} \ No newline at end of file diff --git a/vis/js/default-config.js b/vis/js/default-config.js index fb448eb3f..2550d367a 100644 --- a/vis/js/default-config.js +++ b/vis/js/default-config.js @@ -779,6 +779,69 @@ var config = { scale_by_label: 'Distribution for:', credit_alt: "Created by Open Knowledge Maps", }, + eng_gsheets: { + loading: "Updating and retrieving map. This may take a few seconds, please hold on.", + search_placeholder: "Search within map...", + show_list: "Show list", + hide_list: "Hide list", + intro_label: "[more info]", + intro_icon: "", + relevance: "relevance", + readers: "citations", + year: "date", + authors: "authors", + title: "title", + area: "Area", + backlink: "← Back to overview", + backlink_list: "← Show all documents in area", + backlink_list_streamgraph: "← Show all documents", + backlink_list_streamgraph_stream_selected: "← Show all documents in stream", + keywords: "Keywords", + no_keywords: "n/a", + no_title: "No title", + overview_label: 'Overview of', + streamgraph_label: 'Streamgraph for', + overview_authors_label: 'Overview of the works of', + streamgraph_authors_label: 'Streamgraph for the works of', + articles_label: 'resources', + most_recent_label: 'most recent', + most_relevant_label: 'most relevant', + most_relevant_tooltip: 'To determine the most relevant documents, we use the relevance ranking provided by the source - either BASE or PubMed. Both sources compute the text similarity between your query and the article metadata to establish the relevance ranking. Please consult the FAQ for more information.', + source_label: 'Data source', + resulttype_label: 'Document type', + documenttypes_label: 'Document types', + timestamp_label: 'Last updated', + documenttypes_tooltip: 'The following document types were taken into consideration in the creation of this map (not all of them may appear in the map):', + default_area: "No area", + default_author: "", + default_id: "defaultid", + default_hash: "hashHash", + default_abstract: "No Abstract", + default_published_in: "", + default_readers: 0, + default_url: "", + default_x: 1., + default_y: 1., + default_year: "", + sort_by_label: 'sort by:', + filter_by_label: 'show: ', + all: "any", + open_access: "Open Access", + "Journal Article": "Journal article", + Preprint: "Preprint", + ReFigure: "ReFigure", + Review: "Review", + link: 'link', + items: "items", + comment_by_label: "by", + pdf_not_loaded: "Sorry, we were not able to retrieve the PDF for this publication. You can get it directly from", + pdf_not_loaded_linktext: "this website", + share_button_title: "share this map", + embed_button_title: "Embed this knowledge map on other websites", + embed_button_text: 'Copy', + embed_title: 'embed map', + embed_body_text: 'You can use this code to embed the visualization on your own website or in a dashboard.', + }, }, scale_types: [], From a6ff30c56b502608fa8894d32f028b960829b850 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 27 Oct 2020 22:43:25 +0100 Subject: [PATCH 30/76] activate new persistence route for GSHeets update functionality --- server/services/GSheetUpdateAvailable.php | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/server/services/GSheetUpdateAvailable.php b/server/services/GSheetUpdateAvailable.php index 4cdf11ec9..f430facd5 100644 --- a/server/services/GSheetUpdateAvailable.php +++ b/server/services/GSheetUpdateAvailable.php @@ -13,14 +13,25 @@ $vis_id = library\CommUtils::getParameter($_GET, "vis_id"); $gsheet_last_updated = library\CommUtils::getParameter($_GET, "gsheet_last_updated"); -$backend = isset($_GET["backend"]) ? library\CommUtils::getParameter($_GET, "backend") : "legacy"; +$persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); -if ($backend == "api") { - $return_data = array("status" => "error", "reason" => "Not implemented."); - $jsonData = json_encode($return_data); - library\CommUtils::echoOrCallback($jsonData, $_GET); +if ($persistence_backend == "api") { + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = json_decode($res["result"], true); + $rev_data = json_decode($data["rev_data"], true); + $timestamp_old = $rev_data["last_update"]; + $update_available = ($timestamp_old != $gsheet_last_updated) ? true : false; + $return_data = array("update_available" => $update_available); + $jsonData = json_encode($return_data); + library\CommUtils::echoOrCallback($jsonData, $_GET); + } } else { $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; $rev_data = json_decode($data["rev_data"], true); From 2bdcc481c6feb0c8b2409e05b0819c983e1276a4 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 27 Oct 2020 22:58:42 +0100 Subject: [PATCH 31/76] less logging clutter --- server/workers/services/src/apis/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index f5ce93ef7..b72adfcfc 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -166,7 +166,7 @@ def post(self): persistence_ns.logger.debug("writeRevision") vis_id = payload.get("vis_id") data = payload.get("data") - persistence_ns.logger.debug(data) + persistence_ns.logger.debug(data.keys()) write_revision(vis_id, data) result = {'success': True} headers = {'ContentType': 'application/json'} From 6395bb152ab340f71f1f5c212a6412d92a055bda Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 27 Oct 2020 23:20:43 +0100 Subject: [PATCH 32/76] less logging clutter --- server/workers/services/src/apis/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index b72adfcfc..e670423b4 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -166,7 +166,7 @@ def post(self): persistence_ns.logger.debug("writeRevision") vis_id = payload.get("vis_id") data = payload.get("data") - persistence_ns.logger.debug(data.keys()) + # persistence_ns.logger.debug(data) write_revision(vis_id, data) result = {'success': True} headers = {'ContentType': 'application/json'} From 4e524470e55cda5c1b7fc125e6f7cca8ba640fd4 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 28 Oct 2020 14:20:03 +0100 Subject: [PATCH 33/76] change default param --- examples/triple/data-config_triple.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/triple/data-config_triple.js b/examples/triple/data-config_triple.js index 35de799c5..39deecc49 100644 --- a/examples/triple/data-config_triple.js +++ b/examples/triple/data-config_triple.js @@ -32,7 +32,7 @@ var data_config = { share_modal: false, backend: "api", - persistence_backend: "api", + persistence_backend: "legacy", streamgraph_colors: ["#215A66", "#66214A", "#5D40FB", "#CB40FB", "#40C0FB", "#FB4068" , "#FBB240", "#40FBC8", "#fee4bc", "#bcfeec", "#c6bcfe", "#febcca"], From 5ab4cb2c7074da2afa9ee8441ed89213201ed826 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 28 Oct 2020 22:46:24 +0100 Subject: [PATCH 34/76] fix sha for bootstrap for dropdowns --- examples/triple/index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/triple/index.php b/examples/triple/index.php index 5dbf905e1..322114fdf 100644 --- a/examples/triple/index.php +++ b/examples/triple/index.php @@ -19,7 +19,7 @@ - + From 8d27a11ed55fd069e896c4926ce5ae4dbfbb678c Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Sun, 1 Nov 2020 19:48:09 +0100 Subject: [PATCH 35/76] first working draft of new endpoint --- server/workers/gsheets/src/search_gsheets.py | 93 +++++++++++++++++--- server/workers/services/src/apis/gsheets.py | 41 +++++++++ 2 files changed, 122 insertions(+), 12 deletions(-) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 586588b0f..2e6694b9c 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -84,7 +84,8 @@ class GSheetsClient(object): def __init__(self, redis_store, loglevel="INFO"): # If modifying these scopes, delete the file token.pickle. self.SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly', - 'https://www.googleapis.com/auth/drive.metadata.readonly'] + 'https://www.googleapis.com/auth/drive.metadata.readonly', + 'https://www.googleapis.com/auth/drive'] self.redis_store = redis_store self.register_services() self.last_updated = {} @@ -98,8 +99,12 @@ def __init__(self, redis_store, loglevel="INFO"): def authenticate(self): creds = None - tokenpath = os.path.join(pathlib.Path(__file__).parent.parent, "token.pickle") - credentialspath = os.path.join(pathlib.Path(__file__).parent.parent, "credentials.json") + try: + tokenpath = os.path.join(pathlib.Path(__file__).parent.parent, "token.pickle") + credentialspath = os.path.join(pathlib.Path(__file__).parent.parent, "credentials.json") + except NameError: + tokenpath = os.path.join(os.getcwd(), "token.pickle") + credentialspath = os.path.join(os.getcwd(), "credentials.json") if os.path.exists(tokenpath): with open(tokenpath, 'rb') as token: creds = pickle.load(token) @@ -278,28 +283,92 @@ def update(self, params): sheet_id = params.get('sheet_id') sheet_range = params.get('sheet_range') last_known_update = params.get('last_update') - if not sheet_id in self.last_updated: + if sheet_id not in self.last_updated: self.last_updated[sheet_id] = {} last_change = self.files.get(fileId=sheet_id, fields='modifiedTime').execute().get('modifiedTime') d = parse(last_change) last_update_timestamp_utc = d.strftime("%Y-%m-%d %H:%M:%S %Z") self.last_updated[sheet_id]["timestamp_utc"] = last_update_timestamp_utc sheet_has_changed = self.sheet_has_changed(sheet_id) - if (last_known_update is not None and - last_known_update != self.last_updated[sheet_id]["timestamp_utc"]): + if (last_known_update is not None + and last_known_update != self.last_updated[sheet_id]["timestamp_utc"]): res = self.get_new_mapdata(sheet_id, sheet_range, params) if sheet_has_changed is True: res = self.get_new_mapdata(sheet_id, sheet_range, params) return res + def create_knowledgebase(self, params): + try: + sheet_name = params.get('sheet_name') + project_name = params.get('project_name') + main_curator_email = params.get('main_curator_email') + knowledge_base_template_id = params.get('knowledge_base_template_id') + new_drive = self.create_new_drive(project_name) + new_drive_id = new_drive.get('id') + new_kb = self.duplicate_knowledgebase( + knowledge_base_template_id, sheet_name, + new_drive_id) + self.set_new_kb_permissions(new_drive, new_kb, main_curator_email) + res = {"status": "success"} + except Exception as e: + res = {"status": "error", "msg": str(e)} + return res + + def create_new_drive(self, project_name): + drive_metadata = {'name': project_name} + request_id = str(uuid.uuid4()) + new_drive = self.drive_service.drives().create(body=drive_metadata, + requestId=request_id, + fields='id').execute() + return new_drive + + def duplicate_knowledgebase(self, knowledge_base_template_id, sheet_name, + target_folder_id): + file_metadata = {'name': sheet_name, 'parents': [target_folder_id]} + new_kb = self.files.copy(fileId=knowledge_base_template_id, + body=file_metadata, + supportsAllDrives=True).execute() + return new_kb + + def set_new_kb_permissions(self, new_drive, new_kb, main_curator_email): + new_organizer_permission = { + 'type': 'user', + 'role': 'organizer', + 'emailAddress': main_curator_email + } + permission = self.drive_service.permissions().create( + fileId=new_drive.get('id'), + body=new_organizer_permission, + supportsAllDrives=True + ).execute() + new_fileorganizer_permission = { + 'type': 'user', + 'role': 'writer', + 'emailAddress': main_curator_email + } + permission = self.drive_service.permissions().create( + fileId=new_kb.get('id'), + body=new_fileorganizer_permission, + supportsAllDrives=True + ).execute() + + def run(self): while True: k, params, endpoint = self.next_item() self.logger.debug(k) self.logger.debug(params) - try: - res = self.update(params) - self.redis_store.set(k+"_output", json.dumps(res)) - except Exception as e: - self.logger.error(e) - self.logger.error(params) + if endpoint == "search": + try: + res = self.update(params) + self.redis_store.set(k+"_output", json.dumps(res)) + except Exception as e: + self.logger.error(e) + self.logger.error(params) + if endpoint == "create_kb": + try: + res = self.create_knowledgebase(params) + self.redis_store.set(k+"_output", json.dumps(res)) + except Exception as e: + self.logger.error(e) + self.logger.error(params) diff --git a/server/workers/services/src/apis/gsheets.py b/server/workers/services/src/apis/gsheets.py index 1289c054f..360dd5bfb 100644 --- a/server/workers/services/src/apis/gsheets.py +++ b/server/workers/services/src/apis/gsheets.py @@ -30,6 +30,21 @@ description='timestamp of last known GSheets edit')}) +create_kb_query = gsheets_ns.model("CreateKnowledgebaseQuery", + {"sheet_name": fields.String(example='CoVis', + description='Title of the knowledge base', + required=True), + "project_name": fields.String(example="Collaborative Visualization", + description="Project Name", + required=True), + "main_curator_email": fields.String(example='name@gmail.com', + description='gmail account of main curator', + required=True), + "knowledge_base_template_id": fields.String(example='1q0947856', + description='google drive ID of source template', + required=True)}) + + @gsheets_ns.route('/search') class Search(Resource): @gsheets_ns.doc(responses={200: 'OK', @@ -59,3 +74,29 @@ def post(self): except Exception as e: gsheets_ns.logger.error(e) abort(500, "Problem encountered during processing, sorry.") + + +@gsheets_ns.route('/createKnowledgebase') +class createKnowledgebase(Resource): + @gsheets_ns.doc(responses={200: 'OK', + 400: 'Invalid parameters'}) + @gsheets_ns.expect(create_kb_query) + @gsheets_ns.produces(["application/json"]) + def post(self): + params = request.get_json() + gsheets_ns.logger.debug(params) + k = str(uuid.uuid4()) + d = {"id": k, "params": params, + "endpoint": "create_kb"} + gsheets_ns.logger.debug(d) + redis_store.rpush("gsheets", json.dumps(d)) + result = get_key(redis_store, k) + try: + headers = {} + headers["Content-Type"] = "application/json" + return make_response(result, + 200, + headers) + except Exception as e: + gsheets_ns.logger.error(e) + abort(500, "Problem encountered during processing, sorry.") From 9bed439df9cc29953e5c57886c06018aadebe992 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Sun, 1 Nov 2020 23:47:32 +0100 Subject: [PATCH 36/76] php backend local call --- server/services/createNewGSheet.php | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 server/services/createNewGSheet.php diff --git a/server/services/createNewGSheet.php b/server/services/createNewGSheet.php new file mode 100644 index 000000000..69bf5ff7a --- /dev/null +++ b/server/services/createNewGSheet.php @@ -0,0 +1,40 @@ + $sheet_name, + "project_name" => $project_name, + "main_curator_email" => $main_curator_email, + "knowledge_base_template_id" => $knowledge_base_template_id)); +$res = library\CommUtils::call_api($route, $payload); +if ($res["httpcode"] != 200) { + echo json_encode($res); +} else { + $result = json_decode($res["result"], true); + echo $result; +} + +?> From d86c0671c0f296f27023eb41d11f245d9841fd56 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 10:11:08 +0100 Subject: [PATCH 37/76] robustify validation --- server/workers/gsheets/src/search_gsheets.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 2e6694b9c..6c6378f4c 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -50,10 +50,6 @@ def get_key(store, key): Column('Tags', []), Column('Access', []), Column('Area', []), - Column('Included in map', [InListValidation(["yes", "no"])]), - Column('Ready for publication?', [InListValidation(["yes", "no"])]), - Column('Validation', []), - Column('Curator notes', []), Column('Comment 1', []), Column('Author Comment 1', []), Column('Comment 2', []), @@ -195,7 +191,7 @@ def validate_data(df): df.drop([0, 1, 2], inplace=True) df = df[(df["Ready for publication?"] == "yes") & (df["Included in map"] == "yes")] - errors = schema.validate(df) + errors = schema.validate(df, columns=schema.get_column_names()) errors_index_rows = [e.row for e in errors] error_columns = [e.column for e in errors] error_reasons = [" ".join([str(e.value), str(e.message)]) for e in errors] @@ -352,7 +348,6 @@ def set_new_kb_permissions(self, new_drive, new_kb, main_curator_email): supportsAllDrives=True ).execute() - def run(self): while True: k, params, endpoint = self.next_item() From 748ddbda776d938354d3675b98413b7d9d9e18f5 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 13:40:39 +0100 Subject: [PATCH 38/76] additional context data added --- server/workers/gsheets/src/search_gsheets.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 6c6378f4c..67d486d5f 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -37,6 +37,12 @@ def get_key(store, key): return result +additional_context_fields = [ + "Project name", "Project website", "Topic", + "Main curator name", "Main curator e-mail" +] + + schema = Schema([ Column('ID', []), Column('Title', []), @@ -255,9 +261,14 @@ def create_input_data(df): input_data["text"] = text.to_json(orient='records') return input_data + def get_additional_context_data(self, df): + df.columns = df.iloc[0] + df.drop([0], inplace=True) + return df[additional_context_fields].iloc[0].to_dict() + def get_new_mapdata(self, sheet_id, sheet_range, params): raw = self.get_sheet_content(sheet_id, sheet_range) - clean_df, errors, errors_df = self.validate_data(raw) + clean_df, errors, errors_df = self.validate_data(raw.copy()) input_data = self.create_input_data(clean_df) map_k = str(uuid.uuid4()) map_input = {} @@ -270,6 +281,7 @@ def get_new_mapdata(self, sheet_id, sheet_range, params): res = {} res["data"] = result_df.to_json(orient="records") res["errors"] = errors_df.to_dict(orient="records") + res["additional_context"] = self.get_additional_context_data(raw.copy()) res["sheet_id"] = sheet_id res["last_update"] = self.last_updated[sheet_id]["timestamp_utc"] return res From 0bd0cae3cd34d0b759e04b85aa982646e6390a35 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 14:08:22 +0100 Subject: [PATCH 39/76] additional context fields and backwards compatibility --- server/services/getGSheetsMap.php | 36 ++++++++++++-------- server/workers/gsheets/src/search_gsheets.py | 13 ++++--- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index 7dd252716..6af3d19b7 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -26,13 +26,17 @@ } else { $data = json_decode($res["result"], true); $rev_data = json_decode($data["rev_data"], true); - $return_data = array("context" => array("id" => $data["rev_vis"], - "query" => $data["vis_query"], - "service" => $data["vis_title"], - "timestamp" => $data["rev_timestamp"], - "params" => $data["vis_params"], - "sheet_id" => $rev_data["sheet_id"], - "last_update" => $rev_data["last_update"]), + $context = array("id" => $data["rev_vis"], + "query" => $data["vis_query"], + "service" => $data["vis_title"], + "timestamp" => $data["rev_timestamp"], + "params" => $data["vis_params"], + "sheet_id" => $rev_data["sheet_id"], + "last_update" => $rev_data["last_update"]); + if (isset($rev_data["additional_context"])) { + $context = array_merge($context, $rev_data["additional_context"]); + } + $return_data = array("context" => $context, "data" => $rev_data["data"], "errors" => $rev_data["errors"]); $jsonData = json_encode($return_data); @@ -41,13 +45,17 @@ } else { $data = $persistence->getLastVersion($vis_id, $details = false, $context = true)[0]; $rev_data = json_decode($data["rev_data"], true); - $return_data = array("context" => array("id" => $data["rev_vis"], - "query" => $data["vis_query"], - "service" => $data["vis_title"], - "timestamp" => $data["rev_timestamp"], - "params" => $data["vis_params"], - "sheet_id" => $rev_data["sheet_id"], - "last_update" => $rev_data["last_update"]), + $context = array("id" => $data["rev_vis"], + "query" => $data["vis_query"], + "service" => $data["vis_title"], + "timestamp" => $data["rev_timestamp"], + "params" => $data["vis_params"], + "sheet_id" => $rev_data["sheet_id"], + "last_update" => $rev_data["last_update"]); + if (isset($rev_data["additional_context"])) { + $context = array_merge($context, $rev_data["additional_context"]); + } + $return_data = array("context" => $context, "data" => $rev_data["data"], "errors" => $rev_data["errors"]); $jsonData = json_encode($return_data); diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 67d486d5f..d2d74ab2b 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -262,9 +262,12 @@ def create_input_data(df): return input_data def get_additional_context_data(self, df): - df.columns = df.iloc[0] - df.drop([0], inplace=True) - return df[additional_context_fields].iloc[0].to_dict() + if all(field in df.columns for field in additional_context_fields): + df.columns = df.iloc[0] + df.drop([0], inplace=True) + return df[additional_context_fields].iloc[0].to_dict() + else: + return None def get_new_mapdata(self, sheet_id, sheet_range, params): raw = self.get_sheet_content(sheet_id, sheet_range) @@ -281,7 +284,9 @@ def get_new_mapdata(self, sheet_id, sheet_range, params): res = {} res["data"] = result_df.to_json(orient="records") res["errors"] = errors_df.to_dict(orient="records") - res["additional_context"] = self.get_additional_context_data(raw.copy()) + additional_context = self.get_additional_context_data(raw.copy()) + if additional_context: + res["additional_context"] = additional_context res["sheet_id"] = sheet_id res["last_update"] = self.last_updated[sheet_id]["timestamp_utc"] return res From 23c7a36b4e5876d5629a911f64d3b0e9e3965dff Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 21:47:14 +0100 Subject: [PATCH 40/76] integration of extension --- server/services/getGSheetsMap.php | 4 ++-- server/workers/gsheets/src/search_gsheets.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index 6af3d19b7..4a8dd5001 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -33,7 +33,7 @@ "params" => $data["vis_params"], "sheet_id" => $rev_data["sheet_id"], "last_update" => $rev_data["last_update"]); - if (isset($rev_data["additional_context"])) { + if (array_key_exists("additional_context", $rev_data)) { $context = array_merge($context, $rev_data["additional_context"]); } $return_data = array("context" => $context, @@ -52,7 +52,7 @@ "params" => $data["vis_params"], "sheet_id" => $rev_data["sheet_id"], "last_update" => $rev_data["last_update"]); - if (isset($rev_data["additional_context"])) { + if (array_key_exists("additional_context", $rev_data)) { $context = array_merge($context, $rev_data["additional_context"]); } $return_data = array("context" => $context, diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index d2d74ab2b..7b3f63a5f 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -262,9 +262,9 @@ def create_input_data(df): return input_data def get_additional_context_data(self, df): + df.columns = df.iloc[0] + df.drop([0], inplace=True) if all(field in df.columns for field in additional_context_fields): - df.columns = df.iloc[0] - df.drop([0], inplace=True) return df[additional_context_fields].iloc[0].to_dict() else: return None @@ -288,7 +288,7 @@ def get_new_mapdata(self, sheet_id, sheet_range, params): if additional_context: res["additional_context"] = additional_context res["sheet_id"] = sheet_id - res["last_update"] = self.last_updated[sheet_id]["timestamp_utc"] + res["last_update"] = self.last_updated.get(sheet_id, {}).get("timestamp_utc") return res def update(self, params): From c435373d6574adc10657e00af22fbf3596f56760 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 22:19:17 +0100 Subject: [PATCH 41/76] default params --- server/services/updateGSheetsMap.php | 4 ++-- server/workers/gsheets/src/search_gsheets.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/services/updateGSheetsMap.php b/server/services/updateGSheetsMap.php index 762031bc3..6280ed884 100644 --- a/server/services/updateGSheetsMap.php +++ b/server/services/updateGSheetsMap.php @@ -15,7 +15,7 @@ $longopts = array( "q:", "sheet_id:", - "last_update" + "last_update:" ); $options = getopt($shortopts, $longopts); $dirty_query = $options["q"]; @@ -39,7 +39,7 @@ , false, null, 3 , "area_uri", "subject" , $sheet_id, false - , "api", "api"); + , "api", "legacy"); echo $result; diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 7b3f63a5f..62d9966a0 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -298,7 +298,9 @@ def update(self, params): last_known_update = params.get('last_update') if sheet_id not in self.last_updated: self.last_updated[sheet_id] = {} - last_change = self.files.get(fileId=sheet_id, fields='modifiedTime').execute().get('modifiedTime') + last_change = self.files.get(fileId=sheet_id, + fields='modifiedTime', + supportsAllDrives=True).execute().get('modifiedTime') d = parse(last_change) last_update_timestamp_utc = d.strftime("%Y-%m-%d %H:%M:%S %Z") self.last_updated[sheet_id]["timestamp_utc"] = last_update_timestamp_utc From 2c9c7a484c5acf396d235b88738592c491752518 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 23:21:04 +0100 Subject: [PATCH 42/76] default params --- examples/triple/config_example.php | 1 + examples/triple/search.js | 5 +++-- server/services/searchTRIPLE.php | 4 +++- server/services/updateGSheetsMap.php | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/triple/config_example.php b/examples/triple/config_example.php index 08e7e1f57..5dad8807f 100644 --- a/examples/triple/config_example.php +++ b/examples/triple/config_example.php @@ -2,4 +2,5 @@ $HEADSTART_PATH = "../../"; $WEBSITE_PATH = "http://localhost/triple/"; $SNAPSHOT_PATH = $WEBSITE_PATH . "server/storage/"; +$PERSISTENCE_BACKEND = "legacy"; ?> diff --git a/examples/triple/search.js b/examples/triple/search.js index 2ade374ff..b11b23cb0 100644 --- a/examples/triple/search.js +++ b/examples/triple/search.js @@ -77,6 +77,7 @@ $("#searchform").validate({ var doSubmit = function (data, newWindow, callback) { data += "&today=" + new Date().toLocaleDateString("en-US"); + data += "&persistence_backend=" + data_config["persistence_backend"]; var params = $("#searchform").serializeArray().reduce(function(obj, item) { obj[item.name] = item.value; return obj; @@ -110,9 +111,9 @@ var doSubmit = function (data, newWindow, callback) { + "?query=" + data.query + ((params.hasOwnProperty("language"))?("&lang=" + params.language):("")) + ((params.hasOwnProperty("sg_method"))?("&algo=" + params.sg_method):("")) - + "&file=" + file + + "&file=" + file + "&service=" + data_config.service - + "&service_name=" + service_name + + "&service_name=" + service_name + "&vis_type=" + vis_type; return false; } else { diff --git a/server/services/searchTRIPLE.php b/server/services/searchTRIPLE.php index a04173cca..99f2ce38c 100644 --- a/server/services/searchTRIPLE.php +++ b/server/services/searchTRIPLE.php @@ -9,8 +9,10 @@ $dirty_query = library\CommUtils::getParameter($_POST, "q"); $precomputed_id = (isset($_POST["unique_id"]))?($_POST["unique_id"]):(null); +$persistence_backend = (isset($_POST["persistence_backend"]))?($_POST["persistence_backend"]):("legacy"); $post_params = $_POST; +unset($post_params["persistence_backend"]); $result = search("triple", $dirty_query , $post_params, array("from", "to", "sorting", "vis_type", "language", "limit", "sg_method") @@ -18,7 +20,7 @@ , true, null, 3 , "area_uri", "subject" , $precomputed_id, true - , "api", "api"); + , "api", $persistence_backend); echo $result diff --git a/server/services/updateGSheetsMap.php b/server/services/updateGSheetsMap.php index 762031bc3..3791e39e9 100644 --- a/server/services/updateGSheetsMap.php +++ b/server/services/updateGSheetsMap.php @@ -39,7 +39,7 @@ , false, null, 3 , "area_uri", "subject" , $sheet_id, false - , "api", "api"); + , "api", "legacy"); echo $result; From 85813573e2abde11a7aee20b0a9dd62360cb4b11 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 5 Nov 2020 23:48:06 +0100 Subject: [PATCH 43/76] cleanup --- examples/triple/config_example.php | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/triple/config_example.php b/examples/triple/config_example.php index 5dad8807f..08e7e1f57 100644 --- a/examples/triple/config_example.php +++ b/examples/triple/config_example.php @@ -2,5 +2,4 @@ $HEADSTART_PATH = "../../"; $WEBSITE_PATH = "http://localhost/triple/"; $SNAPSHOT_PATH = $WEBSITE_PATH . "server/storage/"; -$PERSISTENCE_BACKEND = "legacy"; ?> From eab03417087ab2c6a7e7dd3044037693171a6403 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Fri, 6 Nov 2020 00:33:59 +0100 Subject: [PATCH 44/76] param names --- server/workers/gsheets/src/search_gsheets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 62d9966a0..5f9c7850d 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -265,7 +265,10 @@ def get_additional_context_data(self, df): df.columns = df.iloc[0] df.drop([0], inplace=True) if all(field in df.columns for field in additional_context_fields): - return df[additional_context_fields].iloc[0].to_dict() + additional_context = df[additional_context_fields].iloc[0].to_dict() + for k in additional_context_fields: + additional_context[k.lower().replace(" ", "_")] = additional_context.pop(k) + return additional_context else: return None From d001a7ccac3c71d947deb94b2d0076055ba9a48e Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Sat, 7 Nov 2020 14:49:29 +0100 Subject: [PATCH 45/76] prefill new knowledge base additional context --- server/workers/gsheets/src/search_gsheets.py | 22 ++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 5f9c7850d..20ead7810 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -176,8 +176,8 @@ def next_item(self): return k, params, endpoint def get_sheet_content(self, sheet_id, sheet_range): - res = res = self.sheet.values().get(spreadsheetId=sheet_id, - range=sheet_range).execute() + res = self.sheet.values().get(spreadsheetId=sheet_id, + range=sheet_range).execute() raw = pd.DataFrame(res.get('values')) return raw @@ -327,6 +327,7 @@ def create_knowledgebase(self, params): knowledge_base_template_id, sheet_name, new_drive_id) self.set_new_kb_permissions(new_drive, new_kb, main_curator_email) + self.prefill_additional_context(new_kb, params) res = {"status": "success"} except Exception as e: res = {"status": "error", "msg": str(e)} @@ -348,6 +349,23 @@ def duplicate_knowledgebase(self, knowledge_base_template_id, sheet_name, supportsAllDrives=True).execute() return new_kb + def prefill_additional_context(self, new_kb, params): + context_range = 'Resources!Y2:AC2' + value_input_option = 'RAW' # USER_ENTERED + values = [ + [params.get('project_name', ''), + params.get('project_website', ''), + params.get('topic', ''), + params.get('main_curator_name', ''), + params.get('main_curator_email', '')] + ] + body = {'values': values} + result = self.sheet.values().update( + spreadsheetId=new_kb.get('id'), range=context_range, + valueInputOption=value_input_option, body=body).execute() + + + def set_new_kb_permissions(self, new_drive, new_kb, main_curator_email): new_organizer_permission = { 'type': 'user', From 3e4688782bfc4b1cb1f70f58318a8779f679eb98 Mon Sep 17 00:00:00 2001 From: pkraker Date: Mon, 9 Nov 2020 20:35:28 +0100 Subject: [PATCH 46/76] WIP: Add context from map json to index.php --- examples/gsheets/config.example.php | 3 --- examples/gsheets/index.php | 32 +++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/examples/gsheets/config.example.php b/examples/gsheets/config.example.php index 3c12cd570..d1ba22a19 100644 --- a/examples/gsheets/config.example.php +++ b/examples/gsheets/config.example.php @@ -1,9 +1,6 @@ @@ -10,7 +26,7 @@ - Knowledge Map of <?php echo $TOPIC ?> + Knowledge Map of <?php echo $context["topic"] ?> @@ -77,7 +93,7 @@ class: "error-row-top" } function updateCheck(context) { - $.getJSON("server/services/GSheetUpdateAvailable.php?vis_id=&gsheet_last_updated=" + encodeURIComponent(context.last_update), + $.getJSON("server/services/GSheetUpdateAvailable.php?vis_id=&gsheet_last_updated=" + encodeURIComponent(context.last_update), function(output) { if (output.update_available) { $("#reload").addClass("show-reload-button"); @@ -89,7 +105,7 @@ function(output) { function updateMap() { - $.getJSON("server/services/updateGSheetsMap.php?q=covis&sheet_id=", + $.getJSON("server/services/updateGSheetsMap.php?q=covis&sheet_id=", function(output) { }); } @@ -109,22 +125,22 @@ function(output) { - - + + - Knowledge Map of <?php echo $context["topic"] ?> + Knowledge Map of <?php echo $topic ?> @@ -122,7 +136,7 @@ function(output) { check_update = window.setInterval(updateCheck, 6000, e.detail.data.context); }); - + @@ -135,12 +149,12 @@ function(output) { , file: "" }] - data_config.title = "Knowledge map of "; + data_config.title = "Knowledge map of "; data_config.service_name= 'Spreadsheet'; data_config.intro = { title: "About this knowledge map" - , body: "

Knowledge maps provide an instant overview of a topic by showing the main areas at a glance and resources related to each area. This makes it possible to easily identify useful, pertinent information.

Research areas are displayed as bubbles. By clicking on one of the bubbles, you can inspect the resources assigned to it. The size of the bubbles is relative to the number of resources assigned to it. Closeness of bubbles implies subject similarity. The closer two bubbles, the closer they are subject-wise. Centrality of bubbles implies subject similarity with the rest of the map, not importance. The closer a bubble is to the center, the closer it is subject-wise to all the other bubbles in the map.

Content

The content of this knowledge map is curated by as part of " . $context["project_name"] . ""; } else { echo $context["project_name"]; } ?>. Resources are collected and annotated in a spreadsheet, which is then transformed into the knowledge map.

Software

The knowledge map is based on the award-winning software developed by Open Knowledge Maps. For more information and the ability to create knowledge maps based on 250+ million documents, please see openknowledgemaps.org. To get in touch, please e-mail us at info@openknowledgemaps.org

Rights

The curator(s) are solely responsible for the content of the knowledge map. Unless otherwise noted, all content is licensed under a Creative Commons Attribution 4.0 International License. The spreadsheet is made available under CC0 (Public Domain Dedication). The knowledge mapping software is open source and hosted on Github.

" + , body: "

Knowledge maps provide an instant overview of a topic by showing the main areas at a glance and resources related to each area. This makes it possible to easily identify useful, pertinent information.

Research areas are displayed as bubbles. By clicking on one of the bubbles, you can inspect the resources assigned to it. The size of the bubbles is relative to the number of resources assigned to it. Closeness of bubbles implies subject similarity. The closer two bubbles, the closer they are subject-wise. Centrality of bubbles implies subject similarity with the rest of the map, not importance. The closer a bubble is to the center, the closer it is subject-wise to all the other bubbles in the map.

Content

The content of this knowledge map is curated by as part of " . $project_name . ""; } else { echo $project_name; } ?>. Resources are collected and annotated in a spreadsheet, which is then transformed into the knowledge map.

Software

The knowledge map is based on the award-winning software developed by Open Knowledge Maps. For more information and the ability to create knowledge maps based on 250+ million documents, please see openknowledgemaps.org. To get in touch, please e-mail us at info@openknowledgemaps.org

Rights

The curator(s) are solely responsible for the content of the knowledge map. Unless otherwise noted, all content is licensed under a Creative Commons Attribution 4.0 International License. The spreadsheet is made available under CC0 (Public Domain Dedication). The knowledge mapping software is open source and hosted on Github.

" }; $(document).ready(function () { From d895091cc0b28917089284283f21f288cb752718 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 11 Nov 2020 23:53:33 +0100 Subject: [PATCH 54/76] postgres security via pg_hba.conf --- docker-compose.yml | 3 + server/workers/example_pg_hba.conf | 95 +++++++++++++++++++ .../services/src/config/example_settings.py | 13 ++- 3 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 server/workers/example_pg_hba.conf diff --git a/docker-compose.yml b/docker-compose.yml index 402e2e12d..a992da3ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,9 @@ services: POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" volumes: - db-data:/var/lib/postgresql/data + - ./server/workers/pg_hba.conf:/var/lib/postgresql/data/pg_hba.conf + ports: + - '127.0.0.1:5432:5432' pgadmin: image: 'dpage/pgadmin4' diff --git a/server/workers/example_pg_hba.conf b/server/workers/example_pg_hba.conf new file mode 100644 index 000000000..1598392cd --- /dev/null +++ b/server/workers/example_pg_hba.conf @@ -0,0 +1,95 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# CAUTION: Configuring the system for local "trust" authentication +# allows any local user to connect as any PostgreSQL user, including +# the database superuser. If you do not trust all your local users, +# use another authentication method. + + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +# host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust +# Allow replication connections from localhost, by a user with the +# replication privilege. +local replication all trust +host replication all 127.0.0.1/32 trust +host replication all ::1/128 trust + +host all headstart all md5 diff --git a/server/workers/services/src/config/example_settings.py b/server/workers/services/src/config/example_settings.py index d602840e2..29251bb36 100644 --- a/server/workers/services/src/config/example_settings.py +++ b/server/workers/services/src/config/example_settings.py @@ -1,6 +1,13 @@ BEHIND_PROXY = True -SWAGGER_BASEPATH = "" -# change to "production" +POSTGRES = { + 'user': 'user', + 'pw': 'pw', + 'db': 'db', + 'host': '127.0.0.1', + 'port': '5432', +} +SQLALCHEMY_DATABASE_URI = 'postgresql://%(user)s:%(pw)s@%(host)s:%(port)s/%(db)s' % POSTGRES +# SQLALCHEMY_DATABASE_URI = 'sqlite:////var/data/db.sqlite' +SQLALCHEMY_TRACK_MODIFICATIONS = False ENV = "development" -# change to False DEBUG = True From eadf4da4c95716e81cc7071688e1c1bfcc14bc88 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 12 Nov 2020 01:02:29 +0100 Subject: [PATCH 55/76] added getContext api route --- server/services/getContext.php | 34 ++++++++++++--- .../workers/services/src/apis/persistence.py | 41 +++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/server/services/getContext.php b/server/services/getContext.php index 78bd92148..adafe3cfc 100644 --- a/server/services/getContext.php +++ b/server/services/getContext.php @@ -13,12 +13,34 @@ $ini_array = library\Toolkit::loadIni($INI_DIR); $vis_id = library\CommUtils::getParameter($_GET, "vis_id"); +$persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); -$data = $persistence->getContext($vis_id)[0]; -$return_data = array("id" => $data["rev_vis"], "query" => $data["vis_query"], "service" => $data["vis_title"] - , "timestamp" => $data["rev_timestamp"], "params" => $data["vis_params"]); - -$jsonData = json_encode($return_data); -library\CommUtils::echoOrCallback($jsonData, $_GET); +if ($persistence_backend === "api") { + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getContext"; + $payload = json_encode(array("vis_id" => $vis_id)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + library\CommUtils::echoOrCallback($res, $_GET); + } else { + $data = json_decode($res["result"], true); + $return_data = array("id" => $data["rev_vis"], + "query" => $data["vis_query"], + "service" => $data["vis_title"], + "timestamp" => $data["rev_timestamp"], + "params" => $data["vis_params"]); + $jsonData = json_encode($return_data); + library\CommUtils::echoOrCallback($jsonData, $_GET); + } +} else { + $data = $persistence->getContext($vis_id)[0]; + $return_data = array("id" => $data["rev_vis"], + "query" => $data["vis_query"], + "service" => $data["vis_title"], + "timestamp" => $data["rev_timestamp"], + "params" => $data["vis_params"]); + + $jsonData = json_encode($return_data); + library\CommUtils::echoOrCallback($jsonData, $_GET); +} diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 5ba6d509c..7105a49ac 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -109,6 +109,24 @@ def get_revision(vis_id, rev_id, details=False, context=False): return rev.rev_data +def get_context(vis_id): + vis, rev = (db.session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == Visualizations.vis_latest) + ).first() + res = { + "rev_vis": rev.rev_vis, + "vis_query": rev.vis_query, + "vis_title": vis.vis_title, + "rev_timestamp": rev.rev_timestamp, + "vis_params": vis.vis_params + } + return res + + @persistence_ns.route('/existsVisualization') class existsVisualization(Resource): @@ -222,6 +240,29 @@ def post(self): return make_response(result, 200, headers) +@persistence_ns.route('/getContext') +class getContext(Resource): + + @persistence_ns.produces(["application/json"]) + def post(self): + try: + payload = request.get_json() + persistence_ns.logger.debug("getLastVersion") + persistence_ns.logger.debug(payload) + vis_id = payload.get('vis_id') + result = get_context(vis_id) + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), + 200, + headers) + except Exception as e: + result = {'success': False, 'reason': str(e)} + headers = {'ContentType': 'application/json'} + return make_response(jsonify(result), + 500, + headers) + + @persistence_ns.route('/createID') class createID(Resource): From ec7029a0713b1df5bb8384d6ec4f90a0e7b65f2d Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 12 Nov 2020 18:47:38 +0100 Subject: [PATCH 56/76] multiple db wip --- server/workers/services/src/database.py | 3 +++ server/workers/services/src/manage.py | 5 +++-- server/workers/services/src/models.py | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server/workers/services/src/database.py b/server/workers/services/src/database.py index f606adc00..63c9fdfcf 100644 --- a/server/workers/services/src/database.py +++ b/server/workers/services/src/database.py @@ -1,4 +1,7 @@ from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.orm import sessionmaker +from sqlalchemy import create_engine db = SQLAlchemy() +Session = sessionmaker() diff --git a/server/workers/services/src/manage.py b/server/workers/services/src/manage.py index 23a19b74c..acb4cb8a5 100644 --- a/server/workers/services/src/manage.py +++ b/server/workers/services/src/manage.py @@ -1,10 +1,11 @@ from flask_sqlalchemy import SQLAlchemy from app import app -from models import Visualizations, Revisions +from models import Visualizations, Revisions, TripleRevisions, TripleVisualizations from database import db if __name__ == '__main__': db.init_app(app) with app.app_context(): - db.create_all() + db.create_all(bind=None) + db.create_all(bind=['triple', 'gsheets', 'openaire']) diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index 58eacbd3f..13e6e99c8 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -33,3 +33,11 @@ class Revisions(db.Model): def as_dict(self): return {c.name: getattr(self, c.name) for c in self.__table__.columns} + + +class TripleVisualizations(Visualizations): + pass + + +class TripleRevisions(Revisions): + pass From 014fc231b5b1890934f797c898e85cc74d9a458f Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 12 Nov 2020 18:53:30 +0100 Subject: [PATCH 57/76] add parameter to example --- examples/gsheets/index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gsheets/index.php b/examples/gsheets/index.php index 7ad47c5a9..3e0a5a381 100644 --- a/examples/gsheets/index.php +++ b/examples/gsheets/index.php @@ -23,7 +23,7 @@ function checkReturn($object, $field, $default = "") { $protocol = !empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off' ? 'https:' : 'http:'; $headstart_url = $protocol . $SITE_URL . $HEADSTART_PATH; -$context_json = curl_get_contents($headstart_url . "server/services/getContext.php?vis_id=$SHEET_ID"); +$context_json = curl_get_contents($headstart_url . "server/services/getContext.php?vis_id=$SHEET_ID" . "&persistence_backend=$PERSISTENCE_BACKEND"); $context = json_decode($context_json, true); $topic = checkReturn($context, "topic"); From c7cb699ad5808753766236fcda08bdaf53e66793 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Thu, 12 Nov 2020 19:25:37 +0100 Subject: [PATCH 58/76] getContext api route --- examples/gsheets/index.php | 2 +- server/services/getContext.php | 10 ++++++++-- server/workers/services/src/apis/persistence.py | 11 ++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/examples/gsheets/index.php b/examples/gsheets/index.php index 3e0a5a381..3e5e2b7c3 100644 --- a/examples/gsheets/index.php +++ b/examples/gsheets/index.php @@ -23,7 +23,7 @@ function checkReturn($object, $field, $default = "") { $protocol = !empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off' ? 'https:' : 'http:'; $headstart_url = $protocol . $SITE_URL . $HEADSTART_PATH; -$context_json = curl_get_contents($headstart_url . "server/services/getContext.php?vis_id=$SHEET_ID" . "&persistence_backend=$PERSISTENCE_BACKEND"); +$context_json = curl_get_contents($headstart_url . "server/services/getContext.php?vis_id=$SHEET_ID" . "&persistence_backend=$PERSISTENCE_BACKEND" . "&revision_context=true"); $context = json_decode($context_json, true); $topic = checkReturn($context, "topic"); diff --git a/server/services/getContext.php b/server/services/getContext.php index adafe3cfc..c18e739bf 100644 --- a/server/services/getContext.php +++ b/server/services/getContext.php @@ -14,12 +14,13 @@ $vis_id = library\CommUtils::getParameter($_GET, "vis_id"); $persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; +$revision_context = isset($_GET["revision_context"]) ? library\CommUtils::getParameter($_GET, "revision_context") : false; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($persistence_backend === "api") { $route = $ini_array["general"]["api_url"] . "/persistence" . "/getContext"; - $payload = json_encode(array("vis_id" => $vis_id)); + $payload = json_encode(array("vis_id" => $vis_id, "revision_context" => $revision_context)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { library\CommUtils::echoOrCallback($res, $_GET); @@ -30,6 +31,9 @@ "service" => $data["vis_title"], "timestamp" => $data["rev_timestamp"], "params" => $data["vis_params"]); + if (array_key_exists("additional_context", $data)) { + $return_data = array_merge($return_data, $data["additional_context"]); + } $jsonData = json_encode($return_data); library\CommUtils::echoOrCallback($jsonData, $_GET); } @@ -40,7 +44,9 @@ "service" => $data["vis_title"], "timestamp" => $data["rev_timestamp"], "params" => $data["vis_params"]); - + if (array_key_exists("additional_context", $data)) { + $return_data = array_merge($return_data, $data["additional_context"]); + } $jsonData = json_encode($return_data); library\CommUtils::echoOrCallback($jsonData, $_GET); } diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 7105a49ac..2cf214c8a 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -109,7 +109,7 @@ def get_revision(vis_id, rev_id, details=False, context=False): return rev.rev_data -def get_context(vis_id): +def get_context(vis_id, revision_context=False): vis, rev = (db.session .query(Visualizations, Revisions) .select_from(Visualizations, Revisions) @@ -124,6 +124,9 @@ def get_context(vis_id): "rev_timestamp": rev.rev_timestamp, "vis_params": vis.vis_params } + if revision_context == 'true': + data = json.loads(rev.rev_data) + res["additional_context"] = data.get("additional_context", {}) return res @@ -247,10 +250,12 @@ class getContext(Resource): def post(self): try: payload = request.get_json() - persistence_ns.logger.debug("getLastVersion") + persistence_ns.logger.debug("getContext") persistence_ns.logger.debug(payload) vis_id = payload.get('vis_id') - result = get_context(vis_id) + revision_context = payload.get('revision_context', False) + result = get_context(vis_id, revision_context) + persistence_ns.logger.debug(result) headers = {'ContentType': 'application/json'} return make_response(jsonify(result), 200, From 5316487e605b5bf8580307ad8df64f56fa82fb14 Mon Sep 17 00:00:00 2001 From: pkraker Date: Fri, 13 Nov 2020 17:02:07 +0100 Subject: [PATCH 59/76] Improved error handling for gsheets example --- examples/gsheets/data-config.js | 3 ++- examples/gsheets/index.php | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/examples/gsheets/data-config.js b/examples/gsheets/data-config.js index 73a80548e..a9d36f572 100644 --- a/examples/gsheets/data-config.js +++ b/examples/gsheets/data-config.js @@ -22,7 +22,7 @@ var data_config = { show_infolink_areas: false, show_context: true, - create_title_from_context: false, + create_title_from_context: true, show_context_timestamp: true, show_loading_screen: true, @@ -52,4 +52,5 @@ var data_config = { embed_modal: true, share_modal: false, + persistence_backend: "api" }; diff --git a/examples/gsheets/index.php b/examples/gsheets/index.php index 3e5e2b7c3..806dd1827 100644 --- a/examples/gsheets/index.php +++ b/examples/gsheets/index.php @@ -21,16 +21,22 @@ function checkReturn($object, $field, $default = "") { } } +function addScheme($url, $scheme = 'http://') { + return parse_url($url, PHP_URL_SCHEME) === null ? + $scheme . $url : $url; +} + $protocol = !empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off' ? 'https:' : 'http:'; $headstart_url = $protocol . $SITE_URL . $HEADSTART_PATH; -$context_json = curl_get_contents($headstart_url . "server/services/getContext.php?vis_id=$SHEET_ID" . "&persistence_backend=$PERSISTENCE_BACKEND" . "&revision_context=true"); +$context_json = curl_get_contents($headstart_url . "server/services/getContext.php?vis_id=$SHEET_ID&persistence_backend=$PERSISTENCE_BACKEND&revision_context=true"); $context = json_decode($context_json, true); $topic = checkReturn($context, "topic"); $main_curator_name = checkReturn($context, "main_curator_name"); $main_curator_email = checkReturn($context, "main_curator_email"); $project_name = checkReturn($context, "project_name"); -$project_website = checkReturn($context, "project_website", null); +$project_website_raw = checkReturn($context, "project_website", null); +$project_website = ($project_website_raw !== null)?(addScheme($project_website_raw)):(null); ?> @@ -107,7 +113,9 @@ class: "error-row-top" } function updateCheck(context) { - $.getJSON("server/services/GSheetUpdateAvailable.php?vis_id=&persistence_backend=&gsheet_last_updated=" + encodeURIComponent(context.last_update), + let last_update = (typeof context !== "undefined" && context.hasOwnProperty("last_update"))?(context.last_update):(""); + + $.getJSON("server/services/GSheetUpdateAvailable.php?vis_id=&persistence_backend=&gsheet_last_updated=" + encodeURIComponent(last_update), function(output) { if (output.update_available) { $("#reload").addClass("show-reload-button"); @@ -149,12 +157,11 @@ function(output) { , file: "" }] - data_config.title = "Knowledge map of "; data_config.service_name= 'Spreadsheet'; data_config.intro = { title: "About this knowledge map" - , body: "

Knowledge maps provide an instant overview of a topic by showing the main areas at a glance and resources related to each area. This makes it possible to easily identify useful, pertinent information.

Research areas are displayed as bubbles. By clicking on one of the bubbles, you can inspect the resources assigned to it. The size of the bubbles is relative to the number of resources assigned to it. Closeness of bubbles implies subject similarity. The closer two bubbles, the closer they are subject-wise. Centrality of bubbles implies subject similarity with the rest of the map, not importance. The closer a bubble is to the center, the closer it is subject-wise to all the other bubbles in the map.

Content

The content of this knowledge map is curated by as part of " . $project_name . ""; } else { echo $project_name; } ?>. Resources are collected and annotated in a spreadsheet, which is then transformed into the knowledge map.

Software

The knowledge map is based on the award-winning software developed by Open Knowledge Maps. For more information and the ability to create knowledge maps based on 250+ million documents, please see openknowledgemaps.org. To get in touch, please e-mail us at info@openknowledgemaps.org

Rights

The curator(s) are solely responsible for the content of the knowledge map. Unless otherwise noted, all content is licensed under a Creative Commons Attribution 4.0 International License. The spreadsheet is made available under CC0 (Public Domain Dedication). The knowledge mapping software is open source and hosted on Github.

" + , body: "

Knowledge maps provide an instant overview of a topic by showing the main areas at a glance and resources related to each area. This makes it possible to easily identify useful, pertinent information.

Research areas are displayed as bubbles. By clicking on one of the bubbles, you can inspect the resources assigned to it. The size of the bubbles is relative to the number of resources assigned to it. Closeness of bubbles implies subject similarity. The closer two bubbles, the closer they are subject-wise. Centrality of bubbles implies subject similarity with the rest of the map, not importance. The closer a bubble is to the center, the closer it is subject-wise to all the other bubbles in the map.

Content

The content of this knowledge map is curated by as part of " . $project_name . ""; } else { echo $project_name; } ?>. Resources are collected and annotated in a spreadsheet, which is then transformed into the knowledge map.

Software

The knowledge map is based on the award-winning software developed by Open Knowledge Maps. For more information and the ability to create knowledge maps based on 250+ million documents, please see openknowledgemaps.org. To get in touch, please e-mail us at info@openknowledgemaps.org

Rights

The curator(s) are solely responsible for the content of the knowledge map. Unless otherwise noted, all content is licensed under a Creative Commons Attribution 4.0 International License. The spreadsheet is made available under CC0 (Public Domain Dedication). The knowledge mapping software is open source and hosted on Github.

" }; $(document).ready(function () { From 4a7f30dd6e56d5205502e434979db5f9e998b560 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Sun, 15 Nov 2020 12:58:57 +0100 Subject: [PATCH 60/76] gsheets api default values --- examples/gsheets/config.example.php | 2 +- examples/gsheets/data-config.js | 25 +++++++++++++------------ server/services/updateGSheetsMap.php | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/examples/gsheets/config.example.php b/examples/gsheets/config.example.php index cbfae22e1..0dc14c56f 100644 --- a/examples/gsheets/config.example.php +++ b/examples/gsheets/config.example.php @@ -7,5 +7,5 @@ $GA_ENABLED = false; $GA_CODE = "XX-00000000-0"; $COOKIE_DOMAIN = ""; -$PERSISTENCE_BACKEND = "legacy"; +$PERSISTENCE_BACKEND = "api"; ?> diff --git a/examples/gsheets/data-config.js b/examples/gsheets/data-config.js index 73a80548e..55ba7b08f 100644 --- a/examples/gsheets/data-config.js +++ b/examples/gsheets/data-config.js @@ -1,17 +1,18 @@ var data_config = { tag: "visualization", mode: "gsheets", - + persistence_backend: "api", + bubble_min_scale: 1.1, bubble_max_scale: 1.1, - + paper_min_scale: 1, input_format: "json", use_area_uri: true, preview_type: "pdf", use_hypothesis: true, - + show_multiples: false, show_dropdown: false, show_intro: false, @@ -20,36 +21,36 @@ var data_config = { is_title_clickable: false, show_infolink: true, show_infolink_areas: false, - + show_context: true, create_title_from_context: false, show_context_timestamp: true, show_loading_screen: true, - + scale_toolbar: false, - + content_based: true, is_evaluation: true, evaluation_service: ["ga", "matomo"], - + is_force_areas: true, area_force_alpha: 0.03, papers_force_alpha: 0.2, - + language: "eng_gsheets", - + sort_options: ["year", "title", "area"], filter_options: ["all", "open_access"], - + show_keywords: true, hide_keywords_overview: false, show_tags: true, show_comments: true, show_resulttype: true, - + sort_menu_dropdown: true, filter_menu_dropdown: true, - + embed_modal: true, share_modal: false, }; diff --git a/server/services/updateGSheetsMap.php b/server/services/updateGSheetsMap.php index 6280ed884..b0e00196d 100644 --- a/server/services/updateGSheetsMap.php +++ b/server/services/updateGSheetsMap.php @@ -39,7 +39,7 @@ , false, null, 3 , "area_uri", "subject" , $sheet_id, false - , "api", "legacy"); + , "api", "api"); echo $result; From 789687259fe4ec130c220984a19399291163aff2 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 17 Nov 2020 17:18:44 +0100 Subject: [PATCH 61/76] Update README.md --- server/workers/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/workers/README.md b/server/workers/README.md index 90346abd6..311dde0fd 100644 --- a/server/workers/README.md +++ b/server/workers/README.md @@ -92,6 +92,10 @@ Enter container: `docker exec -it VARYINGNAME_pgsql_1 psql -U headstart` Execute command: `CREATE DATABASE databasename;` + +Secure Postgres: +* In `server/workers` duplicate `example_pg_hba.conf` to `pg_hba.conf` and review the settings. The default values should be ok for a default deployment (host connections are only allowed for user "headstart" with an md5-hashed password), but you may want to change access rights. + ### Starting the backend services with docker-compose Following commands have to be executed from the root folder of the repository, where `docker-compose.yml` is located. From edcce11686946f086dc541a4dd2bfaf7b993fe1f Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 18 Nov 2020 00:23:25 +0100 Subject: [PATCH 62/76] multiple databases wip --- docker-compose.yml | 5 +- server/preprocessing/conf/config.ini | 1 + server/workers/README.md | 2 + .../workers/services/src/apis/persistence.py | 93 ++++++++++--------- server/workers/services/src/database.py | 6 +- server/workers/services/src/manage.py | 12 ++- server/workers/services/src/models.py | 8 -- server/workers/tests/test_persistence.py | 8 +- 8 files changed, 74 insertions(+), 61 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a992da3ab..61229cf71 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,8 +18,9 @@ services: image: 'dpage/pgadmin4' ports: - '127.0.0.1:54323:80' - env_file: - - .env + environment: + PGADMIN_DEFAULT_EMAIL: "${PGADMIN_DEFAULT_EMAIL}" + PGADMIN_DEFAULT_PASSWORD: "${PGADMIN_DEFAULT_PASSWORD}" volumes: - ${PGADMIN_VOLUME}:/pgadmin4/data diff --git a/server/preprocessing/conf/config.ini b/server/preprocessing/conf/config.ini index f0e2e2abd..108e946ce 100644 --- a/server/preprocessing/conf/config.ini +++ b/server/preprocessing/conf/config.ini @@ -44,6 +44,7 @@ title = "Visualization" [connection] # Full path to the sqlite datatabase file. Make sure that your webserver has write access to this file. For development purposes, duplicate headstart.sqlite in server/storage/ and rename it to a filename of your choice. Enter the path to this file here. sqlite_db = "/path/to/db.sqlite" +database = "databasename" # Solr endpoint for LinkedCat example linkedcat_solr = "" linkedcat_user = "" diff --git a/server/workers/README.md b/server/workers/README.md index 311dde0fd..0d9e4159a 100644 --- a/server/workers/README.md +++ b/server/workers/README.md @@ -69,6 +69,7 @@ Dataprocessing: Services: * In `server/workers/services/src/config` copy `example_settings.py` to `settings.py` and change the values for `ENV` (`development` or `production`) and `DEBUG` (`TRUE` or `FALSE`). +* In `settings.py` you can also configure databases. TRIPLE ElasticSearch core service: @@ -92,6 +93,7 @@ Enter container: `docker exec -it VARYINGNAME_pgsql_1 psql -U headstart` Execute command: `CREATE DATABASE databasename;` +* In `preprocessing/conf/config_local.ini` change "databasename" to the dev/production database name for the specific integration. This should be in line with the database names provided in `settings.py` Secure Postgres: * In `server/workers` duplicate `example_pg_hba.conf` to `pg_hba.conf` and review the settings. The default values should be ok for a default deployment (host connections are only allowed for user "headstart" with an md5-hashed password), but you may want to change access rights. diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 5ba6d509c..bc1bbc0af 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -6,7 +6,7 @@ from flask_restx import Namespace, Resource, fields from models import Revisions, Visualizations -from database import db +from database import sessions persistence_ns = Namespace("persistence", description="OKMAps persistence operations") @@ -25,9 +25,10 @@ def create_vis_id(params, param_types): return vis_id -def write_revision(vis_id, data, rev_id=None): - - vis = db.session.query(Visualizations).filter_by(vis_id=vis_id).first() +def write_revision(database, vis_id, data, rev_id=None): + Session = sessions.get(database) + session = Session() + vis = session.query(Visualizations).filter_by(vis_id=vis_id).first() if rev_id is None: if vis.vis_latest is None: @@ -45,52 +46,59 @@ def write_revision(vis_id, data, rev_id=None): rev_comment="Visualization created", rev_data=data, vis_query=query) - db.session.add(new_rev) + session.add(new_rev) vis.vis_latest = rev_id - db.session.commit() + session.commit() -def create_visualization(vis_id, vis_title, data, +def create_visualization(database, + vis_id, vis_title, data, vis_clean_query=None, vis_query=None, vis_params=None): - if not exists_visualization(vis_id): + if not exists_visualization(database, vis_id): + Session = sessions.get(database) + session = Session() new_vis = Visualizations( vis_id=vis_id, vis_clean_query=vis_clean_query, vis_query=vis_query, vis_title=vis_title, vis_params=vis_params) - db.session.add(new_vis) - db.session.commit() - write_revision(vis_id, data, 1) + session.add(new_vis) + session.commit() + write_revision(database, vis_id, data, 1) -def exists_visualization(vis_id): - vis = db.session.query(Visualizations).filter_by(vis_id=vis_id).first() +def exists_visualization(database, vis_id): + Session = sessions.get(database) + session = Session() + vis = session.query(Visualizations).filter_by(vis_id=vis_id).first() exists = True if vis else False return exists -def get_last_version(vis_id, details=False, context=False): - return get_revision(vis_id, None, details, context) +def get_last_version(database, vis_id, details=False, context=False): + return get_revision(database, vis_id, None, details, context) -def get_revision(vis_id, rev_id, details=False, context=False): +def get_revision(database, vis_id, rev_id, details=False, context=False): + Session = sessions.get(database) + session = Session() if rev_id is None: - vis, rev = (db.session - .query(Visualizations, Revisions) - .select_from(Visualizations, Revisions) - .filter(Visualizations.vis_id == vis_id) - .filter(Revisions.rev_vis == vis_id) - .filter(Revisions.rev_id == Visualizations.vis_latest) + vis, rev = (session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == Visualizations.vis_latest) ).first() else: - vis, rev = (db.session - .query(Visualizations, Revisions) - .select_from(Visualizations, Revisions) - .filter(Visualizations.vis_id == vis_id) - .filter(Revisions.rev_vis == vis_id) - .filter(Revisions.rev_id == rev_id) + vis, rev = (session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == rev_id) ).first() if context is True: res = { @@ -109,14 +117,14 @@ def get_revision(vis_id, rev_id, details=False, context=False): return rev.rev_data -@persistence_ns.route('/existsVisualization') +@persistence_ns.route('/existsVisualization/') class existsVisualization(Resource): - def post(self): + def post(self, database): payload = request.get_json() persistence_ns.logger.debug("existsVisualization") vis_id = payload.get("vis_id") - exists = exists_visualization(vis_id) + exists = exists_visualization(database, vis_id) # create response headers = {} result = {"exists": exists} @@ -124,10 +132,10 @@ def post(self): return make_response(result, 200, headers) -@persistence_ns.route('/createVisualization') +@persistence_ns.route('/createVisualization/') class createVisualization(Resource): - def post(self): + def post(self, database): try: payload = request.get_json() persistence_ns.logger.debug("createVisualization") @@ -143,7 +151,8 @@ def post(self): persistence_ns.logger.debug(vis_clean_query) persistence_ns.logger.debug(vis_query) persistence_ns.logger.debug(vis_params) - create_visualization(vis_id, vis_title, data, + create_visualization(database, + vis_id, vis_title, data, vis_clean_query, vis_query, vis_params) result = {'success': True} headers = {'ContentType': 'application/json'} @@ -158,18 +167,18 @@ def post(self): headers) -@persistence_ns.route('/writeRevision') +@persistence_ns.route('/writeRevision/') class writeRevision(Resource): @persistence_ns.produces(["application/json"]) - def post(self): + def post(self, database): try: payload = request.get_json() persistence_ns.logger.debug("writeRevision") vis_id = payload.get("vis_id") data = payload.get("data") # persistence_ns.logger.debug(data) - write_revision(vis_id, data) + write_revision(database, vis_id, data) result = {'success': True} headers = {'ContentType': 'application/json'} return make_response(jsonify(result), 200, headers) @@ -179,7 +188,7 @@ def post(self): return make_response(jsonify(result), 500, headers) -@persistence_ns.route('/getLastVersion') +@persistence_ns.route('/getLastVersion/') class getLastVersion(Resource): """ Is actually a call to getRevision but taking the latest one @@ -188,7 +197,7 @@ class getLastVersion(Resource): """ - def post(self): + def post(self, database): try: payload = request.get_json() persistence_ns.logger.debug("getLastVersion") @@ -196,7 +205,7 @@ def post(self): vis_id = payload.get('vis_id') details = payload.get('details') context = payload.get('context') - result = get_last_version(vis_id, details, context) + result = get_last_version(database, vis_id, details, context) headers = {'ContentType': 'application/json'} return make_response(jsonify(result), 200, @@ -209,11 +218,11 @@ def post(self): headers) -@persistence_ns.route('/getRevision') +@persistence_ns.route('/getRevision/') class getRevision(Resource): @persistence_ns.produces(["application/json"]) - def post(self): + def post(self, database): # create response headers = {} diff --git a/server/workers/services/src/database.py b/server/workers/services/src/database.py index 63c9fdfcf..8d8db10b0 100644 --- a/server/workers/services/src/database.py +++ b/server/workers/services/src/database.py @@ -1,7 +1,11 @@ from flask_sqlalchemy import SQLAlchemy from sqlalchemy.orm import sessionmaker from sqlalchemy import create_engine +from config import settings db = SQLAlchemy() -Session = sessionmaker() +sessions = {} +sessions[settings.POSTGRES["db"]] = sessionmaker(bind=create_engine(settings.SQLALCHEMY_DATABASE_URI)) +for data_integration, database in settings.SQLALCHEMY_BINDS.items(): + sessions[data_integration] = sessionmaker(bind=create_engine(database)) diff --git a/server/workers/services/src/manage.py b/server/workers/services/src/manage.py index acb4cb8a5..ccc740947 100644 --- a/server/workers/services/src/manage.py +++ b/server/workers/services/src/manage.py @@ -1,11 +1,15 @@ from flask_sqlalchemy import SQLAlchemy from app import app -from models import Visualizations, Revisions, TripleRevisions, TripleVisualizations -from database import db +from models import Visualizations, Revisions +from database import db, sessions if __name__ == '__main__': db.init_app(app) with app.app_context(): - db.create_all(bind=None) - db.create_all(bind=['triple', 'gsheets', 'openaire']) + for database, Session in sessions.items(): + session = Session() + engine = session.get_bind() + for name, table in Visualizations.metadata.tables.items(): + if not engine.dialect.has_table(engine, name): + table.create(engine) diff --git a/server/workers/services/src/models.py b/server/workers/services/src/models.py index 13e6e99c8..58eacbd3f 100644 --- a/server/workers/services/src/models.py +++ b/server/workers/services/src/models.py @@ -33,11 +33,3 @@ class Revisions(db.Model): def as_dict(self): return {c.name: getattr(self, c.name) for c in self.__table__.columns} - - -class TripleVisualizations(Visualizations): - pass - - -class TripleRevisions(Revisions): - pass diff --git a/server/workers/tests/test_persistence.py b/server/workers/tests/test_persistence.py index 6e53e8c7c..39c414cff 100644 --- a/server/workers/tests/test_persistence.py +++ b/server/workers/tests/test_persistence.py @@ -504,7 +504,7 @@ def test_create_visualization(testcase): payload["vis_clean_query"] = caseid payload["vis_query"] = caseid payload["vis_params"] = json.dumps(testcase["casedata"]["params"]) - res = requests.post("http://localhost/api/persistence/createVisualization", + res = requests.post("http://localhost/api/persistence/createVisualization/test", json=payload) assert res.status_code == 200, res.json().get('reason') @@ -516,7 +516,7 @@ def test_write_revision(testcase): payload = {} payload["vis_id"] = caseid payload["data"] = RESULTS[caseid].to_json(orient='records') - res = requests.post("http://localhost/api/persistence/writeRevision", + res = requests.post("http://localhost/api/persistence/writeRevision/test", json=payload) assert res.status_code == 200, res.json().get('reason') @@ -527,7 +527,7 @@ def test_map_exists(testcase): caseid = testcase["caseid"] payload = {} payload["vis_id"] = caseid - res = requests.post("http://localhost/api/persistence/existsVisualization", + res = requests.post("http://localhost/api/persistence/existsVisualization/test", json=payload) result = res.json() assert result["exists"] is True @@ -540,7 +540,7 @@ def test_map_exists_not(testcase): invalid_id = caseid[2:] payload = {} payload["vis_id"] = invalid_id - res = requests.post("http://localhost/api/persistence/existsVisualization", + res = requests.post("http://localhost/api/persistence/existsVisualization/test", json=payload) result = res.json() assert result["exists"] is False From 773c113083d775ef44005ecd0286565aa451591f Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 18 Nov 2020 10:23:07 +0100 Subject: [PATCH 63/76] new way to choose db sessions --- .../workers/services/src/apis/persistence.py | 23 +++++++++++-------- .../services/src/config/example_settings.py | 11 ++++++++- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index bc1bbc0af..4f293b387 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -6,12 +6,21 @@ from flask_restx import Namespace, Resource, fields from models import Revisions, Visualizations -from database import sessions +from database import db, sessions persistence_ns = Namespace("persistence", description="OKMAps persistence operations") +def select_session(Session=None): + """Select session according to database, + else select session for default database.""" + if Session is not None: + return Session() + else: + return db.session + + def create_vis_id(params, param_types): # create map id ordered_params = OrderedDict() @@ -26,8 +35,7 @@ def create_vis_id(params, param_types): def write_revision(database, vis_id, data, rev_id=None): - Session = sessions.get(database) - session = Session() + session = select_session(sessions.get(database)) vis = session.query(Visualizations).filter_by(vis_id=vis_id).first() if rev_id is None: @@ -56,8 +64,7 @@ def create_visualization(database, vis_clean_query=None, vis_query=None, vis_params=None): if not exists_visualization(database, vis_id): - Session = sessions.get(database) - session = Session() + session = select_session(sessions.get(database)) new_vis = Visualizations( vis_id=vis_id, vis_clean_query=vis_clean_query, @@ -70,8 +77,7 @@ def create_visualization(database, def exists_visualization(database, vis_id): - Session = sessions.get(database) - session = Session() + session = select_session(sessions.get(database)) vis = session.query(Visualizations).filter_by(vis_id=vis_id).first() exists = True if vis else False return exists @@ -82,8 +88,7 @@ def get_last_version(database, vis_id, details=False, context=False): def get_revision(database, vis_id, rev_id, details=False, context=False): - Session = sessions.get(database) - session = Session() + session = select_session(sessions.get(database)) if rev_id is None: vis, rev = (session .query(Visualizations, Revisions) diff --git a/server/workers/services/src/config/example_settings.py b/server/workers/services/src/config/example_settings.py index 29251bb36..8364595be 100644 --- a/server/workers/services/src/config/example_settings.py +++ b/server/workers/services/src/config/example_settings.py @@ -6,8 +6,17 @@ 'host': '127.0.0.1', 'port': '5432', } +TEST = { + 'user': 'testuser', + 'pw': 'testpassword', + 'db': 'test', + 'host': '127.0.0.1', + 'port': '5432', +} SQLALCHEMY_DATABASE_URI = 'postgresql://%(user)s:%(pw)s@%(host)s:%(port)s/%(db)s' % POSTGRES -# SQLALCHEMY_DATABASE_URI = 'sqlite:////var/data/db.sqlite' +SQLALCHEMY_BINDS = { + 'test': 'postgresql://%(user)s:%(pw)s@%(host)s:%(port)s/%(db)s' % TEST +} SQLALCHEMY_TRACK_MODIFICATIONS = False ENV = "development" DEBUG = True From 3970e5cd0ac50d5639dc53f3a8f4d382f14099bf Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 11:38:49 +0100 Subject: [PATCH 64/76] add database paths to service api calls --- server/services/getGSheetsMap.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index 7dd252716..d706125d3 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -16,9 +16,10 @@ $persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); +$database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { From f70ad415fe4b85f45719b654013523afddb107b2 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 11:48:13 +0100 Subject: [PATCH 65/76] add database paths to service api calls --- server/services/GSheetUpdateAvailable.php | 3 ++- server/services/getContext.php | 3 ++- server/services/getGSheetsMap.php | 2 +- server/services/getLatestRevision.php | 3 ++- server/services/search.php | 7 ++++--- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/server/services/GSheetUpdateAvailable.php b/server/services/GSheetUpdateAvailable.php index f430facd5..595effb79 100644 --- a/server/services/GSheetUpdateAvailable.php +++ b/server/services/GSheetUpdateAvailable.php @@ -14,11 +14,12 @@ $vis_id = library\CommUtils::getParameter($_GET, "vis_id"); $gsheet_last_updated = library\CommUtils::getParameter($_GET, "gsheet_last_updated"); $persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; +$database = $ini_array["connection"]["database"]; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($persistence_backend == "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $backend; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getContext.php b/server/services/getContext.php index c18e739bf..f734990f6 100644 --- a/server/services/getContext.php +++ b/server/services/getContext.php @@ -17,9 +17,10 @@ $revision_context = isset($_GET["revision_context"]) ? library\CommUtils::getParameter($_GET, "revision_context") : false; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); +$database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getContext"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getContext" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "revision_context" => $revision_context)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index dc4642818..e7d24998b 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -19,7 +19,7 @@ $database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . $database; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getLatestRevision.php b/server/services/getLatestRevision.php index 4ff0ec2a2..bcecabcce 100644 --- a/server/services/getLatestRevision.php +++ b/server/services/getLatestRevision.php @@ -20,6 +20,7 @@ array("flags" => FILTER_NULL_ON_FAILURE)); $backend = isset($_GET["backend"]) ? library\CommUtils::getParameter($_GET, "backend") : "legacy"; $persistence_backend = isset($_GET["persistence_backend"]) ? library\CommUtils::getParameter($_GET, "persistence_backend") : "legacy"; +$database = $ini_array["connection"]["database"]; $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); @@ -29,7 +30,7 @@ # context data true start if ($persistence_backend === "api") { # get data + context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/search.php b/server/services/search.php index 2979282fa..93c32a048 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -67,6 +67,7 @@ function search($repository, $dirty_query :($dirty_query); $persistence = new \headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); + $database = $ini_array["connection"]["database"]; $settings = $ini_array["general"]; @@ -146,7 +147,7 @@ function search($repository, $dirty_query $vis_title = $repository; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization" . "/" . $database; $payload = json_encode(array("vis_id" => $unique_id)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -161,7 +162,7 @@ function search($repository, $dirty_query if (!$exists) { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization" . "/" . $database; $payload = json_encode(array("vis_id" => $unique_id, "vis_title" => $vis_title, "data" => $input_json, @@ -177,7 +178,7 @@ function search($repository, $dirty_query } } else { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/writeRevision"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/writeRevision" . "/" . $database; $payload = json_encode(array("vis_id" => $unique_id, "data" => $input_json)); $res = library\CommUtils::call_api($route, $payload); From 7a76d61e47e5661d5808f22a1b3b71c768e6481a Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 14:53:08 +0100 Subject: [PATCH 66/76] cleanup --- examples/gsheets/data-config.js | 5 ++--- examples/triple/data-config_triple.js | 2 +- server/services/GSheetUpdateAvailable.php | 2 +- server/services/getLatestRevision.php | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/gsheets/data-config.js b/examples/gsheets/data-config.js index f0abb5642..3c62a0d8c 100644 --- a/examples/gsheets/data-config.js +++ b/examples/gsheets/data-config.js @@ -31,7 +31,7 @@ var data_config = { content_based: true, is_evaluation: true, - evaluation_service: ["ga", "matomo"], + evaluation_service: ["matomo"], is_force_areas: true, area_force_alpha: 0.03, @@ -52,6 +52,5 @@ var data_config = { filter_menu_dropdown: true, embed_modal: true, - share_modal: false, - persistence_backend: "api" + share_modal: false }; diff --git a/examples/triple/data-config_triple.js b/examples/triple/data-config_triple.js index 39deecc49..35de799c5 100644 --- a/examples/triple/data-config_triple.js +++ b/examples/triple/data-config_triple.js @@ -32,7 +32,7 @@ var data_config = { share_modal: false, backend: "api", - persistence_backend: "legacy", + persistence_backend: "api", streamgraph_colors: ["#215A66", "#66214A", "#5D40FB", "#CB40FB", "#40C0FB", "#FB4068" , "#FBB240", "#40FBC8", "#fee4bc", "#bcfeec", "#c6bcfe", "#febcca"], diff --git a/server/services/GSheetUpdateAvailable.php b/server/services/GSheetUpdateAvailable.php index 595effb79..547dbd326 100644 --- a/server/services/GSheetUpdateAvailable.php +++ b/server/services/GSheetUpdateAvailable.php @@ -19,7 +19,7 @@ $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($persistence_backend == "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $backend; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getLatestRevision.php b/server/services/getLatestRevision.php index bcecabcce..7caf2e616 100644 --- a/server/services/getLatestRevision.php +++ b/server/services/getLatestRevision.php @@ -62,7 +62,7 @@ } else { if ($persistence_backend === "api") { # return data without context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -82,7 +82,7 @@ if ($context === true) { if ($persistence_backend === "api") { # get data + context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -115,7 +115,7 @@ } else { if ($persistence_backend === "api") { # get data without context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion"; + $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { From 598085cd82ca00253b0c47e3ea538cf47b5e16d0 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 21:37:13 +0100 Subject: [PATCH 67/76] update manage.py --- server/workers/services/src/manage.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/workers/services/src/manage.py b/server/workers/services/src/manage.py index ccc740947..377bcd21f 100644 --- a/server/workers/services/src/manage.py +++ b/server/workers/services/src/manage.py @@ -8,8 +8,11 @@ db.init_app(app) with app.app_context(): for database, Session in sessions.items(): - session = Session() - engine = session.get_bind() - for name, table in Visualizations.metadata.tables.items(): - if not engine.dialect.has_table(engine, name): - table.create(engine) + try: + session = Session() + engine = session.get_bind() + for name, table in Visualizations.metadata.tables.items(): + if not engine.dialect.has_table(engine, name): + table.create(engine) + except Exception as e: + print(database, e) From 431c6ef68db316eb9eb24c18e46310b32baa4648 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 22:09:57 +0100 Subject: [PATCH 68/76] logging --- examples/triple/search.js | 2 +- server/workers/services/src/apis/persistence.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/triple/search.js b/examples/triple/search.js index b11b23cb0..fabbc0277 100644 --- a/examples/triple/search.js +++ b/examples/triple/search.js @@ -77,7 +77,7 @@ $("#searchform").validate({ var doSubmit = function (data, newWindow, callback) { data += "&today=" + new Date().toLocaleDateString("en-US"); - data += "&persistence_backend=" + data_config["persistence_backend"]; + data += "&persistence_backend=" + data_config.persistence_backend; var params = $("#searchform").serializeArray().reduce(function(obj, item) { obj[item.name] = item.value; return obj; diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index cf10e031e..22c6a5f2f 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -149,8 +149,8 @@ class existsVisualization(Resource): def post(self, database): payload = request.get_json() - persistence_ns.logger.debug("existsVisualization") vis_id = payload.get("vis_id") + persistence_ns.logger.debug("existsVisualization: %s" % vis_id) exists = exists_visualization(database, vis_id) # create response headers = {} @@ -294,6 +294,9 @@ def post(self): params = payload.get("params") param_types = payload.get("param_types") vis_id = create_vis_id(params, param_types) + persistence_ns.logger.debug(params) + persistence_ns.logger.debug(param_types) + persistence_ns.logger.debug(vis_id) # create response headers = {} result = {"unique_id": vis_id} From d71a7c7c0db77d184c163eb4f2c42d121bac1b1e Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 22:44:41 +0100 Subject: [PATCH 69/76] integration cleanup --- docker-compose.yml | 7 ++---- server/services/GSheetUpdateAvailable.php | 2 +- server/services/getContext.php | 2 +- server/services/getGSheetsMap.php | 2 +- server/services/getLastVersion.php | 3 ++- server/services/getLatestRevision.php | 18 ++++++++------ server/services/search.php | 30 ++++++++++++++++------- server/services/searchTRIPLE.php | 2 +- 8 files changed, 40 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 61229cf71..f0bac4daa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,11 +18,8 @@ services: image: 'dpage/pgadmin4' ports: - '127.0.0.1:54323:80' - environment: - PGADMIN_DEFAULT_EMAIL: "${PGADMIN_DEFAULT_EMAIL}" - PGADMIN_DEFAULT_PASSWORD: "${PGADMIN_DEFAULT_PASSWORD}" - volumes: - - ${PGADMIN_VOLUME}:/pgadmin4/data + env_file: + - .env api: build: diff --git a/server/services/GSheetUpdateAvailable.php b/server/services/GSheetUpdateAvailable.php index 547dbd326..50201c2ff 100644 --- a/server/services/GSheetUpdateAvailable.php +++ b/server/services/GSheetUpdateAvailable.php @@ -19,7 +19,7 @@ $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($persistence_backend == "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getContext.php b/server/services/getContext.php index f734990f6..96f6dfd70 100644 --- a/server/services/getContext.php +++ b/server/services/getContext.php @@ -20,7 +20,7 @@ $database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getContext" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getContext/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "revision_context" => $revision_context)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index e7d24998b..831c91fff 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -19,7 +19,7 @@ $database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getLastVersion.php b/server/services/getLastVersion.php index 528cf27dd..0711135ad 100644 --- a/server/services/getLastVersion.php +++ b/server/services/getLastVersion.php @@ -1,5 +1,6 @@ $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -62,13 +62,15 @@ } else { if ($persistence_backend === "api") { # return data without context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { library\CommUtils::echoOrCallback($res, $_GET); } else { - $data = $res; + $data = json_decode($res["result"], true); + $jsonData = json_encode($data); + library\CommUtils::echoOrCallback($jsonData, $_GET); } } else { # return data without context from legacy @@ -82,7 +84,7 @@ if ($context === true) { if ($persistence_backend === "api") { # get data + context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -115,16 +117,18 @@ } else { if ($persistence_backend === "api") { # get data without context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { library\CommUtils::echoOrCallback($res, $_GET); } else { - $data = $res; + $data = json_decode($res["result"], true); + $jsonData = json_encode($data); + library\CommUtils::echoOrCallback($jsonData, $_GET); } } else { - # get data without context from legac + # get data without context from legacy $jsonData = $persistence->getLastVersion($vis_id); } library\CommUtils::echoOrCallback($jsonData[0], $_GET); diff --git a/server/services/search.php b/server/services/search.php index 93c32a048..126216714 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -76,7 +76,7 @@ function search($repository, $dirty_query $params_for_id_creation = ($params_for_id === null)?($params_json):(packParamsJSON($params_for_id, $post_params)); if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/createID"; + $route = $ini_array["general"]["api_url"] . "persistence/" . "createID"; $payload = json_encode(array("params" => $post_params, "param_types" => $param_types)); $res = library\CommUtils::call_api($route, $payload); @@ -94,12 +94,24 @@ function search($repository, $dirty_query $params_json = packParamsJSON($param_types, $post_params); if($retrieve_cached_map) { - $last_version = $persistence->getLastVersion($unique_id, false); - - if ($last_version != null && $last_version != "null" && $last_version != false) { - echo json_encode(array("query" => $query, "id" => $unique_id, "status" => "success")); - return; + if ($persistence_backend === "api") { + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; + $payload = json_encode(array("vis_id" => $unique_id, + "details" => false, + "context" => false)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + echo json_encode($res); + } else { + $last_version = json_decode($res["result"], true); } + } else { + $last_version = $persistence->getLastVersion($unique_id, false); + } + if ($last_version != null && $last_version != "null" && $last_version != false) { + echo json_encode(array("query" => $query, "id" => $unique_id, "status" => "success")); + return; + } } $params_file = tmpfile(); @@ -147,7 +159,7 @@ function search($repository, $dirty_query $vis_title = $repository; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "existsVisualization/" . $database; $payload = json_encode(array("vis_id" => $unique_id)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -162,7 +174,7 @@ function search($repository, $dirty_query if (!$exists) { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "createVisualization/" . $database; $payload = json_encode(array("vis_id" => $unique_id, "vis_title" => $vis_title, "data" => $input_json, @@ -178,7 +190,7 @@ function search($repository, $dirty_query } } else { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/writeRevision" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "writeRevision/" . $database; $payload = json_encode(array("vis_id" => $unique_id, "data" => $input_json)); $res = library\CommUtils::call_api($route, $payload); diff --git a/server/services/searchTRIPLE.php b/server/services/searchTRIPLE.php index 99f2ce38c..ce779b6de 100644 --- a/server/services/searchTRIPLE.php +++ b/server/services/searchTRIPLE.php @@ -9,7 +9,7 @@ $dirty_query = library\CommUtils::getParameter($_POST, "q"); $precomputed_id = (isset($_POST["unique_id"]))?($_POST["unique_id"]):(null); -$persistence_backend = (isset($_POST["persistence_backend"]))?($_POST["persistence_backend"]):("legacy"); +$persistence_backend = (isset($_POST["persistence_backend"]))?($_POST["persistence_backend"]):("api"); $post_params = $_POST; unset($post_params["persistence_backend"]); From a466c12339a59795d0c1aad6f349544816140d0c Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 23 Nov 2020 22:44:41 +0100 Subject: [PATCH 70/76] integration cleanup --- docker-compose.yml | 7 +- server/services/GSheetUpdateAvailable.php | 2 +- server/services/getContext.php | 2 +- server/services/getGSheetsMap.php | 2 +- server/services/getLastVersion.php | 29 ++++++++- server/services/getLatestRevision.php | 18 ++++-- server/services/search.php | 30 ++++++--- server/services/searchTRIPLE.php | 2 +- .../workers/services/src/apis/persistence.py | 64 ++++++++++--------- 9 files changed, 99 insertions(+), 57 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 61229cf71..f0bac4daa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,11 +18,8 @@ services: image: 'dpage/pgadmin4' ports: - '127.0.0.1:54323:80' - environment: - PGADMIN_DEFAULT_EMAIL: "${PGADMIN_DEFAULT_EMAIL}" - PGADMIN_DEFAULT_PASSWORD: "${PGADMIN_DEFAULT_PASSWORD}" - volumes: - - ${PGADMIN_VOLUME}:/pgadmin4/data + env_file: + - .env api: build: diff --git a/server/services/GSheetUpdateAvailable.php b/server/services/GSheetUpdateAvailable.php index 547dbd326..50201c2ff 100644 --- a/server/services/GSheetUpdateAvailable.php +++ b/server/services/GSheetUpdateAvailable.php @@ -19,7 +19,7 @@ $persistence = new headstart\persistence\SQLitePersistence($ini_array["connection"]["sqlite_db"]); if ($persistence_backend == "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getContext.php b/server/services/getContext.php index f734990f6..96f6dfd70 100644 --- a/server/services/getContext.php +++ b/server/services/getContext.php @@ -20,7 +20,7 @@ $database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getContext" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getContext/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "revision_context" => $revision_context)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getGSheetsMap.php b/server/services/getGSheetsMap.php index e7d24998b..831c91fff 100644 --- a/server/services/getGSheetsMap.php +++ b/server/services/getGSheetsMap.php @@ -19,7 +19,7 @@ $database = $ini_array["connection"]["database"]; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { diff --git a/server/services/getLastVersion.php b/server/services/getLastVersion.php index 528cf27dd..bab9974de 100644 --- a/server/services/getLastVersion.php +++ b/server/services/getLastVersion.php @@ -1,5 +1,6 @@ getLastVersion($vis_id, false); +$backend_mapping = array( + "pubmed" => "legacy", + "base" => "legacy", + "openaire" => "legacy", + "triple" => "api", + "gsheets" => "api" +); + +$persistence_backend = $backend_mapping[$service]; +if ($persistence_backend === "api") { + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $service; + $payload = json_encode(array("vis_id" => $vis_id, + "details" => false, + "context" => false)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + echo json_encode($res); + } else { + $last_version = json_decode($res["result"], true); + } +} else { + $last_version = $persistence->getLastVersion($vis_id, false); +} + if ($last_version != null && $last_version != "null" && $last_version != false) { echo json_encode(array("status" => "success", "last_version" => $last_version)); diff --git a/server/services/getLatestRevision.php b/server/services/getLatestRevision.php index 7caf2e616..d9e1c13d1 100644 --- a/server/services/getLatestRevision.php +++ b/server/services/getLatestRevision.php @@ -30,7 +30,7 @@ # context data true start if ($persistence_backend === "api") { # get data + context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -62,13 +62,15 @@ } else { if ($persistence_backend === "api") { # return data without context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { library\CommUtils::echoOrCallback($res, $_GET); } else { - $data = $res; + $data = json_decode($res["result"], true); + $jsonData = json_encode($data); + library\CommUtils::echoOrCallback($jsonData, $_GET); } } else { # return data without context from legacy @@ -82,7 +84,7 @@ if ($context === true) { if ($persistence_backend === "api") { # get data + context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => true)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -115,16 +117,18 @@ } else { if ($persistence_backend === "api") { # get data without context from api - $route = $ini_array["general"]["api_url"] . "/persistence" . "/getLastVersion" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; $payload = json_encode(array("vis_id" => $vis_id, "details" => false, "context" => false)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { library\CommUtils::echoOrCallback($res, $_GET); } else { - $data = $res; + $data = json_decode($res["result"], true); + $jsonData = json_encode($data); + library\CommUtils::echoOrCallback($jsonData, $_GET); } } else { - # get data without context from legac + # get data without context from legacy $jsonData = $persistence->getLastVersion($vis_id); } library\CommUtils::echoOrCallback($jsonData[0], $_GET); diff --git a/server/services/search.php b/server/services/search.php index 93c32a048..126216714 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -76,7 +76,7 @@ function search($repository, $dirty_query $params_for_id_creation = ($params_for_id === null)?($params_json):(packParamsJSON($params_for_id, $post_params)); if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/createID"; + $route = $ini_array["general"]["api_url"] . "persistence/" . "createID"; $payload = json_encode(array("params" => $post_params, "param_types" => $param_types)); $res = library\CommUtils::call_api($route, $payload); @@ -94,12 +94,24 @@ function search($repository, $dirty_query $params_json = packParamsJSON($param_types, $post_params); if($retrieve_cached_map) { - $last_version = $persistence->getLastVersion($unique_id, false); - - if ($last_version != null && $last_version != "null" && $last_version != false) { - echo json_encode(array("query" => $query, "id" => $unique_id, "status" => "success")); - return; + if ($persistence_backend === "api") { + $route = $ini_array["general"]["api_url"] . "persistence/" . "getLastVersion/" . $database; + $payload = json_encode(array("vis_id" => $unique_id, + "details" => false, + "context" => false)); + $res = library\CommUtils::call_api($route, $payload); + if ($res["httpcode"] != 200) { + echo json_encode($res); + } else { + $last_version = json_decode($res["result"], true); } + } else { + $last_version = $persistence->getLastVersion($unique_id, false); + } + if ($last_version != null && $last_version != "null" && $last_version != false) { + echo json_encode(array("query" => $query, "id" => $unique_id, "status" => "success")); + return; + } } $params_file = tmpfile(); @@ -147,7 +159,7 @@ function search($repository, $dirty_query $vis_title = $repository; if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/existsVisualization" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "existsVisualization/" . $database; $payload = json_encode(array("vis_id" => $unique_id)); $res = library\CommUtils::call_api($route, $payload); if ($res["httpcode"] != 200) { @@ -162,7 +174,7 @@ function search($repository, $dirty_query if (!$exists) { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/createVisualization" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "createVisualization/" . $database; $payload = json_encode(array("vis_id" => $unique_id, "vis_title" => $vis_title, "data" => $input_json, @@ -178,7 +190,7 @@ function search($repository, $dirty_query } } else { if ($persistence_backend === "api") { - $route = $ini_array["general"]["api_url"] . "/persistence" . "/writeRevision" . "/" . $database; + $route = $ini_array["general"]["api_url"] . "persistence/" . "writeRevision/" . $database; $payload = json_encode(array("vis_id" => $unique_id, "data" => $input_json)); $res = library\CommUtils::call_api($route, $payload); diff --git a/server/services/searchTRIPLE.php b/server/services/searchTRIPLE.php index 99f2ce38c..ce779b6de 100644 --- a/server/services/searchTRIPLE.php +++ b/server/services/searchTRIPLE.php @@ -9,7 +9,7 @@ $dirty_query = library\CommUtils::getParameter($_POST, "q"); $precomputed_id = (isset($_POST["unique_id"]))?($_POST["unique_id"]):(null); -$persistence_backend = (isset($_POST["persistence_backend"]))?($_POST["persistence_backend"]):("legacy"); +$persistence_backend = (isset($_POST["persistence_backend"]))?($_POST["persistence_backend"]):("api"); $post_params = $_POST; unset($post_params["persistence_backend"]); diff --git a/server/workers/services/src/apis/persistence.py b/server/workers/services/src/apis/persistence.py index 22c6a5f2f..3b5fbb426 100644 --- a/server/workers/services/src/apis/persistence.py +++ b/server/workers/services/src/apis/persistence.py @@ -89,37 +89,41 @@ def get_last_version(database, vis_id, details=False, context=False): def get_revision(database, vis_id, rev_id, details=False, context=False): session = select_session(sessions.get(database)) - if rev_id is None: - vis, rev = (session - .query(Visualizations, Revisions) - .select_from(Visualizations, Revisions) - .filter(Visualizations.vis_id == vis_id) - .filter(Revisions.rev_vis == vis_id) - .filter(Revisions.rev_id == Visualizations.vis_latest) - ).first() - else: - vis, rev = (session - .query(Visualizations, Revisions) - .select_from(Visualizations, Revisions) - .filter(Visualizations.vis_id == vis_id) - .filter(Revisions.rev_vis == vis_id) - .filter(Revisions.rev_id == rev_id) - ).first() - if context is True: - res = { - "rev_vis": rev.rev_vis, - "vis_query": rev.vis_query, - "vis_title": vis.vis_title, - "rev_timestamp": rev.rev_timestamp, - "vis_params": vis.vis_params, - "rev_data": rev.rev_data - } - return res - else: - if details is True: - return rev.as_dict() + try: + if rev_id is None: + vis, rev = (session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == Visualizations.vis_latest) + ).first() + else: + vis, rev = (session + .query(Visualizations, Revisions) + .select_from(Visualizations, Revisions) + .filter(Visualizations.vis_id == vis_id) + .filter(Revisions.rev_vis == vis_id) + .filter(Revisions.rev_id == rev_id) + ).first() + if context is True: + res = { + "rev_vis": rev.rev_vis, + "vis_query": rev.vis_query, + "vis_title": vis.vis_title, + "rev_timestamp": rev.rev_timestamp, + "vis_params": vis.vis_params, + "rev_data": rev.rev_data + } + return res else: - return rev.rev_data + if details is True: + return rev.as_dict() + else: + return rev.rev_data + except TypeError: + persistence_ns.logger.info("Vis ID not found: %s in database %s" % (vis_id, database)) + return None def get_context(database, vis_id, revision_context=False): From ae5415cefb20925aea62d04ffb19183838d176ef Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Mon, 30 Nov 2020 13:51:16 +0100 Subject: [PATCH 71/76] add title and activate snapshots --- .../services/snapshot/data-config_gsheets.js | 85 ++----------------- server/workers/gsheets/src/search_gsheets.py | 7 ++ 2 files changed, 12 insertions(+), 80 deletions(-) diff --git a/server/services/snapshot/data-config_gsheets.js b/server/services/snapshot/data-config_gsheets.js index efd4e438e..eb4e69a2f 100644 --- a/server/services/snapshot/data-config_gsheets.js +++ b/server/services/snapshot/data-config_gsheets.js @@ -2,13 +2,13 @@ var data_config = { tag: "visualization", mode: "gsheets", service_name: 'CoVis database', + persistence_backend: "api", bubble_min_scale: 1.1, bubble_max_scale: 1.1, paper_min_scale: 1, - title: "Knowledge map of COVID-19 research", input_format: "json", base_unit: "questions", use_area_uri: true, @@ -25,7 +25,7 @@ var data_config = { show_infolink_areas: false, show_context: true, - create_title_from_context: false, + create_title_from_context: true, show_context_timestamp: true, show_loading_screen: true, @@ -33,14 +33,12 @@ var data_config = { content_based: true, is_evaluation: true, - evaluation_service: ["ga", "matomo"], + evaluation_service: ["matomo"], is_force_areas: true, area_force_alpha: 0.03, papers_force_alpha: 0.2, - language: "eng_covis", - sort_options: ["year", "title", "area"], filter_options: ["all", "Dataset", "Journal Article", "Preprint", "ReFigure", "Review"], filter_field: "resulttype", @@ -54,79 +52,6 @@ var data_config = { sort_menu_dropdown: true, filter_menu_dropdown: true, - share_modal: true, - hashtags_twitter_card: "COVID19,openscience,discovery", - - intro: { - title:"Knowledge Map of COVID-19 research curated by experts" - , body: "

About Covis

CoVis provides a curated knowledge map of seminal works on COVID-19 from eight critical areas of biomedical research. This collection is not meant to be exhaustive, but rather a single reference point for definitive research in key areas of coronavirus and COVID-19 research. CoVis makes it easier to get started on the topic - but also helps you to stay up-to-date. The knowledge map is constantly evolving thanks to the collective editing of subject-matter experts.

Read our FAQs to find out more.

Data Source curated by ReFigure

The articles, datasets and ReFigures in CoVis are curated by an editorial team led by immunologists and ReFigure founders Dr. Girija Goyal and Dr. James Akin. Given the fast pace of research and the limited historical data on COVID-19, many findings are under debate and only understandable after reading several reports from different sources. Team ReFigure creates visual, easy to understand, annotated, figure collections which provide analyses and consensus on key issues.

Find out more about the curation process and the methodology for paper inclusion in our FAQs. We invite subject-matter experts to help us with our efforts. If you would like to contribute to CoVis, please get in touch.

Software created by Open Knowledge Maps

The resources selected for inclusion in CoVis are visualized in a knowledge map. Knowledge maps provide an instant overview of a topic by showing the main areas at a glance, and resources related to each area. This makes it possible to easily identify useful, pertinent information. The knowledge map is based on award-winning software developed by Open Knowledge Maps.

In the knowledge map, research areas are displayed as bubbles. By clicking on one of the bubbles, you can inspect the resources assigned to it; open access papers can be directly viewed and downloaded within the interface. Find out more about knowledge maps and their properties in our FAQs.

" - }, - - localization: { - eng_covis: { - loading: "Updating and retrieving map. This may take a few seconds, please hold on.", - search_placeholder: "Search within map...", - show_list: "Show list", - hide_list: "Hide list", - intro_label: "curated by experts", - intro_icon: "", - relevance: "relevance", - readers: "citations", - year: "date", - authors: "authors", - title: "title", - area: "Area", - backlink: "← Back to overview", - backlink_list: "← Show all documents in area", - backlink_list_streamgraph: "← Show all documents", - backlink_list_streamgraph_stream_selected: "← Show all documents in stream", - keywords: "Keywords", - no_keywords: "n/a", - no_title: "No title", - overview_label: 'Overview of', - streamgraph_label: 'Streamgraph for', - overview_authors_label: 'Overview of the works of', - streamgraph_authors_label: 'Streamgraph for the works of', - articles_label: 'resources and collections', - most_recent_label: 'most recent', - most_relevant_label: 'most relevant', - most_relevant_tooltip: 'To determine the most relevant documents, we use the relevance ranking provided by the source - either BASE or PubMed. Both sources compute the text similarity between your query and the article metadata to establish the relevance ranking. Please consult the FAQ for more information.', - source_label: 'Data source', - resulttype_label: 'Document type', - documenttypes_label: 'Document types', - timestamp_label: 'Last updated', - documenttypes_tooltip: 'The following document types were taken into consideration in the creation of this map (not all of them may appear in the map):', - default_area: "No area", - default_author: "", - default_id: "defaultid", - default_hash: "hashHash", - default_abstract: "No Abstract", - default_published_in: "", - default_readers: 0, - default_url: "", - default_x: 1., - default_y: 1., - default_year: "", - sort_by_label: 'sort by:', - filter_by_label: 'show: ', - all: "any", - open_access: "Open Access", - Dataset: "Dataset", - "Journal Article": "Journal article", - List: "List", - Preprint: "Preprint", - ReFigure: "ReFigure", - Review: "Review", - link: 'link', - items: "items", - comment_by_label: "by", - pdf_not_loaded: "Sorry, we were not able to retrieve the PDF for this publication. You can get it directly from", - pdf_not_loaded_linktext: "this website", - share_button_title: "share this map", - embed_button_title: "Embed this knowledge map on other websites", - embed_button_text: 'Copy', - embed_title: 'embed map', - embed_body_text: 'You can use this code to embed the visualization on your own website or in a dashboard.', - }, - } + share_modal: false, + hashtags_twitter_card: "COVID19,openscience,discovery" }; diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 58f578292..2fa59a7e3 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -272,6 +272,10 @@ def get_additional_context_data(self, df): else: return None + def get_spreadsheet_title(self, sheet_id): + res = self.sheet.get(spreadsheetId=sheet_id, fields='properties/title').execute() + return res.get('properties').get('title') + def get_new_mapdata(self, sheet_id, sheet_range, params): raw = self.get_sheet_content(sheet_id, sheet_range) clean_df, errors, errors_df = self.validate_data(raw.copy()) @@ -290,6 +294,9 @@ def get_new_mapdata(self, sheet_id, sheet_range, params): additional_context = self.get_additional_context_data(raw.copy()) if additional_context: res["additional_context"] = additional_context + else: + # inject CoVis multi-map title from sheet title + res["additional_context"]["query"] = self.get_spreadsheet_title(sheet_id) res["sheet_id"] = sheet_id res["last_update"] = self.last_updated.get(sheet_id, {}).get("timestamp_utc") return res From 6ab9788d4e46c52a06a4600854db64511585b43c Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 1 Dec 2020 14:36:12 +0100 Subject: [PATCH 72/76] bugfix --- server/workers/gsheets/src/search_gsheets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 2fa59a7e3..0ae916b41 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -296,6 +296,7 @@ def get_new_mapdata(self, sheet_id, sheet_range, params): res["additional_context"] = additional_context else: # inject CoVis multi-map title from sheet title + res["additional_context"] = {} res["additional_context"]["query"] = self.get_spreadsheet_title(sheet_id) res["sheet_id"] = sheet_id res["last_update"] = self.last_updated.get(sheet_id, {}).get("timestamp_utc") From 9d6386eb50c82f2248a1ef209851fab5ecab3152 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Tue, 1 Dec 2020 15:27:52 +0100 Subject: [PATCH 73/76] additional rights --- server/workers/gsheets/src/search_gsheets.py | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 0ae916b41..5775d05fa 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -375,6 +375,18 @@ def prefill_additional_context(self, new_kb, params): def set_new_kb_permissions(self, new_drive, new_kb, main_curator_email): + # set folder rights for okmaps + new_domain_permission = { + 'type': 'domain', + 'role': 'organizer', + 'domain': 'openknowledgemaps.org' + } + permission = self.drive_service.permissions().create( + fileId=new_drive.get('id'), + body=new_domain_permission, + supportsAllDrives=True + ).execute() + # set folder rights for main curator new_organizer_permission = { 'type': 'user', 'role': 'organizer', @@ -385,6 +397,7 @@ def set_new_kb_permissions(self, new_drive, new_kb, main_curator_email): body=new_organizer_permission, supportsAllDrives=True ).execute() + # set file rights for main curator new_fileorganizer_permission = { 'type': 'user', 'role': 'writer', @@ -395,6 +408,17 @@ def set_new_kb_permissions(self, new_drive, new_kb, main_curator_email): body=new_fileorganizer_permission, supportsAllDrives=True ).execute() + # set file rights for info okmaps + new_fileorganizer_permission = { + 'type': 'user', + 'role': 'writer', + 'emailAddress': 'info@openknowledgemaps.org' + } + permission = self.drive_service.permissions().create( + fileId=new_kb.get('id'), + body=new_fileorganizer_permission, + supportsAllDrives=True + ).execute() def run(self): while True: From 210600e7d55767fd4851e04b5a841b70dffb17b7 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 2 Dec 2020 12:37:34 +0100 Subject: [PATCH 74/76] config cleanup --- examples/triple/config_example.php | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/triple/config_example.php b/examples/triple/config_example.php index 08e7e1f57..8cbd5fdd1 100644 --- a/examples/triple/config_example.php +++ b/examples/triple/config_example.php @@ -2,4 +2,5 @@ $HEADSTART_PATH = "../../"; $WEBSITE_PATH = "http://localhost/triple/"; $SNAPSHOT_PATH = $WEBSITE_PATH . "server/storage/"; +$PERSISTENCE_BACKEND = "api"; ?> From ce80641aa9e2f30b70cc960c74d447e49516f3c6 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 2 Dec 2020 13:40:58 +0100 Subject: [PATCH 75/76] cleanup --- server/workers/services/src/apis/request_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/workers/services/src/apis/request_validators.py b/server/workers/services/src/apis/request_validators.py index b8f8a1993..3cf80448f 100644 --- a/server/workers/services/src/apis/request_validators.py +++ b/server/workers/services/src/apis/request_validators.py @@ -21,7 +21,7 @@ class SearchParamSchema(Schema): unique_id = fields.Str() raw = fields.Boolean() sg_method = fields.Str() - vis_id = fields.Str() + vis_id = fields.Str(default=None) @pre_load def fix_years(self, in_data, **kwargs): From f11c83c34907febd7bfe400adbb5bd168609d545 Mon Sep 17 00:00:00 2001 From: Christopher Kittel Date: Wed, 2 Dec 2020 17:30:56 +0100 Subject: [PATCH 76/76] automatic map title in gsheets-example --- server/workers/gsheets/src/search_gsheets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/workers/gsheets/src/search_gsheets.py b/server/workers/gsheets/src/search_gsheets.py index 5775d05fa..15ecd9055 100644 --- a/server/workers/gsheets/src/search_gsheets.py +++ b/server/workers/gsheets/src/search_gsheets.py @@ -294,6 +294,7 @@ def get_new_mapdata(self, sheet_id, sheet_range, params): additional_context = self.get_additional_context_data(raw.copy()) if additional_context: res["additional_context"] = additional_context + res["additional_context"]["query"] = additional_context["topic"] else: # inject CoVis multi-map title from sheet title res["additional_context"] = {}