From 4db3c874a0b0648d7cdaa9110dcc28966671d9eb Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 26 Mar 2019 13:52:40 +0100 Subject: [PATCH 1/7] Add handover to schemas --- beacon_api/schemas/info.json | 29 ++++++++++++++++ beacon_api/schemas/response.json | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/beacon_api/schemas/info.json b/beacon_api/schemas/info.json index 280d49dd..0d989895 100644 --- a/beacon_api/schemas/info.json +++ b/beacon_api/schemas/info.json @@ -826,6 +826,35 @@ }, "info": { "type": "object" + }, + "beaconHandover": { + "type": "array", + "required": [ + "handoverType", + "url" + ], + "properties": { + "handoverType": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + } } } } diff --git a/beacon_api/schemas/response.json b/beacon_api/schemas/response.json index 186ee381..c0f39be1 100644 --- a/beacon_api/schemas/response.json +++ b/beacon_api/schemas/response.json @@ -108,6 +108,35 @@ } } }, + "beaconHandover": { + "type": "array", + "required": [ + "handoverType", + "url" + ], + "properties": { + "handoverType": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, "datasetAlleleResponses": { "type": "array", "items": { @@ -129,6 +158,35 @@ } ] }, + "datasetHandover": { + "type": "array", + "required": [ + "handoverType", + "url" + ], + "properties": { + "handoverType": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, "referenceBases": { "type": "string", "pattern": "^([ACGT]+)$" From d5ea7bc53f5f9318bc2fd1268ae51f573905a757 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Wed, 27 Mar 2019 08:06:51 +0100 Subject: [PATCH 2/7] Add simple handover --- beacon_api/__init__.py | 4 ++++ beacon_api/api/info.py | 8 +++++--- beacon_api/api/query.py | 7 +++++-- beacon_api/conf/__init__.py | 9 +++++++++ beacon_api/conf/config.ini | 18 ++++++++++++++++++ beacon_api/utils/data_query.py | 28 ++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/beacon_api/__init__.py b/beacon_api/__init__.py index 770c7305..77e24208 100644 --- a/beacon_api/__init__.py +++ b/beacon_api/__init__.py @@ -12,6 +12,10 @@ __author__ = CONFIG_INFO.author __license__ = CONFIG_INFO.license __copyright__ = CONFIG_INFO.copyright +__handover_drs__ = CONFIG_INFO.handover_drs +__handover_datasets__ = CONFIG_INFO.handover_datasets +__handover_beacon__ = CONFIG_INFO.handover_beacon +__handover_base__ = CONFIG_INFO.handover_base __apiVersion__ = CONFIG_INFO.apiVersion __beaconId__ = CONFIG_INFO.beaconId diff --git a/beacon_api/api/info.py b/beacon_api/api/info.py index 17c209cf..5c1eceec 100644 --- a/beacon_api/api/info.py +++ b/beacon_api/api/info.py @@ -6,10 +6,10 @@ .. note:: See ``beacon_api`` root folder ``__init__.py`` for changing values used here. """ -from .. import __apiVersion__, __title__, __version__, __description__, __url__, __alturl__ +from .. import __apiVersion__, __title__, __version__, __description__, __url__, __alturl__, __handover_beacon__ from .. import __createtime__, __updatetime__, __org_id__, __org_name__, __org_description__ from .. import __org_address__, __org_logoUrl__, __org_welcomeUrl__, __org_info__, __org_contactUrl__ -from ..utils.data_query import fetch_dataset_metadata +from ..utils.data_query import fetch_dataset_metadata, make_handover from aiocache import cached from aiocache.serializers import JsonSerializer @@ -21,6 +21,7 @@ async def beacon_info(host, pool): :return beacon_info: A dict that contain information about the ``Beacon`` endpoint. """ beacon_dataset = await fetch_dataset_metadata(pool) + beaconhandover = make_handover(__handover_beacon__, [x['id'] for x in beacon_dataset]) # If one sets up a beacon it is recommended to adjust these sample requests sample_allele_request = [{ "alternateBases": "G", @@ -73,7 +74,8 @@ async def beacon_info(host, pool): 'updateDateTime': __updatetime__, 'datasets': beacon_dataset, 'sampleAlleleRequests': sample_allele_request, - 'info': {"achievement": "World's first 1.0 Beacon"} + 'info': {"achievement": "World's first 1.0 Beacon"}, + 'beaconHandover': beaconhandover } return beacon_info diff --git a/beacon_api/api/query.py b/beacon_api/api/query.py index bfee7f88..545d80e7 100644 --- a/beacon_api/api/query.py +++ b/beacon_api/api/query.py @@ -6,8 +6,8 @@ """ from ..utils.logging import LOG -from .. import __apiVersion__ -from ..utils.data_query import filter_exists, find_datasets, fetch_datasets_access +from .. import __apiVersion__, __handover_beacon__ +from ..utils.data_query import filter_exists, find_datasets, fetch_datasets_access, make_handover from .exceptions import BeaconUnauthorised, BeaconForbidden, BeaconBadRequest @@ -63,6 +63,7 @@ async def query_request_handler(params): """ LOG.info(f'{params[1]} request to beacon endpoint "/query"') request = params[2] + # Fills the Beacon variable with the found data. alleleRequest = {'referenceName': request.get("referenceName"), 'referenceBases': request.get("referenceBases"), @@ -99,6 +100,7 @@ async def query_request_handler(params): request.get("referenceBases"), alternate, accessible_datasets, access_type, request.get("includeDatasetResponses", "NONE")) + beaconhandover = make_handover(__handover_beacon__, [x['datasetId'] for x in datasets]) beacon_response = {'beaconId': '.'.join(reversed(params[4].split('.'))), 'apiVersion': __apiVersion__, 'exists': any([x['exists'] for x in datasets]), @@ -107,6 +109,7 @@ async def query_request_handler(params): # otherwise schema validation will fail # "error": None, 'alleleRequest': alleleRequest, + 'beaconHandover': beaconhandover, 'datasetAlleleResponses': filter_exists(request.get("includeDatasetResponses", "NONE"), datasets)} return beacon_response diff --git a/beacon_api/conf/__init__.py b/beacon_api/conf/__init__.py index 412a30b5..00a10283 100644 --- a/beacon_api/conf/__init__.py +++ b/beacon_api/conf/__init__.py @@ -5,6 +5,11 @@ from collections import namedtuple +def parse_drspaths(paths): + """Parse handover configuration.""" + return [p.strip().split(',', 2) for p in paths.split('\n') if p.split()] + + def parse_config_file(path): """Parse configuration file.""" config = ConfigParser() @@ -15,6 +20,10 @@ def parse_config_file(path): 'author': config.get('beacon_general_info', 'author'), 'license': config.get('beacon_general_info', 'license'), 'copyright': config.get('beacon_general_info', 'copyright'), + 'handover_drs': config.get('handover_info', 'drs'), + 'handover_datasets': parse_drspaths(config.get('handover_info', 'dataset_paths')), + 'handover_beacon': parse_drspaths(config.get('handover_info', 'beacon_paths')), + 'handover_base': int(config.get('handover_info', 'handover_base')), 'apiVersion': config.get('beacon_api_info', 'apiVersion'), 'beaconId': config.get('beacon_api_info', 'beaconId'), 'description': config.get('beacon_api_info', 'description'), diff --git a/beacon_api/conf/config.ini b/beacon_api/conf/config.ini index 12033caa..cef05933 100644 --- a/beacon_api/conf/config.ini +++ b/beacon_api/conf/config.ini @@ -39,6 +39,24 @@ alturl= createtime=2018-07-25T00:00:00Z +[handover_info] +# The base url for all handovers +drs=https://examplebrowser.org + +# Make the handovers 1- or 0-based +handover_base = 1 + +# Handovers for datasets +dataset_paths= + Variants,browse the variants matched by the query,dataset/{dataset}/browser/variant/{chr}-{start}-{ref}-{alt} + Region,browse data of the region matched by the query,dataset/{dataset}/browser/region/{chr}-{start}-{end} + Data,retrieve information of the datasets,dataset/{dataset}/browser + +# Handovers for general beacon +beacon_paths= + Project,retrieve information about the datasets,dataset/{dataset} + + [organisation_info] # Globally unique identifier for organisation that hosts this Beacon service org_id=fi.csc diff --git a/beacon_api/utils/data_query.py b/beacon_api/utils/data_query.py index 8b7dd6ac..b0a2a473 100644 --- a/beacon_api/utils/data_query.py +++ b/beacon_api/utils/data_query.py @@ -4,6 +4,7 @@ from .logging import LOG from ..api.exceptions import BeaconServerError from ..conf.config import DB_SCHEMA +from .. import __handover_drs__, __handover_datasets__, __handover_base__ # def transform_record(record, variantCount): @@ -58,6 +59,32 @@ def transform_metadata(record): return response +def add_handover(record): + """Add handover to a dataset response.""" + response = dict(record) + response["datasetHandover"] = make_handover(__handover_datasets__, [record['datasetId']], + record['referenceName'], record['start'], + record['end'], record['referenceBases'], + record['alternateBases'], record['variantType']) + return response + + +def make_handover(paths, datasetIds, chr='', start=0, end=0, ref='', alt='', variant=''): + """Create one handover for each path (specified in config).""" + alt = alt if alt else variant + handovers = [] + start = start + __handover_base__ + end = end + __handover_base__ + for label, desc, path in paths: + for dataset in set(datasetIds): + handovers.append({"handoverType": {"id": "CUSTOM", "label": label}, + "description": desc, + "url": __handover_drs__ + path.format(dataset=dataset, chr=chr, start=start, + end=end, ref=ref, alt=alt)}) + + return handovers + + async def fetch_datasets_access(db_pool, datasets): """Retrieve CONTROLLED datasets.""" public = [] @@ -183,6 +210,7 @@ async def fetch_filtered_dataset(db_pool, assembly_id, position, chromosome, ref endMin_pos, endMax_pos) LOG.info(f"Query for dataset(s): {datasets} that are {access_type} matching conditions.") for record in list(db_response): + record = add_handover(record) processed = transform_misses(record) if misses else transform_record(record) datasets.append(processed) return datasets From 1cb456463c334a6ba4e099405692ad819403c93e Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 26 Mar 2019 13:57:47 +0100 Subject: [PATCH 3/7] Add tests for handover --- tests/test_data_query.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_data_query.py b/tests/test_data_query.py index 528b5e69..0c6a8c98 100644 --- a/tests/test_data_query.py +++ b/tests/test_data_query.py @@ -1,6 +1,7 @@ import asynctest +from unittest import mock from beacon_api.utils.data_query import filter_exists, transform_record -from beacon_api.utils.data_query import transform_misses, transform_metadata, find_datasets +from beacon_api.utils.data_query import transform_misses, transform_metadata, find_datasets, add_handover, make_handover from datetime import datetime # from beacon_api.utils.data_query import fetch_dataset_metadata from beacon_api.utils.data_query import handle_wildcard @@ -110,6 +111,26 @@ def test_transform_metadata(self): result = transform_metadata(record) self.assertEqual(result, response) + def test_add_handover(self): + """Test that add handover.""" + # Test that the handover actually is added + handovers = [{"handover1": "info"}, {"handover2": "url"}] + record = {"datasetId": "test", "referenceName": "22", "referenceBases": "A", + "alternateBases": "C", "start": 10, "end": 11, "variantType": "SNP"} + with mock.patch('beacon_api.utils.data_query.make_handover', return_value=handovers): + result = add_handover(record) + record['datasetHandover'] = handovers + self.assertEqual(result, record) + + def test_make_handover(self): + """Test make handover.""" + paths = [('lab1', 'desc1', 'path1'), ('lab2', 'desc2', 'path2')] + result = make_handover(paths, ['id1', 'id2', 'id1']) + # The number of handovers = number of paths * number of unique datasets + self.assertEqual(len(result), 4) + self.assertIn("path1", result[0]["url"]) + self.assertEqual(result[0]["description"], 'desc1') + @asynctest.mock.patch('beacon_api.utils.data_query.fetch_filtered_dataset') async def test_find_datasets(self, mock_filtered): """Test find datasets.""" From db5b57324537f4d14625a66a51455a60aa181c61 Mon Sep 17 00:00:00 2001 From: Teemu Kataja Date: Mon, 1 Apr 2019 10:14:59 +0300 Subject: [PATCH 4/7] fix integration test --- deploy/test/auth_test.ini | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/deploy/test/auth_test.ini b/deploy/test/auth_test.ini index a0ff34b5..f222507c 100644 --- a/deploy/test/auth_test.ini +++ b/deploy/test/auth_test.ini @@ -39,6 +39,24 @@ alturl=https://ega-archive.org/ createtime=2018-07-25T00:00:00Z +[handover_info] +# The base url for all handovers +drs=https://examplebrowser.org + +# Make the handovers 1- or 0-based +handover_base = 1 + +# Handovers for datasets +dataset_paths= + Variants,browse the variants matched by the query,dataset/{dataset}/browser/variant/{chr}-{start}-{ref}-{alt} + Region,browse data of the region matched by the query,dataset/{dataset}/browser/region/{chr}-{start}-{end} + Data,retrieve information of the datasets,dataset/{dataset}/browser + +# Handovers for general beacon +beacon_paths= + Project,retrieve information about the datasets,dataset/{dataset} + + [organisation_info] # Globally unique identifier for organisation that hosts this Beacon service org_id=CSC From f61529a466157af8a373463dbd43dd2cd284e1ad Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 1 Apr 2019 10:54:55 +0200 Subject: [PATCH 5/7] Make handover optional --- beacon_api/api/info.py | 6 ++++-- beacon_api/api/query.py | 6 +++--- beacon_api/conf/__init__.py | 8 ++++---- beacon_api/utils/data_query.py | 9 +++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/beacon_api/api/info.py b/beacon_api/api/info.py index 5c1eceec..f7a911dd 100644 --- a/beacon_api/api/info.py +++ b/beacon_api/api/info.py @@ -9,6 +9,7 @@ from .. import __apiVersion__, __title__, __version__, __description__, __url__, __alturl__, __handover_beacon__ from .. import __createtime__, __updatetime__, __org_id__, __org_name__, __org_description__ from .. import __org_address__, __org_logoUrl__, __org_welcomeUrl__, __org_info__, __org_contactUrl__ +from .. import __handover_drs__ from ..utils.data_query import fetch_dataset_metadata, make_handover from aiocache import cached from aiocache.serializers import JsonSerializer @@ -21,7 +22,7 @@ async def beacon_info(host, pool): :return beacon_info: A dict that contain information about the ``Beacon`` endpoint. """ beacon_dataset = await fetch_dataset_metadata(pool) - beaconhandover = make_handover(__handover_beacon__, [x['id'] for x in beacon_dataset]) + # If one sets up a beacon it is recommended to adjust these sample requests sample_allele_request = [{ "alternateBases": "G", @@ -75,7 +76,8 @@ async def beacon_info(host, pool): 'datasets': beacon_dataset, 'sampleAlleleRequests': sample_allele_request, 'info': {"achievement": "World's first 1.0 Beacon"}, - 'beaconHandover': beaconhandover } + if __handover_drs__: + beacon_info['beaconHandover'] = make_handover(__handover_beacon__, [x['id'] for x in beacon_dataset]) return beacon_info diff --git a/beacon_api/api/query.py b/beacon_api/api/query.py index 545d80e7..e02d858d 100644 --- a/beacon_api/api/query.py +++ b/beacon_api/api/query.py @@ -6,7 +6,7 @@ """ from ..utils.logging import LOG -from .. import __apiVersion__, __handover_beacon__ +from .. import __apiVersion__, __handover_beacon__, __handover_drs__ from ..utils.data_query import filter_exists, find_datasets, fetch_datasets_access, make_handover from .exceptions import BeaconUnauthorised, BeaconForbidden, BeaconBadRequest @@ -100,7 +100,6 @@ async def query_request_handler(params): request.get("referenceBases"), alternate, accessible_datasets, access_type, request.get("includeDatasetResponses", "NONE")) - beaconhandover = make_handover(__handover_beacon__, [x['datasetId'] for x in datasets]) beacon_response = {'beaconId': '.'.join(reversed(params[4].split('.'))), 'apiVersion': __apiVersion__, 'exists': any([x['exists'] for x in datasets]), @@ -109,7 +108,8 @@ async def query_request_handler(params): # otherwise schema validation will fail # "error": None, 'alleleRequest': alleleRequest, - 'beaconHandover': beaconhandover, 'datasetAlleleResponses': filter_exists(request.get("includeDatasetResponses", "NONE"), datasets)} + if __handover_drs__: + beacon_response['beaconHandover'] = make_handover(__handover_beacon__, [x['datasetId'] for x in datasets]) return beacon_response diff --git a/beacon_api/conf/__init__.py b/beacon_api/conf/__init__.py index 00a10283..6a764d17 100644 --- a/beacon_api/conf/__init__.py +++ b/beacon_api/conf/__init__.py @@ -20,10 +20,10 @@ def parse_config_file(path): 'author': config.get('beacon_general_info', 'author'), 'license': config.get('beacon_general_info', 'license'), 'copyright': config.get('beacon_general_info', 'copyright'), - 'handover_drs': config.get('handover_info', 'drs'), - 'handover_datasets': parse_drspaths(config.get('handover_info', 'dataset_paths')), - 'handover_beacon': parse_drspaths(config.get('handover_info', 'beacon_paths')), - 'handover_base': int(config.get('handover_info', 'handover_base')), + 'handover_drs': config.get('handover_info', 'drs', fallback=''), + 'handover_datasets': parse_drspaths(config.get('handover_info', 'dataset_paths', fallback='')), + 'handover_beacon': parse_drspaths(config.get('handover_info', 'beacon_paths', fallback='')), + 'handover_base': int(config.get('handover_info', 'handover_base', fallback=0)), 'apiVersion': config.get('beacon_api_info', 'apiVersion'), 'beaconId': config.get('beacon_api_info', 'beaconId'), 'description': config.get('beacon_api_info', 'description'), diff --git a/beacon_api/utils/data_query.py b/beacon_api/utils/data_query.py index b0a2a473..c237cb80 100644 --- a/beacon_api/utils/data_query.py +++ b/beacon_api/utils/data_query.py @@ -62,10 +62,11 @@ def transform_metadata(record): def add_handover(record): """Add handover to a dataset response.""" response = dict(record) - response["datasetHandover"] = make_handover(__handover_datasets__, [record['datasetId']], - record['referenceName'], record['start'], - record['end'], record['referenceBases'], - record['alternateBases'], record['variantType']) + if __handover_drs__: + response["datasetHandover"] = make_handover(__handover_datasets__, [record['datasetId']], + record['referenceName'], record['start'], + record['end'], record['referenceBases'], + record['alternateBases'], record['variantType']) return response From ab00f1e2b75066174d983a81f0b15f8bb3b30331 Mon Sep 17 00:00:00 2001 From: Teemu Kataja Date: Tue, 2 Apr 2019 08:27:02 +0300 Subject: [PATCH 6/7] disable handover from master by default --- beacon_api/conf/config.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beacon_api/conf/config.ini b/beacon_api/conf/config.ini index cef05933..9278c72a 100644 --- a/beacon_api/conf/config.ini +++ b/beacon_api/conf/config.ini @@ -41,7 +41,8 @@ createtime=2018-07-25T00:00:00Z [handover_info] # The base url for all handovers -drs=https://examplebrowser.org +# if this url is empty or commented, handover feature is disabled +#drs=https://examplebrowser.org # Make the handovers 1- or 0-based handover_base = 1 From 03d361ae3daeffc0752762c4d43bbcd9fd73850a Mon Sep 17 00:00:00 2001 From: Teemu Kataja Date: Tue, 2 Apr 2019 09:12:06 +0300 Subject: [PATCH 7/7] move handover check from handover function to dataset filter function to keep old tests working(if drs is commented from config), and reduce dict transformations --- beacon_api/utils/data_query.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/beacon_api/utils/data_query.py b/beacon_api/utils/data_query.py index c237cb80..4461c555 100644 --- a/beacon_api/utils/data_query.py +++ b/beacon_api/utils/data_query.py @@ -59,14 +59,12 @@ def transform_metadata(record): return response -def add_handover(record): +def add_handover(response): """Add handover to a dataset response.""" - response = dict(record) - if __handover_drs__: - response["datasetHandover"] = make_handover(__handover_datasets__, [record['datasetId']], - record['referenceName'], record['start'], - record['end'], record['referenceBases'], - record['alternateBases'], record['variantType']) + response["datasetHandover"] = make_handover(__handover_datasets__, [response['datasetId']], + response['referenceName'], response['start'], + response['end'], response['referenceBases'], + response['alternateBases'], response['variantType']) return response @@ -211,8 +209,10 @@ async def fetch_filtered_dataset(db_pool, assembly_id, position, chromosome, ref endMin_pos, endMax_pos) LOG.info(f"Query for dataset(s): {datasets} that are {access_type} matching conditions.") for record in list(db_response): - record = add_handover(record) processed = transform_misses(record) if misses else transform_record(record) + if __handover_drs__: + # If handover feature is enabled, add handover object to response + processed = add_handover(processed) datasets.append(processed) return datasets except Exception as e: