diff --git a/Makefile b/Makefile index cc0875a9..659af782 100644 --- a/Makefile +++ b/Makefile @@ -45,10 +45,16 @@ unit-tests: ## integration-tests: Run integration tests. .PHONY: integration-tests -integration-tests: export CACHE_URL = redis://localhost:6379 integration-tests: test_path = tests/integration integration-tests: - @coverage run -m pytest $(test_path) + @CACHE_URL=redis://localhost:6379 \ + PROVIDER='' \ + TEST_DATASET='' \ + API_ROOT_URL=https://platform.localhost.ai \ + DATA_API_ROOT_URL=https://data.localhost.ai \ + TEST_API_KEY='' \ + TEST_BEARER_TOKEN='' \ + coverage run -m pytest --vcr-record=none $(test_path) ## coverage: Display code coverage in the console. .PHONY: coverage diff --git a/requirements-test.txt b/requirements-test.txt index 8f13da73..eb01cd6c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,5 @@ freezegun==1.0.0 pytest==7.1.2 pytest-httpx==0.20.0 pytest-mock==3.3.1 +pytest-vcr~=1.0.2 requests-mock==1.8.0 diff --git a/setup.cfg b/setup.cfg index bc0a44fe..b70a1bbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.09 +fail_under = 98.44 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 94292f04..7a3619e7 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -2,7 +2,6 @@ import functools import json import logging -import posixpath from typing import Callable, List, Optional import httpx @@ -154,14 +153,16 @@ class DataApiVersions: class UserApiSdk: def __init__( self, - platform_api_url: str, + platform_v1_url: str, + platform_v2_url: str, data_api_url: str, api_key: str, app_key: str, - logger: logging.Logger, + logger: logging.Logger = logging.getLogger(), timeout: int = 30, ): - self._platform_api_url = platform_api_url + self._platform_v1_url = platform_v1_url + self._platform_v2_url = platform_v2_url self._data_api_url = data_api_url self._headers = { "Authorization": f"API {api_key}", @@ -172,17 +173,17 @@ def __init__( def __enter__(self): data_cli = httpx.Client( - base_url=posixpath.join(self._data_api_url, "api/v1"), + base_url=self._data_api_url, headers=self._headers, timeout=self._timeout, ) platform_v1_cli = httpx.Client( - base_url=posixpath.join(self._platform_api_url, "v1"), + base_url=self._platform_v1_url, headers=self._headers, timeout=self._timeout, ) platform_v2_cli = httpx.Client( - base_url=posixpath.join(self._platform_api_url, "v2"), + base_url=self._platform_v2_url, headers=self._headers, timeout=self._timeout, ) diff --git a/src/corva/configuration.py b/src/corva/configuration.py index ebbbd7a4..42552747 100644 --- a/src/corva/configuration.py +++ b/src/corva/configuration.py @@ -1,4 +1,5 @@ import datetime +import os import pydantic @@ -25,3 +26,21 @@ class Settings(pydantic.BaseSettings): SETTINGS = Settings() + + +def get_test_api_key() -> str: + """Api key for testing""" + + return os.environ['TEST_API_KEY'] + + +def get_test_bearer() -> str: + """Bearer token for testing""" + + return os.environ['TEST_BEARER_TOKEN'] + + +def get_test_dataset() -> str: + """Dataset for testing""" + + return os.environ['TEST_DATASET'] diff --git a/tests/integration/cassettes/TestUserApiSdk.test_get.yaml b/tests/integration/cassettes/TestUserApiSdk.test_get.yaml new file mode 100644 index 00000000..7a341e9f --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_get.yaml @@ -0,0 +1,289 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-b307dd45"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"325393","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-b307dd45","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:38 GMT + ETag: + - W/"091ea5d8aea25ce573aabb2cc6e565ef" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 33879c04-be0f-4457-ba80-74891ef992ba + X-Runtime: + - '0.136328' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"325393","type":"well","attributes":{"asset_id":89513687},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:38 GMT + ETag: + - W/"cdb0cbdaaae6aca210587ce8dfdfbb2b" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 3cba62ba-04d5-4d93-ad86-b38aa900701c + X-Runtime: + - '0.064769' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:39 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 12}, + {"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 11}, {"asset_id": + 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 13}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '300' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"inserted_ids":["62839173c5745c7f7606c6fa","62839173c5745c7f7606c6fb","62839173c5745c7f7606c6fc","62839173c5745c7f7606c6fd"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '157' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:39 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-17 12:13:37+00:00 + method: GET + uri: null + response: + content: '[{"_id":"62839173c5745c7f7606c6fc","timestamp":11},{"_id":"62839173c5745c7f7606c6fb","timestamp":12}]' + headers: + Connection: + - keep-alive + Content-Length: + - '101' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:40 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:40 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - bf08403a-c9de-43e3-a365-bbda6b009563 + X-Runtime: + - '0.118524' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":4}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:40 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 00aa7461..89d16e4c 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -1,4 +1,10 @@ +import contextlib +import datetime +import json import logging +import posixpath +import uuid +from typing import Iterable import httpx import pytest @@ -14,7 +20,8 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture): caplog.handler.setFormatter(logging.Formatter('%(message)s')) sdk = corva.api_adapter.UserApiSdk( - platform_api_url='', + platform_v1_url='', + platform_v2_url='', data_api_url='', api_key='', app_key='', @@ -31,7 +38,7 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture): "'https://' protocol.", 'request': { 'method': 'GET', - 'url': 'api/v1/whatever', + 'url': '/whatever', 'headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', @@ -53,7 +60,8 @@ def test_unsuccessful_response( httpx_mock.add_response(status_code=400) sdk = corva.api_adapter.UserApiSdk( - platform_api_url='', + platform_v1_url='', + platform_v2_url='', data_api_url="https://test_url", api_key='', app_key='', @@ -67,7 +75,7 @@ def test_unsuccessful_response( expected = { 'message': "Request failed - Client error '400 Bad Request' for url " - "'https://test_url/api/v1/'\nFor more information check: " + "'https://test_url/'\nFor more information check: " "https://httpstatuses.com/400", 'response': { 'code': 400, @@ -77,7 +85,7 @@ def test_unsuccessful_response( }, 'request': { 'method': 'GET', - 'url': 'https://test_url/api/v1/', + 'url': 'https://test_url/', 'headers': { 'host': 'test_url', 'accept': '*/*', @@ -92,3 +100,173 @@ def test_unsuccessful_response( } assert actual == expected + + +def vcr_before_record_request(request): + request.uri = None + + return request + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + # Replace the Authorization request header + "filter_headers": ["Authorization", 'host'], + "before_record_request": vcr_before_record_request, + } + + +@pytest.fixture(scope='module') +def platform_v1_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.API_ROOT_URL, 'v1') + + +@pytest.fixture(scope='module') +def platform_v2_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.API_ROOT_URL, 'v2') + + +@pytest.fixture(scope='module') +def data_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.DATA_API_ROOT_URL, 'api/v1') + + +@pytest.fixture(scope='module') +def headers() -> dict: + return {'Authorization': f'Bearer {corva.configuration.get_test_bearer()}'} + + +@pytest.fixture(scope='module') +def data(data_url: str, headers: dict) -> Iterable[httpx.Client]: + with httpx.Client(base_url=data_url, headers=headers) as data: + yield data + + +@pytest.fixture(scope='module') +def provider() -> str: + return corva.configuration.SETTINGS.PROVIDER + + +@pytest.fixture(scope='module') +def dataset() -> str: + return corva.configuration.get_test_dataset() + + +@pytest.fixture(scope='module') +def app_key() -> str: + now = datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0) + return f'python-sdk-autotest-{now}' + + +@contextlib.contextmanager +def _setup( + platform: httpx.Client, data: httpx.Client, provider: str, dataset: str +) -> Iterable[int]: + response = platform.post( + url='wells', + json={ + 'well': {'name': f'deleteme-python-sdk-autotest-{str(uuid.uuid4())[:8]}'} + }, + ) + response.raise_for_status() + well_id = int(response.json()['data']['id']) + + response = platform.get(url=f'wells/{well_id}?fields[]=well.asset_id') + response.raise_for_status() + asset_id = response.json()['data']['attributes']['asset_id'] + + data.delete( + f'data/{provider}/{dataset}/', + params={'query': json.dumps({'asset_id': asset_id})}, + ).raise_for_status() + + try: + yield asset_id + finally: + platform.delete(f'wells/{well_id}').raise_for_status() + + data.delete( + f'data/{provider}/{dataset}/', + params={'query': json.dumps({'asset_id': asset_id})}, + ).raise_for_status() + + +@pytest.fixture(scope='module') +def setup_( + platform_v2_url: str, data: httpx.Client, headers: dict, provider: str, dataset: str +) -> Iterable[int]: + with httpx.Client(base_url=platform_v2_url, headers=headers) as platform: + yield _setup(platform=platform, data=data, provider=provider, dataset=dataset) + + +@pytest.fixture(scope='function') +def sdk( + platform_v1_url: str, platform_v2_url: str, data_url: str, app_key: str +) -> Iterable[corva.api_adapter.UserApiSdk]: + sdk = corva.api_adapter.UserApiSdk( + platform_v1_url=platform_v1_url, + platform_v2_url=platform_v2_url, + data_api_url=data_url, + api_key=corva.configuration.get_test_api_key(), + app_key=app_key, + logger=logging.getLogger(), + ) + + with sdk: + yield sdk + + +class TestUserApiSdk: + @pytest.mark.vcr + def test_get( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + data: httpx.Client, + ): + with setup_ as asset_id: + data.post( + f'data/{provider}/{dataset}/', + json=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 10, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 12, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 11, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 13, + }, + ], + ).raise_for_status() + + data = sdk.data.v1.get( + provider=provider, + dataset=dataset, + query={'asset_id': asset_id}, + sort={'timestamp': 1}, + limit=2, + skip=1, + fields='timestamp', + ) + + assert len(data) == 2 + assert set(datum['timestamp'] for datum in data) == {11, 12}