From 90f774e3a3d5cb56cffbf047e355a982100b4d65 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:31:17 +0300 Subject: [PATCH 01/18] Do not add suffixes to urls in `UserApiSdk` --- src/corva/api_adapter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 94292f04..437a2711 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -2,7 +2,6 @@ import functools import json import logging -import posixpath from typing import Callable, List, Optional import httpx @@ -154,14 +153,16 @@ class DataApiVersions: class UserApiSdk: def __init__( self, - platform_api_url: str, + platform_v1_url: str, + platform_v2_url: str, data_api_url: str, api_key: str, app_key: str, logger: logging.Logger, timeout: int = 30, ): - self._platform_api_url = platform_api_url + self._platform_v1_url = platform_v1_url + self._platform_v2_url = platform_v2_url self._data_api_url = data_api_url self._headers = { "Authorization": f"API {api_key}", @@ -172,17 +173,17 @@ def __init__( def __enter__(self): data_cli = httpx.Client( - base_url=posixpath.join(self._data_api_url, "api/v1"), + base_url=self._data_api_url, headers=self._headers, timeout=self._timeout, ) platform_v1_cli = httpx.Client( - base_url=posixpath.join(self._platform_api_url, "v1"), + base_url=self._platform_v1_url, headers=self._headers, timeout=self._timeout, ) platform_v2_cli = httpx.Client( - base_url=posixpath.join(self._platform_api_url, "v2"), + base_url=self._platform_v2_url, headers=self._headers, timeout=self._timeout, ) From 2a57c8f94dc6e9aa1c87fa38db7e2b18232b47f3 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:31:28 +0300 Subject: [PATCH 02/18] Added default `logger` value --- src/corva/api_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 437a2711..7a3619e7 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -158,7 +158,7 @@ def __init__( data_api_url: str, api_key: str, app_key: str, - logger: logging.Logger, + logger: logging.Logger = logging.getLogger(), timeout: int = 30, ): self._platform_v1_url = platform_v1_url From 3e1b799c4981bcf826a84c8604f371149f3fbdbf Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:33:38 +0300 Subject: [PATCH 03/18] Added TestUserApiSdk.test_get --- Makefile | 12 +- requirements-test.txt | 1 + src/corva/configuration.py | 19 ++ .../cassettes/TestUserApiSdk.test_get.yaml | 289 ++++++++++++++++++ tests/integration/test_api.py | 191 +++++++++++- 5 files changed, 503 insertions(+), 9 deletions(-) create mode 100644 tests/integration/cassettes/TestUserApiSdk.test_get.yaml diff --git a/Makefile b/Makefile index cc0875a9..43ead767 100644 --- a/Makefile +++ b/Makefile @@ -39,16 +39,20 @@ test: up-cache unit-tests integration-tests down-cache ## unit-tests: Run unit tests. .PHONY: unit-tests -unit-tests: test_path = tests/unit unit-tests: - @coverage run -m pytest $(test_path) + @coverage run -m pytest tests/unit ## integration-tests: Run integration tests. .PHONY: integration-tests integration-tests: export CACHE_URL = redis://localhost:6379 -integration-tests: test_path = tests/integration +integration-tests: export PROVIDER ?= +integration-tests: export TEST_DATASET ?= +integration-tests: export API_ROOT_URL ?= +integration-tests: export DATA_API_ROOT_URL ?= +integration-tests: export TEST_API_KEY ?= +integration-tests: export TEST_BEARER_TOKEN ?= integration-tests: - @coverage run -m pytest $(test_path) + @coverage run -m pytest --vcr-record=none tests/integration ## coverage: Display code coverage in the console. .PHONY: coverage diff --git a/requirements-test.txt b/requirements-test.txt index 8f13da73..eb01cd6c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,5 @@ freezegun==1.0.0 pytest==7.1.2 pytest-httpx==0.20.0 pytest-mock==3.3.1 +pytest-vcr~=1.0.2 requests-mock==1.8.0 diff --git a/src/corva/configuration.py b/src/corva/configuration.py index ebbbd7a4..42552747 100644 --- a/src/corva/configuration.py +++ b/src/corva/configuration.py @@ -1,4 +1,5 @@ import datetime +import os import pydantic @@ -25,3 +26,21 @@ class Settings(pydantic.BaseSettings): SETTINGS = Settings() + + +def get_test_api_key() -> str: + """Api key for testing""" + + return os.environ['TEST_API_KEY'] + + +def get_test_bearer() -> str: + """Bearer token for testing""" + + return os.environ['TEST_BEARER_TOKEN'] + + +def get_test_dataset() -> str: + """Dataset for testing""" + + return os.environ['TEST_DATASET'] diff --git a/tests/integration/cassettes/TestUserApiSdk.test_get.yaml b/tests/integration/cassettes/TestUserApiSdk.test_get.yaml new file mode 100644 index 00000000..7a341e9f --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_get.yaml @@ -0,0 +1,289 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-b307dd45"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"325393","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-b307dd45","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:38 GMT + ETag: + - W/"091ea5d8aea25ce573aabb2cc6e565ef" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 33879c04-be0f-4457-ba80-74891ef992ba + X-Runtime: + - '0.136328' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"325393","type":"well","attributes":{"asset_id":89513687},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:38 GMT + ETag: + - W/"cdb0cbdaaae6aca210587ce8dfdfbb2b" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 3cba62ba-04d5-4d93-ad86-b38aa900701c + X-Runtime: + - '0.064769' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:39 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 12}, + {"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 11}, {"asset_id": + 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 13}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '300' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"inserted_ids":["62839173c5745c7f7606c6fa","62839173c5745c7f7606c6fb","62839173c5745c7f7606c6fc","62839173c5745c7f7606c6fd"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '157' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:39 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-17 12:13:37+00:00 + method: GET + uri: null + response: + content: '[{"_id":"62839173c5745c7f7606c6fc","timestamp":11},{"_id":"62839173c5745c7f7606c6fb","timestamp":12}]' + headers: + Connection: + - keep-alive + Content-Length: + - '101' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:40 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:40 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - bf08403a-c9de-43e3-a365-bbda6b009563 + X-Runtime: + - '0.118524' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":4}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:40 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 00aa7461..13670f4a 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -1,4 +1,10 @@ +import contextlib +import datetime +import json import logging +import posixpath +import uuid +from typing import Iterable import httpx import pytest @@ -14,7 +20,8 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture): caplog.handler.setFormatter(logging.Formatter('%(message)s')) sdk = corva.api_adapter.UserApiSdk( - platform_api_url='', + platform_v1_url='', + platform_v2_url='', data_api_url='', api_key='', app_key='', @@ -31,7 +38,7 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture): "'https://' protocol.", 'request': { 'method': 'GET', - 'url': 'api/v1/whatever', + 'url': '/whatever', 'headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', @@ -53,7 +60,8 @@ def test_unsuccessful_response( httpx_mock.add_response(status_code=400) sdk = corva.api_adapter.UserApiSdk( - platform_api_url='', + platform_v1_url='', + platform_v2_url='', data_api_url="https://test_url", api_key='', app_key='', @@ -67,7 +75,7 @@ def test_unsuccessful_response( expected = { 'message': "Request failed - Client error '400 Bad Request' for url " - "'https://test_url/api/v1/'\nFor more information check: " + "'https://test_url/'\nFor more information check: " "https://httpstatuses.com/400", 'response': { 'code': 400, @@ -77,7 +85,7 @@ def test_unsuccessful_response( }, 'request': { 'method': 'GET', - 'url': 'https://test_url/api/v1/', + 'url': 'https://test_url/', 'headers': { 'host': 'test_url', 'accept': '*/*', @@ -92,3 +100,176 @@ def test_unsuccessful_response( } assert actual == expected + + +def vcr_before_record_request(request): + request.uri = None + + return request + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + # Replace the Authorization request header + "filter_headers": ["Authorization", 'host'], + "before_record_request": vcr_before_record_request, + } + + +@pytest.fixture(scope='module') +def platform_v1_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.API_ROOT_URL, 'v1') + + +@pytest.fixture(scope='module') +def platform_v2_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.API_ROOT_URL, 'v2') + + +@pytest.fixture(scope='module') +def data_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.DATA_API_ROOT_URL, 'api/v1') + + +@pytest.fixture(scope='module') +def headers() -> dict: + return {'Authorization': f'Bearer {corva.configuration.get_test_bearer()}'} + + +@pytest.fixture(scope='module') +def data(data_url: str, headers: dict) -> Iterable[httpx.Client]: + with httpx.Client(base_url=data_url, headers=headers) as data: + yield data + + +@pytest.fixture(scope='module') +def provider() -> str: + return corva.configuration.SETTINGS.PROVIDER + + +@pytest.fixture(scope='module') +def dataset() -> str: + return corva.configuration.get_test_dataset() + + +@pytest.fixture(scope='module') +def app_key() -> str: + now = datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0) + return f'python-sdk-autotest-{now}' + + +@contextlib.contextmanager +def _setup( + platform: httpx.Client, data: httpx.Client, provider: str, dataset: str +) -> Iterable[int]: + response = platform.post( + url='wells', + json={ + 'well': {'name': f'deleteme-python-sdk-autotest-{str(uuid.uuid4())[:8]}'} + }, + ) + response.raise_for_status() + well_id = int(response.json()['data']['id']) + + response = platform.get(url=f'wells/{well_id}?fields[]=well.asset_id') + response.raise_for_status() + asset_id = response.json()['data']['attributes']['asset_id'] + + response = data.delete( + f'data/{provider}/{dataset}/', + params={'query': json.dumps({'asset_id': asset_id})}, + ) + response.raise_for_status() + + try: + yield asset_id + finally: + response = platform.delete(f'wells/{well_id}') + response.raise_for_status() + + response = data.delete( + f'data/{provider}/{dataset}/', + params={'query': json.dumps({'asset_id': asset_id})}, + ) + response.raise_for_status() + + +@pytest.fixture(scope='module') +def setup_( + platform_v2_url: str, data: httpx.Client, headers: dict, provider: str, dataset: str +) -> Iterable[int]: + with httpx.Client(base_url=platform_v2_url, headers=headers) as platform: + yield _setup(platform=platform, data=data, provider=provider, dataset=dataset) + + +@pytest.fixture(scope='function') +def sdk( + platform_v1_url: str, platform_v2_url: str, data_url: str, app_key: str +) -> Iterable[corva.api_adapter.UserApiSdk]: + sdk = corva.api_adapter.UserApiSdk( + platform_v1_url=platform_v1_url, + platform_v2_url=platform_v2_url, + data_api_url=data_url, + api_key=corva.configuration.get_test_api_key(), + app_key=app_key, + logger=logging.getLogger(), + ) + + with sdk: + yield sdk + + +class TestUserApiSdk: + @pytest.mark.vcr + def test_get( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + data: httpx.Client, + ): + with setup_ as asset_id: + data.post( + f'data/{provider}/{dataset}/', + json=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 10, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 12, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 11, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 13, + }, + ], + ).raise_for_status() + + data = sdk.data.v1.get( + provider=provider, + dataset=dataset, + query={'asset_id': asset_id}, + sort={'timestamp': 1}, + limit=2, + skip=1, + fields='timestamp', + ) + + assert len(data) == 2 + assert set(datum['timestamp'] for datum in data) == {11, 12} From 0e7b97990f3f32f8a2a2344f0689dc207d1588ff Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:45:52 +0300 Subject: [PATCH 04/18] Use `coverage` instead of `test` in `all` target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 43ead767..59194f7d 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ black = black --skip-string-normalization $(srcs) ## all: Run linter and tests. .PHONY: all -all: lint test +all: lint coverage ## help: Show this help. .PHONY: help From c73978dcb2ceec516460f236e704477914b9962d Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:46:00 +0300 Subject: [PATCH 05/18] Increased code coverage --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bc0a44fe..b70a1bbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.09 +fail_under = 98.44 skip_covered = True show_missing = True exclude_lines = From 6b2809fe25e05d31407ad75d124b5480026e32ee Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:49:42 +0300 Subject: [PATCH 06/18] Added env vars for integration tests to `test` step --- .github/workflows/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 43e127b5..f49e4dab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,6 +38,13 @@ jobs: - name: Run run: make coverage + env: + PROVIDER: ${{ secrets.PROVIDER }} + TEST_DATASET: ${{ secrets.TEST_DATASET }} + API_ROOT_URL: ${{ secrets.API_ROOT_URL }} + DATA_API_ROOT_URL: ${{ secrets.DATA_API_ROOT_URL }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + TEST_BEARER_TOKEN: ${{ secrets.TEST_BEARER_TOKEN }} deploy: name: Build and publish to PyPI From 33920406e7a66eafe80006e7c48758c157c40760 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:56:50 +0300 Subject: [PATCH 07/18] Fix CI --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 59194f7d..707b1d1f 100644 --- a/Makefile +++ b/Makefile @@ -45,12 +45,12 @@ unit-tests: ## integration-tests: Run integration tests. .PHONY: integration-tests integration-tests: export CACHE_URL = redis://localhost:6379 -integration-tests: export PROVIDER ?= -integration-tests: export TEST_DATASET ?= -integration-tests: export API_ROOT_URL ?= -integration-tests: export DATA_API_ROOT_URL ?= -integration-tests: export TEST_API_KEY ?= -integration-tests: export TEST_BEARER_TOKEN ?= +integration-tests: export PROVIDER = +integration-tests: export TEST_DATASET = +integration-tests: export API_ROOT_URL = https://platform.localhost.ai +integration-tests: export DATA_API_ROOT_URL = https://data.localhost.ai +integration-tests: export TEST_API_KEY = +integration-tests: export TEST_BEARER_TOKEN = integration-tests: @coverage run -m pytest --vcr-record=none tests/integration From 3f91e44c7ba8cc5108662c949e1ab0ef0413c8d5 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 15:57:08 +0300 Subject: [PATCH 08/18] Deleted env vars from `test` step --- .github/workflows/main.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f49e4dab..43e127b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,13 +38,6 @@ jobs: - name: Run run: make coverage - env: - PROVIDER: ${{ secrets.PROVIDER }} - TEST_DATASET: ${{ secrets.TEST_DATASET }} - API_ROOT_URL: ${{ secrets.API_ROOT_URL }} - DATA_API_ROOT_URL: ${{ secrets.DATA_API_ROOT_URL }} - TEST_API_KEY: ${{ secrets.TEST_API_KEY }} - TEST_BEARER_TOKEN: ${{ secrets.TEST_BEARER_TOKEN }} deploy: name: Build and publish to PyPI From 0401187daedfe92ef867dbd4b2b30998207b0b55 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 17 May 2022 17:28:29 +0300 Subject: [PATCH 09/18] Raise for status immediately --- tests/integration/test_api.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 13670f4a..89d16e4c 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -176,23 +176,20 @@ def _setup( response.raise_for_status() asset_id = response.json()['data']['attributes']['asset_id'] - response = data.delete( + data.delete( f'data/{provider}/{dataset}/', params={'query': json.dumps({'asset_id': asset_id})}, - ) - response.raise_for_status() + ).raise_for_status() try: yield asset_id finally: - response = platform.delete(f'wells/{well_id}') - response.raise_for_status() + platform.delete(f'wells/{well_id}').raise_for_status() - response = data.delete( + data.delete( f'data/{provider}/{dataset}/', params={'query': json.dumps({'asset_id': asset_id})}, - ) - response.raise_for_status() + ).raise_for_status() @pytest.fixture(scope='module') From 50e9328c3a5759d80d87d7c301fc9832fbfc79ba Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 16:20:24 +0300 Subject: [PATCH 10/18] Added DataApiV1Sdk.insert --- setup.cfg | 2 +- src/corva/api_adapter.py | 23 +- .../cassettes/TestUserApiSdk.test_insert.yaml | 315 ++++++++++++++++++ tests/integration/test_api.py | 86 ++++- 4 files changed, 409 insertions(+), 17 deletions(-) create mode 100644 tests/integration/cassettes/TestUserApiSdk.test_insert.yaml diff --git a/setup.cfg b/setup.cfg index b70a1bbf..addc8de2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.44 +fail_under = 98.45 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 7a3619e7..d531c1dc 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -2,7 +2,7 @@ import functools import json import logging -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Sequence import httpx import yaml @@ -128,6 +128,27 @@ def get( return data + def insert(self, provider: str, dataset: str, *, documents: Sequence[dict]) -> dict: + """Inserts data using the endpoint POST 'data/{provider}/{dataset}/'. + + Args: + provider: company name owning the dataset. + dataset: dataset name. + documents: data to insert. + + Raises: + httpx.HTTPStatusError: if request was unsuccessful. + + Returns: + Data from dataset. + """ + + response = self.http.post(url=f"data/{provider}/{dataset}/", json=documents) + + response.raise_for_status() + + return response.json() + class PlatformApiV1Sdk: def __init__(self, client: httpx.Client): diff --git a/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml new file mode 100644 index 00000000..c1d658a9 --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml @@ -0,0 +1,315 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-dd55ea8a"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"326390","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-dd55ea8a","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 18 May 2022 13:10:07 GMT + ETag: + - W/"2c6cd5798156037b6539b51077f3828c" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 317799f8-50e1-48ba-9cc5-52f283604ba1 + X-Runtime: + - '0.156188' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"326390","type":"well","attributes":{"asset_id":83776921},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 18 May 2022 13:10:08 GMT + ETag: + - W/"90948c5d20ba8334b7bcb3e4413b14f6" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 76035711-c879-46c3-8659-70e034a23ecc + X-Runtime: + - '0.064434' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 13:10:08 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '[]' + headers: + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 13:10:09 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 83776921, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 83776921, "version": 1, "data": {"k": "v"}, "timestamp": 11}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '150' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-18 13:10:06+00:00 + method: POST + uri: null + response: + content: '{"inserted_ids":["6284f0323cc8096aa7e5c8f8","6284f0323cc8096aa7e5c8f9"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 13:10:10 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '[{"_id":"6284f0323cc8096aa7e5c8f8","company_id":80,"asset_id":83776921,"version":1,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k":"v"},"timestamp":10,"app_key":"big-data-energy.oleksii_test_task_app"},{"_id":"6284f0323cc8096aa7e5c8f9","company_id":80,"asset_id":83776921,"version":1,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k":"v"},"timestamp":11,"app_key":"big-data-energy.oleksii_test_task_app"}]' + headers: + Connection: + - keep-alive + Content-Length: + - '457' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 13:10:10 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 18 May 2022 13:10:10 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - c0433dcd-cedb-4cd1-861f-4cad8b69b5d2 + X-Runtime: + - '0.189032' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":2}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 13:10:10 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 89d16e4c..411b0421 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -112,7 +112,7 @@ def vcr_before_record_request(request): def vcr_config(): return { # Replace the Authorization request header - "filter_headers": ["Authorization", 'host'], + "filter_headers": ["Authorization", "host"], "before_record_request": vcr_before_record_request, } @@ -174,7 +174,7 @@ def _setup( response = platform.get(url=f'wells/{well_id}?fields[]=well.asset_id') response.raise_for_status() - asset_id = response.json()['data']['attributes']['asset_id'] + asset_id: int = response.json()['data']['attributes']['asset_id'] data.delete( f'data/{provider}/{dataset}/', @@ -192,7 +192,7 @@ def _setup( ).raise_for_status() -@pytest.fixture(scope='module') +@pytest.fixture(scope='function') def setup_( platform_v2_url: str, data: httpx.Client, headers: dict, provider: str, dataset: str ) -> Iterable[int]: @@ -213,8 +213,7 @@ def sdk( logger=logging.getLogger(), ) - with sdk: - yield sdk + yield sdk class TestUserApiSdk: @@ -258,15 +257,72 @@ def test_get( ], ).raise_for_status() - data = sdk.data.v1.get( - provider=provider, - dataset=dataset, - query={'asset_id': asset_id}, - sort={'timestamp': 1}, - limit=2, - skip=1, - fields='timestamp', + with sdk as s: + collection = s.data.v1.get( + provider=provider, + dataset=dataset, + query={'asset_id': asset_id}, + sort={'timestamp': 1}, + limit=2, + skip=1, + fields='timestamp', + ) + + assert len(collection) == 2 + assert set(doc['timestamp'] for doc in collection) == {11, 12} + + @pytest.mark.vcr + def test_insert( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + data: httpx.Client, + ): + with setup_ as asset_id: + response = data.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps({'asset_id': asset_id}), + "sort": json.dumps({'timestamp': 1}), + "limit": 1, + }, + ) + response.raise_for_status() + collection = response.json() + + assert not collection + + with sdk as s: + s.data.v1.insert( + provider=provider, + dataset=dataset, + documents=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 10, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 11, + }, + ], + ) + + response = data.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps({'asset_id': asset_id}), + "sort": json.dumps({'timestamp': 1}), + "limit": 3, + }, ) + response.raise_for_status() + collection = response.json() - assert len(data) == 2 - assert set(datum['timestamp'] for datum in data) == {11, 12} + assert len(collection) == 2 From a18242a609b45a91da2d4757fbfea6df4efe90e6 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 16:20:32 +0300 Subject: [PATCH 11/18] Fixed docstr --- src/corva/api_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index d531c1dc..3c6cc7e0 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -105,7 +105,7 @@ def get( fields: comma separated list of fields to return. Example: "_id,data". Raises: - requests.HTTPError: if request was unsuccessful. + httpx.HTTPStatusError: if request was unsuccessful. Returns: Data from dataset. From f043c07c01459451a23f4dcafe7fbc1d4642469f Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 16:23:48 +0300 Subject: [PATCH 12/18] Rollback Makefile changes --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 707b1d1f..0b0ac86a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ black = black --skip-string-normalization $(srcs) ## all: Run linter and tests. .PHONY: all -all: lint coverage +all: lint test ## help: Show this help. .PHONY: help @@ -39,11 +39,13 @@ test: up-cache unit-tests integration-tests down-cache ## unit-tests: Run unit tests. .PHONY: unit-tests +unit-tests: test_path = tests/unit unit-tests: - @coverage run -m pytest tests/unit + @coverage run -m pytest $(test_path) ## integration-tests: Run integration tests. .PHONY: integration-tests +integration-tests: test_path = tests/integration integration-tests: export CACHE_URL = redis://localhost:6379 integration-tests: export PROVIDER = integration-tests: export TEST_DATASET = @@ -52,7 +54,7 @@ integration-tests: export DATA_API_ROOT_URL = https://data.localhost.ai integration-tests: export TEST_API_KEY = integration-tests: export TEST_BEARER_TOKEN = integration-tests: - @coverage run -m pytest --vcr-record=none tests/integration + @coverage run -m pytest --vcr-record=none $(test_path) ## coverage: Display code coverage in the console. .PHONY: coverage From 60d9a1c8cffd387b062cab9051b36c410a92f14e Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 16:28:49 +0300 Subject: [PATCH 13/18] Set env variables inline in `integration-tests` --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 0b0ac86a..659af782 100644 --- a/Makefile +++ b/Makefile @@ -46,15 +46,15 @@ unit-tests: ## integration-tests: Run integration tests. .PHONY: integration-tests integration-tests: test_path = tests/integration -integration-tests: export CACHE_URL = redis://localhost:6379 -integration-tests: export PROVIDER = -integration-tests: export TEST_DATASET = -integration-tests: export API_ROOT_URL = https://platform.localhost.ai -integration-tests: export DATA_API_ROOT_URL = https://data.localhost.ai -integration-tests: export TEST_API_KEY = -integration-tests: export TEST_BEARER_TOKEN = integration-tests: - @coverage run -m pytest --vcr-record=none $(test_path) + @CACHE_URL=redis://localhost:6379 \ + PROVIDER='' \ + TEST_DATASET='' \ + API_ROOT_URL=https://platform.localhost.ai \ + DATA_API_ROOT_URL=https://data.localhost.ai \ + TEST_API_KEY='' \ + TEST_BEARER_TOKEN='' \ + coverage run -m pytest --vcr-record=none $(test_path) ## coverage: Display code coverage in the console. .PHONY: coverage From 136bbb270624ea6a3b393e4ed14fade959b6e041 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 18:48:42 +0300 Subject: [PATCH 14/18] Added `DataApiV1Sdk.replace` method --- Makefile | 1 + setup.cfg | 2 +- src/corva/api_adapter.py | 25 ++ src/corva/configuration.py | 6 + .../TestUserApiSdk.test_replace.yaml | 320 ++++++++++++++++++ tests/integration/test_api.py | 63 +++- 6 files changed, 415 insertions(+), 2 deletions(-) create mode 100644 tests/integration/cassettes/TestUserApiSdk.test_replace.yaml diff --git a/Makefile b/Makefile index 659af782..fe5a560e 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ integration-tests: DATA_API_ROOT_URL=https://data.localhost.ai \ TEST_API_KEY='' \ TEST_BEARER_TOKEN='' \ + TEST_COMPANY_ID='-1' \ coverage run -m pytest --vcr-record=none $(test_path) ## coverage: Display code coverage in the console. diff --git a/setup.cfg b/setup.cfg index addc8de2..bfe79078 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.45 +fail_under = 98.47 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 3c6cc7e0..4c11908b 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -149,6 +149,31 @@ def insert(self, provider: str, dataset: str, *, documents: Sequence[dict]) -> d return response.json() + def replace(self, provider: str, dataset: str, id_: str, *, document: dict) -> dict: + """Replaces all document data. + + Replace all document data using the endpoint PUT + 'data/{provider}/{dataset}/{id}/'. + + Args: + provider: company name owning the dataset. + dataset: dataset name. + id_: document id to replace. + document: new document data. + + Raises: + httpx.HTTPStatusError: if request was unsuccessful. + + Returns: + Updated document. + """ + + response = self.http.put(url=f"data/{provider}/{dataset}/{id_}/", json=document) + + response.raise_for_status() + + return response.json() + class PlatformApiV1Sdk: def __init__(self, client: httpx.Client): diff --git a/src/corva/configuration.py b/src/corva/configuration.py index 42552747..128180cc 100644 --- a/src/corva/configuration.py +++ b/src/corva/configuration.py @@ -44,3 +44,9 @@ def get_test_dataset() -> str: """Dataset for testing""" return os.environ['TEST_DATASET'] + + +def get_test_company_id() -> int: + """Company id for testing""" + + return int(os.environ['TEST_COMPANY_ID']) diff --git a/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml b/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml new file mode 100644 index 00000000..5b92e644 --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml @@ -0,0 +1,320 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-49ed84f5"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"326537","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-49ed84f5","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 18 May 2022 14:25:27 GMT + ETag: + - W/"2b4c6d4d3a0bae2cda001a44dce7a587" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 96e8dba2-f6db-49be-8db0-3cad5456dd6e + X-Runtime: + - '0.135435' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"326537","type":"well","attributes":{"asset_id":32946351},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 18 May 2022 14:25:28 GMT + ETag: + - W/"39909209f7c32816963aac957c249f15" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 1503989c-86a3-4287-8313-59da4abc76eb + X-Runtime: + - '0.078469' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 14:25:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 32946351, "version": 1, "data": {"k1": "v1"}, "timestamp": + 10}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '77' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"inserted_ids":["628501d93cc8096aa7e5d248"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '76' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 14:25:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"data": {"k2": "v2"}, "version": 2, "company_id": 80, "timestamp": 11, + "asset_id": 32946351}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '93' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-18 14:25:27+00:00 + method: PUT + uri: null + response: + content: '{"_id":"628501d93cc8096aa7e5d248","company_id":80,"asset_id":32946351,"version":2,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.oleksii_test_task_app"}' + headers: + Connection: + - keep-alive + Content-Length: + - '229' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 14:25:30 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"_id":"628501d93cc8096aa7e5d248","company_id":80,"asset_id":32946351,"version":2,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.oleksii_test_task_app"}' + headers: + Connection: + - keep-alive + Content-Length: + - '229' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 14:25:30 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 18 May 2022 14:25:31 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - f5e6dbb9-b87f-41b6-8541-8231a6e4640f + X-Runtime: + - '0.136116' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":1}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Wed, 18 May 2022 14:25:31 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 411b0421..b262218e 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -153,6 +153,11 @@ def dataset() -> str: return corva.configuration.get_test_dataset() +@pytest.fixture(scope='module') +def company_id() -> int: + return corva.configuration.get_test_company_id() + + @pytest.fixture(scope='module') def app_key() -> str: now = datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0) @@ -204,13 +209,20 @@ def setup_( def sdk( platform_v1_url: str, platform_v2_url: str, data_url: str, app_key: str ) -> Iterable[corva.api_adapter.UserApiSdk]: + import sys + + logger = logging.getLogger(name='l') + logger.setLevel('INFO') + handler = logging.StreamHandler(stream=sys.stdout) + logger.addHandler(handler) + sdk = corva.api_adapter.UserApiSdk( platform_v1_url=platform_v1_url, platform_v2_url=platform_v2_url, data_api_url=data_url, api_key=corva.configuration.get_test_api_key(), app_key=app_key, - logger=logging.getLogger(), + logger=logger, ) yield sdk @@ -326,3 +338,52 @@ def test_insert( collection = response.json() assert len(collection) == 2 + + @pytest.mark.vcr + def test_replace( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + company_id: int, + data: httpx.Client, + ): + with setup_ as asset_id: + response = data.post( + f'data/{provider}/{dataset}/', + json=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k1": "v1"}, + "timestamp": 10, + }, + ], + ) + response.raise_for_status() + id_ = response.json()['inserted_ids'][0] + + with sdk as s: + s.data.v1.replace( + provider=provider, + dataset=dataset, + id_=id_, + document={ + 'data': {'k2': 'v2'}, + 'version': 2, + 'company_id': company_id, + 'timestamp': 11, + 'asset_id': asset_id, + }, + ) + + response = data.get( + url=f"data/{provider}/{dataset}/{id_}/", + ) + response.raise_for_status() + document_after_replace = response.json() + + assert document_after_replace['data'] == {'k2': 'v2'} + assert document_after_replace['version'] == 2 + assert document_after_replace['timestamp'] == 11 From 56773e1a07d97a24af30b1116d34ffc5652e317c Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 18:59:55 +0300 Subject: [PATCH 15/18] Added `InsertResult` to `DataApiV1Sdk.insert` --- src/corva/__init__.py | 1 + src/corva/api_adapter.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/corva/__init__.py b/src/corva/__init__.py index 49dd5732..1bc1f0b5 100644 --- a/src/corva/__init__.py +++ b/src/corva/__init__.py @@ -1,4 +1,5 @@ from .api import Api +from .api_adapter import InsertResult from .handlers import scheduled, stream, task from .logger import CORVA_LOGGER as Logger from .models.scheduled.scheduled import ( diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 3c6cc7e0..3a79aca5 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -5,6 +5,7 @@ from typing import Callable, List, Optional, Sequence import httpx +import pydantic import yaml @@ -71,6 +72,12 @@ def wrapper(request: httpx.Request, *args, **kwargs): return wrapper +class InsertResult(pydantic.BaseModel): + inserted_ids: List[str] + failed_count: int + messages: List[str] + + class DataApiV1Sdk: def __init__(self, client: httpx.Client): self.http = client @@ -128,7 +135,9 @@ def get( return data - def insert(self, provider: str, dataset: str, *, documents: Sequence[dict]) -> dict: + def insert( + self, provider: str, dataset: str, *, documents: Sequence[dict] + ) -> InsertResult: """Inserts data using the endpoint POST 'data/{provider}/{dataset}/'. Args: @@ -140,14 +149,14 @@ def insert(self, provider: str, dataset: str, *, documents: Sequence[dict]) -> d httpx.HTTPStatusError: if request was unsuccessful. Returns: - Data from dataset. + Insert result. """ response = self.http.post(url=f"data/{provider}/{dataset}/", json=documents) response.raise_for_status() - return response.json() + return InsertResult.parse_obj(response.json()) class PlatformApiV1Sdk: From 0a0ee5e616336a0a1514f8a6a48dcc5c464128cd Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Wed, 18 May 2022 19:02:08 +0300 Subject: [PATCH 16/18] Deleted debug logging in tests --- tests/integration/test_api.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index b262218e..73fcd46f 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -209,20 +209,13 @@ def setup_( def sdk( platform_v1_url: str, platform_v2_url: str, data_url: str, app_key: str ) -> Iterable[corva.api_adapter.UserApiSdk]: - import sys - - logger = logging.getLogger(name='l') - logger.setLevel('INFO') - handler = logging.StreamHandler(stream=sys.stdout) - logger.addHandler(handler) - sdk = corva.api_adapter.UserApiSdk( platform_v1_url=platform_v1_url, platform_v2_url=platform_v2_url, data_api_url=data_url, api_key=corva.configuration.get_test_api_key(), app_key=app_key, - logger=logger, + logger=logging.getLogger(), ) yield sdk From ab049d34a4787ab2499462c314160b2394eb236c Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Mon, 23 May 2022 14:40:45 +0300 Subject: [PATCH 17/18] Regenerated test_insert.yaml with generic app name and dataset --- .../cassettes/TestUserApiSdk.test_insert.yaml | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml index c1d658a9..7725a8c6 100644 --- a/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml +++ b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"well": {"name": "deleteme-python-sdk-autotest-dd55ea8a"}}' + body: '{"well": {"name": "deleteme-python-sdk-autotest-eb81d18c"}}' headers: accept: - '*/*' @@ -17,7 +17,7 @@ interactions: method: POST uri: null response: - content: '{"data":{"id":"326390","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-dd55ea8a","status":"unknown","state":"planned"},"relationships":{}}}' + content: '{"data":{"id":"328567","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-eb81d18c","status":"unknown","state":"planned"},"relationships":{}}}' headers: Cache-Control: - max-age=0, private, must-revalidate @@ -28,9 +28,9 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 18 May 2022 13:10:07 GMT + - Mon, 23 May 2022 11:39:27 GMT ETag: - - W/"2c6cd5798156037b6539b51077f3828c" + - W/"ec3494f3333667f241a2e1d1b41205ae" Referrer-Policy: - strict-origin-when-cross-origin Server: @@ -50,9 +50,9 @@ interactions: X-Rack-CORS: - miss; no-origin X-Request-Id: - - 317799f8-50e1-48ba-9cc5-52f283604ba1 + - 3462393c-fe38-4cd3-aca1-a4d4a7b722ee X-Runtime: - - '0.156188' + - '0.105799' X-XSS-Protection: - 1; mode=block http_version: HTTP/1.1 @@ -71,7 +71,7 @@ interactions: method: GET uri: null response: - content: '{"data":{"id":"326390","type":"well","attributes":{"asset_id":83776921},"relationships":{}}}' + content: '{"data":{"id":"328567","type":"well","attributes":{"asset_id":53781610},"relationships":{}}}' headers: Cache-Control: - max-age=0, private, must-revalidate @@ -82,9 +82,9 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 18 May 2022 13:10:08 GMT + - Mon, 23 May 2022 11:39:27 GMT ETag: - - W/"90948c5d20ba8334b7bcb3e4413b14f6" + - W/"c412a0007563bca3e870f857bfe4cf1e" Referrer-Policy: - strict-origin-when-cross-origin Server: @@ -104,9 +104,9 @@ interactions: X-Rack-CORS: - miss; no-origin X-Request-Id: - - 76035711-c879-46c3-8659-70e034a23ecc + - 099e239c-b7af-452d-9340-9456471dbcf9 X-Runtime: - - '0.064434' + - '0.059369' X-XSS-Protection: - 1; mode=block http_version: HTTP/1.1 @@ -134,7 +134,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 13:10:08 GMT + - Mon, 23 May 2022 11:39:27 GMT server: - uvicorn http_version: HTTP/1.1 @@ -162,14 +162,14 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 13:10:09 GMT + - Mon, 23 May 2022 11:39:28 GMT server: - uvicorn http_version: HTTP/1.1 status_code: 200 - request: - body: '[{"asset_id": 83776921, "version": 1, "data": {"k": "v"}, "timestamp": - 10}, {"asset_id": 83776921, "version": 1, "data": {"k": "v"}, "timestamp": 11}]' + body: '[{"asset_id": 53781610, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 53781610, "version": 1, "data": {"k": "v"}, "timestamp": 11}]' headers: accept: - '*/*' @@ -184,11 +184,11 @@ interactions: user-agent: - python-httpx/0.22.0 x-corva-app: - - python-sdk-autotest-2022-05-18 13:10:06+00:00 + - python-sdk-autotest-2022-05-23 11:39:26+00:00 method: POST uri: null response: - content: '{"inserted_ids":["6284f0323cc8096aa7e5c8f8","6284f0323cc8096aa7e5c8f9"],"failed_count":0,"messages":[]}' + content: '{"inserted_ids":["628b7271d6aff355ccea8b61","628b7271d6aff355ccea8b62"],"failed_count":0,"messages":[]}' headers: Connection: - keep-alive @@ -197,7 +197,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 13:10:10 GMT + - Mon, 23 May 2022 11:39:29 GMT server: - uvicorn http_version: HTTP/1.1 @@ -216,16 +216,16 @@ interactions: method: GET uri: null response: - content: '[{"_id":"6284f0323cc8096aa7e5c8f8","company_id":80,"asset_id":83776921,"version":1,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k":"v"},"timestamp":10,"app_key":"big-data-energy.oleksii_test_task_app"},{"_id":"6284f0323cc8096aa7e5c8f9","company_id":80,"asset_id":83776921,"version":1,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k":"v"},"timestamp":11,"app_key":"big-data-energy.oleksii_test_task_app"}]' + content: '[{"_id":"628b7271d6aff355ccea8b61","company_id":80,"asset_id":53781610,"version":1,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k":"v"},"timestamp":10,"app_key":"big-data-energy.python_sdk_app_for_autotests"},{"_id":"628b7271d6aff355ccea8b62","company_id":80,"asset_id":53781610,"version":1,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k":"v"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}]' headers: Connection: - keep-alive Content-Length: - - '457' + - '475' Content-Type: - application/json Date: - - Wed, 18 May 2022 13:10:10 GMT + - Mon, 23 May 2022 11:39:29 GMT server: - uvicorn http_version: HTTP/1.1 @@ -255,7 +255,7 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 18 May 2022 13:10:10 GMT + - Mon, 23 May 2022 11:39:29 GMT ETag: - W/"4df20f95e824b2af44a61642d88daaf0" Referrer-Policy: @@ -277,9 +277,9 @@ interactions: X-Rack-CORS: - miss; no-origin X-Request-Id: - - c0433dcd-cedb-4cd1-861f-4cad8b69b5d2 + - 8424ca9f-13a6-4ac7-9f70-836a3011124f X-Runtime: - - '0.189032' + - '0.099032' X-XSS-Protection: - 1; mode=block http_version: HTTP/1.1 @@ -307,7 +307,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 13:10:10 GMT + - Mon, 23 May 2022 11:39:29 GMT server: - uvicorn http_version: HTTP/1.1 From 3220624040e699603b34d84e6e6d14dddab51ea1 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Mon, 23 May 2022 14:41:27 +0300 Subject: [PATCH 18/18] Regenerated test_replace.yaml with generic app name and dataset --- .../TestUserApiSdk.test_replace.yaml | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml b/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml index 5b92e644..149b542d 100644 --- a/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml +++ b/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"well": {"name": "deleteme-python-sdk-autotest-49ed84f5"}}' + body: '{"well": {"name": "deleteme-python-sdk-autotest-d4a1214c"}}' headers: accept: - '*/*' @@ -17,7 +17,7 @@ interactions: method: POST uri: null response: - content: '{"data":{"id":"326537","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-49ed84f5","status":"unknown","state":"planned"},"relationships":{}}}' + content: '{"data":{"id":"328568","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-d4a1214c","status":"unknown","state":"planned"},"relationships":{}}}' headers: Cache-Control: - max-age=0, private, must-revalidate @@ -28,9 +28,9 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 18 May 2022 14:25:27 GMT + - Mon, 23 May 2022 11:41:15 GMT ETag: - - W/"2b4c6d4d3a0bae2cda001a44dce7a587" + - W/"11b6d1d9eb9720313d1fd72196df9595" Referrer-Policy: - strict-origin-when-cross-origin Server: @@ -50,9 +50,9 @@ interactions: X-Rack-CORS: - miss; no-origin X-Request-Id: - - 96e8dba2-f6db-49be-8db0-3cad5456dd6e + - ce585d4e-d4dc-47bb-9f2a-f109e6a3a59b X-Runtime: - - '0.135435' + - '0.101889' X-XSS-Protection: - 1; mode=block http_version: HTTP/1.1 @@ -71,7 +71,7 @@ interactions: method: GET uri: null response: - content: '{"data":{"id":"326537","type":"well","attributes":{"asset_id":32946351},"relationships":{}}}' + content: '{"data":{"id":"328568","type":"well","attributes":{"asset_id":30719716},"relationships":{}}}' headers: Cache-Control: - max-age=0, private, must-revalidate @@ -82,9 +82,9 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 18 May 2022 14:25:28 GMT + - Mon, 23 May 2022 11:41:15 GMT ETag: - - W/"39909209f7c32816963aac957c249f15" + - W/"b161ea51687ef8fd4b47fd4dc1ca754d" Referrer-Policy: - strict-origin-when-cross-origin Server: @@ -104,9 +104,9 @@ interactions: X-Rack-CORS: - miss; no-origin X-Request-Id: - - 1503989c-86a3-4287-8313-59da4abc76eb + - 05e9b6f6-ab30-4478-8f0d-dfe7e7679ac7 X-Runtime: - - '0.078469' + - '0.044065' X-XSS-Protection: - 1; mode=block http_version: HTTP/1.1 @@ -134,13 +134,13 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 14:25:29 GMT + - Mon, 23 May 2022 11:41:16 GMT server: - uvicorn http_version: HTTP/1.1 status_code: 200 - request: - body: '[{"asset_id": 32946351, "version": 1, "data": {"k1": "v1"}, "timestamp": + body: '[{"asset_id": 30719716, "version": 1, "data": {"k1": "v1"}, "timestamp": 10}]' headers: accept: @@ -158,7 +158,7 @@ interactions: method: POST uri: null response: - content: '{"inserted_ids":["628501d93cc8096aa7e5d248"],"failed_count":0,"messages":[]}' + content: '{"inserted_ids":["628b72dcd6aff355ccea8b78"],"failed_count":0,"messages":[]}' headers: Connection: - keep-alive @@ -167,14 +167,14 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 14:25:29 GMT + - Mon, 23 May 2022 11:41:16 GMT server: - uvicorn http_version: HTTP/1.1 status_code: 200 - request: body: '{"data": {"k2": "v2"}, "version": 2, "company_id": 80, "timestamp": 11, - "asset_id": 32946351}' + "asset_id": 30719716}' headers: accept: - '*/*' @@ -189,20 +189,20 @@ interactions: user-agent: - python-httpx/0.22.0 x-corva-app: - - python-sdk-autotest-2022-05-18 14:25:27+00:00 + - python-sdk-autotest-2022-05-23 11:41:14+00:00 method: PUT uri: null response: - content: '{"_id":"628501d93cc8096aa7e5d248","company_id":80,"asset_id":32946351,"version":2,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.oleksii_test_task_app"}' + content: '{"_id":"628b72dcd6aff355ccea8b78","company_id":80,"asset_id":30719716,"version":2,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}' headers: Connection: - keep-alive Content-Length: - - '229' + - '238' Content-Type: - application/json Date: - - Wed, 18 May 2022 14:25:30 GMT + - Mon, 23 May 2022 11:41:17 GMT server: - uvicorn http_version: HTTP/1.1 @@ -221,16 +221,16 @@ interactions: method: GET uri: null response: - content: '{"_id":"628501d93cc8096aa7e5d248","company_id":80,"asset_id":32946351,"version":2,"provider":"big-data-energy","collection":"corva-sdk-autotest","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.oleksii_test_task_app"}' + content: '{"_id":"628b72dcd6aff355ccea8b78","company_id":80,"asset_id":30719716,"version":2,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}' headers: Connection: - keep-alive Content-Length: - - '229' + - '238' Content-Type: - application/json Date: - - Wed, 18 May 2022 14:25:30 GMT + - Mon, 23 May 2022 11:41:17 GMT server: - uvicorn http_version: HTTP/1.1 @@ -260,7 +260,7 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 18 May 2022 14:25:31 GMT + - Mon, 23 May 2022 11:41:18 GMT ETag: - W/"4df20f95e824b2af44a61642d88daaf0" Referrer-Policy: @@ -282,9 +282,9 @@ interactions: X-Rack-CORS: - miss; no-origin X-Request-Id: - - f5e6dbb9-b87f-41b6-8541-8231a6e4640f + - db6d7a71-cce6-4279-8050-eb7c52cecd26 X-Runtime: - - '0.136116' + - '0.073215' X-XSS-Protection: - 1; mode=block http_version: HTTP/1.1 @@ -312,7 +312,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 18 May 2022 14:25:31 GMT + - Mon, 23 May 2022 11:41:18 GMT server: - uvicorn http_version: HTTP/1.1