diff --git a/setup.cfg b/setup.cfg index b70a1bbf..addc8de2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.44 +fail_under = 98.45 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/__init__.py b/src/corva/__init__.py index 49dd5732..1bc1f0b5 100644 --- a/src/corva/__init__.py +++ b/src/corva/__init__.py @@ -1,4 +1,5 @@ from .api import Api +from .api_adapter import InsertResult from .handlers import scheduled, stream, task from .logger import CORVA_LOGGER as Logger from .models.scheduled.scheduled import ( diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 7a3619e7..3a79aca5 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -2,9 +2,10 @@ import functools import json import logging -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Sequence import httpx +import pydantic import yaml @@ -71,6 +72,12 @@ def wrapper(request: httpx.Request, *args, **kwargs): return wrapper +class InsertResult(pydantic.BaseModel): + inserted_ids: List[str] + failed_count: int + messages: List[str] + + class DataApiV1Sdk: def __init__(self, client: httpx.Client): self.http = client @@ -105,7 +112,7 @@ def get( fields: comma separated list of fields to return. Example: "_id,data". Raises: - requests.HTTPError: if request was unsuccessful. + httpx.HTTPStatusError: if request was unsuccessful. Returns: Data from dataset. @@ -128,6 +135,29 @@ def get( return data + def insert( + self, provider: str, dataset: str, *, documents: Sequence[dict] + ) -> InsertResult: + """Inserts data using the endpoint POST 'data/{provider}/{dataset}/'. + + Args: + provider: company name owning the dataset. + dataset: dataset name. + documents: data to insert. + + Raises: + httpx.HTTPStatusError: if request was unsuccessful. + + Returns: + Insert result. + """ + + response = self.http.post(url=f"data/{provider}/{dataset}/", json=documents) + + response.raise_for_status() + + return InsertResult.parse_obj(response.json()) + class PlatformApiV1Sdk: def __init__(self, client: httpx.Client): diff --git a/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml new file mode 100644 index 00000000..7725a8c6 --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml @@ -0,0 +1,315 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-eb81d18c"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"328567","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-eb81d18c","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:39:27 GMT + ETag: + - W/"ec3494f3333667f241a2e1d1b41205ae" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 3462393c-fe38-4cd3-aca1-a4d4a7b722ee + X-Runtime: + - '0.105799' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"328567","type":"well","attributes":{"asset_id":53781610},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:39:27 GMT + ETag: + - W/"c412a0007563bca3e870f857bfe4cf1e" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 099e239c-b7af-452d-9340-9456471dbcf9 + X-Runtime: + - '0.059369' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:27 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '[]' + headers: + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:28 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 53781610, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 53781610, "version": 1, "data": {"k": "v"}, "timestamp": 11}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '150' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-23 11:39:26+00:00 + method: POST + uri: null + response: + content: '{"inserted_ids":["628b7271d6aff355ccea8b61","628b7271d6aff355ccea8b62"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '[{"_id":"628b7271d6aff355ccea8b61","company_id":80,"asset_id":53781610,"version":1,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k":"v"},"timestamp":10,"app_key":"big-data-energy.python_sdk_app_for_autotests"},{"_id":"628b7271d6aff355ccea8b62","company_id":80,"asset_id":53781610,"version":1,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k":"v"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}]' + headers: + Connection: + - keep-alive + Content-Length: + - '475' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:39:29 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 8424ca9f-13a6-4ac7-9f70-836a3011124f + X-Runtime: + - '0.099032' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":2}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 89d16e4c..411b0421 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -112,7 +112,7 @@ def vcr_before_record_request(request): def vcr_config(): return { # Replace the Authorization request header - "filter_headers": ["Authorization", 'host'], + "filter_headers": ["Authorization", "host"], "before_record_request": vcr_before_record_request, } @@ -174,7 +174,7 @@ def _setup( response = platform.get(url=f'wells/{well_id}?fields[]=well.asset_id') response.raise_for_status() - asset_id = response.json()['data']['attributes']['asset_id'] + asset_id: int = response.json()['data']['attributes']['asset_id'] data.delete( f'data/{provider}/{dataset}/', @@ -192,7 +192,7 @@ def _setup( ).raise_for_status() -@pytest.fixture(scope='module') +@pytest.fixture(scope='function') def setup_( platform_v2_url: str, data: httpx.Client, headers: dict, provider: str, dataset: str ) -> Iterable[int]: @@ -213,8 +213,7 @@ def sdk( logger=logging.getLogger(), ) - with sdk: - yield sdk + yield sdk class TestUserApiSdk: @@ -258,15 +257,72 @@ def test_get( ], ).raise_for_status() - data = sdk.data.v1.get( - provider=provider, - dataset=dataset, - query={'asset_id': asset_id}, - sort={'timestamp': 1}, - limit=2, - skip=1, - fields='timestamp', + with sdk as s: + collection = s.data.v1.get( + provider=provider, + dataset=dataset, + query={'asset_id': asset_id}, + sort={'timestamp': 1}, + limit=2, + skip=1, + fields='timestamp', + ) + + assert len(collection) == 2 + assert set(doc['timestamp'] for doc in collection) == {11, 12} + + @pytest.mark.vcr + def test_insert( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + data: httpx.Client, + ): + with setup_ as asset_id: + response = data.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps({'asset_id': asset_id}), + "sort": json.dumps({'timestamp': 1}), + "limit": 1, + }, + ) + response.raise_for_status() + collection = response.json() + + assert not collection + + with sdk as s: + s.data.v1.insert( + provider=provider, + dataset=dataset, + documents=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 10, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 11, + }, + ], + ) + + response = data.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps({'asset_id': asset_id}), + "sort": json.dumps({'timestamp': 1}), + "limit": 3, + }, ) + response.raise_for_status() + collection = response.json() - assert len(data) == 2 - assert set(datum['timestamp'] for datum in data) == {11, 12} + assert len(collection) == 2