From 33e61eb0bedd871ce192c1c8b0ee5901f3b0e7ce Mon Sep 17 00:00:00 2001
From: Oleksii Symon <oleksii.symon@corva.ai>
Date: Mon, 9 May 2022 15:03:48 +0300
Subject: [PATCH 1/6] Added Api skeleton

---
 setup.py                 |   2 +
 src/corva/api_adapter.py | 300 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 302 insertions(+)
 create mode 100644 src/corva/api_adapter.py

diff --git a/setup.py b/setup.py
index a92b2880..20f653bc 100644
--- a/setup.py
+++ b/setup.py
@@ -43,6 +43,8 @@
         "pydantic >=1.8.2, <2.0.0",
         "redis >=3.5.3, <4.0.0",
         "requests >=2.25.0, <3.0.0",
+        "httpx >=0.22.0, <0.23.0",
+        "PyYAML >=6.0, <6.1"
     ],
     python_requires='>=3.8, <4.0',
     license='The Unlicense',
diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py
new file mode 100644
index 00000000..152b59d2
--- /dev/null
+++ b/src/corva/api_adapter.py
@@ -0,0 +1,300 @@
+# What's needed?
+# 1. Api SDK Request retries
+# 2. Api SDK Request timeouts
+# 3. Use api as standalone object
+# 4. Request/Response error logging
+import dataclasses
+import functools
+import json
+import logging
+import posixpath
+from typing import Callable, List, Optional
+
+import httpx
+import yaml
+
+
+def _httpx_headers_to_dict(headers: httpx.Headers) -> dict:
+    return json.loads(
+        repr(headers)  # use built-in `repr` as it obfuscates sensitive headers
+        .strip("Headers()")  # strip obsolete data
+        .replace(
+            "'", '"'
+        )  # replace single quotes with double ones to get proper json string
+    )
+
+
+def _failed_request_msg(
+    msg: str,
+    request: httpx.Request,
+    response: Optional[httpx.Response],
+) -> str:
+    data = {"message": f"Request failed - {msg}"}
+
+    if response:
+        # log response first, so there is less chance it gets truncated
+        # and users are able to see server error message
+        data["response"] = {
+            "code": response.status_code,
+            "reason": response.reason_phrase,
+            "headers": _httpx_headers_to_dict(response.headers),
+            "content": str(response.content),
+        }
+
+    data["request"] = {
+        "method": request.method,
+        "url": str(request.url),
+        "headers": _httpx_headers_to_dict(request.headers),
+        "content": str(request.content),
+    }
+
+    # use yaml because it is much more readable in logs
+    return yaml.dump(data, sort_keys=False)
+
+
+def logging_send(func: Callable, *, logger: logging.Logger) -> Callable:
+    @functools.wraps(func)
+    def wrapper(request: httpx.Request, *args, **kwargs):
+        try:
+            response = func(request, *args, **kwargs)
+        except httpx.HTTPError as exc:
+            # Response was not received at all
+            logger.error(
+                _failed_request_msg(msg=str(exc), request=request, response=None)
+            )
+            raise
+
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            # Response has unsuccessful status
+            logger.error(
+                _failed_request_msg(msg=str(exc), request=request, response=response)
+            )
+
+        return response
+
+    return wrapper
+
+
+# ===============  User stuff  ===============
+
+# ================================================
+
+
+class DataApiV1Sdk:
+    def __init__(self, client: httpx.Client):
+        self.http = client
+
+    def get(
+        self,
+        provider: str,
+        dataset: str,
+        *,
+        query: dict,
+        sort: dict,
+        limit: int,
+        skip: int = 0,
+        fields: Optional[str] = None,
+    ) -> List[dict]:
+        """Fetches data from the endpoint GET 'data/{provider}/{dataset}/'.
+
+        Args:
+            provider: company name owning the dataset.
+            dataset: dataset name.
+            query: search conditions. Example: {"asset_id": 123} - will fetch data
+                for asset with id 123.
+            sort: sort conditions. Example: {"timestamp": 1} - will sort data
+                in ascending order by timestamp.
+            limit: number of data points to fecth.
+                Recommendation for setting the limit:
+                    1. The bigger ↑ each data point is - the smaller ↓ the limit;
+                    2. The smaller ↓ each data point is - the bigger ↑ the limit.
+            skip: exclude from the response the first N items of the dataset.
+            fields: comma separated list of fields to return. Example: "_id,data".
+
+        Raises:
+            requests.HTTPError: if request was unsuccessful.
+
+        Returns:
+            Data from dataset.
+        """
+
+        response = self.http.get(
+            url=f"data/{provider}/{dataset}/",
+            params={
+                "query": json.dumps(query),
+                "sort": json.dumps(sort),
+                "fields": fields,
+                "limit": limit,
+                "skip": skip,
+            },
+        )
+
+        response.raise_for_status()
+
+        data = list(response.json())
+
+        return data
+
+    def get_one(self):
+        ...
+
+    def get_aggregate(self):
+        ...
+
+    def get_aggregate_pipeline(self):
+        ...
+
+    def set(self):
+        ...
+
+    def delete(self):
+        ...
+
+    def update(self):
+        ...
+
+    def update_partial(self):
+        ...
+
+
+class PlatformApiV1Sdk:
+    def __init__(self, client: httpx.Client):
+        self.http = client
+
+    # .....
+    # other high-level methods here
+    # .....
+
+
+class PlatformApiV2Sdk:
+    def __init__(self, client: httpx.Client):
+        self.http = client
+
+    # .....
+    # other high-level methods here
+    # .....
+
+
+# ================================================
+
+
+@dataclasses.dataclass(frozen=True)
+class PlatformApiVersions:
+    v1: PlatformApiV1Sdk
+    v2: PlatformApiV2Sdk
+
+
+@dataclasses.dataclass(frozen=True)
+class DataApiVersions:
+    v1: DataApiV1Sdk
+
+
+class UserApiSdk:
+    def __init__(
+        self,
+        platform_api_url: str,
+        data_api_url: str,
+        api_key: str,
+        app_key: str,
+        logger: logging.Logger,
+        timeout: int = 30,
+    ):
+        self._platform_api_url = platform_api_url
+        self._data_api_url = data_api_url
+        self._headers = {
+            "Authorization": f"API {api_key}",
+            "X-Corva-App": app_key,
+        }
+        self._logger = logger
+        self._timeout = timeout
+
+    def __enter__(self):
+        data_cli = httpx.Client(
+            base_url=posixpath.join(self._data_api_url, "api/v1"),
+            headers=self._headers,
+            timeout=self._timeout,
+        )
+        platform_v1_cli = httpx.Client(
+            base_url=posixpath.join(self._platform_api_url, "v1"),
+            headers=self._headers,
+            timeout=self._timeout,
+        )
+        platform_v2_cli = httpx.Client(
+            base_url=posixpath.join(self._platform_api_url, "v2"),
+            headers=self._headers,
+            timeout=self._timeout,
+        )
+
+        data_cli.send = logging_send(func=data_cli.send, logger=self._logger)
+        platform_v1_cli.send = logging_send(
+            func=platform_v1_cli.send, logger=self._logger
+        )
+        platform_v2_cli.send = logging_send(
+            func=platform_v2_cli.send, logger=self._logger
+        )
+
+        self.data = DataApiVersions(v1=DataApiV1Sdk(client=data_cli))
+        self.platform = PlatformApiVersions(
+            v1=PlatformApiV1Sdk(client=platform_v1_cli),
+            v2=PlatformApiV2Sdk(client=platform_v2_cli),
+        )
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.data.v1.http.close()
+        self.platform.v1.http.close()
+        self.platform.v2.http.close()
+
+
+# with UserApiSdk(..., ..., ..., ..., ...) as sdk:
+#     sdk.data.v1.http.get()
+#     ...
+
+
+def app(api: UserApiSdk):
+    # Objective: users should get clear, easy to use Api interface without low level details that
+    # they currently get in api object.
+
+    # Currently users get a low level api adapter:
+    #   - adapter contains low-level functions of no use for user
+    #   - adapter shuffles functions for all apis (not clear which one is called)
+    #   - user needs to specify api and version through path param
+
+    # New api sdk:
+    #   - has "dot" acess to api type and version (e.g., api.data.v1..., api.platform.v1...)
+    #   - has frequently used api method represented as functions (e.g., api.data.v1.get_dataset)
+    #   - not frequently used api method can be called using http client (e.g., api.data.v1.http.get...)
+    #   - has correct type hints for all methods (intead of *args and **kwargs currently used)
+
+    # Platform api calls
+
+    # before
+    #   - not clear which api is called
+    api.get("/v1/data/provider/dataset")
+    # after
+    api.platform.v1.http.get("data/provider/dataset")
+
+    # before
+    #   - not clear which api is called
+    api.post("/v2/rigs")
+    # after
+    api.platform.v2.http.post("rigs")
+
+    # Specialized functions
+
+    # before
+    #   - not clear which api is called
+    api.get_dataset()
+    # after
+    api.data.v1.get_dataset()
+
+    # Data api calls
+
+    # before
+    #   - not clear which api is called
+    api.delete("/api/v1/data/provider/dataset/")
+    # after
+    api.data.v1.http.delete("data/provider/dataset/")

From 98fe9498d9d17fce420aac4e88e4031b5f6c5578 Mon Sep 17 00:00:00 2001
From: Oleksii Symon <oleksii.symon@corva.ai>
Date: Mon, 9 May 2022 15:04:00 +0300
Subject: [PATCH 2/6] Added api logging tests

---
 requirements-test.txt         |  3 +-
 tests/integration/test_api.py | 87 +++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/test_api.py

diff --git a/requirements-test.txt b/requirements-test.txt
index 5b6e9aa7..8f13da73 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,5 +1,6 @@
 coverage==5.3
 freezegun==1.0.0
-pytest==6.1.2
+pytest==7.1.2
+pytest-httpx==0.20.0
 pytest-mock==3.3.1
 requests-mock==1.8.0
diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py
new file mode 100644
index 00000000..5548ba8e
--- /dev/null
+++ b/tests/integration/test_api.py
@@ -0,0 +1,87 @@
+import inspect
+import logging
+
+import httpx
+import pytest
+import pytest_httpx
+
+import corva.api_adapter
+import corva.configuration
+
+
+class TestLogsFailedRequests:
+    def test_no_response(self, caplog: pytest.LogCaptureFixture):
+        sdk = corva.api_adapter.UserApiSdk(
+            platform_api_url='',
+            data_api_url='',
+            api_key='',
+            app_key='',
+            logger=logging.getLogger(),
+        )
+
+        with sdk as s, pytest.raises(httpx.HTTPError):
+            s.data.v1.http.get(url='whatever')
+
+        actual = caplog.text.strip()
+        expected = inspect.cleandoc(
+            """
+            ERROR    root:api_adapter.py:62 message: Request failed - Request URL is missing an 'http://' or 'https://' protocol.
+            request:
+              method: GET
+              url: api/v1/whatever
+              headers:
+                accept: '*/*'
+                accept-encoding: gzip, deflate
+                connection: keep-alive
+                user-agent: python-httpx/0.22.0
+                authorization: '[secure]'
+                x-corva-app: ''
+              content: b''
+              """
+        ).strip()
+
+        assert expected == actual
+
+    def test_unsuccessful_response(
+        self, caplog: pytest.LogCaptureFixture, httpx_mock: pytest_httpx.HTTPXMock
+    ):
+        httpx_mock.add_response(status_code=400)
+
+        sdk = corva.api_adapter.UserApiSdk(
+            platform_api_url='',
+            data_api_url="https://test_url",
+            api_key='',
+            app_key='',
+            logger=logging.getLogger(),
+        )
+
+        with sdk as s:
+            s.data.v1.http.get(url='')
+
+        actual = caplog.text.strip()
+        expected = inspect.cleandoc(
+            """
+            ERROR    root:api_adapter.py:71 message: 'Request failed - Client error ''400 Bad Request'' for url ''https://test_url/api/v1/''
+
+              For more information check: https://httpstatuses.com/400'
+            response:
+              code: 400
+              reason: Bad Request
+              headers: {}
+              content: b''
+            request:
+              method: GET
+              url: https://test_url/api/v1/
+              headers:
+                host: test_url
+                accept: '*/*'
+                accept-encoding: gzip, deflate
+                connection: keep-alive
+                user-agent: python-httpx/0.22.0
+                authorization: '[secure]'
+                x-corva-app: ''
+              content: b''
+            """
+        ).strip()
+
+        assert actual == expected

From 80ffcf26d3bd21a671c76e93b295aeb532f110f6 Mon Sep 17 00:00:00 2001
From: Oleksii Symon <oleksii.symon@corva.ai>
Date: Mon, 9 May 2022 15:14:07 +0300
Subject: [PATCH 3/6] Cleaned-up api adapter file

---
 src/corva/api_adapter.py | 88 ----------------------------------------
 1 file changed, 88 deletions(-)

diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py
index 152b59d2..0b7a461d 100644
--- a/src/corva/api_adapter.py
+++ b/src/corva/api_adapter.py
@@ -1,8 +1,3 @@
-# What's needed?
-# 1. Api SDK Request retries
-# 2. Api SDK Request timeouts
-# 3. Use api as standalone object
-# 4. Request/Response error logging
 import dataclasses
 import functools
 import json
@@ -137,48 +132,16 @@ def get(
 
         return data
 
-    def get_one(self):
-        ...
-
-    def get_aggregate(self):
-        ...
-
-    def get_aggregate_pipeline(self):
-        ...
-
-    def set(self):
-        ...
-
-    def delete(self):
-        ...
-
-    def update(self):
-        ...
-
-    def update_partial(self):
-        ...
-
 
 class PlatformApiV1Sdk:
     def __init__(self, client: httpx.Client):
         self.http = client
 
-    # .....
-    # other high-level methods here
-    # .....
-
 
 class PlatformApiV2Sdk:
     def __init__(self, client: httpx.Client):
         self.http = client
 
-    # .....
-    # other high-level methods here
-    # .....
-
-
-# ================================================
-
 
 @dataclasses.dataclass(frozen=True)
 class PlatformApiVersions:
@@ -247,54 +210,3 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.data.v1.http.close()
         self.platform.v1.http.close()
         self.platform.v2.http.close()
-
-
-# with UserApiSdk(..., ..., ..., ..., ...) as sdk:
-#     sdk.data.v1.http.get()
-#     ...
-
-
-def app(api: UserApiSdk):
-    # Objective: users should get clear, easy to use Api interface without low level details that
-    # they currently get in api object.
-
-    # Currently users get a low level api adapter:
-    #   - adapter contains low-level functions of no use for user
-    #   - adapter shuffles functions for all apis (not clear which one is called)
-    #   - user needs to specify api and version through path param
-
-    # New api sdk:
-    #   - has "dot" acess to api type and version (e.g., api.data.v1..., api.platform.v1...)
-    #   - has frequently used api method represented as functions (e.g., api.data.v1.get_dataset)
-    #   - not frequently used api method can be called using http client (e.g., api.data.v1.http.get...)
-    #   - has correct type hints for all methods (intead of *args and **kwargs currently used)
-
-    # Platform api calls
-
-    # before
-    #   - not clear which api is called
-    api.get("/v1/data/provider/dataset")
-    # after
-    api.platform.v1.http.get("data/provider/dataset")
-
-    # before
-    #   - not clear which api is called
-    api.post("/v2/rigs")
-    # after
-    api.platform.v2.http.post("rigs")
-
-    # Specialized functions
-
-    # before
-    #   - not clear which api is called
-    api.get_dataset()
-    # after
-    api.data.v1.get_dataset()
-
-    # Data api calls
-
-    # before
-    #   - not clear which api is called
-    api.delete("/api/v1/data/provider/dataset/")
-    # after
-    api.data.v1.http.delete("data/provider/dataset/")

From e0d7b8ecd943ad125fda1456a36ae775ceaed594 Mon Sep 17 00:00:00 2001
From: Oleksii Symon <oleksii.symon@corva.ai>
Date: Mon, 9 May 2022 15:15:21 +0300
Subject: [PATCH 4/6] Cleaned-up api adapter file

---
 src/corva/api_adapter.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py
index 0b7a461d..9173256d 100644
--- a/src/corva/api_adapter.py
+++ b/src/corva/api_adapter.py
@@ -72,11 +72,6 @@ def wrapper(request: httpx.Request, *args, **kwargs):
     return wrapper
 
 
-# ===============  User stuff  ===============
-
-# ================================================
-
-
 class DataApiV1Sdk:
     def __init__(self, client: httpx.Client):
         self.http = client

From bca7f5a305ac101df9a664742c362f7549c3b55f Mon Sep 17 00:00:00 2001
From: Oleksii Symon <oleksii.symon@corva.ai>
Date: Tue, 10 May 2022 11:14:35 +0300
Subject: [PATCH 5/6] Lint fix

---
 setup.py                      |  2 +-
 tests/integration/test_api.py | 91 +++++++++++++++++++----------------
 2 files changed, 50 insertions(+), 43 deletions(-)

diff --git a/setup.py b/setup.py
index 20f653bc..1fc93821 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@
         "redis >=3.5.3, <4.0.0",
         "requests >=2.25.0, <3.0.0",
         "httpx >=0.22.0, <0.23.0",
-        "PyYAML >=6.0, <6.1"
+        "PyYAML >=6.0, <6.1",
     ],
     python_requires='>=3.8, <4.0',
     license='The Unlicense',
diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py
index 5548ba8e..00aa7461 100644
--- a/tests/integration/test_api.py
+++ b/tests/integration/test_api.py
@@ -1,9 +1,9 @@
-import inspect
 import logging
 
 import httpx
 import pytest
 import pytest_httpx
+import yaml
 
 import corva.api_adapter
 import corva.configuration
@@ -11,6 +11,8 @@
 
 class TestLogsFailedRequests:
     def test_no_response(self, caplog: pytest.LogCaptureFixture):
+        caplog.handler.setFormatter(logging.Formatter('%(message)s'))
+
         sdk = corva.api_adapter.UserApiSdk(
             platform_api_url='',
             data_api_url='',
@@ -22,29 +24,32 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture):
         with sdk as s, pytest.raises(httpx.HTTPError):
             s.data.v1.http.get(url='whatever')
 
-        actual = caplog.text.strip()
-        expected = inspect.cleandoc(
-            """
-            ERROR    root:api_adapter.py:62 message: Request failed - Request URL is missing an 'http://' or 'https://' protocol.
-            request:
-              method: GET
-              url: api/v1/whatever
-              headers:
-                accept: '*/*'
-                accept-encoding: gzip, deflate
-                connection: keep-alive
-                user-agent: python-httpx/0.22.0
-                authorization: '[secure]'
-                x-corva-app: ''
-              content: b''
-              """
-        ).strip()
+        actual = yaml.safe_load(caplog.text)
+
+        expected = {
+            'message': "Request failed - Request URL is missing an 'http://' or "
+            "'https://' protocol.",
+            'request': {
+                'method': 'GET',
+                'url': 'api/v1/whatever',
+                'headers': {
+                    'accept': '*/*',
+                    'accept-encoding': 'gzip, deflate',
+                    'connection': 'keep-alive',
+                    'user-agent': 'python-httpx/0.22.0',
+                    'authorization': '[secure]',
+                    'x-corva-app': '',
+                },
+                'content': "b''",
+            },
+        }
 
         assert expected == actual
 
     def test_unsuccessful_response(
         self, caplog: pytest.LogCaptureFixture, httpx_mock: pytest_httpx.HTTPXMock
     ):
+        caplog.handler.setFormatter(logging.Formatter('%(message)s'))
         httpx_mock.add_response(status_code=400)
 
         sdk = corva.api_adapter.UserApiSdk(
@@ -58,30 +63,32 @@ def test_unsuccessful_response(
         with sdk as s:
             s.data.v1.http.get(url='')
 
-        actual = caplog.text.strip()
-        expected = inspect.cleandoc(
-            """
-            ERROR    root:api_adapter.py:71 message: 'Request failed - Client error ''400 Bad Request'' for url ''https://test_url/api/v1/''
+        actual = yaml.safe_load(caplog.text)
 
-              For more information check: https://httpstatuses.com/400'
-            response:
-              code: 400
-              reason: Bad Request
-              headers: {}
-              content: b''
-            request:
-              method: GET
-              url: https://test_url/api/v1/
-              headers:
-                host: test_url
-                accept: '*/*'
-                accept-encoding: gzip, deflate
-                connection: keep-alive
-                user-agent: python-httpx/0.22.0
-                authorization: '[secure]'
-                x-corva-app: ''
-              content: b''
-            """
-        ).strip()
+        expected = {
+            'message': "Request failed - Client error '400 Bad Request' for url "
+            "'https://test_url/api/v1/'\nFor more information check: "
+            "https://httpstatuses.com/400",
+            'response': {
+                'code': 400,
+                'reason': 'Bad Request',
+                'headers': {},
+                'content': "b''",
+            },
+            'request': {
+                'method': 'GET',
+                'url': 'https://test_url/api/v1/',
+                'headers': {
+                    'host': 'test_url',
+                    'accept': '*/*',
+                    'accept-encoding': 'gzip, deflate',
+                    'connection': 'keep-alive',
+                    'user-agent': 'python-httpx/0.22.0',
+                    'authorization': '[secure]',
+                    'x-corva-app': '',
+                },
+                'content': "b''",
+            },
+        }
 
         assert actual == expected

From 35e3d0d55f69da7d4dfa44d8a31be3c77bdef858 Mon Sep 17 00:00:00 2001
From: Oleksii Symon <oleksii.symon@corva.ai>
Date: Tue, 10 May 2022 11:19:32 +0300
Subject: [PATCH 6/6] Fixed docstr

---
 src/corva/api_adapter.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py
index 9173256d..94292f04 100644
--- a/src/corva/api_adapter.py
+++ b/src/corva/api_adapter.py
@@ -101,6 +101,8 @@ def get(
                     1. The bigger ↑ each data point is - the smaller ↓ the limit;
                     2. The smaller ↓ each data point is - the bigger ↑ the limit.
             skip: exclude from the response the first N items of the dataset.
+                Note: skip should only be used for small amounts of data, having a
+                large skip will lead to very slow queries.
             fields: comma separated list of fields to return. Example: "_id,data".
 
         Raises: