From 5ae448baee291b9df18cd8e354d48e3bb332f0fe Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 28 Sep 2023 16:06:30 +0300 Subject: [PATCH 1/7] adds Recommendations backend boilerplate class --- .../tests/appnexus/test_backends.py | 8 +++++++ .../automation/utils/appnexus/backends.py | 21 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 contentcuration/automation/tests/appnexus/test_backends.py diff --git a/contentcuration/automation/tests/appnexus/test_backends.py b/contentcuration/automation/tests/appnexus/test_backends.py new file mode 100644 index 0000000000..7c8b5d1892 --- /dev/null +++ b/contentcuration/automation/tests/appnexus/test_backends.py @@ -0,0 +1,8 @@ +from automation.utils.appnexus.backends import Recommendations +from django.test import TestCase + + +class RecommendationsTestCase(TestCase): + def test_recommendations_backend_initialization(self): + recomendations = Recommendations() + assert isinstance(recomendations.get_instance(), Recommendations) diff --git a/contentcuration/automation/utils/appnexus/backends.py b/contentcuration/automation/utils/appnexus/backends.py index b4b1e77ab4..5c0bf8e360 100644 --- a/contentcuration/automation/utils/appnexus/backends.py +++ b/contentcuration/automation/utils/appnexus/backends.py @@ -1 +1,20 @@ -# Implementation of ML, GCS Backend etc. +from .base import Backend + + +class Recommendations(Backend): + + def connect(self) -> None: + return super().connect() + + def make_request(self, url: str, params=None): + return super().make_request(url, params) + + def request(self) -> None: + return super().request() + + def response(self) -> None: + return super().response() + + @classmethod + def _create_instance(cls) -> 'Recommendations': + return cls() From 051968a1f666840476b7d83a0bb0e7cb8a0b8258 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 28 Sep 2023 16:19:34 +0300 Subject: [PATCH 2/7] updates test --- contentcuration/automation/tests/appnexus/test_backends.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contentcuration/automation/tests/appnexus/test_backends.py b/contentcuration/automation/tests/appnexus/test_backends.py index 7c8b5d1892..0e04c6a544 100644 --- a/contentcuration/automation/tests/appnexus/test_backends.py +++ b/contentcuration/automation/tests/appnexus/test_backends.py @@ -3,6 +3,7 @@ class RecommendationsTestCase(TestCase): - def test_recommendations_backend_initialization(self): + def test_backend_initialization(self): recomendations = Recommendations() - assert isinstance(recomendations.get_instance(), Recommendations) + self.assertIsNotNone(recomendations) + self.assertIsInstance(recomendations.get_instance(), Recommendations) From ea5396a135541c88ff425a10248b05abdc7b504c Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 28 Sep 2023 21:47:59 +0300 Subject: [PATCH 3/7] Refactors file locations --- .../automation/tests/appnexus/test_base.py | 4 +-- .../automation/utils/appnexus/APILayer.md | 12 +++---- .../automation/utils/appnexus/adapters.py | 1 - .../automation/utils/appnexus/backends.py | 1 - .../automation/utils/appnexus/base.py | 34 +++++++++++++++++-- .../contentcuration/utils/recommendations.py | 24 +++++++++++++ 6 files changed, 63 insertions(+), 13 deletions(-) delete mode 100644 contentcuration/automation/utils/appnexus/adapters.py delete mode 100644 contentcuration/automation/utils/appnexus/backends.py create mode 100644 contentcuration/contentcuration/utils/recommendations.py diff --git a/contentcuration/automation/tests/appnexus/test_base.py b/contentcuration/automation/tests/appnexus/test_base.py index dec6104d9e..23b4f495bf 100644 --- a/contentcuration/automation/tests/appnexus/test_base.py +++ b/contentcuration/automation/tests/appnexus/test_base.py @@ -6,8 +6,8 @@ class MockBackend(Backend): def connect(self) -> None: return super().connect() - def make_request(self, url: str, params=None): - return super().make_request(url, params) + def make_request(self, request): + return super().make_request(request) def request(self) -> None: return super().request() diff --git a/contentcuration/automation/utils/appnexus/APILayer.md b/contentcuration/automation/utils/appnexus/APILayer.md index a197fb13be..4e82e5b3f3 100644 --- a/contentcuration/automation/utils/appnexus/APILayer.md +++ b/contentcuration/automation/utils/appnexus/APILayer.md @@ -52,7 +52,7 @@ Different backends can now be created by implementing the base `Backend` class: ```python # Implement CONCRETE CLASS using ABSTRACT Backend class CLASS GCS IMPLEMENTS Backend: - METHOD make_request(params): + METHOD make_request(request): # make request to Google Cloud Storage services METHOD connect(params): @@ -62,7 +62,7 @@ CLASS GCS IMPLEMENTS Backend: # initialize a GCS Backend instance CLASS ML IMPLEMENTS Backend: - METHOD make_request(params): + METHOD make_request(request): # make request to DeepLearning models hosted as service METHOD connect(params): @@ -128,18 +128,18 @@ With this `Adapter` class in place, we can create Adapter that are able interact ```python CLASS Recommendation INHERITS ADAPTER: - METHOD generateEmbeddings(self, params) -> Boolean + METHOD generateEmbeddings(self, request) -> Boolean # [ Implementation ] - METHOD getRecommendation(self, params) -> Array + METHOD getRecommendation(self, request) -> Array # [ Implementation ] CLASS Transcription INHERITS ADAPTER: - METHOD generateCaption(self, params) -> Array + METHOD generateCaption(self, request) -> Array # [ Implementation ] CLASS OtherAdapter INHERITS ADAPTER: - METHOD someOperation(self, params) -> Any + METHOD someOperation(self, request) -> Any # Operation that any backend wants ``` diff --git a/contentcuration/automation/utils/appnexus/adapters.py b/contentcuration/automation/utils/appnexus/adapters.py deleted file mode 100644 index c94b712ddf..0000000000 --- a/contentcuration/automation/utils/appnexus/adapters.py +++ /dev/null @@ -1 +0,0 @@ -# A file to implement adapters like Recommendation, Transcription etc diff --git a/contentcuration/automation/utils/appnexus/backends.py b/contentcuration/automation/utils/appnexus/backends.py deleted file mode 100644 index b4b1e77ab4..0000000000 --- a/contentcuration/automation/utils/appnexus/backends.py +++ /dev/null @@ -1 +0,0 @@ -# Implementation of ML, GCS Backend etc. diff --git a/contentcuration/automation/utils/appnexus/base.py b/contentcuration/automation/utils/appnexus/base.py index 95ddef0b4f..1c7f36ac8f 100644 --- a/contentcuration/automation/utils/appnexus/base.py +++ b/contentcuration/automation/utils/appnexus/base.py @@ -1,10 +1,37 @@ from abc import ABC from abc import abstractmethod from builtins import NotImplementedError -from typing import Dict from typing import Union +class RecommendationsBackendRequest(object): + pass + + +class RecommedationsRequest(RecommendationsBackendRequest): + def __init__(self) -> None: + super().__init__() + + +class EmbeddingsRequest(RecommendationsBackendRequest): + def __init__(self) -> None: + super().__init__() + + +class RecommendationsBackendResponse(object): + pass + + +class RecommendationsResponse(RecommendationsBackendResponse): + def __init__(self) -> None: + pass + + +class EmbeddingsResponse(RecommendationsBackendResponse): + def __init__(self) -> None: + pass + + class Backend(ABC): """ An abstract base class for backend interfaces that also implements the singleton pattern """ _instance = None @@ -20,8 +47,8 @@ def connect(self) -> None: pass @abstractmethod - def make_request(self, url: str, params=None) -> Union[bytes, str, Dict]: - """ Makes an HTTP request to a given URL using the specified method. """ + def make_request(self, request) -> Union[RecommendationsResponse, EmbeddingsResponse]: + """ Make a request based on "request" """ pass @abstractmethod @@ -59,6 +86,7 @@ class Adapter: This class should be inherited by adapter classes that facilitate interaction with different backend implementations. """ + def __init__(self, backend: Backend) -> None: self.backend = backend diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py new file mode 100644 index 0000000000..34597ce008 --- /dev/null +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -0,0 +1,24 @@ +from automation.utils.appnexus.base import Adapter +from automation.utils.appnexus.base import Backend +from automation.utils.appnexus.base import BackendFactory +from automation.utils.appnexus.base import EmbeddingsRequest +from automation.utils.appnexus.base import EmbeddingsResponse +from automation.utils.appnexus.base import RecommedationsRequest +from automation.utils.appnexus.base import RecommendationsResponse + + +class RecommendationsBackendFatory(BackendFactory): + def create_backend(self) -> Backend: + # Return backend based on some setting. + return super().create_backend() + + +class RecommendationsAdapter(Adapter): + + def generate_embedding(self, text) -> EmbeddingsResponse: + request = EmbeddingsRequest() + return self.backend.make_request(request) + + def get_recommendations(self) -> RecommendationsResponse: + request = RecommedationsRequest() + return self.backend.make_request(request) From 608be48d44c44a79669c908655c8504ab0fa38c9 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 28 Sep 2023 22:08:02 +0300 Subject: [PATCH 4/7] Refactors boilerplate --- .../automation/utils/appnexus/base.py | 31 ++---------------- .../contentcuration/utils/recommendations.py | 32 ++++++++++++++++--- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/contentcuration/automation/utils/appnexus/base.py b/contentcuration/automation/utils/appnexus/base.py index 1c7f36ac8f..aca3e7b776 100644 --- a/contentcuration/automation/utils/appnexus/base.py +++ b/contentcuration/automation/utils/appnexus/base.py @@ -1,37 +1,10 @@ from abc import ABC from abc import abstractmethod from builtins import NotImplementedError +from typing import Dict from typing import Union -class RecommendationsBackendRequest(object): - pass - - -class RecommedationsRequest(RecommendationsBackendRequest): - def __init__(self) -> None: - super().__init__() - - -class EmbeddingsRequest(RecommendationsBackendRequest): - def __init__(self) -> None: - super().__init__() - - -class RecommendationsBackendResponse(object): - pass - - -class RecommendationsResponse(RecommendationsBackendResponse): - def __init__(self) -> None: - pass - - -class EmbeddingsResponse(RecommendationsBackendResponse): - def __init__(self) -> None: - pass - - class Backend(ABC): """ An abstract base class for backend interfaces that also implements the singleton pattern """ _instance = None @@ -47,7 +20,7 @@ def connect(self) -> None: pass @abstractmethod - def make_request(self, request) -> Union[RecommendationsResponse, EmbeddingsResponse]: + def make_request(self, request) -> Union[bytes, str, Dict]: """ Make a request based on "request" """ pass diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index 34597ce008..a6b5e3ffc2 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -1,10 +1,34 @@ from automation.utils.appnexus.base import Adapter from automation.utils.appnexus.base import Backend from automation.utils.appnexus.base import BackendFactory -from automation.utils.appnexus.base import EmbeddingsRequest -from automation.utils.appnexus.base import EmbeddingsResponse -from automation.utils.appnexus.base import RecommedationsRequest -from automation.utils.appnexus.base import RecommendationsResponse + + +class RecommendationsBackendRequest(object): + pass + + +class RecommedationsRequest(RecommendationsBackendRequest): + def __init__(self) -> None: + super().__init__() + + +class EmbeddingsRequest(RecommendationsBackendRequest): + def __init__(self) -> None: + super().__init__() + + +class RecommendationsBackendResponse(object): + pass + + +class RecommendationsResponse(RecommendationsBackendResponse): + def __init__(self) -> None: + pass + + +class EmbeddingsResponse(RecommendationsBackendResponse): + def __init__(self) -> None: + pass class RecommendationsBackendFatory(BackendFactory): From ba921bd70bdee46373f90a6a6d19dca69cd447b5 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 28 Sep 2023 23:18:07 +0300 Subject: [PATCH 5/7] Refactors boilerplate --- .../automation/utils/appnexus/backends.py | 20 ------------------ .../tests/utils/test_recommendations.py} | 3 ++- .../contentcuration/utils/recommendations.py | 21 +++++++++++++++++++ 3 files changed, 23 insertions(+), 21 deletions(-) delete mode 100644 contentcuration/automation/utils/appnexus/backends.py rename contentcuration/{automation/tests/appnexus/test_backends.py => contentcuration/tests/utils/test_recommendations.py} (80%) diff --git a/contentcuration/automation/utils/appnexus/backends.py b/contentcuration/automation/utils/appnexus/backends.py deleted file mode 100644 index 5c0bf8e360..0000000000 --- a/contentcuration/automation/utils/appnexus/backends.py +++ /dev/null @@ -1,20 +0,0 @@ -from .base import Backend - - -class Recommendations(Backend): - - def connect(self) -> None: - return super().connect() - - def make_request(self, url: str, params=None): - return super().make_request(url, params) - - def request(self) -> None: - return super().request() - - def response(self) -> None: - return super().response() - - @classmethod - def _create_instance(cls) -> 'Recommendations': - return cls() diff --git a/contentcuration/automation/tests/appnexus/test_backends.py b/contentcuration/contentcuration/tests/utils/test_recommendations.py similarity index 80% rename from contentcuration/automation/tests/appnexus/test_backends.py rename to contentcuration/contentcuration/tests/utils/test_recommendations.py index 0e04c6a544..d410651de1 100644 --- a/contentcuration/automation/tests/appnexus/test_backends.py +++ b/contentcuration/contentcuration/tests/utils/test_recommendations.py @@ -1,6 +1,7 @@ -from automation.utils.appnexus.backends import Recommendations from django.test import TestCase +from contentcuration.utils.recommendations import Recommendations + class RecommendationsTestCase(TestCase): def test_backend_initialization(self): diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index a6b5e3ffc2..3b243c80a9 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -1,3 +1,5 @@ +from typing import Union + from automation.utils.appnexus.base import Adapter from automation.utils.appnexus.base import Backend from automation.utils.appnexus.base import BackendFactory @@ -46,3 +48,22 @@ def generate_embedding(self, text) -> EmbeddingsResponse: def get_recommendations(self) -> RecommendationsResponse: request = RecommedationsRequest() return self.backend.make_request(request) + + +class Recommendations(Backend): + + def connect(self) -> None: + return super().connect() + + def make_request(self, request) -> Union[EmbeddingsResponse, RecommendationsResponse]: + return super().make_request(request) + + def request(self) -> None: + return super().request() + + def response(self) -> None: + return super().response() + + @classmethod + def _create_instance(cls) -> 'Recommendations': + return cls() From 3d8cf19a206f8ce49e834e38d2d73b901e188dc5 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Fri, 29 Sep 2023 13:07:51 +0300 Subject: [PATCH 6/7] updates typo in factory class --- contentcuration/contentcuration/utils/recommendations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index 3b243c80a9..676686bd6f 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -33,7 +33,7 @@ def __init__(self) -> None: pass -class RecommendationsBackendFatory(BackendFactory): +class RecommendationsBackendFactory(BackendFactory): def create_backend(self) -> Backend: # Return backend based on some setting. return super().create_backend() From 7a197291a7a3d51f38c363a6aa1f85638f4e6a90 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Fri, 29 Sep 2023 13:27:53 +0300 Subject: [PATCH 7/7] Refactors boilerplate to remove ambiguity --- .../automation/tests/appnexus/test_base.py | 6 ---- .../automation/utils/appnexus/base.py | 32 +++++++------------ .../contentcuration/utils/recommendations.py | 12 +++---- 3 files changed, 15 insertions(+), 35 deletions(-) diff --git a/contentcuration/automation/tests/appnexus/test_base.py b/contentcuration/automation/tests/appnexus/test_base.py index 23b4f495bf..cd3d5c0047 100644 --- a/contentcuration/automation/tests/appnexus/test_base.py +++ b/contentcuration/automation/tests/appnexus/test_base.py @@ -9,12 +9,6 @@ def connect(self) -> None: def make_request(self, request): return super().make_request(request) - def request(self) -> None: - return super().request() - - def response(self) -> None: - return super().response() - @classmethod def _create_instance(cls) -> 'MockBackend': return cls() diff --git a/contentcuration/automation/utils/appnexus/base.py b/contentcuration/automation/utils/appnexus/base.py index aca3e7b776..ab9e6d5096 100644 --- a/contentcuration/automation/utils/appnexus/base.py +++ b/contentcuration/automation/utils/appnexus/base.py @@ -1,8 +1,16 @@ from abc import ABC from abc import abstractmethod from builtins import NotImplementedError -from typing import Dict -from typing import Union + + +class BackendRequest(object): + """ Class that should be inherited by specific backend for its requests""" + pass + + +class BackendResponse(object): + """ Class that should be inherited by specific backend for its responses""" + pass class Backend(ABC): @@ -20,20 +28,10 @@ def connect(self) -> None: pass @abstractmethod - def make_request(self, request) -> Union[bytes, str, Dict]: + def make_request(self, request) -> BackendResponse: """ Make a request based on "request" """ pass - @abstractmethod - def request(self) -> None: - """ Blueprint for the request object. """ - pass - - @abstractmethod - def response(self) -> None: - """ Blueprint for the response object. """ - pass - @classmethod def get_instance(cls) -> 'Backend': """ Returns existing instance, if not then create one. """ @@ -62,11 +60,3 @@ class Adapter: def __init__(self, backend: Backend) -> None: self.backend = backend - - def request(self): - """ Forward the request to the chosen Backend """ - return self.backend.request() - - def response(self): - """ Forward the response to the chosen Backend """ - return self.backend.response() diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index 676686bd6f..448ac07c97 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -3,9 +3,11 @@ from automation.utils.appnexus.base import Adapter from automation.utils.appnexus.base import Backend from automation.utils.appnexus.base import BackendFactory +from automation.utils.appnexus.base import BackendRequest +from automation.utils.appnexus.base import BackendResponse -class RecommendationsBackendRequest(object): +class RecommendationsBackendRequest(BackendRequest): pass @@ -19,7 +21,7 @@ def __init__(self) -> None: super().__init__() -class RecommendationsBackendResponse(object): +class RecommendationsBackendResponse(BackendResponse): pass @@ -58,12 +60,6 @@ def connect(self) -> None: def make_request(self, request) -> Union[EmbeddingsResponse, RecommendationsResponse]: return super().make_request(request) - def request(self) -> None: - return super().request() - - def response(self) -> None: - return super().response() - @classmethod def _create_instance(cls) -> 'Recommendations': return cls()