MAINT: Remove dependency on direct Mongo plugin

dserver_dependency_graph_plugin/__init__.py

@@ -33,7 +33,7 @@
 from dservercore import AuthenticationError, ExtensionABC
 from dservercore.sql_models import DatasetSchema
 from dservercore.utils import _preprocess_privileges
-from dserver_direct_mongo_plugin.utils import _dict_to_mongo_query
+from .utils import _dict_to_mongo_query
 
 from .schemas import DependencyKeysSchema
 
@@ -246,7 +246,8 @@ def dependency_graph_by_user_and_uuid(username, uuid, dependency_keys=Config.DEP
     mongo_aggregation = query_dependency_graph(pre_query=pre_query,
                                                post_query=post_query,
                                                dependency_keys=dependency_keys,
-                                               mongo_dependency_view=dependency_view)
+                                               mongo_dependency_view=dependency_view,
+                                               mongo_collection=current_app.config['MONGO_COLLECTION'])
     logger.debug("Constructed mongo aggregation: {}".format(mongo_aggregation))
     cx = DependencyGraphExtension.db[current_app.config['MONGO_COLLECTION']].aggregate(mongo_aggregation)
 

dserver_dependency_graph_plugin/config.py

@@ -8,6 +8,11 @@
 
 
 class Config(object):
+    # MongoDB connection settings
+    # These are required for the dependency graph plugin to connect to MongoDB
+    MONGO_URI = os.environ.get("MONGO_URI")
+    MONGO_DB = os.environ.get("MONGO_DB")
+    MONGO_COLLECTION = os.environ.get("MONGO_COLLECTION")
     # If enabled, the underlying database will offer dependency graph views on
     # the server's default collection. Those views offer on-the-fly-generated
     # collections of undirected per-dataset adjacency lists in order to

dserver_dependency_graph_plugin/graph.py

@@ -1,15 +1,14 @@
 """Aggregation pipelines for graph operations."""
 
-from dserver_dependency_graph_plugin.config import Config as dependency_graph_plugin_config
-from dserver_direct_mongo_plugin.config import Config as direct_mongo_plugin_config
+from .config import Config
 
 # a regular expression to filter valid v4 UUIDs
 UUID_v4_REGEX = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[4][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}'
 
 
 # most of those 'functions' are pretty static and just wrapped in function
 # definitions for convenience.
-def unwind_dependencies(dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS):
+def unwind_dependencies(dependency_keys=Config.DEPENDENCY_KEYS):
     """Create parallel aggregation pipelines for unwinding all configured dependency keys."""
 
     parallel_aggregations = []
@@ -41,7 +40,7 @@ def unwind_dependencies(dependency_keys=dependency_graph_plugin_config.DEPENDENC
     return parallel_aggregations
 
 
-def merge_dependencies(dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS):
+def merge_dependencies(dependency_keys=Config.DEPENDENCY_KEYS):
     """Aggregate (directed) dependency graph edges.
 
     All configured dependency keys are merged in a key-agnostic 'dependencies'
@@ -117,7 +116,7 @@ def group_inverse_dependencies():
     return aggregation
 
 
-def build_undirected_adjecency_lists(dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS):
+def build_undirected_adjecency_lists(dependency_keys=Config.DEPENDENCY_KEYS):
     """Aggregate undirected adjacency lists."""
     aggregation = [
         *merge_dependencies(dependency_keys),
@@ -200,8 +199,8 @@ def build_undirected_adjecency_lists(dependency_keys=dependency_graph_plugin_con
 # behavior would be to yield all redundant dataset entries for a uuid.
 def query_dependency_graph(mongo_dependency_view,
                            pre_query, post_query=None,
-                           dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS,
-                           mongo_collection=direct_mongo_plugin_config.MONGO_COLLECTION):
+                           dependency_keys=Config.DEPENDENCY_KEYS,
+                           mongo_collection=None):
     """Aggregation pipeline for querying dependency view on datasets collection.
 
     :param pre_query: selects all documents for whicht to query the dependency graph.

dserver_dependency_graph_plugin/utils.py

@@ -0,0 +1,117 @@
+"""Utility functions for MongoDB query construction.
+
+These functions were originally part of dserver-direct-mongo-plugin but are
+copied here to decouple the dependency-graph-plugin from that package.
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+VALID_MONGO_QUERY_KEYS = (
+    "free_text",
+    "creator_usernames",
+    "base_uris",
+    "uuids",
+    "tags",
+)
+
+MONGO_QUERY_LIST_KEYS = (
+    "creator_usernames",
+    "base_uris",
+    "uuids",
+    "tags",
+)
+
+
+def _dict_to_mongo(query_dict):
+    """Convert a query dictionary to a MongoDB query.
+
+    :param query_dict: Dictionary with query parameters
+    :returns: MongoDB query dictionary
+    """
+    def _sanitise(query_dict):
+        for key in list(query_dict.keys()):
+            if key not in VALID_MONGO_QUERY_KEYS:
+                del query_dict[key]
+        for lk in MONGO_QUERY_LIST_KEYS:
+            if lk in query_dict:
+                if len(query_dict[lk]) == 0:
+                    del query_dict[lk]
+
+    def _deal_with_possible_or_statment(a_list, key):
+        if len(a_list) == 1:
+            return {key: a_list[0]}
+        else:
+            return {"$or": [{key: v} for v in a_list]}
+
+    def _deal_with_possible_and_statement(a_list, key):
+        if len(a_list) == 1:
+            return {key: a_list[0]}
+        else:
+            return {key: {"$all": a_list}}
+
+    _sanitise(query_dict)
+
+    sub_queries = []
+    if "free_text" in query_dict:
+        sub_queries.append({"$text": {"$search": query_dict["free_text"]}})
+    if "creator_usernames" in query_dict:
+        sub_queries.append(
+            _deal_with_possible_or_statment(
+                query_dict["creator_usernames"], "creator_username"
+            )
+        )
+    if "base_uris" in query_dict:
+        sub_queries.append(
+            _deal_with_possible_or_statment(query_dict["base_uris"], "base_uri")
+        )
+    if "uuids" in query_dict:
+        sub_queries.append(_deal_with_possible_or_statment(query_dict["uuids"], "uuid"))
+    if "tags" in query_dict:
+        sub_queries.append(
+            _deal_with_possible_and_statement(query_dict["tags"], "tags")
+        )
+
+    if len(sub_queries) == 0:
+        return {}
+    elif len(sub_queries) == 1:
+        return sub_queries[0]
+    else:
+        return {"$and": [q for q in sub_queries]}
+
+
+def _dict_to_mongo_query(query_dict):
+    """Construct mongo query, allowing embedding of a raw mongo query.
+
+    Converts a query dictionary to a MongoDB query format. If the query_dict
+    contains a 'query' key with a dict value, that raw MongoDB query is
+    merged with the constructed query.
+
+    :param query_dict: Dictionary with query parameters. May contain:
+        - free_text: Text search string
+        - creator_usernames: List of creator usernames
+        - base_uris: List of base URIs
+        - uuids: List of UUIDs
+        - tags: List of tags
+        - query: Raw MongoDB query dict (optional)
+    :returns: MongoDB query dictionary
+    """
+    if "query" in query_dict and isinstance(query_dict["query"], dict):
+        raw_mongo = query_dict["query"]
+        del query_dict["query"]
+    else:
+        raw_mongo = {}
+
+    mongo_query = _dict_to_mongo(query_dict)
+
+    if len(raw_mongo) > 0 and len(mongo_query) == 0:
+        mongo_query = raw_mongo
+    elif len(raw_mongo) > 0 and len(mongo_query) == 1 and "$and" in mongo_query:
+        mongo_query["$and"].append(raw_mongo)
+    elif len(raw_mongo) > 0:
+        mongo_query = {"$and": [mongo_query, raw_mongo]}
+
+    logger.debug("Constructed mongo query: {}".format(mongo_query))
+    return mongo_query

dserver_dependency_graph_plugin/version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.4.3.dev2'
+__version_tuple__ = version_tuple = (0, 4, 3, 'dev2')
+
+__commit_id__ = commit_id = 'g8c317d2de'

pyproject.toml

@@ -1,21 +1,22 @@
 [build-system]
-requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"]
-build-backend = "setuptools.build_meta"
+requires = ["flit_scm"]
+build-backend = "flit_scm:buildapi"
 
 [project]
 name = "dserver-dependency-graph-plugin"
 description = "dserver plugin for receiving s3 notifications on updated objects."
 readme = "README.rst"
-license = {file = "LICENSE"}
+license = {text = "MIT"}
 authors = [
     {name = "Johannes L. Hörmann", email = "johannes.laurin@gmail.com"},
 ]
 dynamic = ["version"]
+requires-python = ">=3.8"
 dependencies = [
-        "dtoolcore>=3.18.0",
-        "dservercore>=0.20.0",
-        "dserver-direct-mongo-plugin",
-    ]
+    "dtoolcore>=3.18.0",
+    "dservercore>=0.20.0",
+    "pymongo",
+]
 
 [project.optional-dependencies]
 test = [
@@ -31,13 +32,17 @@ Documentation = "https://github.com/livMatS/dserver-dependency-graph-plugin/blob
 Repository = "https://github.com/livMatS/dserver-dependency-graph-plugin"
 Changelog = "https://github.com/livMatS/dserver-dependency-graph-plugin/blob/main/CHANGELOG.rst"
 
+[project.entry-points."dservercore.extension"]
+DependencyGraphExtension = "dserver_dependency_graph_plugin:DependencyGraphExtension"
+
+[tool.flit.module]
+name = "dserver_dependency_graph_plugin"
+
 [tool.setuptools_scm]
 version_scheme = "guess-next-dev"
 local_scheme = "no-local-version"
 write_to = "dserver_dependency_graph_plugin/version.py"
 
-[tool.setuptools]
-packages = ["dserver_dependency_graph_plugin"]
-
-[project.entry-points."dservercore.extension"]
-"DependencyGraphExtension" = "dserver_dependency_graph_plugin:DependencyGraphExtension"
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=dserver_dependency_graph_plugin --cov-report=term-missing"