From cf0167da5e0766f6301631ecfcf3a98cec58748b Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 10:26:37 +0100 Subject: [PATCH 01/13] ENH: Health check route --- dservercore/config_routes.py | 13 ++++++++++++- dservercore/schemas.py | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dservercore/config_routes.py b/dservercore/config_routes.py index 2191a91..8c9de8d 100644 --- a/dservercore/config_routes.py +++ b/dservercore/config_routes.py @@ -13,7 +13,7 @@ import dservercore import dservercore.utils_auth from dservercore.blueprint import Blueprint -from dservercore.schemas import ConfigSchema, VersionSchema +from dservercore.schemas import ConfigSchema, HealthSchema, VersionSchema from dservercore.utils import versions_to_dict, obj_to_lowercase_key_dict @@ -45,3 +45,14 @@ def server_versions(): This does not require authorization.""" return jsonify({"versions": versions_to_dict()}) + + +@bp.route("/health", methods=["GET"]) +@bp.response(200, HealthSchema) +def health(): + """Health check endpoint for container orchestration. + + Returns a simple status indicating the service is running. + This does not require authorization.""" + + return jsonify({"status": "healthy"}), 200 diff --git a/dservercore/schemas.py b/dservercore/schemas.py index 12df225..451e775 100644 --- a/dservercore/schemas.py +++ b/dservercore/schemas.py @@ -13,6 +13,10 @@ ) +class HealthSchema(Schema): + status = String() + + class ConfigSchema(Schema): config = Dict(keys=String(), values=Raw()) From d98351682ef979f32dd03049e619fa9a5e023dfa Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 15:47:18 +0100 Subject: [PATCH 02/13] ENH: API routes for modifying tags --- dservercore/tags_routes.py | 107 +++++++++++++++++++++++++++++++++---- dservercore/utils.py | 38 +++++++++++++ 2 files changed, 136 insertions(+), 9 deletions(-) diff --git a/dservercore/tags_routes.py b/dservercore/tags_routes.py index fb4a807..92d6760 100644 --- a/dservercore/tags_routes.py +++ b/dservercore/tags_routes.py @@ -1,41 +1,42 @@ -"""Route for retrieving tags of a dataset""" +"""Routes for retrieving and modifying tags of a dataset""" from flask import ( abort, jsonify, - current_app + current_app, + request ) from dservercore.utils_auth import ( jwt_required, get_jwt_identity, ) -from dservercore import UnknownURIError +from dservercore import UnknownURIError, AuthorizationError from dservercore.blueprint import Blueprint from dservercore.schemas import TagSchema import dservercore.utils_auth from dservercore.utils import ( url_suffix_to_uri, - get_tags_from_uri_by_user + get_tags_from_uri_by_user, + set_tags_for_uri_by_user ) bp = Blueprint("tags", __name__, url_prefix="/tags") + @bp.route("/", methods=["GET"]) @bp.response(200, TagSchema) -@bp.alt_response(1, description=2) +@bp.alt_response(401, description="Unauthorized") @bp.alt_response(403, description="No permissions") @bp.alt_response(404, description="Not found") @jwt_required() -def manifest(uri): - """Request the dataset manifest.""" +def get_tags(uri): + """Request the dataset tags.""" username = get_jwt_identity() if not dservercore.utils_auth.user_exists(username): - # Unregistered users should see 401. abort(401) uri = url_suffix_to_uri(uri) if not dservercore.utils_auth.may_access(username, uri): - # Authorization errors should return 400. abort(403) try: @@ -47,3 +48,91 @@ def manifest(uri): return {"tags": tags} +@bp.route("/", methods=["PUT"]) +@bp.arguments(TagSchema) +@bp.response(200, TagSchema) +@bp.alt_response(401, description="Unauthorized") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def set_tags(data, uri): + """Set the dataset tags (replaces all existing tags).""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + + try: + tags = set_tags_for_uri_by_user(username, uri, data.get("tags", [])) + except AuthorizationError: + abort(403) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + + return {"tags": tags} + + +@bp.route("//", methods=["POST"]) +@bp.response(200, TagSchema) +@bp.alt_response(401, description="Unauthorized") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def add_tag(uri, tag): + """Add a single tag to the dataset.""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + + try: + # Get existing tags + existing_tags = get_tags_from_uri_by_user(username, uri) + # Add new tag if not already present + if tag not in existing_tags: + existing_tags.append(tag) + # Set updated tags + tags = set_tags_for_uri_by_user(username, uri, existing_tags) + except AuthorizationError: + abort(403) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + + return {"tags": tags} + + +@bp.route("//", methods=["DELETE"]) +@bp.response(200, TagSchema) +@bp.alt_response(401, description="Unauthorized") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def delete_tag(uri, tag): + """Remove a single tag from the dataset.""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + + try: + # Get existing tags + existing_tags = get_tags_from_uri_by_user(username, uri) + # Remove tag if present + if tag in existing_tags: + existing_tags.remove(tag) + # Set updated tags + tags = set_tags_for_uri_by_user(username, uri, existing_tags) + except AuthorizationError: + abort(403) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + + return {"tags": tags} + + diff --git a/dservercore/utils.py b/dservercore/utils.py index 6d296f2..b1e9c3c 100644 --- a/dservercore/utils.py +++ b/dservercore/utils.py @@ -1068,3 +1068,41 @@ def get_annotations_from_uri_by_user(username, uri): raise (AuthorizationError()) return current_app.retrieve.get_annotations(uri) + + +def set_tags_for_uri_by_user(username, uri, tags): + """Set tags for a dataset. + + :param username: username + :param uri: dataset URI + :param tags: list of tags to set + :returns: updated list of tags + :raises: AuthenticationError if user is invalid. + AuthorizationError if the user has not got permissions to modify + content in the base URI + UnknownBaseURIError if the base URI has not been registered. + UnknownURIError if the URI is not available to the user. + """ + user = get_user_obj(username) + + base_uri_str = uri.rsplit("/", 1)[0] + base_uri = _get_base_uri_obj(base_uri_str) + if base_uri is None: + raise (UnknownBaseURIError()) + + # Check if user has register permissions (write access) for this base URI + if base_uri not in user.register_base_uris: + raise (AuthorizationError()) + + # Update tags in both search and retrieve plugins + if hasattr(current_app.search, "set_tags"): + current_app.search.set_tags(uri, tags) + else: + logger.warning("Search plugin has no method 'set_tags'") + + if hasattr(current_app.retrieve, "set_tags"): + current_app.retrieve.set_tags(uri, tags) + else: + logger.warning("Retrieve plugin has no method 'set_tags'") + + return tags From b0a5aaf10024e13745e0e975fa1664c90cba64ba Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 15:59:47 +0100 Subject: [PATCH 03/13] ENH: Annotation routes --- dservercore/annotations_routes.py | 105 +++++++++++++++++++++++++++--- dservercore/schemas.py | 6 +- dservercore/utils.py | 86 ++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 11 deletions(-) diff --git a/dservercore/annotations_routes.py b/dservercore/annotations_routes.py index 886d82d..f005b46 100644 --- a/dservercore/annotations_routes.py +++ b/dservercore/annotations_routes.py @@ -1,21 +1,25 @@ -"""Route for retrieving dataset annotations by URI.""" +"""Routes for retrieving and modifying dataset annotations by URI.""" from flask import ( abort, jsonify, - current_app + current_app, + request ) from dservercore.utils_auth import ( jwt_required, get_jwt_identity, ) -from dservercore import UnknownURIError +from dservercore import UnknownURIError, AuthorizationError from dservercore.blueprint import Blueprint -from dservercore.schemas import AnnotationSchema +from dservercore.schemas import AnnotationSchema, SingleAnnotationSchema import dservercore.utils_auth from dservercore.utils import ( url_suffix_to_uri, - get_annotations_from_uri_by_user + get_annotations_from_uri_by_user, + set_annotations_for_uri_by_user, + set_annotation_for_uri_by_user, + delete_annotation_for_uri_by_user ) bp = Blueprint("annotations", __name__, url_prefix="/annotations") @@ -23,21 +27,19 @@ @bp.route("/", methods=["GET"]) @bp.response(200, AnnotationSchema) -@bp.alt_response(401, description="Not registered") +@bp.alt_response(401, description="Unauthorized") @bp.alt_response(403, description="No permissions") -@bp.alt_response(400, description="Unknown URI") +@bp.alt_response(404, description="Not found") @jwt_required() -def annotations(uri): +def get_annotations(uri): """Request the dataset annotations.""" username = get_jwt_identity() if not dservercore.utils_auth.user_exists(username): - # Unregistered users should see 401. abort(401) uri = url_suffix_to_uri(uri) if not dservercore.utils_auth.may_access(username, uri): - # Authorization errors should return 403. abort(403) try: @@ -46,4 +48,87 @@ def annotations(uri): current_app.logger.info("UnknownURIError") abort(404) + return {"annotations": annotations} + + +@bp.route("/", methods=["PUT"]) +@bp.arguments(AnnotationSchema) +@bp.response(200, AnnotationSchema) +@bp.alt_response(401, description="Unauthorized") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def set_annotations(data, uri): + """Set all dataset annotations (replaces existing annotations).""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + + try: + annotations = set_annotations_for_uri_by_user( + username, uri, data.get("annotations", {}) + ) + except AuthorizationError: + abort(403) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + + return {"annotations": annotations} + + +@bp.route("//", methods=["PUT"]) +@bp.arguments(SingleAnnotationSchema) +@bp.response(200, AnnotationSchema) +@bp.alt_response(401, description="Unauthorized") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def set_annotation(data, uri, annotation_name): + """Set a single annotation (creates or updates).""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + + try: + annotations = set_annotation_for_uri_by_user( + username, uri, annotation_name, data.get("value") + ) + except AuthorizationError: + abort(403) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + + return {"annotations": annotations} + + +@bp.route("//", methods=["DELETE"]) +@bp.response(200, AnnotationSchema) +@bp.alt_response(401, description="Unauthorized") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def delete_annotation(uri, annotation_name): + """Delete a single annotation.""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + + try: + annotations = delete_annotation_for_uri_by_user( + username, uri, annotation_name + ) + except AuthorizationError: + abort(403) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + return {"annotations": annotations} \ No newline at end of file diff --git a/dservercore/schemas.py b/dservercore/schemas.py index 451e775..15c0130 100644 --- a/dservercore/schemas.py +++ b/dservercore/schemas.py @@ -44,7 +44,11 @@ class ManifestSchema(Schema): # Define a schema for the response class AnnotationSchema(Schema): - annotations = Dict(keys=String(), values=String()) + annotations = Dict(keys=String(), values=Raw()) + + +class SingleAnnotationSchema(Schema): + value = Raw() class TagSchema(Schema): diff --git a/dservercore/utils.py b/dservercore/utils.py index b1e9c3c..799b187 100644 --- a/dservercore/utils.py +++ b/dservercore/utils.py @@ -1106,3 +1106,89 @@ def set_tags_for_uri_by_user(username, uri, tags): logger.warning("Retrieve plugin has no method 'set_tags'") return tags + + +def set_annotations_for_uri_by_user(username, uri, annotations): + """Set all annotations for a dataset (replaces existing annotations). + + :param username: username + :param uri: dataset URI + :param annotations: dictionary of annotations to set + :returns: updated annotations dictionary + :raises: AuthenticationError if user is invalid. + AuthorizationError if the user has not got permissions to modify + content in the base URI + UnknownBaseURIError if the base URI has not been registered. + UnknownURIError if the URI is not available to the user. + """ + user = get_user_obj(username) + + base_uri_str = uri.rsplit("/", 1)[0] + base_uri = _get_base_uri_obj(base_uri_str) + if base_uri is None: + raise (UnknownBaseURIError()) + + # Check if user has register permissions (write access) for this base URI + if base_uri not in user.register_base_uris: + raise (AuthorizationError()) + + # Update annotations in both search and retrieve plugins + if hasattr(current_app.search, "set_annotations"): + current_app.search.set_annotations(uri, annotations) + else: + logger.warning("Search plugin has no method 'set_annotations'") + + if hasattr(current_app.retrieve, "set_annotations"): + current_app.retrieve.set_annotations(uri, annotations) + else: + logger.warning("Retrieve plugin has no method 'set_annotations'") + + return annotations + + +def set_annotation_for_uri_by_user(username, uri, annotation_name, value): + """Set a single annotation for a dataset. + + :param username: username + :param uri: dataset URI + :param annotation_name: name of the annotation + :param value: value to set for the annotation + :returns: updated annotations dictionary + :raises: AuthenticationError if user is invalid. + AuthorizationError if the user has not got permissions to modify + content in the base URI + UnknownBaseURIError if the base URI has not been registered. + UnknownURIError if the URI is not available to the user. + """ + # Get existing annotations + existing_annotations = get_annotations_from_uri_by_user(username, uri) + + # Update the specific annotation + existing_annotations[annotation_name] = value + + # Set all annotations + return set_annotations_for_uri_by_user(username, uri, existing_annotations) + + +def delete_annotation_for_uri_by_user(username, uri, annotation_name): + """Delete a single annotation from a dataset. + + :param username: username + :param uri: dataset URI + :param annotation_name: name of the annotation to delete + :returns: updated annotations dictionary + :raises: AuthenticationError if user is invalid. + AuthorizationError if the user has not got permissions to modify + content in the base URI + UnknownBaseURIError if the base URI has not been registered. + UnknownURIError if the URI is not available to the user. + """ + # Get existing annotations + existing_annotations = get_annotations_from_uri_by_user(username, uri) + + # Remove the annotation if it exists + if annotation_name in existing_annotations: + del existing_annotations[annotation_name] + + # Set all annotations + return set_annotations_for_uri_by_user(username, uri, existing_annotations) From 6b24cf75fde2e0b0c6f0efc8d44b20ed1d3ae24b Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 20:31:05 +0100 Subject: [PATCH 04/13] MAINT: Removed unused legacy code --- MANIFEST.in | 1 - dservercore/templates/index.html | 9 --------- 2 files changed, 10 deletions(-) delete mode 100644 dservercore/templates/index.html diff --git a/MANIFEST.in b/MANIFEST.in index 2fdd34e..a5021c6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include README.rst include LICENSE -include dserver/templates/* diff --git a/dservercore/templates/index.html b/dservercore/templates/index.html deleted file mode 100644 index 7be1599..0000000 --- a/dservercore/templates/index.html +++ /dev/null @@ -1,9 +0,0 @@ - - - dserver - - -

dservercore

-

{{ num_datasets }} registered datasets.

- - From 57c09ca978a1cb3a8d7fd378371d7797fe770806 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 21:01:13 +0100 Subject: [PATCH 05/13] BUG: Update tags and annotations in storage --- dservercore/utils.py | 113 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/dservercore/utils.py b/dservercore/utils.py index 799b187..49b3d3d 100644 --- a/dservercore/utils.py +++ b/dservercore/utils.py @@ -12,6 +12,7 @@ from flask_smorest.pagination import PaginationParameters from sqlalchemy.sql import exists +import dtoolcore import dtoolcore.utils from dservercore import ( @@ -1070,6 +1071,108 @@ def get_annotations_from_uri_by_user(username, uri): return current_app.retrieve.get_annotations(uri) +def _update_tags_in_storage(uri, tags): + """Update tags in the actual storage backend using dtoolcore. + + :param uri: dataset URI + :param tags: list of tags to set + """ + try: + # Load the dataset + dataset = dtoolcore.DataSet.from_uri(uri) + storage_broker = dataset._storage_broker + + # Check if storage broker supports tag operations + if not hasattr(storage_broker, 'put_text') or not hasattr(storage_broker, 'delete_key'): + logger.debug(f"Storage broker for {uri} does not support tag operations") + return + + # Get the dataset UUID from the URI + uuid = uri.rsplit("/", 1)[1] + prefix = uuid + "/" + + # Get existing tags from storage + existing_tags = set() + try: + if hasattr(storage_broker, 'list_tags'): + existing_tags = set(storage_broker.list_tags()) + except Exception: + pass # No existing tags or method not available + + new_tags = set(tags) + + # Delete tags that are no longer present + for tag in existing_tags - new_tags: + logger.debug(f"Deleting tag '{tag}' from storage for {uri}") + try: + storage_broker.delete_key(prefix + "tags/" + tag) + except Exception as e: + logger.warning(f"Failed to delete tag '{tag}': {e}") + + # Add new tags + for tag in new_tags - existing_tags: + logger.debug(f"Adding tag '{tag}' to storage for {uri}") + storage_broker.put_text(prefix + "tags/" + tag, "") + + logger.info(f"Updated tags in storage for {uri}") + + except Exception as e: + # Log but don't fail - database update succeeded + logger.warning(f"Failed to update tags in storage for {uri}: {e}") + + +def _update_annotations_in_storage(uri, annotations): + """Update annotations in the actual storage backend using dtoolcore. + + :param uri: dataset URI + :param annotations: dictionary of annotations to set + """ + try: + # Load the dataset + dataset = dtoolcore.DataSet.from_uri(uri) + storage_broker = dataset._storage_broker + + # Check if storage broker supports annotation operations + if not hasattr(storage_broker, 'put_text') or not hasattr(storage_broker, 'delete_key'): + logger.debug(f"Storage broker for {uri} does not support annotation operations") + return + + # Get the dataset UUID from the URI + uuid = uri.rsplit("/", 1)[1] + prefix = uuid + "/" + + # Get existing annotation names from storage + existing_annotations = set() + try: + if hasattr(storage_broker, 'list_annotation_names'): + existing_annotations = set(storage_broker.list_annotation_names()) + except Exception: + pass # No existing annotations or method not available + + new_annotations = set(annotations.keys()) + + # Delete annotations that are no longer present + for annotation_name in existing_annotations - new_annotations: + logger.debug(f"Deleting annotation '{annotation_name}' from storage for {uri}") + try: + storage_broker.delete_key(prefix + "annotations/" + annotation_name + ".json") + except Exception as e: + logger.warning(f"Failed to delete annotation '{annotation_name}': {e}") + + # Add/update annotations + for annotation_name, value in annotations.items(): + logger.debug(f"Setting annotation '{annotation_name}' in storage for {uri}") + storage_broker.put_text( + prefix + "annotations/" + annotation_name + ".json", + json.dumps(value, indent=2) + ) + + logger.info(f"Updated annotations in storage for {uri}") + + except Exception as e: + logger.warning(f"Failed to update annotations in storage for {uri}: {e}") + + def set_tags_for_uri_by_user(username, uri, tags): """Set tags for a dataset. @@ -1094,7 +1197,7 @@ def set_tags_for_uri_by_user(username, uri, tags): if base_uri not in user.register_base_uris: raise (AuthorizationError()) - # Update tags in both search and retrieve plugins + # Update tags in both search and retrieve plugins (database) if hasattr(current_app.search, "set_tags"): current_app.search.set_tags(uri, tags) else: @@ -1105,6 +1208,9 @@ def set_tags_for_uri_by_user(username, uri, tags): else: logger.warning("Retrieve plugin has no method 'set_tags'") + # Update tags in actual storage backend + _update_tags_in_storage(uri, tags) + return tags @@ -1132,7 +1238,7 @@ def set_annotations_for_uri_by_user(username, uri, annotations): if base_uri not in user.register_base_uris: raise (AuthorizationError()) - # Update annotations in both search and retrieve plugins + # Update annotations in both search and retrieve plugins (database) if hasattr(current_app.search, "set_annotations"): current_app.search.set_annotations(uri, annotations) else: @@ -1143,6 +1249,9 @@ def set_annotations_for_uri_by_user(username, uri, annotations): else: logger.warning("Retrieve plugin has no method 'set_annotations'") + # Update annotations in actual storage backend + _update_annotations_in_storage(uri, annotations) + return annotations From 3e99eecbf22bd29e321fe3a34f6d7a0c8ddc3e6a Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 21:09:56 +0100 Subject: [PATCH 06/13] MAINT: Don't use storage broker directly for setting tags and annotations --- dservercore/utils.py | 51 +++++++------------------------------------- 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/dservercore/utils.py b/dservercore/utils.py index 49b3d3d..0038a10 100644 --- a/dservercore/utils.py +++ b/dservercore/utils.py @@ -1080,39 +1080,23 @@ def _update_tags_in_storage(uri, tags): try: # Load the dataset dataset = dtoolcore.DataSet.from_uri(uri) - storage_broker = dataset._storage_broker - - # Check if storage broker supports tag operations - if not hasattr(storage_broker, 'put_text') or not hasattr(storage_broker, 'delete_key'): - logger.debug(f"Storage broker for {uri} does not support tag operations") - return - - # Get the dataset UUID from the URI - uuid = uri.rsplit("/", 1)[1] - prefix = uuid + "/" # Get existing tags from storage - existing_tags = set() - try: - if hasattr(storage_broker, 'list_tags'): - existing_tags = set(storage_broker.list_tags()) - except Exception: - pass # No existing tags or method not available - + existing_tags = set(dataset.list_tags()) new_tags = set(tags) # Delete tags that are no longer present for tag in existing_tags - new_tags: logger.debug(f"Deleting tag '{tag}' from storage for {uri}") try: - storage_broker.delete_key(prefix + "tags/" + tag) + dataset.delete_tag(tag) except Exception as e: logger.warning(f"Failed to delete tag '{tag}': {e}") - # Add new tags + # Add new tags (put_tag includes name validation) for tag in new_tags - existing_tags: logger.debug(f"Adding tag '{tag}' to storage for {uri}") - storage_broker.put_text(prefix + "tags/" + tag, "") + dataset.put_tag(tag) logger.info(f"Updated tags in storage for {uri}") @@ -1130,42 +1114,23 @@ def _update_annotations_in_storage(uri, annotations): try: # Load the dataset dataset = dtoolcore.DataSet.from_uri(uri) - storage_broker = dataset._storage_broker - - # Check if storage broker supports annotation operations - if not hasattr(storage_broker, 'put_text') or not hasattr(storage_broker, 'delete_key'): - logger.debug(f"Storage broker for {uri} does not support annotation operations") - return - - # Get the dataset UUID from the URI - uuid = uri.rsplit("/", 1)[1] - prefix = uuid + "/" # Get existing annotation names from storage - existing_annotations = set() - try: - if hasattr(storage_broker, 'list_annotation_names'): - existing_annotations = set(storage_broker.list_annotation_names()) - except Exception: - pass # No existing annotations or method not available - + existing_annotations = set(dataset.list_annotation_names()) new_annotations = set(annotations.keys()) # Delete annotations that are no longer present for annotation_name in existing_annotations - new_annotations: logger.debug(f"Deleting annotation '{annotation_name}' from storage for {uri}") try: - storage_broker.delete_key(prefix + "annotations/" + annotation_name + ".json") + dataset.delete_annotation(annotation_name) except Exception as e: logger.warning(f"Failed to delete annotation '{annotation_name}': {e}") - # Add/update annotations + # Add/update annotations (put_annotation includes name validation) for annotation_name, value in annotations.items(): logger.debug(f"Setting annotation '{annotation_name}' in storage for {uri}") - storage_broker.put_text( - prefix + "annotations/" + annotation_name + ".json", - json.dumps(value, indent=2) - ) + dataset.put_annotation(annotation_name, value) logger.info(f"Updated annotations in storage for {uri}") From 839f4e91f9d2d9bfe2455fdfdcf161f31c55cf8a Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 22:26:08 +0100 Subject: [PATCH 07/13] ENH: Modifying README --- dservercore/readme_routes.py | 40 ++++++++++++++++++++--- dservercore/schemas.py | 4 +++ dservercore/utils.py | 62 ++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/dservercore/readme_routes.py b/dservercore/readme_routes.py index 784bac4..82ea882 100644 --- a/dservercore/readme_routes.py +++ b/dservercore/readme_routes.py @@ -1,8 +1,9 @@ -"""Route for retrieving the readme of a dataset""" +"""Routes for retrieving and updating the readme of a dataset""" from flask import ( abort, jsonify, - current_app + current_app, + request, ) from dservercore.utils_auth import ( jwt_required, @@ -11,11 +12,12 @@ from dservercore import UnknownURIError from dservercore.blueprint import Blueprint -from dservercore.schemas import ReadmeSchema +from dservercore.schemas import ReadmeSchema, ReadmeRequestSchema import dservercore.utils_auth from dservercore.utils import ( url_suffix_to_uri, - get_readme_from_uri_by_user + get_readme_from_uri_by_user, + set_readme_for_uri_by_user, ) bp = Blueprint("readmes", __name__, url_prefix="/readmes") @@ -45,4 +47,32 @@ def readme(uri): current_app.logger.info("UnknownURIError") abort(404) - return {"readme": readme} \ No newline at end of file + return {"readme": readme} + + +@bp.route("/", methods=["PUT"]) +@bp.arguments(ReadmeRequestSchema) +@bp.response(200, ReadmeSchema) +@bp.alt_response(401, description="Not registered") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def set_readme(request_data, uri): + """Update the dataset readme.""" + username = get_jwt_identity() + if not dservercore.utils_auth.user_exists(username): + abort(401) + + uri = url_suffix_to_uri(uri) + if not dservercore.utils_auth.may_access(username, uri): + abort(403) + + readme_content = request_data.get("readme", "") + + try: + set_readme_for_uri_by_user(username, uri, readme_content) + except UnknownURIError: + current_app.logger.info("UnknownURIError") + abort(404) + + return {"readme": readme_content} \ No newline at end of file diff --git a/dservercore/schemas.py b/dservercore/schemas.py index 15c0130..8628370 100644 --- a/dservercore/schemas.py +++ b/dservercore/schemas.py @@ -36,6 +36,10 @@ class ReadmeSchema(Schema): readme = String() +class ReadmeRequestSchema(Schema): + readme = String(required=True) + + class ManifestSchema(Schema): items = Dict(keys=String, values=Nested(ItemSchema)) hash_function = String() diff --git a/dservercore/utils.py b/dservercore/utils.py index 0038a10..5fa5474 100644 --- a/dservercore/utils.py +++ b/dservercore/utils.py @@ -1266,3 +1266,65 @@ def delete_annotation_for_uri_by_user(username, uri, annotation_name): # Set all annotations return set_annotations_for_uri_by_user(username, uri, existing_annotations) + + +def _update_readme_in_storage(uri, content): + """Update README in the actual storage backend using dtoolcore. + + :param uri: dataset URI + :param content: README content string + """ + try: + # Load the dataset + dataset = dtoolcore.DataSet.from_uri(uri) + + # Update the README using put_readme + logger.debug(f"Updating README in storage for {uri}") + dataset.put_readme(content) + + logger.info(f"Updated README in storage for {uri}") + + except Exception as e: + # Log but don't fail - database update succeeded + logger.warning(f"Failed to update README in storage for {uri}: {e}") + + +def set_readme_for_uri_by_user(username, uri, content): + """Set README content for a dataset. + + :param username: username + :param uri: dataset URI + :param content: README content string + :returns: updated README content + :raises: AuthenticationError if user is invalid. + AuthorizationError if the user has not got permissions to modify + content in the base URI + UnknownBaseURIError if the base URI has not been registered. + UnknownURIError if the URI is not available to the user. + """ + user = get_user_obj(username) + + base_uri_str = uri.rsplit("/", 1)[0] + base_uri = _get_base_uri_obj(base_uri_str) + if base_uri is None: + raise (UnknownBaseURIError()) + + # Check if user has register permissions (write access) for this base URI + if base_uri not in user.register_base_uris: + raise (AuthorizationError()) + + # Update README in both search and retrieve plugins (database) + if hasattr(current_app.search, "set_readme"): + current_app.search.set_readme(uri, content) + else: + logger.warning("Search plugin has no method 'set_readme'") + + if hasattr(current_app.retrieve, "set_readme"): + current_app.retrieve.set_readme(uri, content) + else: + logger.warning("Retrieve plugin has no method 'set_readme'") + + # Update README in actual storage backend + _update_readme_in_storage(uri, content) + + return content From 924a8669fc6ae7196e40b83517c52685d1a9d3c6 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 23:25:46 +0100 Subject: [PATCH 08/13] BUILD: Switched build system to flit --- pyproject.toml | 57 ++++++++++++++++++++++++++++---------------------- setup.cfg | 10 --------- 2 files changed, 32 insertions(+), 35 deletions(-) delete mode 100644 setup.cfg diff --git a/pyproject.toml b/pyproject.toml index 58b339c..2058751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,32 +1,32 @@ [build-system] -requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"] -build-backend = "setuptools.build_meta" +requires = ["flit_scm"] +build-backend = "flit_scm:buildapi" [project] name = "dservercore" description = "Web API to register/lookup/search for dtool dataset metadata" readme = "README.rst" -license = {file = "LICENSE"} +license = {text = "MIT"} authors = [ {name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"} ] dynamic = ["version"] +requires-python = ">=3.8" dependencies = [ - "setuptools", - "flask<3", - "pymongo", - "alembic", - "flask-sqlalchemy", - "flask-migrate", - "flask-pymongo", - "flask-marshmallow", - "flask-smorest", - "marshmallow-sqlalchemy", - "flask-cors", - "dtoolcore>=3.18.0", - "flask-jwt-extended[asymmetric_crypto]>=4.6.0", - "pyyaml" - ] + "flask<3", + "pymongo", + "alembic", + "flask-sqlalchemy", + "flask-migrate", + "flask-pymongo", + "flask-marshmallow", + "flask-smorest", + "marshmallow-sqlalchemy", + "flask-cors", + "dtoolcore>=3.18.0", + "flask-jwt-extended[asymmetric_crypto]>=4.6.0", + "pyyaml" +] [project.optional-dependencies] test = [ @@ -45,16 +45,23 @@ Documentation = "https://dservercore.readthedocs.io" Repository = "https://github.com/jic-dtool/dservercore" Changelog = "https://github.com/jic-dtool/dservercore/blob/main/CHANGELOG.rst" +[project.entry-points."flask.commands"] +base_uri = "dservercore.cli:base_uri_cli" +user = "dservercore.cli:user_cli" +config = "dservercore.cli:config_cli" +dataset = "dservercore.cli:dataset_cli" + +[tool.flit.module] +name = "dservercore" + [tool.setuptools_scm] version_scheme = "guess-next-dev" local_scheme = "no-local-version" write_to = "dservercore/version.py" -[tool.setuptools] -packages = ["dservercore"] +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=dservercore --cov-report=term-missing" -[project.entry-points."flask.commands"] -"base_uri" = "dservercore.cli:base_uri_cli" -"user" = "dservercore.cli:user_cli" -"config" = "dservercore.cli:config_cli" -"dataset" = "dservercore.cli:dataset_cli" +[tool.flake8] +exclude = ["venv*", "env*", ".tox", ".git", "*.egg", "build", "docs", "migrations", "jwt-spike"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 52056ef..0000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -exclude=venv*,env*,.tox,.git,*.egg,build,docs,migrations,jwt-spike - -[tool:pytest] -testpaths = tests -#addopts = --cov=dservercore --cov-report=term-missing --disable-warnings -addopts = --cov=dservercore --cov-report=term-missing - -[cov:run] -source = dserver From 7d4ac82e4861fc2c240b8bab2a1637ce7b62f305 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Mon, 8 Dec 2025 09:10:30 +0100 Subject: [PATCH 09/13] DOC: Updated CHANGELOG and README --- CHANGELOG.rst | 40 ++++++++++++++++++++++++++++++++ README.rst | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b01752b..0f78d62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,46 @@ CHANGELOG This project uses `semantic versioning `_. This change log uses principles from `keep a changelog `_. +[0.23.0] - 2025-12-08 +--------------------- + +Added +^^^^^ + +- Health check endpoint ``GET /config/health`` for container orchestration + (does not require authorization) +- Tag manipulation routes: + + - ``GET /tags/`` - Get dataset tags + - ``PUT /tags/`` - Set all dataset tags (replaces existing) + - ``POST /tags//`` - Add a single tag + - ``DELETE /tags//`` - Remove a single tag + +- Annotation manipulation routes: + + - ``GET /annotations/`` - Get dataset annotations + - ``PUT /annotations/`` - Set all annotations (replaces existing) + - ``PUT /annotations//`` - Set a single annotation + - ``DELETE /annotations//`` - Delete a single annotation + +- README manipulation routes: + + - ``GET /readmes/`` - Get dataset README + - ``PUT /readmes/`` - Update dataset README + +Changed +^^^^^^^ + +- Switched build system to flit +- Tags and annotations are now updated both in the database and in storage + (previously only updated in database) + +Removed +^^^^^^^ + +- Removed unused legacy code + + [0.22.0] ------------ diff --git a/README.rst b/README.rst index be992f7..41bcfd8 100644 --- a/README.rst +++ b/README.rst @@ -504,6 +504,57 @@ URI ``s3://dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db``:: http://localhost:5000/manifests/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db +Modifying dataset tags +~~~~~~~~~~~~~~~~~~~~~~ + +Add a single tag to a dataset:: + + $ curl -H "$HEADER" -X POST \ + http://localhost:5000/tags/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db/new-tag + +Replace all tags on a dataset:: + + $ curl -H "$HEADER" -H "Content-Type: application/json" \ + -X PUT -d '{"tags": ["tag1", "tag2"]}' \ + http://localhost:5000/tags/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db + +Remove a tag from a dataset:: + + $ curl -H "$HEADER" -X DELETE \ + http://localhost:5000/tags/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db/old-tag + + +Modifying dataset annotations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set a single annotation on a dataset:: + + $ curl -H "$HEADER" -H "Content-Type: application/json" \ + -X PUT -d '{"value": "some value"}' \ + http://localhost:5000/annotations/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db/my-annotation + +Replace all annotations on a dataset:: + + $ curl -H "$HEADER" -H "Content-Type: application/json" \ + -X PUT -d '{"annotations": {"key1": "value1", "key2": 42}}' \ + http://localhost:5000/annotations/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db + +Delete an annotation from a dataset:: + + $ curl -H "$HEADER" -X DELETE \ + http://localhost:5000/annotations/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db/my-annotation + + +Modifying dataset README +~~~~~~~~~~~~~~~~~~~~~~~~ + +Update the README of a dataset:: + + $ curl -H "$HEADER" -H "Content-Type: application/json" \ + -X PUT -d '{"readme": "---\ndescription: Updated README content\n"}' \ + http://localhost:5000/readmes/s3/dtool-demo/ba92a5fa-d3b4-4f10-bcb9-947f62e652db + + Getting information about one's own permissions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -727,6 +778,19 @@ and extension plugins with their versions, i.e.:: This request does not require any authorization. +The request:: + + $ curl http://localhost:5000/config/health + +will return a simple health check status for container orchestration:: + + { + "status": "healthy" + } + +This request does not require any authorization and can be used for +Kubernetes liveness/readiness probes or Docker health checks. + Creating a plugin ----------------- From 35c55eb4ef41bdd93e92449e6d497f760d31422a Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 14 Dec 2025 19:23:12 +0100 Subject: [PATCH 10/13] ENH: User management endpoints --- dservercore/user_routes.py | 152 ++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 2 deletions(-) diff --git a/dservercore/user_routes.py b/dservercore/user_routes.py index 0ff1ddd..799e120 100644 --- a/dservercore/user_routes.py +++ b/dservercore/user_routes.py @@ -16,7 +16,13 @@ from dservercore.sort import SortParameters, ASCENDING, DESCENDING from dservercore.schemas import SummarySchema from dservercore.sql_models import User, UserSchema, UserWithPermissionsSchema -from dservercore.utils import register_user, delete_user, summary_of_datasets_by_user +from dservercore.utils import ( + register_user, + delete_user, + summary_of_datasets_by_user, + get_permission_info, + register_permissions, +) bp = Blueprint("users", __name__, url_prefix="/users") @@ -166,4 +172,146 @@ def user_summary_get(username): abort(404) summary = summary_of_datasets_by_user(username) - return summary \ No newline at end of file + return summary + + +@bp.route("//search/", methods=["POST"]) +@bp.response(200, UserWithPermissionsSchema) +@bp.alt_response(401, description="Not registered") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="User or base URI not found") +@bp.alt_response(409, description="Permission already exists") +@jwt_required() +def user_search_permission_post(username, base_uri): + """Grant search permission to a user on a base URI. + + The user in the Authorization token needs to be admin. + """ + identity = get_jwt_identity() + + if not dservercore.utils_auth.user_exists(identity): + abort(401) + + if not dservercore.utils_auth.has_admin_rights(identity): + abort(403) + + if not dservercore.utils.base_uri_exists(base_uri): + abort(404) + + if not dservercore.utils_auth.user_exists(username): + abort(404) + + permissions = get_permission_info(base_uri) + + if username in permissions["users_with_search_permissions"]: + abort(409) + + permissions["users_with_search_permissions"].append(username) + register_permissions(base_uri, permissions) + + return dservercore.utils.get_user_info(username) + + +@bp.route("//search/", methods=["DELETE"]) +@bp.response(200, UserWithPermissionsSchema) +@bp.alt_response(401, description="Not registered") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="User or base URI not found") +@jwt_required() +def user_search_permission_delete(username, base_uri): + """Revoke search permission from a user on a base URI. + + The user in the Authorization token needs to be admin. + """ + identity = get_jwt_identity() + + if not dservercore.utils_auth.user_exists(identity): + abort(401) + + if not dservercore.utils_auth.has_admin_rights(identity): + abort(403) + + if not dservercore.utils.base_uri_exists(base_uri): + abort(404) + + if not dservercore.utils_auth.user_exists(username): + abort(404) + + permissions = get_permission_info(base_uri) + + if username in permissions["users_with_search_permissions"]: + permissions["users_with_search_permissions"].remove(username) + register_permissions(base_uri, permissions) + + return dservercore.utils.get_user_info(username) + + +@bp.route("//register/", methods=["POST"]) +@bp.response(200, UserWithPermissionsSchema) +@bp.alt_response(401, description="Not registered") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="User or base URI not found") +@bp.alt_response(409, description="Permission already exists") +@jwt_required() +def user_register_permission_post(username, base_uri): + """Grant register permission to a user on a base URI. + + The user in the Authorization token needs to be admin. + """ + identity = get_jwt_identity() + + if not dservercore.utils_auth.user_exists(identity): + abort(401) + + if not dservercore.utils_auth.has_admin_rights(identity): + abort(403) + + if not dservercore.utils.base_uri_exists(base_uri): + abort(404) + + if not dservercore.utils_auth.user_exists(username): + abort(404) + + permissions = get_permission_info(base_uri) + + if username in permissions["users_with_register_permissions"]: + abort(409) + + permissions["users_with_register_permissions"].append(username) + register_permissions(base_uri, permissions) + + return dservercore.utils.get_user_info(username) + + +@bp.route("//register/", methods=["DELETE"]) +@bp.response(200, UserWithPermissionsSchema) +@bp.alt_response(401, description="Not registered") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="User or base URI not found") +@jwt_required() +def user_register_permission_delete(username, base_uri): + """Revoke register permission from a user on a base URI. + + The user in the Authorization token needs to be admin. + """ + identity = get_jwt_identity() + + if not dservercore.utils_auth.user_exists(identity): + abort(401) + + if not dservercore.utils_auth.has_admin_rights(identity): + abort(403) + + if not dservercore.utils.base_uri_exists(base_uri): + abort(404) + + if not dservercore.utils_auth.user_exists(username): + abort(404) + + permissions = get_permission_info(base_uri) + + if username in permissions["users_with_register_permissions"]: + permissions["users_with_register_permissions"].remove(username) + register_permissions(base_uri, permissions) + + return dservercore.utils.get_user_info(username) \ No newline at end of file From 9460ab74d149893b82649aadeb93e1860f077af5 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 14 Dec 2025 19:39:44 +0100 Subject: [PATCH 11/13] TST: Tests for user routes --- tests/conftest.py | 30 +++-- tests/test_user_routes.py | 270 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1468e5c..b649446 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -93,10 +93,10 @@ def tmp_app(request): "SECRET_KEY": "secret", "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", - "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/", + "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "RETRIEVE_MONGO_DB": tmp_mongo_db_name, "RETRIEVE_MONGO_COLLECTION": "datasets", - "SEARCH_MONGO_URI": "mongodb://localhost:27017/", + "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "SEARCH_MONGO_DB": tmp_mongo_db_name, "SEARCH_MONGO_COLLECTION": "datasets", "SQLALCHEMY_TRACK_MODIFICATIONS": False, @@ -105,6 +105,9 @@ def tmp_app(request): "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", + "MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), + "MONGO_DB": tmp_mongo_db_name, + "MONGO_COLLECTION": "dependencies", } app = create_app(config) @@ -151,10 +154,10 @@ def tmp_app_with_users(request): "SECRET_KEY": "secret", "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", - "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/", + "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "RETRIEVE_MONGO_DB": tmp_mongo_db_name, "RETRIEVE_MONGO_COLLECTION": "datasets", - "SEARCH_MONGO_URI": "mongodb://localhost:27017/", + "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "SEARCH_MONGO_DB": tmp_mongo_db_name, "SEARCH_MONGO_COLLECTION": "datasets", "SQLALCHEMY_TRACK_MODIFICATIONS": False, @@ -163,6 +166,9 @@ def tmp_app_with_users(request): "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", + "MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), + "MONGO_DB": tmp_mongo_db_name, + "MONGO_COLLECTION": "dependencies", } app = create_app(config) @@ -224,10 +230,10 @@ def tmp_app_with_data(request): "OPENAPI_VERSION": '3.0.2', "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", - "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/", + "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "RETRIEVE_MONGO_DB": tmp_mongo_db_name, "RETRIEVE_MONGO_COLLECTION": "datasets", - "SEARCH_MONGO_URI": "mongodb://localhost:27017/", + "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "SEARCH_MONGO_DB": tmp_mongo_db_name, "SEARCH_MONGO_COLLECTION": "datasets", "SQLALCHEMY_TRACK_MODIFICATIONS": False, @@ -236,6 +242,9 @@ def tmp_app_with_data(request): "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", + "MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), + "MONGO_DB": tmp_mongo_db_name, + "MONGO_COLLECTION": "dependencies", } app = create_app(config) @@ -348,14 +357,17 @@ def tmp_cli_runner(request): "OPENAPI_VERSION": '3.0.2', "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", - "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/", + "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "RETRIEVE_MONGO_DB": tmp_mongo_db_name, "RETRIEVE_MONGO_COLLECTION": "datasets", - "SEARCH_MONGO_URI": "mongodb://localhost:27017/", + "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), "SEARCH_MONGO_DB": tmp_mongo_db_name, "SEARCH_MONGO_COLLECTION": "datasets", "SQLALCHEMY_TRACK_MODIFICATIONS": False, - "SECRET_KEY": "dev" + "SECRET_KEY": "dev", + "MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), + "MONGO_DB": tmp_mongo_db_name, + "MONGO_COLLECTION": "dependencies", } app = create_app(config) diff --git a/tests/test_user_routes.py b/tests/test_user_routes.py index 326be66..ad4d4ee 100644 --- a/tests/test_user_routes.py +++ b/tests/test_user_routes.py @@ -489,3 +489,273 @@ def test_dataset_summary_route( headers=headers ) assert r.status_code == 404 + + +def test_grant_search_permission_route( + tmp_app_with_users_client, + snowwhite_token, + grumpy_token, + noone_token, + sleepy_token): + """Test granting search permission via POST.""" + + from dservercore.sql_models import UserWithPermissionsSchema + + # 1 - Admin grants search permission to sleepy on s3://snow-white + # sleepy already has search permission, so we need to first check + # the initial state then test with a new base_uri + headers = dict(Authorization="Bearer " + snowwhite_token) + + # First, register a new base URI for testing + from dservercore.utils import register_base_uri + register_base_uri("s3://test-bucket") + + # Grant search permission - should succeed with 200 + r = tmp_app_with_users_client.post( + "/users/sleepy/search/s3://test-bucket", + headers=headers, + json={} + ) + assert r.status_code == 200 + + user_response = r.json + assert len(UserWithPermissionsSchema().validate(user_response)) == 0 + assert "s3://test-bucket" in user_response["search_permissions_on_base_uris"] + + # 2 - Try to grant the same permission again - should return 409 Conflict + r = tmp_app_with_users_client.post( + "/users/sleepy/search/s3://test-bucket", + headers=headers, + json={} + ) + assert r.status_code == 409 + + # 3 - Non-admin (grumpy) tries to grant permission - should return 403 + headers = dict(Authorization="Bearer " + grumpy_token) + r = tmp_app_with_users_client.post( + "/users/sleepy/search/s3://test-bucket", + headers=headers, + json={} + ) + assert r.status_code == 403 + + # 4 - Unregistered user (noone) tries to grant permission - should return 401 + headers = dict(Authorization="Bearer " + noone_token) + r = tmp_app_with_users_client.post( + "/users/sleepy/search/s3://test-bucket", + headers=headers, + json={} + ) + assert r.status_code == 401 + + # 5 - Admin tries to grant permission to non-existent user - should return 404 + headers = dict(Authorization="Bearer " + snowwhite_token) + r = tmp_app_with_users_client.post( + "/users/nonexistent/search/s3://test-bucket", + headers=headers, + json={} + ) + assert r.status_code == 404 + + # 6 - Admin tries to grant permission on non-existent base URI - should return 404 + r = tmp_app_with_users_client.post( + "/users/sleepy/search/s3://nonexistent-bucket", + headers=headers, + json={} + ) + assert r.status_code == 404 + + +def test_revoke_search_permission_route( + tmp_app_with_users_client, + snowwhite_token, + grumpy_token, + noone_token, + sleepy_token): + """Test revoking search permission via DELETE.""" + + from dservercore.sql_models import UserWithPermissionsSchema + + # Initial state: grumpy and sleepy have search permissions on s3://snow-white + headers = dict(Authorization="Bearer " + snowwhite_token) + + # 1 - Admin revokes search permission from sleepy + r = tmp_app_with_users_client.delete( + "/users/sleepy/search/s3://snow-white", + headers=headers + ) + assert r.status_code == 200 + + user_response = r.json + assert len(UserWithPermissionsSchema().validate(user_response)) == 0 + assert "s3://snow-white" not in user_response["search_permissions_on_base_uris"] + + # 2 - Revoking permission that doesn't exist should still succeed (idempotent) + r = tmp_app_with_users_client.delete( + "/users/sleepy/search/s3://snow-white", + headers=headers + ) + assert r.status_code == 200 + + # 3 - Non-admin (grumpy) tries to revoke permission - should return 403 + headers = dict(Authorization="Bearer " + grumpy_token) + r = tmp_app_with_users_client.delete( + "/users/sleepy/search/s3://snow-white", + headers=headers + ) + assert r.status_code == 403 + + # 4 - Unregistered user (noone) tries to revoke permission - should return 401 + headers = dict(Authorization="Bearer " + noone_token) + r = tmp_app_with_users_client.delete( + "/users/sleepy/search/s3://snow-white", + headers=headers + ) + assert r.status_code == 401 + + # 5 - Admin tries to revoke permission from non-existent user - should return 404 + headers = dict(Authorization="Bearer " + snowwhite_token) + r = tmp_app_with_users_client.delete( + "/users/nonexistent/search/s3://snow-white", + headers=headers + ) + assert r.status_code == 404 + + # 6 - Admin tries to revoke permission on non-existent base URI - should return 404 + r = tmp_app_with_users_client.delete( + "/users/sleepy/search/s3://nonexistent-bucket", + headers=headers + ) + assert r.status_code == 404 + + +def test_grant_register_permission_route( + tmp_app_with_users_client, + snowwhite_token, + grumpy_token, + noone_token, + sleepy_token): + """Test granting register permission via POST.""" + + from dservercore.sql_models import UserWithPermissionsSchema + + headers = dict(Authorization="Bearer " + snowwhite_token) + + # 1 - Admin grants register permission to sleepy on s3://snow-white + # sleepy does not have register permission initially + r = tmp_app_with_users_client.post( + "/users/sleepy/register/s3://snow-white", + headers=headers, + json={} + ) + assert r.status_code == 200 + + user_response = r.json + assert len(UserWithPermissionsSchema().validate(user_response)) == 0 + assert "s3://snow-white" in user_response["register_permissions_on_base_uris"] + + # 2 - Try to grant the same permission again - should return 409 Conflict + r = tmp_app_with_users_client.post( + "/users/sleepy/register/s3://snow-white", + headers=headers, + json={} + ) + assert r.status_code == 409 + + # 3 - Non-admin (grumpy) tries to grant permission - should return 403 + headers = dict(Authorization="Bearer " + grumpy_token) + r = tmp_app_with_users_client.post( + "/users/sleepy/register/s3://snow-white", + headers=headers, + json={} + ) + assert r.status_code == 403 + + # 4 - Unregistered user (noone) tries to grant permission - should return 401 + headers = dict(Authorization="Bearer " + noone_token) + r = tmp_app_with_users_client.post( + "/users/sleepy/register/s3://snow-white", + headers=headers, + json={} + ) + assert r.status_code == 401 + + # 5 - Admin tries to grant permission to non-existent user - should return 404 + headers = dict(Authorization="Bearer " + snowwhite_token) + r = tmp_app_with_users_client.post( + "/users/nonexistent/register/s3://snow-white", + headers=headers, + json={} + ) + assert r.status_code == 404 + + # 6 - Admin tries to grant permission on non-existent base URI - should return 404 + r = tmp_app_with_users_client.post( + "/users/sleepy/register/s3://nonexistent-bucket", + headers=headers, + json={} + ) + assert r.status_code == 404 + + +def test_revoke_register_permission_route( + tmp_app_with_users_client, + snowwhite_token, + grumpy_token, + noone_token, + sleepy_token): + """Test revoking register permission via DELETE.""" + + from dservercore.sql_models import UserWithPermissionsSchema + + # Initial state: grumpy has register permission on s3://snow-white + headers = dict(Authorization="Bearer " + snowwhite_token) + + # 1 - Admin revokes register permission from grumpy + r = tmp_app_with_users_client.delete( + "/users/grumpy/register/s3://snow-white", + headers=headers + ) + assert r.status_code == 200 + + user_response = r.json + assert len(UserWithPermissionsSchema().validate(user_response)) == 0 + assert "s3://snow-white" not in user_response["register_permissions_on_base_uris"] + + # 2 - Revoking permission that doesn't exist should still succeed (idempotent) + r = tmp_app_with_users_client.delete( + "/users/grumpy/register/s3://snow-white", + headers=headers + ) + assert r.status_code == 200 + + # 3 - Non-admin (sleepy) tries to revoke permission - should return 403 + headers = dict(Authorization="Bearer " + sleepy_token) + r = tmp_app_with_users_client.delete( + "/users/grumpy/register/s3://snow-white", + headers=headers + ) + assert r.status_code == 403 + + # 4 - Unregistered user (noone) tries to revoke permission - should return 401 + headers = dict(Authorization="Bearer " + noone_token) + r = tmp_app_with_users_client.delete( + "/users/grumpy/register/s3://snow-white", + headers=headers + ) + assert r.status_code == 401 + + # 5 - Admin tries to revoke permission from non-existent user - should return 404 + headers = dict(Authorization="Bearer " + snowwhite_token) + r = tmp_app_with_users_client.delete( + "/users/nonexistent/register/s3://snow-white", + headers=headers + ) + assert r.status_code == 404 + + # 6 - Admin tries to revoke permission on non-existent base URI - should return 404 + r = tmp_app_with_users_client.delete( + "/users/grumpy/register/s3://nonexistent-bucket", + headers=headers + ) + assert r.status_code == 404 From 17d76c4956a09c02d1b2582f8144d67ef47c4d90 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Mon, 15 Dec 2025 07:35:15 +0100 Subject: [PATCH 12/13] MAINT: Changed user permissions getting and setting --- dservercore/sql_models.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/dservercore/sql_models.py b/dservercore/sql_models.py index 794ba99..5497043 100644 --- a/dservercore/sql_models.py +++ b/dservercore/sql_models.py @@ -145,8 +145,38 @@ class Meta: class UserWithPermissionsSchema(UserSchema): - register_permissions_on_base_uris = fields.List(fields.String) - search_permissions_on_base_uris = fields.List(fields.String) + """User schema including permission fields. + + This schema handles two cases: + - User model objects: returned directly from paginated queries (e.g., GET /users) + - Dict objects: returned from user.as_dict() via get_user_info() (e.g., GET /users/) + + The User model stores permissions as relationships (search_base_uris, register_base_uris) + to BaseURI objects, while the API returns them as lists of base URI strings. The as_dict() + method performs this conversion, but raw model objects need explicit field serialization. + """ + search_permissions_on_base_uris = fields.Method("get_search_permissions") + register_permissions_on_base_uris = fields.Method("get_register_permissions") + + def get_search_permissions(self, obj): + """Serialize search permissions as list of base URI strings.""" + if isinstance(obj, User): + # Raw User model from paginated query - extract from relationship + return [u.base_uri for u in obj.search_base_uris] + elif isinstance(obj, dict): + # Dict from user.as_dict() - field already converted + return obj.get("search_permissions_on_base_uris", []) + return [] + + def get_register_permissions(self, obj): + """Serialize register permissions as list of base URI strings.""" + if isinstance(obj, User): + # Raw User model from paginated query - extract from relationship + return [u.base_uri for u in obj.register_base_uris] + elif isinstance(obj, dict): + # Dict from user.as_dict() - field already converted + return obj.get("register_permissions_on_base_uris", []) + return [] class DatasetSchema(ma.SQLAlchemyAutoSchema): From 321f41701e81090357457206ef57e719bfdea010 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Mon, 15 Dec 2025 14:19:45 +0100 Subject: [PATCH 13/13] ENH: Added display_name to user --- dservercore/me_routes.py | 37 +++++++++++++++++++++++++----- dservercore/schemas.py | 7 +++++- dservercore/sql_models.py | 45 ++++++++++++++++++++---------------- dservercore/user_routes.py | 47 +++++++++++++++++++++++++++++++++++++- dservercore/utils.py | 15 ++++++++---- dservercore/utils_auth.py | 9 ++++++++ 6 files changed, 127 insertions(+), 33 deletions(-) diff --git a/dservercore/me_routes.py b/dservercore/me_routes.py index 79274ee..c889925 100644 --- a/dservercore/me_routes.py +++ b/dservercore/me_routes.py @@ -1,19 +1,18 @@ """Routes for me (information on currently authenticated user)""" -from flask import ( - abort, - jsonify, -) +from flask import abort + from dservercore.utils_auth import ( jwt_required, get_jwt_identity, ) +import dservercore import dservercore.utils import dservercore.utils_auth from dservercore.blueprint import Blueprint -from dservercore.sql_models import UserSchema, UserWithPermissionsSchema -from dservercore.schemas import SummarySchema +from dservercore.sql_models import User, UserWithPermissionsSchema +from dservercore.schemas import SummarySchema, MeUpdateSchema from dservercore.utils import summary_of_datasets_by_user @@ -34,6 +33,32 @@ def me_get(): return dservercore.utils.get_user_info(identity) +@bp.route("", methods=["PATCH"]) +@bp.arguments(MeUpdateSchema) +@bp.response(200, UserWithPermissionsSchema) +@bp.alt_response(401, description="Not registered") +@jwt_required() +def me_patch(data: MeUpdateSchema): + """Update the current user's profile. + + Currently supports updating: + - display_name: string (can be null to clear) + """ + identity = get_jwt_identity() + + if not dservercore.utils_auth.user_exists(identity): + abort(401) + + user = User.query.filter_by(username=identity).first() + + if "display_name" in data: + user.display_name = data.get("display_name") + + dservercore.sql_db.session.commit() + + return dservercore.utils.get_user_info(identity) + + @bp.route("/summary", methods=["GET"]) @bp.response(200, SummarySchema) @bp.alt_response(401, description="Not registered") diff --git a/dservercore/schemas.py b/dservercore/schemas.py index 8628370..0d36e9d 100644 --- a/dservercore/schemas.py +++ b/dservercore/schemas.py @@ -96,4 +96,9 @@ class SummarySchema(Schema): size_in_bytes_per_base_uri = Dict(keys=String, values=Integer) tags = List(String) datasets_per_tag = Dict(keys=String, values=Integer) - size_in_bytes_per_tag = Dict(keys=String, values=Integer) \ No newline at end of file + size_in_bytes_per_tag = Dict(keys=String, values=Integer) + + +class MeUpdateSchema(Schema): + """Schema for updating the current user's profile (PATCH /me).""" + display_name = String(load_default=None, allow_none=True) \ No newline at end of file diff --git a/dservercore/sql_models.py b/dservercore/sql_models.py index 5497043..e9674f7 100644 --- a/dservercore/sql_models.py +++ b/dservercore/sql_models.py @@ -1,6 +1,6 @@ """Database models and derived schemas""" import datetime -from marshmallow import fields +from marshmallow import fields, pre_dump import dtoolcore.utils from dservercore import ma from dservercore import sql_db as db @@ -39,6 +39,7 @@ def _deserialize(self, value, attr, data, **kwargs): class User(db.Model): id = db.Column(db.Integer, primary_key=True) username = db.Column(db.String(64), index=True, unique=True) + display_name = db.Column(db.String(255), nullable=True) # Optional human-readable name is_admin = db.Column(db.Boolean(), nullable=False, default=False) search_base_uris = db.relationship( "BaseURI", secondary=search_permissions, back_populates="search_users" @@ -54,6 +55,7 @@ def as_dict(self): """Return user using dictionary representation.""" return { "username": self.username, + "display_name": self.display_name, "is_admin": self.is_admin, "search_permissions_on_base_uris": [ u.base_uri for u in self.search_base_uris @@ -144,6 +146,12 @@ class Meta: exclude = ("id",) +class UserUpdateSchema(ma.Schema): + """Schema for partial user updates (PATCH).""" + is_admin = fields.Boolean(load_default=None) + display_name = fields.String(load_default=None, allow_none=True) + + class UserWithPermissionsSchema(UserSchema): """User schema including permission fields. @@ -154,29 +162,26 @@ class UserWithPermissionsSchema(UserSchema): The User model stores permissions as relationships (search_base_uris, register_base_uris) to BaseURI objects, while the API returns them as lists of base URI strings. The as_dict() method performs this conversion, but raw model objects need explicit field serialization. + + The @pre_dump hook converts User model objects to dicts before serialization, ensuring + the permission relationships are properly transformed to string lists. """ - search_permissions_on_base_uris = fields.Method("get_search_permissions") - register_permissions_on_base_uris = fields.Method("get_register_permissions") + search_permissions_on_base_uris = fields.List(fields.String()) + register_permissions_on_base_uris = fields.List(fields.String()) - def get_search_permissions(self, obj): - """Serialize search permissions as list of base URI strings.""" - if isinstance(obj, User): - # Raw User model from paginated query - extract from relationship - return [u.base_uri for u in obj.search_base_uris] - elif isinstance(obj, dict): - # Dict from user.as_dict() - field already converted - return obj.get("search_permissions_on_base_uris", []) - return [] + @pre_dump + def convert_user_to_dict(self, obj, **kwargs): + """Convert User model objects to dicts before serialization. + + This is needed because User model objects have permission relationships + (search_base_uris, register_base_uris) that point to BaseURI objects, + not the string lists expected by the API response. - def get_register_permissions(self, obj): - """Serialize register permissions as list of base URI strings.""" + Dict objects (from user.as_dict()) pass through unchanged. + """ if isinstance(obj, User): - # Raw User model from paginated query - extract from relationship - return [u.base_uri for u in obj.register_base_uris] - elif isinstance(obj, dict): - # Dict from user.as_dict() - field already converted - return obj.get("register_permissions_on_base_uris", []) - return [] + return obj.as_dict() + return obj class DatasetSchema(ma.SQLAlchemyAutoSchema): diff --git a/dservercore/user_routes.py b/dservercore/user_routes.py index 799e120..f2d94be 100644 --- a/dservercore/user_routes.py +++ b/dservercore/user_routes.py @@ -9,13 +9,14 @@ ) from flask_smorest.pagination import PaginationParameters +import dservercore import dservercore.utils import dservercore.utils_auth from dservercore.blueprint import Blueprint from dservercore.sort import SortParameters, ASCENDING, DESCENDING from dservercore.schemas import SummarySchema -from dservercore.sql_models import User, UserSchema, UserWithPermissionsSchema +from dservercore.sql_models import User, UserSchema, UserUpdateSchema, UserWithPermissionsSchema from dservercore.utils import ( register_user, delete_user, @@ -124,6 +125,50 @@ def user_put(data: UserSchema, username): return "", success_code +@bp.route("/", methods=["PATCH"]) +@bp.arguments(UserUpdateSchema) +@bp.response(200, UserWithPermissionsSchema) +@bp.alt_response(401, description="Not registered") +@bp.alt_response(403, description="No permissions") +@bp.alt_response(404, description="Not found") +@jwt_required() +def user_patch(data: UserUpdateSchema, username): + """Partially update a user in dserver. + + Only provided fields will be updated. Supports updating: + - is_admin: boolean + - display_name: string (can be null to clear) + + The user in the Authorization token needs to be admin. + """ + identity = get_jwt_identity() + + if not dservercore.utils_auth.user_exists(identity): + abort(401) + + if not dservercore.utils_auth.has_admin_rights(identity): + abort(403) + + if not dservercore.utils_auth.user_exists(username): + abort(404) + + # Get the user object and update only provided fields + user = User.query.filter_by(username=username).first() + + if data.get("is_admin") is not None: + user.is_admin = data["is_admin"] + + if "display_name" in data and data.get("display_name") is not None: + user.display_name = data["display_name"] + elif data.get("display_name") is None and "display_name" in data: + # Explicitly setting to None/null clears the display_name + user.display_name = None + + dservercore.sql_db.session.commit() + + return dservercore.utils.get_user_info(username) + + @bp.route("/", methods=["DELETE"]) @bp.response(200) @bp.alt_response(401, description="Not registered") diff --git a/dservercore/utils.py b/dservercore/utils.py index 5fa5474..f196c6a 100644 --- a/dservercore/utils.py +++ b/dservercore/utils.py @@ -223,12 +223,13 @@ def register_user(username, data): Example input structure:: - {"is_admin": True}, + {"is_admin": True, "display_name": "John Doe"}, If a user is missing in the system it is skipped. The ``is_admin`` status - defaults to False. + defaults to False. The ``display_name`` is optional. """ is_admin = data.get("is_admin", False) + display_name = data.get("display_name") # User already exists, update if sql_db.session.query(exists().where(User.username == username)).scalar(): @@ -236,8 +237,10 @@ def register_user(username, data): sql_db.session.query(User).filter_by(username=username).all() ): sqlalch_user_obj.is_admin = is_admin + if "display_name" in data: + sqlalch_user_obj.display_name = display_name else: # user does not exist yet, create - user = User(username=username, is_admin=is_admin) + user = User(username=username, is_admin=is_admin, display_name=display_name) sql_db.session.add(user) sql_db.session.commit() @@ -326,14 +329,14 @@ def update_users(users): Example input structure:: [ - {"username": "magic.mirror", "is_admin": True}, + {"username": "magic.mirror", "is_admin": True, "display_name": "Magic Mirror"}, {"username": "snow.white", "is_admin": False}, {"username": "dopey"}, {"username": "sleepy"}, ] If a user is missing in the system it is skipped. The ``is_admin`` status - defaults to False. + defaults to False. The ``display_name`` is only updated if provided. """ for user in users: username = user["username"] @@ -343,6 +346,8 @@ def update_users(users): sql_db.session.query(User).filter_by(username=username).all() ): # NOQA sqlalch_user_obj.is_admin = is_admin + if "display_name" in user: + sqlalch_user_obj.display_name = user.get("display_name") sql_db.session.commit() diff --git a/dservercore/utils_auth.py b/dservercore/utils_auth.py index 88e9f10..81dbf58 100644 --- a/dservercore/utils_auth.py +++ b/dservercore/utils_auth.py @@ -11,6 +11,7 @@ from flask_jwt_extended import jwt_required as flask_jwt_required from flask_jwt_extended import get_jwt_identity as flask_get_jwt_identity +from flask_jwt_extended import get_jwt as flask_get_jwt def jwt_required(*jwt_required_args, **jwt_required_kwargs): @@ -34,6 +35,14 @@ def get_jwt_identity(): return flask_get_jwt_identity() +def get_jwt(): + """Return JWT payload or empty dict if JWT authorisation disabled.""" + if current_app.config.get("DISABLE_JWT_AUTHORISATION"): + return {} + else: + return flask_get_jwt() + + def _get_user_obj(username): return User.query.filter_by(username=username).first()