diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index cf149b154..9d910f20f 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -41,7 +41,12 @@ from app.routers.files import add_file_entry, add_local_file_entry, remove_file_entry from app.routers.licenses import delete_license from app.search.connect import delete_document_by_id -from app.search.index import index_dataset, index_file +from app.search.index import ( + index_dataset, + index_file, + index_folder, + remove_folder_index, +) from beanie import PydanticObjectId from beanie.odm.operators.update.general import Inc from beanie.operators import And, Or @@ -341,6 +346,13 @@ async def edit_dataset( # Update entry to the dataset index await index_dataset(es, DatasetOut(**dataset.dict()), update=True) + + # Update folders index since its using dataset downloads and status to index + async for folder in FolderDB.find( + FolderDB.dataset_id == PydanticObjectId(dataset_id) + ): + await index_folder(es, FolderOut(**folder.dict()), update=True) + return dataset.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -379,6 +391,13 @@ async def patch_dataset( # Update entry to the dataset index await index_dataset(es, DatasetOut(**dataset.dict()), update=True) + + # Update folders index since its using dataset downloads and status to index + async for folder in FolderDB.find( + FolderDB.dataset_id == PydanticObjectId(dataset_id) + ): + await index_folder(es, FolderOut(**folder.dict()), update=True) + return dataset.dict() @@ -423,6 +442,7 @@ async def add_folder( dataset_id: str, folder_in: FolderIn, user=Depends(get_current_user), + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(Authorization("uploader")), ): if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: @@ -436,6 +456,7 @@ async def add_folder( **folder_in.dict(), creator=user, dataset_id=PydanticObjectId(dataset_id) ) await new_folder.insert() + await index_folder(es, FolderOut(**new_folder.dict())) return new_folder.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -595,9 +616,11 @@ async def _delete_nested_folders(parent_folder_id): await remove_file_entry(file.id, fs, es) await _delete_nested_folders(subfolder.id) await subfolder.delete() + await remove_folder_index(subfolder.id, es) await _delete_nested_folders(folder_id) await folder.delete() + await remove_folder_index(folder.id, es) return {"deleted": folder_id} else: raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") @@ -623,6 +646,7 @@ async def patch_folder( dataset_id: str, folder_id: str, folder_info: FolderPatch, + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), user=Depends(get_current_user), allow: bool = Depends(Authorization("editor")), ): @@ -640,6 +664,8 @@ async def patch_folder( folder.parent_folder = folder_info.parent_folder folder.modified = datetime.datetime.utcnow() await folder.save() + await index_folder(es, FolderOut(**folder.dict()), update=True) + return folder.dict() else: raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") @@ -894,6 +920,7 @@ async def create_dataset_from_zip( @router.get("/{dataset_id}/download", response_model=DatasetOut) async def download_dataset( dataset_id: str, + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), user=Depends(get_current_user), fs: Minio = Depends(dependencies.get_fs), allow: bool = Depends(Authorization("viewer")), @@ -1043,6 +1070,15 @@ async def download_dataset( response.headers["Content-Disposition"] = "attachment; filename=%s" % zip_name # Increment download count await dataset.update(Inc({DatasetDB.downloads: 1})) + + # reindex + await index_dataset(es, DatasetOut(**dataset.dict()), update=True) + # Update folders index since its using dataset downloads and status to index + async for folder in FolderDB.find( + FolderDB.dataset_id == PydanticObjectId(dataset_id) + ): + await index_folder(es, FolderOut(**folder.dict()), update=True) + return response raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py index cb7e61988..c9cd12b75 100644 --- a/backend/app/routers/files.py +++ b/backend/app/routers/files.py @@ -290,6 +290,7 @@ async def download_file( file_id: str, version: Optional[int] = None, increment: Optional[bool] = True, + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), fs: Minio = Depends(dependencies.get_fs), allow: bool = Depends(FileAuthorization("viewer")), ): @@ -340,6 +341,10 @@ async def download_file( if increment: # Increment download count await file.update(Inc({FileDB.downloads: 1})) + + # reindex + await index_file(es, FileOut(**file.dict()), update=True) + return response else: @@ -351,6 +356,7 @@ async def download_file_url( file_id: str, version: Optional[int] = None, expires_in_seconds: Optional[int] = 3600, + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), external_fs: Minio = Depends(dependencies.get_external_fs), allow: bool = Depends(FileAuthorization("viewer")), ): @@ -392,6 +398,9 @@ async def download_file_url( # Increment download count await file.update(Inc({FileDB.downloads: 1})) + # reindex + await index_file(es, FileOut(**file.dict()), update=True) + # return presigned url return {"presigned_url": presigned_url} else: diff --git a/backend/app/routers/public_datasets.py b/backend/app/routers/public_datasets.py index 1b31acf78..c3138e3c7 100644 --- a/backend/app/routers/public_datasets.py +++ b/backend/app/routers/public_datasets.py @@ -14,10 +14,12 @@ from app.models.folders import FolderDB, FolderDBViewList, FolderOut from app.models.metadata import MetadataDB, MetadataDefinitionDB, MetadataOut from app.models.pages import Paged, _construct_page_metadata, _get_page_query +from app.search.index import index_dataset, index_folder from beanie import PydanticObjectId from beanie.odm.operators.update.general import Inc from beanie.operators import And, Or from bson import ObjectId, json_util +from elasticsearch import Elasticsearch from fastapi import APIRouter, Depends, Form, HTTPException from fastapi.responses import StreamingResponse from fastapi.security import HTTPBearer @@ -217,6 +219,7 @@ async def get_dataset_metadata( @router.get("/{dataset_id}/download", response_model=DatasetOut) async def download_dataset( dataset_id: str, + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), fs: Minio = Depends(dependencies.get_fs), ): if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: @@ -370,6 +373,15 @@ async def download_dataset( ) # Increment download count await dataset.update(Inc({DatasetDB.downloads: 1})) + + # reindex + await index_dataset(es, DatasetOut(**dataset.dict()), update=True) + # Update folders index since its using dataset downloads and status to index + async for folder in FolderDB.find( + FolderDB.dataset_id == PydanticObjectId(dataset_id) + ): + await index_folder(es, FolderOut(**folder.dict()), update=True) + return response else: raise HTTPException( diff --git a/backend/app/search/index.py b/backend/app/search/index.py index 96fb70f84..60c512e0c 100644 --- a/backend/app/search/index.py +++ b/backend/app/search/index.py @@ -1,16 +1,18 @@ -from typing import List, Optional +from typing import List, Optional, Union from app.config import settings from app.models.authorization import AuthorizationDB -from app.models.datasets import DatasetOut +from app.models.datasets import DatasetDB, DatasetOut from app.models.files import FileDB, FileOut +from app.models.folders import FolderOut from app.models.metadata import MetadataDB from app.models.search import ElasticsearchEntry from app.models.thumbnails import ThumbnailDB -from app.search.connect import insert_record, update_record +from app.search.connect import delete_document_by_id, insert_record, update_record from beanie import PydanticObjectId from bson import ObjectId from elasticsearch import Elasticsearch, NotFoundError +from fastapi import HTTPException async def index_dataset( @@ -112,6 +114,50 @@ async def index_file( insert_record(es, settings.elasticsearch_index, doc, file.id) +async def index_folder( + es: Elasticsearch, + folder: FolderOut, + user_ids: Optional[List[str]] = None, + update: bool = False, +): + """Create or update an Elasticsearch entry for the folder.""" + # find dataset this folder belongs to + if ( + dataset := await DatasetDB.find_one( + DatasetDB.id == PydanticObjectId(folder.dataset_id) + ) + ) is not None: + downloads = dataset.downloads + status = dataset.status + else: + raise HTTPException( + status_code=404, detail="Orphan folder doesn't belong to any dataset." + ) + + doc = ElasticsearchEntry( + resource_type="folder", + name=folder.name, + creator=folder.creator.email, + created=folder.created, + dataset_id=str(folder.dataset_id), + folder_id=str(folder.id), + downloads=downloads, + status=status, + ).dict() + + if update: + try: + update_record(es, settings.elasticsearch_index, {"doc": doc}, folder.id) + except NotFoundError: + insert_record(es, settings.elasticsearch_index, doc, folder.id) + else: + insert_record(es, settings.elasticsearch_index, doc, folder.id) + + +async def remove_folder_index(folderId: Union[str, ObjectId], es: Elasticsearch): + delete_document_by_id(es, settings.elasticsearch_index, str(folderId)) + + async def index_thumbnail( es: Elasticsearch, thumbnail_id: str, diff --git a/frontend/src/components/search/PublicSearch.tsx b/frontend/src/components/search/PublicSearch.tsx index 62fe810a2..deb16fa3b 100644 --- a/frontend/src/components/search/PublicSearch.tsx +++ b/frontend/src/components/search/PublicSearch.tsx @@ -164,18 +164,19 @@ export function PublicSearch() { {luceneOn ? ( } + sortBy="desc" /> ) : ( { return ; }} + sortBy="desc" /> )} diff --git a/frontend/src/components/search/PublicSearchResult.tsx b/frontend/src/components/search/PublicSearchResult.tsx index c2d1fc412..3c49fb170 100644 --- a/frontend/src/components/search/PublicSearchResult.tsx +++ b/frontend/src/components/search/PublicSearchResult.tsx @@ -14,6 +14,7 @@ import { parseDate } from "../../utils/common"; import { theme } from "../../theme"; import parse from "html-react-parser"; +import FolderIcon from "@mui/icons-material/Folder"; // Function to parse the elastic search parameter // If it contains HTML tags like , it removes them @@ -52,6 +53,28 @@ function buildDatasetResult(item) { ); } +function buildFolderResult(item) { + return ( + <> + + + + + + {parseString(item.name)} + + + Created by {parseString(item.creator)} at {parseDate(item.created)} + + + + ); +} + function buildFileResult(item) { return ( <> @@ -93,7 +116,11 @@ export function PublicSearchResult(props) { {item.resource_type === "dataset" ? buildDatasetResult(item) - : buildFileResult(item)} + : item.resource_type === "file" + ? buildFileResult(item) + : item.resource_type === "folder" + ? buildFolderResult(item) + : null} ))} diff --git a/frontend/src/components/search/Search.tsx b/frontend/src/components/search/Search.tsx index f24ce91d3..e8773c3ac 100644 --- a/frontend/src/components/search/Search.tsx +++ b/frontend/src/components/search/Search.tsx @@ -204,18 +204,19 @@ export function Search() { {luceneOn ? ( } + sortBy="desc" /> ) : ( { return ; }} + sortBy="desc" /> )} diff --git a/frontend/src/components/search/SearchResult.tsx b/frontend/src/components/search/SearchResult.tsx index ba7cf0bf6..cad4b786d 100644 --- a/frontend/src/components/search/SearchResult.tsx +++ b/frontend/src/components/search/SearchResult.tsx @@ -10,6 +10,7 @@ import { import { Link } from "react-router-dom"; import DatasetIcon from "@mui/icons-material/Dataset"; import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile"; +import FolderIcon from "@mui/icons-material/Folder"; import { parseDate } from "../../utils/common"; import { theme } from "../../theme"; @@ -77,6 +78,28 @@ function buildFileResult(item) { ); } +function buildFolderResult(item) { + return ( + <> + + + + + + {parseString(item.name)} + + + Created by {parseString(item.creator)} at {parseDate(item.created)} + + + + ); +} + export function SearchResult(props) { const { data } = props; @@ -93,7 +116,11 @@ export function SearchResult(props) { {item.resource_type === "dataset" ? buildDatasetResult(item) - : buildFileResult(item)} + : item.resource_type === "file" + ? buildFileResult(item) + : item.resource_type === "folder" + ? buildFolderResult(item) + : null} ))} diff --git a/openapi.json b/openapi.json index 1a084dc4e..d37775bf9 100644 --- a/openapi.json +++ b/openapi.json @@ -2060,6 +2060,67 @@ } ] }, + "put": { + "tags": [ + "metadata" + ], + "summary": "Update Metadata Definition", + "operationId": "update_metadata_definition_api_v2_metadata_definition__metadata_definition_id__put", + "parameters": [ + { + "required": true, + "schema": { + "title": "Metadata Definition Id", + "type": "string" + }, + "name": "metadata_definition_id", + "in": "path" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MetadataDefinitionIn" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MetadataDefinitionOut" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "security": [ + { + "OAuth2AuthorizationCodeBearer": [] + }, + { + "APIKeyHeader": [] + }, + { + "APIKeyCookie": [] + } + ] + }, "delete": { "tags": [ "metadata" @@ -11858,6 +11919,11 @@ "items": { "$ref": "#/components/schemas/MetadataField" } + }, + "modified": { + "title": "Modified", + "type": "string", + "format": "date-time" } }, "description": "This describes a metadata object with a short name and description, predefined set of fields, and context.\nThese provide a shorthand for use by listeners as well as a source for building GUI widgets to add new entries.\n\nExample: {\n \"name\" : \"LatLon\",\n \"description\" : \"A set of Latitude/Longitude coordinates\",\n \"required_for_items\": {\n \"datasets\": false,\n \"files\": false\n },\n \"context\" : [\n {\n \"longitude\" : \"https://schema.org/longitude\",\n \"latitude\" : \"https://schema.org/latitude\"\n },\n ],\n \"fields\" : [\n {\n \"name\" : \"longitude\",\n \"list\" : false,\n \"widgetType\": \"TextField\",\n \"config\": {\n \"type\" : \"float\"\n },\n \"required\" : true\n },\n {\n \"name\" : \"latitude\",\n \"list\" : false,\n \"widgetType\": \"TextField\",\n \"config\": {\n \"type\" : \"float\"\n },\n \"required\" : true\n }\n ]\n}" @@ -11916,6 +11982,11 @@ "$ref": "#/components/schemas/MetadataField" } }, + "modified": { + "title": "Modified", + "type": "string", + "format": "date-time" + }, "id": { "title": "Id", "type": "string",