diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index c06937db7..ad236d41b 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -229,8 +229,15 @@ async def get_datasets( datasets = [] if mine: for doc in ( - await db["datasets"] - .find({"author.email": user_id}) + await db["datasets_view"] + .find( + { + "$and": [ + {"author.email": user_id}, + {"auth": {"$elemMatch": {"user_id": {"$eq": user_id}}}}, + ] + } + ) .sort([("created", pymongo.DESCENDING)]) .skip(skip) .limit(limit) @@ -239,8 +246,15 @@ async def get_datasets( datasets.append(DatasetOut.from_mongo(doc)) else: for doc in ( - await db["datasets"] - .find() + await db["datasets_view"] + .find( + { + "$or": [ + {"author.email": user_id}, + {"auth": {"$elemMatch": {"user_id": {"$eq": user_id}}}}, + ] + } + ) .sort([("created", pymongo.DESCENDING)]) .skip(skip) .limit(limit) @@ -263,6 +277,7 @@ async def get_dataset(dataset_id: str, db: MongoClient = Depends(dependencies.ge async def get_dataset_files( dataset_id: str, folder_id: Optional[str] = None, + user_id=Depends(get_user), db: MongoClient = Depends(dependencies.get_db), skip: int = 0, limit: int = 10, @@ -270,11 +285,21 @@ async def get_dataset_files( files = [] if folder_id is not None: for f in ( - await db["files"] + await db["files_view"] .find( { - "dataset_id": ObjectId(dataset_id), - "folder_id": ObjectId(folder_id), + "$and": [ + { + "dataset_id": ObjectId(dataset_id), + "folder_id": ObjectId(folder_id), + }, + { + "$or": [ + {"creator.email": user_id}, + {"auth": {"$elemMatch": {"user_id": {"$eq": user_id}}}}, + ] + }, + ] } ) .skip(skip) @@ -284,11 +309,21 @@ async def get_dataset_files( files.append(FileOut.from_mongo(f)) else: for f in ( - await db["files"] + await db["files_view"] .find( { - "dataset_id": ObjectId(dataset_id), - "folder_id": None, + "$and": [ + { + "dataset_id": ObjectId(dataset_id), + "folder_id": None, + }, + { + "$or": [ + {"creator.email": user_id}, + {"auth": {"$elemMatch": {"user_id": {"$eq": user_id}}}}, + ] + }, + ] } ) .skip(skip) @@ -468,6 +503,7 @@ async def add_folder( async def get_dataset_folders( dataset_id: str, parent_folder: Optional[str] = None, + user_id=Depends(get_user), db: MongoClient = Depends(dependencies.get_db), ): folders = [] @@ -479,8 +515,18 @@ async def get_dataset_folders( else: async for f in db["folders"].find( { - "dataset_id": ObjectId(dataset_id), - "parent_folder": ObjectId(parent_folder), + "$and": [ + { + "dataset_id": ObjectId(dataset_id), + "parent_folder": ObjectId(parent_folder), + }, + { + "$or": [ + {"author.email": user_id}, + {"auth": {"$elemMatch": {"user_id": {"$eq": user_id}}}}, + ] + }, + ] } ): folders.append(FolderDB.from_mongo(f)) diff --git a/backend/app/routers/jobs.py b/backend/app/routers/jobs.py index 6a32731c3..ece605218 100644 --- a/backend/app/routers/jobs.py +++ b/backend/app/routers/jobs.py @@ -8,15 +8,15 @@ from app import dependencies from app.models.listeners import EventListenerJob, EventListenerJobUpdate -from app.keycloak_auth import get_current_user +from app.keycloak_auth import get_current_user, get_user router = APIRouter() @router.get("", response_model=List[EventListenerJob]) async def get_all_job_summary( + current_user_id=Depends(get_user), db: MongoClient = Depends(dependencies.get_db), - user=Depends(get_current_user), listener_id: Optional[str] = None, status: Optional[str] = None, user_id: Optional[str] = None, @@ -39,7 +39,14 @@ async def get_all_job_summary( limit -- restrict number of records to be returned (i.e. for pagination) """ jobs = [] - filters = [] + filters = [ + { + "$or": [ + {"creator.email": current_user_id}, + {"auth": {"$elemMatch": {"user_id": {"$eq": current_user_id}}}}, + ] + } + ] if listener_id is not None: filters.append({"listener_id": listener_id}) if status is not None: @@ -62,13 +69,11 @@ async def get_all_job_summary( if dataset_id is not None: filters.append({"resource_ref.collection": "dataset"}) filters.append({"resource_ref.resource_id": ObjectId(dataset_id)}) - if len(filters) == 0: - query = {} - else: - query = {"$and": filters} + + query = {"$and": filters} for doc in ( - await db["listener_jobs"] + await db["listener_jobs_view"] .find(query) .skip(skip) .limit(limit) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 67b5ce586..eedfdc5c0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,6 +26,7 @@ services: - clowder2 volumes: - mongo:/data/db + - ./scripts/mongoviews/mongo-init-dev.js:/docker-entrypoint-initdb.d/mongo-init.js:ro minio1: <<: *minio-common diff --git a/docker-compose.yml b/docker-compose.yml index 14b6c60c1..0d739c513 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,7 +109,9 @@ services: - clowder2 volumes: - mongo:/data/db -# environment: + - ./scripts/mongoviews/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro + + # environment: # MONGO_INITDB_ROOT_USERNAME: root # MONGO_INITDB_ROOT_PASSWORD: example diff --git a/scripts/mongoviews/README.md b/scripts/mongoviews/README.md new file mode 100644 index 000000000..678a87b1d --- /dev/null +++ b/scripts/mongoviews/README.md @@ -0,0 +1,14 @@ +# How to Import Views from Query Script + +- Open Studio 3T +- Find the Views section and right click +- Select **Add Views** + ![add_view](img/add_view.png) +- Select the collectino you want to base your view on. E.g. If you are importing **datasets_view.js**, you need to + select datasets collection + ![base_collectin](img/base_collection.png) +- In the new interface, click the folder icon and select the view query script. e.g. **datasets_view.js** + ![import](img/buttons.png) +- Click the **triangle "run"** button to test. +- If everything looks good, click the **create view** button, save it with the name following `collection name` + + `_view` pattern. diff --git a/scripts/mongoviews/datasets_view.js b/scripts/mongoviews/datasets_view.js new file mode 100644 index 000000000..192e913bb --- /dev/null +++ b/scripts/mongoviews/datasets_view.js @@ -0,0 +1,23 @@ +db.getCollection("datasets").aggregate( + + // Pipeline + [ + // Stage 1 + { + $lookup: { + "from" : "authorization", + "localField" : "_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ], + + // Options + { + + } + + // Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/ + +); \ No newline at end of file diff --git a/scripts/mongoviews/files_view.js b/scripts/mongoviews/files_view.js new file mode 100644 index 000000000..e64488d7b --- /dev/null +++ b/scripts/mongoviews/files_view.js @@ -0,0 +1,23 @@ +db.getCollection("files").aggregate( + + // Pipeline + [ + // Stage 1 + { + $lookup: { + "from" : "authorization", + "localField" : "dataset_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ], + + // Options + { + + } + + // Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/ + +); \ No newline at end of file diff --git a/scripts/mongoviews/folders_view.js b/scripts/mongoviews/folders_view.js new file mode 100644 index 000000000..a5b5376f5 --- /dev/null +++ b/scripts/mongoviews/folders_view.js @@ -0,0 +1,23 @@ +db.getCollection("folders").aggregate( + + // Pipeline + [ + // Stage 1 + { + $lookup: { + "from" : "authorization", + "localField" : "dataset_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ], + + // Options + { + + } + + // Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/ + +); \ No newline at end of file diff --git a/scripts/mongoviews/img/add_view.png b/scripts/mongoviews/img/add_view.png new file mode 100644 index 000000000..dfc466631 Binary files /dev/null and b/scripts/mongoviews/img/add_view.png differ diff --git a/scripts/mongoviews/img/base_collection.png b/scripts/mongoviews/img/base_collection.png new file mode 100644 index 000000000..7888c6fae Binary files /dev/null and b/scripts/mongoviews/img/base_collection.png differ diff --git a/scripts/mongoviews/img/buttons.png b/scripts/mongoviews/img/buttons.png new file mode 100644 index 000000000..060a2401f Binary files /dev/null and b/scripts/mongoviews/img/buttons.png differ diff --git a/scripts/mongoviews/img/save.png b/scripts/mongoviews/img/save.png new file mode 100644 index 000000000..be9982a17 Binary files /dev/null and b/scripts/mongoviews/img/save.png differ diff --git a/scripts/mongoviews/listener_job_updates_view.js b/scripts/mongoviews/listener_job_updates_view.js new file mode 100644 index 000000000..b89f95dc0 --- /dev/null +++ b/scripts/mongoviews/listener_job_updates_view.js @@ -0,0 +1,116 @@ +db.getCollection("listener_job_updates").aggregate( + + // Pipeline + [ + // Stage 1 + { + $lookup: // Equality Match + { + from: "listener_jobs", + localField: "job_id", + foreignField: "_id", + as: "listener_job_details" + } + + // Uncorrelated Subqueries + // (supported as of MongoDB 3.6) + // { + // from: "", + // let: { : , …, : }, + // pipeline: [ ], + // as: "" + // } + + // Correlated Subqueries + // (supported as of MongoDB 5.0) + // { + // from: "", + // localField: "", + // foreignField: "", + // let: { : , …, : }, + // pipeline: [ ], + // as: "" + // } + }, + + // Stage 2 + { + $facet: { + "extraction_on_dataset" : [ + { + "$match" : { + "listener_job_details.resource_ref.collection" : { + "$eq" : "dataset" + } + } + }, + { + "$lookup" : { + "from" : "authorization", + "localField" : "listener_job_details.resource_ref.resource_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ], + "extraction_on_file" : [ + { + "$match" : { + "listener_job_details.resource_ref.collection" : { + "$eq" : "file" + } + } + }, + { + "$lookup" : { + "from" : "files", + "localField" : "listener_job_details.resource_ref.resource_id", + "foreignField" : "_id", + "as" : "file_details" + } + }, + { + "$lookup" : { + "from" : "authorization", + "localField" : "file_details.dataset_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ] + } + }, + + // Stage 3 + { + $project: { + "all" : { + "$concatArrays" : [ + "$extraction_on_dataset", + "$extraction_on_file" + ] + } + } + }, + + // Stage 4 + { + $unwind: "$all" + }, + + // Stage 5 + { + $replaceRoot: { + "newRoot" : "$all" + } + } + ], + + // Options + { + + } + + // Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/ + +); \ No newline at end of file diff --git a/scripts/mongoviews/listener_jobs_view.js b/scripts/mongoviews/listener_jobs_view.js new file mode 100644 index 000000000..9cb9770f5 --- /dev/null +++ b/scripts/mongoviews/listener_jobs_view.js @@ -0,0 +1,85 @@ +db.getCollection("listener_jobs").aggregate( + + // Pipeline + [ + // Stage 1 + { + $facet: { + "extraction_on_dataset" : [ + { + "$match" : { + "resource_ref.collection" : { + "$eq" : "dataset" + } + } + }, + { + "$lookup" : { + "from" : "authorization", + "localField" : "resource_ref.resource_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ], + "extraction_on_file" : [ + { + "$match" : { + "resource_ref.collection" : { + "$eq" : "file" + } + } + }, + { + "$lookup" : { + "from" : "files", + "localField" : "resource_ref.resource_id", + "foreignField" : "_id", + "as" : "file_details" + } + }, + { + "$lookup" : { + "from" : "authorization", + "localField" : "file_details.dataset_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ] + } + }, + + // Stage 2 + { + $project: { + "all" : { + "$concatArrays" : [ + "$extraction_on_dataset", + "$extraction_on_file" + ] + } + } + }, + + // Stage 3 + { + $unwind: "$all" + }, + + // Stage 4 + { + $replaceRoot: { + "newRoot" : "$all" + } + } + ], + + // Options + { + + } + + // Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/ + +); \ No newline at end of file diff --git a/scripts/mongoviews/metadata_view.js b/scripts/mongoviews/metadata_view.js new file mode 100644 index 000000000..9507d17ce --- /dev/null +++ b/scripts/mongoviews/metadata_view.js @@ -0,0 +1,85 @@ +db.getCollection("metadata").aggregate( + + // Pipeline + [ + // Stage 1 + { + $facet: { + "metadata_on_dataset" : [ + { + "$match" : { + "resource.collection" : { + "$eq" : "datasets" + } + } + }, + { + "$lookup" : { + "from" : "authorization", + "localField" : "resource.resource_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ], + "metadata_on_file" : [ + { + "$match" : { + "resource.collection" : { + "$eq" : "files" + } + } + }, + { + "$lookup" : { + "from" : "files", + "localField" : "resource.resource_id", + "foreignField" : "_id", + "as" : "file_details" + } + }, + { + "$lookup" : { + "from" : "authorization", + "localField" : "file_details.dataset_id", + "foreignField" : "dataset_id", + "as" : "auth" + } + } + ] + } + }, + + // Stage 2 + { + $project: { + "all" : { + "$concatArrays" : [ + "$metadata_on_dataset", + "$metadata_on_file" + ] + } + } + }, + + // Stage 3 + { + $unwind: "$all" + }, + + // Stage 4 + { + $replaceRoot: { + "newRoot" : "$all" + } + } + ], + + // Options + { + + } + + // Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/ + +); \ No newline at end of file diff --git a/scripts/mongoviews/mongo-init-dev.js b/scripts/mongoviews/mongo-init-dev.js new file mode 100644 index 000000000..d12fa6f5e --- /dev/null +++ b/scripts/mongoviews/mongo-init-dev.js @@ -0,0 +1,275 @@ +function init(databaseName) { + db = db.getSiblingDB(databaseName); + + db.createCollection("authorization"); + db.createCollection("datasets"); + db.createCollection("feeds"); + db.createCollection("file_versions"); + db.createCollection("files"); + db.createCollection("folders"); + db.createCollection("listener_job_updates"); + db.createCollection("listener_jobs"); + db.createCollection("listeners"); + db.createCollection("metadata"); + db.createCollection("metadata.definitions"); + db.createCollection("tokens"); + db.createCollection("users"); + + db.createView("datasets_view", "datasets", + [ + { + $lookup: { + "from": "authorization", + "localField": "_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + ); + + db.createView("files_view", "files", + [ + { + $lookup: { + "from": "authorization", + "localField": "dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + ); + + db.createView("folders_view", "folders", + [ + { + $lookup: { + "from": "authorization", + "localField": "dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + ); + + db.createView("listener_job_updates_view", "listener_job_updates", + [ + { + $lookup: // Equality Match + { + from: "listener_jobs", + localField: "job_id", + foreignField: "_id", + as: "listener_job_details" + } + }, + { + $facet: { + "extraction_on_dataset": [ + { + "$match": { + "listener_job_details.resource_ref.collection": { + "$eq": "dataset" + } + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "listener_job_details.resource_ref.resource_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + "extraction_on_file": [ + { + "$match": { + "listener_job_details.resource_ref.collection": { + "$eq": "file" + } + } + }, + { + "$lookup": { + "from": "files", + "localField": "listener_job_details.resource_ref.resource_id", + "foreignField": "_id", + "as": "file_details" + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "file_details.dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + } + }, + { + $project: { + "all": { + "$concatArrays": [ + "$extraction_on_dataset", + "$extraction_on_file" + ] + } + } + }, + { + $unwind: "$all" + }, + { + $replaceRoot: { + "newRoot": "$all" + } + } + ], + ); + + db.createView("listener_jobs_view", "listener_jobs", + [ + { + $facet: { + "extraction_on_dataset": [ + { + "$match": { + "resource_ref.collection": { + "$eq": "dataset" + } + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "resource_ref.resource_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + "extraction_on_file": [ + { + "$match": { + "resource_ref.collection": { + "$eq": "file" + } + } + }, + { + "$lookup": { + "from": "files", + "localField": "resource_ref.resource_id", + "foreignField": "_id", + "as": "file_details" + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "file_details.dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + } + }, + { + $project: { + "all": { + "$concatArrays": [ + "$extraction_on_dataset", + "$extraction_on_file" + ] + } + } + }, + { + $unwind: "$all" + }, + { + $replaceRoot: { + "newRoot": "$all" + } + } + ], + ); + + db.createView("metadata_view", "metadata", + [ + { + $facet: { + "metadata_on_dataset": [ + { + "$match": { + "resource.collection": { + "$eq": "datasets" + } + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "resource.resource_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + "metadata_on_file": [ + { + "$match": { + "resource.collection": { + "$eq": "files" + } + } + }, + { + "$lookup": { + "from": "files", + "localField": "resource.resource_id", + "foreignField": "_id", + "as": "file_details" + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "file_details.dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + } + }, + { + $project: { + "all": { + "$concatArrays": [ + "$metadata_on_dataset", + "$metadata_on_file" + ] + } + } + }, + { + $unwind: "$all" + }, + { + $replaceRoot: { + "newRoot": "$all" + } + } + ], + ); +} + +init("clowder2"); +init("clowder-tests"); diff --git a/scripts/mongoviews/mongo-init.js b/scripts/mongoviews/mongo-init.js new file mode 100644 index 000000000..12096fbe2 --- /dev/null +++ b/scripts/mongoviews/mongo-init.js @@ -0,0 +1,274 @@ +function init(databaseName) { + db = db.getSiblingDB(databaseName); + + db.createCollection("authorization"); + db.createCollection("datasets"); + db.createCollection("feeds"); + db.createCollection("file_versions"); + db.createCollection("files"); + db.createCollection("folders"); + db.createCollection("listener_job_updates"); + db.createCollection("listener_jobs"); + db.createCollection("listeners"); + db.createCollection("metadata"); + db.createCollection("metadata.definitions"); + db.createCollection("tokens"); + db.createCollection("users"); + + db.createView("datasets_view", "datasets", + [ + { + $lookup: { + "from": "authorization", + "localField": "_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + ); + + db.createView("files_view", "files", + [ + { + $lookup: { + "from": "authorization", + "localField": "dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + ); + + db.createView("folders_view", "folders", + [ + { + $lookup: { + "from": "authorization", + "localField": "dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + ); + + db.createView("listener_job_updates_view", "listener_job_updates", + [ + { + $lookup: // Equality Match + { + from: "listener_jobs", + localField: "job_id", + foreignField: "_id", + as: "listener_job_details" + } + }, + { + $facet: { + "extraction_on_dataset": [ + { + "$match": { + "listener_job_details.resource_ref.collection": { + "$eq": "dataset" + } + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "listener_job_details.resource_ref.resource_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + "extraction_on_file": [ + { + "$match": { + "listener_job_details.resource_ref.collection": { + "$eq": "file" + } + } + }, + { + "$lookup": { + "from": "files", + "localField": "listener_job_details.resource_ref.resource_id", + "foreignField": "_id", + "as": "file_details" + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "file_details.dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + } + }, + { + $project: { + "all": { + "$concatArrays": [ + "$extraction_on_dataset", + "$extraction_on_file" + ] + } + } + }, + { + $unwind: "$all" + }, + { + $replaceRoot: { + "newRoot": "$all" + } + } + ], + ); + + db.createView("listener_jobs_view", "listener_jobs", + [ + { + $facet: { + "extraction_on_dataset": [ + { + "$match": { + "resource_ref.collection": { + "$eq": "dataset" + } + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "resource_ref.resource_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + "extraction_on_file": [ + { + "$match": { + "resource_ref.collection": { + "$eq": "file" + } + } + }, + { + "$lookup": { + "from": "files", + "localField": "resource_ref.resource_id", + "foreignField": "_id", + "as": "file_details" + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "file_details.dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + } + }, + { + $project: { + "all": { + "$concatArrays": [ + "$extraction_on_dataset", + "$extraction_on_file" + ] + } + } + }, + { + $unwind: "$all" + }, + { + $replaceRoot: { + "newRoot": "$all" + } + } + ], + ); + + db.createView("metadata_view", "metadata", + [ + { + $facet: { + "metadata_on_dataset": [ + { + "$match": { + "resource.collection": { + "$eq": "datasets" + } + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "resource.resource_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ], + "metadata_on_file": [ + { + "$match": { + "resource.collection": { + "$eq": "files" + } + } + }, + { + "$lookup": { + "from": "files", + "localField": "resource.resource_id", + "foreignField": "_id", + "as": "file_details" + } + }, + { + "$lookup": { + "from": "authorization", + "localField": "file_details.dataset_id", + "foreignField": "dataset_id", + "as": "auth" + } + } + ] + } + }, + { + $project: { + "all": { + "$concatArrays": [ + "$metadata_on_dataset", + "$metadata_on_file" + ] + } + } + }, + { + $unwind: "$all" + }, + { + $replaceRoot: { + "newRoot": "$all" + } + } + ], + ); +} + +init("clowder2");