From 5c0c5f2b6377ec1794a9021f0131b3f20f98efc8 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Mon, 17 Oct 2022 09:42:49 -0500 Subject: [PATCH 1/4] Add trash checks for Datasets, Collections --- app/services/CollectionService.scala | 2 ++ app/services/DatasetService.scala | 2 ++ app/services/ElasticsearchPlugin.scala | 12 ++++++++---- app/services/mongodb/MongoDBCollectionService.scala | 4 ++++ app/services/mongodb/MongoDBDatasetService.scala | 4 ++++ 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/app/services/CollectionService.scala b/app/services/CollectionService.scala index 5f3c3971e..3715921ee 100644 --- a/app/services/CollectionService.scala +++ b/app/services/CollectionService.scala @@ -252,4 +252,6 @@ trait CollectionService { def getMetrics(): Iterator[Collection] + def isInTrash(id: UUID): Boolean + } diff --git a/app/services/DatasetService.scala b/app/services/DatasetService.scala index 071a707bf..1949c3ae3 100644 --- a/app/services/DatasetService.scala +++ b/app/services/DatasetService.scala @@ -389,5 +389,7 @@ trait DatasetService { def getTrashedIds(): List[UUID] + def isInTrash(id: UUID): Boolean + def recursiveArchive(dataset: Dataset, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User]) } diff --git a/app/services/ElasticsearchPlugin.scala b/app/services/ElasticsearchPlugin.scala index 25b7c8d9d..fb49dc298 100644 --- a/app/services/ElasticsearchPlugin.scala +++ b/app/services/ElasticsearchPlugin.scala @@ -230,10 +230,14 @@ class ElasticsearchPlugin(application: Application) extends Plugin { resource.resourceType match { case ResourceRef.file => if (Permission.checkPermission(user, Permission.ViewFile, resource)) filesFound += resource.id - case ResourceRef.dataset => if (Permission.checkPermission(user, Permission.ViewDataset, resource)) - datasetsFound += resource.id - case ResourceRef.collection => if (Permission.checkPermission(user, Permission.ViewDataset, resource)) - collectionsFound += resource.id + case ResourceRef.dataset => { + if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !datasets.isInTrash(resource.id)) + datasetsFound += resource.id + } + case ResourceRef.collection => { + if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !collections.isInTrash(resource.id)) + collectionsFound += resource.id + } case _ => {} } }) diff --git a/app/services/mongodb/MongoDBCollectionService.scala b/app/services/mongodb/MongoDBCollectionService.scala index 6c9c58b70..340e86a8b 100644 --- a/app/services/mongodb/MongoDBCollectionService.scala +++ b/app/services/mongodb/MongoDBCollectionService.scala @@ -1116,6 +1116,10 @@ class MongoDBCollectionService @Inject() ( Collection.find(MongoDBObject("trash" -> false)).toIterator } + def isInTrash(id: UUID): Boolean = { + Collection.findOne(MongoDBObject("trash" -> true, "_id" -> new ObjectId(id.stringify))).isDefined + } + private def isSubCollectionIdInCollection(subCollectionId: UUID, collection: Collection) : Boolean = { if (collection.child_collection_ids.contains(subCollectionId)){ return true diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala index 15de0db8d..aab880e8f 100644 --- a/app/services/mongodb/MongoDBDatasetService.scala +++ b/app/services/mongodb/MongoDBDatasetService.scala @@ -1682,6 +1682,10 @@ class MongoDBDatasetService @Inject() ( trashedIds.toList } + def isInTrash(id: UUID): Boolean = { + Dataset.findOne(MongoDBObject("trash" -> true, "_id" -> new ObjectId(id.stringify))).isDefined + } + /** * Recursively submit requests to archive or unarchive the contents of the given dataset. * NOTE: "parameters" includes "operation", which supports both archiving and unarchiving From 8565dac432488cc99d6aa8706bbdf2b7bf341855 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Mon, 17 Oct 2022 14:16:36 -0500 Subject: [PATCH 2/4] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85bd19a86..cb549742b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased + +### Fixed + +- Search results are checked to verify nothing has been put in trash before display [#377](https://github.com/clowder-framework/clowder/issues/377) ## 1.21.0 - 2022-08-23 From 32261ea963f443292f08f1300369b7f1abd5517f Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Mon, 17 Oct 2022 14:24:26 -0500 Subject: [PATCH 3/4] Remove entries from Elasticsearch --- app/api/Collections.scala | 4 ++++ app/api/Datasets.scala | 5 +++++ app/api/Files.scala | 3 +++ 3 files changed, 12 insertions(+) diff --git a/app/api/Collections.scala b/app/api/Collections.scala index bbfb8ba29..f917b91b4 100644 --- a/app/api/Collections.scala +++ b/app/api/Collections.scala @@ -157,6 +157,10 @@ class Collections @Inject() (datasets: DatasetService, case Some(collection) => { val useTrash = play.api.Play.configuration.getBoolean("useTrash").getOrElse(false) if (!useTrash || (useTrash && collection.trash)){ + Logger.debug("Deleting collection from indexes " + collectionId) + current.plugin[ElasticsearchPlugin].foreach { + _.delete(collectionId.stringify) + } events.addObjectEvent(request.user , collection.id, collection.name, EventType.DELETE_COLLECTION.toString) collections.delete(collectionId) current.plugin[AdminsNotifierPlugin].foreach { diff --git a/app/api/Datasets.scala b/app/api/Datasets.scala index c0fe3fcbc..76abd4e20 100644 --- a/app/api/Datasets.scala +++ b/app/api/Datasets.scala @@ -2039,6 +2039,11 @@ class Datasets @Inject()( def deleteDatasetHelper(id: UUID, request: UserRequest[AnyContent]) = { datasets.get(id) match { case Some(dataset) => { + Logger.debug("Deleting dataset from indexes " + id) + current.plugin[ElasticsearchPlugin].foreach { + _.delete(id.stringify) + } + //remove dataset from RDF triple store if triple store is used configuration.getString("userdfSPARQLStore").getOrElse("no") match { case "yes" => rdfsparql.removeDatasetFromGraphs(id) diff --git a/app/api/Files.scala b/app/api/Files.scala index 18c0602b8..db737dfde 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -1697,6 +1697,9 @@ class Files @Inject()( current.plugin[VersusPlugin].foreach { _.removeFromIndexes(id) } + current.plugin[ElasticsearchPlugin].foreach { + _.delete(id.stringify) + } Logger.debug("Deleting file: " + file.filename) files.removeFile(id, Utils.baseUrl(request), request.apiKey, request.user) From cd31e2adb5df2becd337de44f2e295f93bf47dbb Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Mon, 17 Oct 2022 14:34:18 -0500 Subject: [PATCH 4/4] check if files are in trash --- app/services/ElasticsearchPlugin.scala | 6 ++++-- app/services/FileService.scala | 2 ++ app/services/mongodb/MongoDBFileService.scala | 9 +++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/services/ElasticsearchPlugin.scala b/app/services/ElasticsearchPlugin.scala index fb49dc298..4c2b0c386 100644 --- a/app/services/ElasticsearchPlugin.scala +++ b/app/services/ElasticsearchPlugin.scala @@ -228,8 +228,10 @@ class ElasticsearchPlugin(application: Application) extends Plugin { // Check permissions for each resource results.foreach(resource => { resource.resourceType match { - case ResourceRef.file => if (Permission.checkPermission(user, Permission.ViewFile, resource)) - filesFound += resource.id + case ResourceRef.file => { + if (Permission.checkPermission(user, Permission.ViewFile, resource) && !files.isInTrash(resource.id)) + filesFound += resource.id + } case ResourceRef.dataset => { if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !datasets.isInTrash(resource.id)) datasetsFound += resource.id diff --git a/app/services/FileService.scala b/app/services/FileService.scala index 9558e7daf..6d474379d 100644 --- a/app/services/FileService.scala +++ b/app/services/FileService.scala @@ -248,4 +248,6 @@ trait FileService { def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[File] + def isInTrash(id: UUID): Boolean + } diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index 52daf75ed..6d2766cf4 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -1237,6 +1237,15 @@ class MongoDBFileService @Inject() ( until.foreach(t => query = query ++ ("uploadDate" $lte Parsers.fromISO8601(t))) FileDAO.find(query) } + + def isInTrash(id: UUID): Boolean = { + var foundTrash = false + datasets.findByFileIdAllContain(id).foreach(ds => { + if (ds.trash) + foundTrash = true + }) + foundTrash + } } object FileDAO extends ModelCompanion[File, ObjectId] {