From 5836d5add09e7931cce3d9120bcb9f8edcd53ae1 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Fri, 17 Sep 2021 09:40:39 -0500 Subject: [PATCH 01/11] add private registration endpoint --- app/api/Extractions.scala | 61 ++++++++++++++++++++++++++++++++++++- app/models/Extraction.scala | 3 +- conf/routes | 1 + 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 0ee8701c9..3cdef8200 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -421,7 +421,6 @@ class Extractions @Inject()( } def addExtractorInfo() = AuthenticatedAction(parse.json) { implicit request => - // If repository is of type object, change it into an array. // This is for backward compatibility with requests from existing extractors. var requestJson = request.body \ "repository" match { @@ -471,6 +470,66 @@ class Extractions @Inject()( ) } + def addPrivateExtractorInfo(extractorKey: String) = AuthenticatedAction(parse.json) { implicit request => + // If repository is of type object, change it into an array. + // This is for backward compatibility with requests from existing extractors. + var requestJson = request.body \ "repository" match { + case rep: JsObject => request.body.as[JsObject] ++ Json.obj("repository" -> Json.arr(rep)) + case _ => request.body + } + + request.user match { + case Some(usr) => { + // Validate document + val extractionInfoResult = requestJson.validate[ExtractorInfo] + + // Update database + extractionInfoResult.fold( + errors => { + BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors))) + }, + info => { + // This is private extractor, so need to use the extractor_key provided + val uniqueName = info.name + extractorKey + val categories = "PRIVATE" :: info.categories + val updatedInfo = info.copy(name=uniqueName, categories=categories) + + extractors.updateExtractorInfo(updatedInfo) match { + case Some(u) => { + // Create/assign any default labels for this extractor + u.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") + } + extractors.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(u.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name)) + val updatedLabel = extractors.updateExtractorsLabel(label) + } + } + } + }) + + Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id))) + } + case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) + } + } + ) + } + case None => BadRequest(Json.obj("status" -> "KO", "message" -> "User is required for private extractors.")) + } + } + def submitFileToExtractor(file_id: UUID) = PermissionAction(Permission.EditFile, Some(ResourceRef(ResourceRef.file, file_id)))(parse.json) { implicit request => Logger.debug(s"Submitting file for extraction with body $request.body") diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index e6b35b503..b28264b90 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -127,10 +127,11 @@ case class ExtractorInfo( * PUBLISH - intended to publish files or datasets to external repositories * WORKFLOW - primarily manages workflows, submits external jobs, triggers other extractors, e.g. extractors-rulechecker * SILENT - if in this category, extractor will not send common status messages (e.g. STARTED) + * PRIVATE - registered as PrivateExtractor, accessible only to the registering user */ object ExtractorCategory extends Enumeration { type ExtractorCategory = Value - val EXTRACT, CONVERT, ARCHIVE, PUBLISH, WORKFLOW, SILENT = Value + val EXTRACT, CONVERT, ARCHIVE, PUBLISH, WORKFLOW, SILENT, PRIVATE = Value } object ExtractorInfo { diff --git a/conf/routes b/conf/routes index 519ee1cc1..736b281c0 100644 --- a/conf/routes +++ b/conf/routes @@ -426,6 +426,7 @@ POST /api/files/:id/sendUnarchiveRequest GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty) GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String) POST /api/extractors @api.Extractions.addExtractorInfo() +POST /api/extractors/private/:key @api.Extractions.addPrivateExtractorInfo(key: String) POST /api/extractors/labels @api.Extractions.createExtractorsLabel() PUT /api/extractors/labels/:id @api.Extractions.updateExtractorsLabel(id: UUID) DELETE /api/extractors/labels/:id @api.Extractions.deleteExtractorsLabel(id: UUID) From 6e313dd86b2e156b8e34d04cd40ba00b8cf46eb7 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Fri, 17 Sep 2021 09:54:08 -0500 Subject: [PATCH 02/11] add user filter --- app/api/Extractions.scala | 5 +-- app/controllers/Extractors.scala | 32 ++++++++++++------- app/controllers/Spaces.scala | 3 +- app/models/Extraction.scala | 6 ++-- app/services/ExtractorService.scala | 2 +- .../mongodb/MongoDBExtractorService.scala | 25 +++++++++------ conf/routes | 2 +- 7 files changed, 47 insertions(+), 28 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 3cdef8200..5e991c651 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -404,8 +404,9 @@ class Extractions @Inject()( Ok(jarr) } - def listExtractors(categories: List[String]) = AuthenticatedAction { implicit request => - Ok(Json.toJson(extractors.listExtractorsInfo(categories))) + def listExtractors(categories: List[String], space: Option[UUID]) = AuthenticatedAction { implicit request => + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + Ok(Json.toJson(extractors.listExtractorsInfo(categories, userid))) } def getExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala index b3e63e948..276949c95 100644 --- a/app/controllers/Extractors.scala +++ b/app/controllers/Extractors.scala @@ -56,9 +56,10 @@ class Extractors @Inject() (extractions: ExtractionService, */ def selectExtractors() = AuthenticatedAction { implicit request => implicit val user = request.user - + val userid = request.user.map(u => Some(u.id)).getOrElse(None) // Filter extractors by user filters necessary - var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty) + // TODO: Filter by multiple spaces + var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty, userid) val selectedExtractors: List[String] = extractorService.getEnabledExtractors() val groups = extractions.groupByType(extractions.findAll()) val allLabels = extractorService.listExtractorsLabels() @@ -166,7 +167,7 @@ class Extractors @Inject() (extractions: ExtractionService, def manageLabels = ServerAdminAction { implicit request => implicit val user = request.user val categories = List[String]("EXTRACT") - val extractors = extractorService.listExtractorsInfo(categories) + val extractors = extractorService.listExtractorsInfo(categories, None) val labels = extractorService.listExtractorsLabels() Ok(views.html.extractorLabels(labels, extractors)) @@ -211,7 +212,8 @@ class Extractors @Inject() (extractions: ExtractionService, def showExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => implicit val user = request.user - val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName) targetExtractor match { case Some(extractor) => { val labels = extractorService.getLabelsForExtractor(extractor.name) @@ -223,6 +225,7 @@ class Extractors @Inject() (extractions: ExtractionService, def showExtractorMetrics(extractorName: String) = AuthenticatedAction { implicit request => implicit val user = request.user + val userid = request.user.map(u => Some(u.id)).getOrElse(None) val dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS") val todaydate = dateFormatter.format(new java.util.Date()) @@ -299,7 +302,7 @@ class Extractors @Inject() (extractions: ExtractionService, } Logger.warn("last 10 average: " + lastTenAverage) - val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName) + val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName) targetExtractor match { case Some(extractor) => Ok(views.html.extractorMetrics(extractorName, average.toString, lastTenAverage.toString, lastweeksubmitted, lastmonthsubmitted)) case None => InternalServerError("Extractor Info not found: " + extractorName) @@ -308,11 +311,12 @@ class Extractors @Inject() (extractions: ExtractionService, def submitFileExtraction(file_id: UUID) = PermissionAction(Permission.EditFile, Some(ResourceRef(ResourceRef.file, file_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) - val extractors = all_extractors.filter(!_.process.file.isEmpty) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) fileService.get(file_id) match { - case Some(file) => { + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) + val extractors = all_extractors.filter(!_.process.file.isEmpty) + val foldersContainingFile = folders.findByFileId(file.id).sortBy(_.name) var folderHierarchy = new ListBuffer[Folder]() if(foldersContainingFile.length > 0) { @@ -352,7 +356,8 @@ class Extractors @Inject() (extractions: ExtractionService, def submitSelectedExtractions(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) val extractors = all_extractors.filter(!_.process.file.isEmpty) datasets.get(ds_id) match { case Some(dataset) => { @@ -372,10 +377,13 @@ class Extractors @Inject() (extractions: ExtractionService, def submitDatasetExtraction(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) - val extractors = all_extractors.filter(!_.process.dataset.isEmpty) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) datasetService.get(ds_id) match { - case Some(ds) => Ok(views.html.extractions.submitDatasetExtraction(extractors, ds)) + case Some(ds) => { + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) + val extractors = all_extractors.filter(!_.process.dataset.isEmpty) + Ok(views.html.extractions.submitDatasetExtraction(extractors, ds)) + } case None => InternalServerError("Dataset not found") } } diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index f95d5b07b..afc8059e7 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -92,10 +92,11 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS def selectExtractors(id: UUID) = AuthenticatedAction { implicit request => implicit val user = request.user + val userid = request.user.map(u => Some(u.id)).getOrElse(None) spaces.get(id) match { case Some(s) => { // get list of registered extractors - val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty) + val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty, userid) // list of extractors enabled globally val globalSelections: List[String] = extractors.getEnabledExtractors() // get list of extractors registered with a specific space diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index b28264b90..20b506b44 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -117,7 +117,8 @@ case class ExtractorInfo( defaultLabels: List[String] = List[String](), process: ExtractorProcessTriggers = new ExtractorProcessTriggers(), categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString), - parameters: JsValue = JsObject(Seq()) + parameters: JsValue = JsObject(Seq()), + users: List[UUID] =List[UUID]() ) /** what are the categories of the extractor? @@ -171,7 +172,8 @@ object ExtractorInfo { (JsPath \ "labels").read[List[String]].orElse(Reads.pure(List.empty)) and (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and - (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) + (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and + (JsPath \ "users").read[List[UUID]].orElse(Reads.pure(List.empty)) )(ExtractorInfo.apply _) } diff --git a/app/services/ExtractorService.scala b/app/services/ExtractorService.scala index 75acf38e1..d468f714d 100644 --- a/app/services/ExtractorService.scala +++ b/app/services/ExtractorService.scala @@ -35,7 +35,7 @@ trait ExtractorService { def dropAllExtractorStatusCollection() - def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] + def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] def getExtractorInfo(extractorName: String): Option[ExtractorInfo] diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 039d7df06..dbca9bee7 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -1,6 +1,6 @@ package services.mongodb -import javax.inject.Singleton +import javax.inject.{Inject, Singleton} import com.mongodb.casbah.Imports._ import com.mongodb.casbah.WriteConcern import com.mongodb.casbah.commons.MongoDBObject @@ -12,11 +12,12 @@ import play.api.Play.current import play.api.libs.json.{JsArray, JsNumber, JsObject, JsString, JsValue, Json} import services._ import services.mongodb.MongoContext.context - import org.bson.types.ObjectId @Singleton -class MongoDBExtractorService extends ExtractorService { +class MongoDBExtractorService @Inject() ( + users: MongoDBUserService + ) extends ExtractorService { def getExtractorServerIPList() = { var listServersIPs = List[String]() @@ -169,20 +170,26 @@ class MongoDBExtractorService extends ExtractorService { } } - def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] = { + def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] = { var list_queue = List[ExtractorInfo]() val allDocs = ExtractorInfoDAO.findAll().sort(orderBy = MongoDBObject("name" -> -1)) for (doc <- allDocs) { - // If no categories are specified, return all extractor names - var category_match = categories.isEmpty - if (!category_match) { + // If no filters are specified, return all extractor names + var filter_match = (categories.isEmpty && doc.users.isEmpty) + if (!filter_match) { // Otherwise check if any extractor categories overlap requested categories (force uppercase) + val user_match = user match { + case Some(u) => doc.users.contains(u) || doc.users.isEmpty + case None => doc.users.isEmpty // If no user filter in registered extractor, everyone can see + } val upper_categories = categories.map(cat => cat.toUpperCase) - category_match = doc.categories.intersect(upper_categories).length > 0 + val category_match = doc.categories.intersect(upper_categories).length > 0 + + filter_match = (category_match && user_match) } - if (category_match) + if (filter_match) list_queue = doc :: list_queue } diff --git a/conf/routes b/conf/routes index 736b281c0..778c38a41 100644 --- a/conf/routes +++ b/conf/routes @@ -423,7 +423,7 @@ POST /api/files/:id/sendUnarchiveRequest # ---------------------------------------------------------------------- # EXTRACTORS ENDPOINTS # ---------------------------------------------------------------------- -GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty) +GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty, , space: Option[UUID] ?= None) GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String) POST /api/extractors @api.Extractions.addExtractorInfo() POST /api/extractors/private/:key @api.Extractions.addPrivateExtractorInfo(key: String) From b36660a0dc0bd7d0efb2f1624357e0a21152e2a0 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Tue, 28 Sep 2021 13:59:26 -0500 Subject: [PATCH 03/11] unique key + perms --- app/models/Extraction.scala | 9 +++++---- app/services/mongodb/MongoDBExtractorService.scala | 13 ++++++++----- app/views/updateExtractors.scala.html | 4 ++-- conf/routes | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index 20b506b44..f2cd7cb95 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -118,7 +118,8 @@ case class ExtractorInfo( process: ExtractorProcessTriggers = new ExtractorProcessTriggers(), categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString), parameters: JsValue = JsObject(Seq()), - users: List[UUID] =List[UUID]() + unique_key: String = "", + permissions: List[ResourceRef] =List[ResourceRef]() ) /** what are the categories of the extractor? @@ -128,11 +129,10 @@ case class ExtractorInfo( * PUBLISH - intended to publish files or datasets to external repositories * WORKFLOW - primarily manages workflows, submits external jobs, triggers other extractors, e.g. extractors-rulechecker * SILENT - if in this category, extractor will not send common status messages (e.g. STARTED) - * PRIVATE - registered as PrivateExtractor, accessible only to the registering user */ object ExtractorCategory extends Enumeration { type ExtractorCategory = Value - val EXTRACT, CONVERT, ARCHIVE, PUBLISH, WORKFLOW, SILENT, PRIVATE = Value + val EXTRACT, CONVERT, ARCHIVE, PUBLISH, WORKFLOW, SILENT = Value } object ExtractorInfo { @@ -173,7 +173,8 @@ object ExtractorInfo { (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and - (JsPath \ "users").read[List[UUID]].orElse(Reads.pure(List.empty)) + (JsPath \ "unique_key").read[String].orElse(Reads.pure("")) and + (JsPath \ "permissions").read[List[ResourceRef]].orElse(Reads.pure(List.empty)) )(ExtractorInfo.apply _) } diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index dbca9bee7..9992ce25d 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -171,21 +171,24 @@ class MongoDBExtractorService @Inject() ( } def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] = { + Logger.info("listing: "+categories.toString) var list_queue = List[ExtractorInfo]() val allDocs = ExtractorInfoDAO.findAll().sort(orderBy = MongoDBObject("name" -> -1)) for (doc <- allDocs) { // If no filters are specified, return all extractor names - var filter_match = (categories.isEmpty && doc.users.isEmpty) + var filter_match = (categories.isEmpty && doc.permissions.isEmpty) if (!filter_match) { // Otherwise check if any extractor categories overlap requested categories (force uppercase) val user_match = user match { - case Some(u) => doc.users.contains(u) || doc.users.isEmpty - case None => doc.users.isEmpty // If no user filter in registered extractor, everyone can see + case Some(u) => { + val rr = new ResourceRef('user, u) + doc.permissions.contains(rr) || doc.permissions.isEmpty + } + case None => doc.permissions.isEmpty // If no user filter in registered extractor, everyone can see } val upper_categories = categories.map(cat => cat.toUpperCase) - val category_match = doc.categories.intersect(upper_categories).length > 0 - + val category_match = categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 filter_match = (category_match && user_match) } diff --git a/app/views/updateExtractors.scala.html b/app/views/updateExtractors.scala.html index 198f709c3..932713ed1 100644 --- a/app/views/updateExtractors.scala.html +++ b/app/views/updateExtractors.scala.html @@ -152,7 +152,7 @@

Extractor Catalog

Name Authors Version - Maturity + Unique Key All Jobs @if(showOptional("ratings")) { Rating } @@ -187,7 +187,7 @@

Extractor Catalog

- @extractor.maturity + @extractor.unique_key diff --git a/conf/routes b/conf/routes index 778c38a41..a950d1e5d 100644 --- a/conf/routes +++ b/conf/routes @@ -423,7 +423,7 @@ POST /api/files/:id/sendUnarchiveRequest # ---------------------------------------------------------------------- # EXTRACTORS ENDPOINTS # ---------------------------------------------------------------------- -GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty, , space: Option[UUID] ?= None) +GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty, space: Option[UUID] ?= None) GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String) POST /api/extractors @api.Extractions.addExtractorInfo() POST /api/extractors/private/:key @api.Extractions.addPrivateExtractorInfo(key: String) From 9f8827ab68d2f7b9d39df23d9caf42f340197199 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Tue, 19 Oct 2021 05:05:37 -0500 Subject: [PATCH 04/11] latest updates --- app/api/Extractions.scala | 90 +++++-------- app/controllers/Extractors.scala | 4 +- app/models/Extraction.scala | 1 + app/services/ExtractorRoutingService.scala | 2 +- app/services/ExtractorService.scala | 4 +- app/services/MessageService.scala | 123 +++++++++++++----- .../mongodb/MongoDBExtractorService.scala | 57 +++++--- conf/routes | 9 +- 8 files changed, 167 insertions(+), 123 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 5e991c651..3ec0f3b3b 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -409,19 +409,19 @@ class Extractions @Inject()( Ok(Json.toJson(extractors.listExtractorsInfo(categories, userid))) } - def getExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => - extractors.getExtractorInfo(extractorName) match { + def getExtractorInfo(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => + extractors.getExtractorInfo(extractorName, extractor_key) match { case Some(info) => Ok(Json.toJson(info)) case None => NotFound(Json.obj("status" -> "KO", "message" -> "Extractor info not found")) } } - def deleteExtractor(extractorName: String) = ServerAdminAction { implicit request => - extractors.deleteExtractor(extractorName) + def deleteExtractor(extractorName: String, extractor_key: Option[String]) = ServerAdminAction { implicit request => + extractors.deleteExtractor(extractorName, extractor_key) Ok(toJson(Map("status" -> "success"))) } - def addExtractorInfo() = AuthenticatedAction(parse.json) { implicit request => + def addExtractorInfo(extractor_key: Option[String], user: Option[String]) = AuthenticatedAction(parse.json) { implicit request => // If repository is of type object, change it into an array. // This is for backward compatibility with requests from existing extractors. var requestJson = request.body \ "repository" match { @@ -435,67 +435,42 @@ class Extractions @Inject()( // Update database extractionInfoResult.fold( errors => { + Logger.info("Couldn't parse info") BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors))) }, info => { - extractors.updateExtractorInfo(info) match { - case Some(u) => { - // Create/assign any default labels for this extractor - u.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") + // TODO: Include version number in queue name as well + // Check private extractor flags + val submissionInfo: Option[ExtractorInfo] = extractor_key match { + case Some(ek) => { + user match { + case None => { + Logger.error("Extractors with a private key must also specify a user email.") + None } - extractors.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name)) - } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(u.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name)) - val updatedLabel = extractors.updateExtractorsLabel(label) + case Some(userEmail) => { + userservice.findByEmail(userEmail) match { + case Some(u) => { + val perms = List(new ResourceRef('user, u.id)) + Some(info.copy(unique_key=ek, permissions = perms)) + } + case None => { + Logger.error("No user found with email "+userEmail) + None } } } - }) - - Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id))) + } } - case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) + case None => Some(info) } - } - ) - } - def addPrivateExtractorInfo(extractorKey: String) = AuthenticatedAction(parse.json) { implicit request => - // If repository is of type object, change it into an array. - // This is for backward compatibility with requests from existing extractors. - var requestJson = request.body \ "repository" match { - case rep: JsObject => request.body.as[JsObject] ++ Json.obj("repository" -> Json.arr(rep)) - case _ => request.body - } + // TODO: Check user permissions if the extractor_key has already been registered - request.user match { - case Some(usr) => { - // Validate document - val extractionInfoResult = requestJson.validate[ExtractorInfo] - - // Update database - extractionInfoResult.fold( - errors => { - BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors))) - }, - info => { - // This is private extractor, so need to use the extractor_key provided - val uniqueName = info.name + extractorKey - val categories = "PRIVATE" :: info.categories - val updatedInfo = info.copy(name=uniqueName, categories=categories) - - extractors.updateExtractorInfo(updatedInfo) match { + submissionInfo match { + case None => BadRequest("Extractors with a private key must also specify a non-anonymous user.") + case Some(subInfo) => { + extractors.updateExtractorInfo(subInfo) match { case Some(u) => { // Create/assign any default labels for this extractor u.defaultLabels.foreach(labelStr => { @@ -525,10 +500,9 @@ class Extractions @Inject()( case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) } } - ) + } } - case None => BadRequest(Json.obj("status" -> "KO", "message" -> "User is required for private extractors.")) - } + ) } def submitFileToExtractor(file_id: UUID) = PermissionAction(Permission.EditFile, Some(ResourceRef(ResourceRef.file, diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala index 276949c95..6f3a52285 100644 --- a/app/controllers/Extractors.scala +++ b/app/controllers/Extractors.scala @@ -39,9 +39,9 @@ class Extractors @Inject() (extractions: ExtractionService, /** * Gets a map of all updates from all jobs given to this extractor. */ - def showJobHistory(extractorName: String) = AuthenticatedAction { implicit request => + def showJobHistory(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => implicit val user = request.user - extractorService.getExtractorInfo(extractorName) match { + extractorService.getExtractorInfo(extractorName, extractor_key) match { case None => NotFound(s"No extractor found with name=${extractorName}") case Some(info) => { val allExtractions = extractions.findAll() diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index f2cd7cb95..f192acab4 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -78,6 +78,7 @@ case class ExtractorDetail( * * @param id id internal to the system * @param name lower case, no spaces, can use dashes + * @param uniqueName name+suffix to uniquely identify extractor for private use e.g. clowder.extractor.v2.johndoe123 * @param version the version, for example 1.3.5 * @param updated date when this information was last updated * @param description short description of what the extractor does diff --git a/app/services/ExtractorRoutingService.scala b/app/services/ExtractorRoutingService.scala index 545eb4194..18b469773 100644 --- a/app/services/ExtractorRoutingService.scala +++ b/app/services/ExtractorRoutingService.scala @@ -76,7 +76,7 @@ class ExtractorRoutingService { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) extractorIds.flatMap(exId => - extractorsService.getExtractorInfo(exId)).filter(exInfo => + extractorsService.getExtractorInfo(exId, None)).filter(exInfo => resourceType match { case ResourceType.dataset => containsOperation(exInfo.process.dataset, operation) diff --git a/app/services/ExtractorService.scala b/app/services/ExtractorService.scala index d468f714d..fb2defe0d 100644 --- a/app/services/ExtractorService.scala +++ b/app/services/ExtractorService.scala @@ -37,11 +37,11 @@ trait ExtractorService { def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] - def getExtractorInfo(extractorName: String): Option[ExtractorInfo] + def getExtractorInfo(extractorName: String, extractorKey: Option[String]): Option[ExtractorInfo] def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] - def deleteExtractor(extractorName: String) + def deleteExtractor(extractorName: String, extractorKey: Option[String]) def listExtractorsLabels(): List[ExtractorsLabel] diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala index 5eb95d557..f6bdbd673 100644 --- a/app/services/MessageService.scala +++ b/app/services/MessageService.scala @@ -305,6 +305,7 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { case statusBody: String => Logger.debug("Received extractor heartbeat: " + statusBody) val json = Json.parse(statusBody) + Logger.debug(json.toString) // TODO store running extractors ids val id = UUID((json \ "id").as[String]) val queue = (json \ "queue").as[String] @@ -315,51 +316,101 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { // Update database extractionInfoResult.fold( - errors => { - Logger.debug("Received extractor heartbeat with bad format: " + extractor_info) - }, + errors => Logger.debug("Received extractor heartbeat with bad format: " + extractor_info), info => { - extractorsService.getExtractorInfo(info.name) match { - case Some(infoFromDB) => { - // TODO only update if new semantic version is greater than old semantic version - if (infoFromDB.version != info.version) { - // TODO keep older versions of extractor info instead of just the latest one - extractorsService.updateExtractorInfo(info) - Logger.info("Updated extractor definition for " + info.name) + info.unique_key match { + case "" => { + extractorsService.getExtractorInfo(info.name, None) match { + case Some(infoFromDB) => { + Logger.info("update info, blank key") + // TODO only update if new semantic version is greater than old semantic version + if (infoFromDB.version != info.version) { + // TODO keep older versions of extractor info instead of just the latest one + extractorsService.updateExtractorInfo(info) + Logger.info("Updated extractor definition for " + info.name) + } + } + case None => { + extractorsService.updateExtractorInfo(info) match { + case None => {} + case Some(eInfo) => { + Logger.info("add info, no key") + // Create (if needed) and assign default labels + eInfo.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") + } + extractorsService.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(eInfo.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) + val updatedLabel = extractorsService.updateExtractorsLabel(label) + } + } + } + }) + } + } + + Logger.info(s"New extractor ${info.name} registered from heartbeat") + } } } - case None => { - extractorsService.updateExtractorInfo(info) match { - case None => {} - case Some(eInfo) => { - // Create (if needed) and assign default labels - eInfo.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") - } - extractorsService.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) - } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(eInfo.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) - val updatedLabel = extractorsService.updateExtractorsLabel(label) + case ek => { + extractorsService.getExtractorInfo(info.name, Some(ek)) match { + case Some(infoFromDB) => { + Logger.info("update info with key") + // TODO only update if new semantic version is greater than old semantic version + if (infoFromDB.version != info.version) { + // TODO keep older versions of extractor info instead of just the latest one + extractorsService.updateExtractorInfo(info) + Logger.info("Updated extractor definition for " + info.name) + } + } + case None => { + extractorsService.updateExtractorInfo(info) match { + case None => {} + case Some(eInfo) => { + Logger.info("updated info, with key") + // Create (if needed) and assign default labels + eInfo.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") + } + extractorsService.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(eInfo.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) + val updatedLabel = extractorsService.updateExtractorsLabel(label) + } + } } - } + }) } - }) + } + + Logger.info(s"New extractor ${info.name} registered from heartbeat") } } - - Logger.info(s"New extractor ${info.name} registered from heartbeat") } } + } ) } diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 9992ce25d..1764d673e 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -199,25 +199,48 @@ class MongoDBExtractorService @Inject() ( list_queue } - def getExtractorInfo(extractorName: String): Option[ExtractorInfo] = { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) + def getExtractorInfo(extractorName: String, extractorKey: Option[String]): Option[ExtractorInfo] = { + extractorKey match { + case Some(ek) => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) + case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) + } } def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] = { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match { - case Some(old) => { - val updated = e.copy(id = old.id) - ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) - Some(updated) + // TODO: Make this account for version as well + e.unique_key match { + case "" => { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match { + case Some(old) => { + val updated = e.copy(id = old.id) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) + Some(updated) + } + case None => { + ExtractorInfoDAO.save(e) + Some(e) + } + } } - case None => { - ExtractorInfoDAO.save(e) - Some(e) + case ek => { + Logger.info("using key lookup on "+ek) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> ek)) match { + case Some(old) => { + val updated = e.copy(id = old.id) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) + Some(updated) + } + case None => { + ExtractorInfoDAO.save(e) + Some(e) + } + } } } + } - def deleteExtractor(extractorName: String) { + def deleteExtractor(extractorName: String, extractorKey: Option[String]) { ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) match { case Some(extractor) => { ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name)) @@ -256,15 +279,11 @@ class MongoDBExtractorService @Inject() ( def getLabelsForExtractor(extractorName: String): List[ExtractorsLabel] = { var results = List[ExtractorsLabel]() - ExtractorInfoDAO.findOne(MongoDBObject("name"->extractorName)) match { - case Some(info) => { - ExtractorsLabelDAO.findAll().foreach(label => { - if (label.extractors.contains(extractorName)) { - results = results ++ List[ExtractorsLabel](label) - } - }) + ExtractorsLabelDAO.findAll().foreach(label => { + if (label.extractors.contains(extractorName) && !results.contains(label)) { + results = results ++ List[ExtractorsLabel](label) } - } + }) results } } diff --git a/conf/routes b/conf/routes index a950d1e5d..9d45fcf6d 100644 --- a/conf/routes +++ b/conf/routes @@ -183,7 +183,7 @@ GET /extractors/labels GET /extractors/:extractorName @controllers.Extractors.showExtractorInfo(extractorName) GET /extractors/:extractorName/metrics @controllers.Extractors.showExtractorMetrics(extractorName) GET /extractors/:extractorName/logs @controllers.Extractors.showExtractorLog(extractorName) -GET /extractors/:extractorName/history @controllers.Extractors.showJobHistory(extractorName) +GET /extractors/:extractorName/history @controllers.Extractors.showJobHistory(extractorName, extractor_key: Option[String] ?= None) # ---------------------------------------------------------------------- @@ -424,9 +424,8 @@ POST /api/files/:id/sendUnarchiveRequest # EXTRACTORS ENDPOINTS # ---------------------------------------------------------------------- GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty, space: Option[UUID] ?= None) -GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String) -POST /api/extractors @api.Extractions.addExtractorInfo() -POST /api/extractors/private/:key @api.Extractions.addPrivateExtractorInfo(key: String) +GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String, extractor_key: Option[String] ?= None) +POST /api/extractors @api.Extractions.addExtractorInfo(extractor_key: Option[String] ?= None, user: Option[String] ?= None) POST /api/extractors/labels @api.Extractions.createExtractorsLabel() PUT /api/extractors/labels/:id @api.Extractions.updateExtractorsLabel(id: UUID) DELETE /api/extractors/labels/:id @api.Extractions.deleteExtractorsLabel(id: UUID) @@ -448,7 +447,7 @@ GET /api/extractions/:id/status GET /api/extractions/:id/metadata @api.Extractions.fetch(id:UUID) GET /api/extractions/:id/statuses @api.Extractions.checkExtractionsStatuses(id:UUID) -DELETE /api/extractions/:extractorName/delete @api.Extractions.deleteExtractor(extractorName : String) +DELETE /api/extractions/:extractorName/delete @api.Extractions.deleteExtractor(extractorName : String, extractor_key: Option[String] ?= None) DELETE /api/files/:file_id/extractions/:msg_id @api.Extractions.cancelFileExtractionSubmission(file_id:UUID, msg_id: UUID) DELETE /api/datasets/:ds_id/extractions/:msg_id @api.Extractions.cancelDatasetExtractionSubmission(ds_id:UUID, msg_id: UUID) From 0e9bafdc006b455b163b853bbce65d910ba594a2 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Wed, 20 Oct 2021 08:47:16 -0500 Subject: [PATCH 05/11] make extractor key an option --- app/api/Extractions.scala | 2 +- app/models/Extraction.scala | 4 +- app/services/MessageService.scala | 157 +++++++++--------- .../mongodb/MongoDBExtractorService.scala | 6 +- 4 files changed, 81 insertions(+), 88 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 3ec0f3b3b..f39430b3b 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -452,7 +452,7 @@ class Extractions @Inject()( userservice.findByEmail(userEmail) match { case Some(u) => { val perms = List(new ResourceRef('user, u.id)) - Some(info.copy(unique_key=ek, permissions = perms)) + Some(info.copy(unique_key=Some(ek), permissions=perms)) } case None => { Logger.error("No user found with email "+userEmail) diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index f192acab4..d8d4d1be1 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -119,7 +119,7 @@ case class ExtractorInfo( process: ExtractorProcessTriggers = new ExtractorProcessTriggers(), categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString), parameters: JsValue = JsObject(Seq()), - unique_key: String = "", + unique_key: Option[String] = None, permissions: List[ResourceRef] =List[ResourceRef]() ) @@ -174,7 +174,7 @@ object ExtractorInfo { (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and - (JsPath \ "unique_key").read[String].orElse(Reads.pure("")) and + (JsPath \ "unique_key").read[Option[String]] and (JsPath \ "permissions").read[List[ResourceRef]].orElse(Reads.pure(List.empty)) )(ExtractorInfo.apply _) } diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala index f6bdbd673..3f50f3384 100644 --- a/app/services/MessageService.scala +++ b/app/services/MessageService.scala @@ -298,6 +298,7 @@ class EventFilter(channel: Channel, queue: String) extends Actor { * @param queue */ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { + val users: UserService = DI.injector.getInstance(classOf[UserService]) val extractions: ExtractionService = DI.injector.getInstance(classOf[ExtractionService]) val extractorsService: ExtractorService = DI.injector.getInstance(classOf[ExtractorService]) @@ -314,103 +315,97 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { // Validate document val extractionInfoResult = extractor_info.validate[ExtractorInfo] + // Determine if there is a user associated with this request + val owner = (json \ "owner").as[String] + val user: Option[User] = if (owner.length > 0) { + users.findByEmail(owner) + } else { + None + } + // Update database extractionInfoResult.fold( errors => Logger.debug("Received extractor heartbeat with bad format: " + extractor_info), info => { - info.unique_key match { - case "" => { - extractorsService.getExtractorInfo(info.name, None) match { - case Some(infoFromDB) => { - Logger.info("update info, blank key") - // TODO only update if new semantic version is greater than old semantic version - if (infoFromDB.version != info.version) { - // TODO keep older versions of extractor info instead of just the latest one - extractorsService.updateExtractorInfo(info) - Logger.info("Updated extractor definition for " + info.name) - } - } - case None => { - extractorsService.updateExtractorInfo(info) match { - case None => {} - case Some(eInfo) => { - Logger.info("add info, no key") - // Create (if needed) and assign default labels - eInfo.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") - } - extractorsService.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) - } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(eInfo.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) - val updatedLabel = extractorsService.updateExtractorsLabel(label) - } - } - } - }) - } - } + if (info.unique_key.isDefined && user.isEmpty) { + Logger.error("Unique extractor keys must have a user associated with them.") - Logger.info(s"New extractor ${info.name} registered from heartbeat") - } - } + } else { + val userRRef = ResourceRef('user, user.get.id) + // We just verified that if we have a key, we have a user, so we can add the user ID to permissions + val infoWithPerms = info.unique_key match { + case Some(ek) => if info.copy(permissions=List() :+ userRRef) + case None => info } - case ek => { - extractorsService.getExtractorInfo(info.name, Some(ek)) match { - case Some(infoFromDB) => { - Logger.info("update info with key") - // TODO only update if new semantic version is greater than old semantic version - if (infoFromDB.version != info.version) { - // TODO keep older versions of extractor info instead of just the latest one - extractorsService.updateExtractorInfo(info) - Logger.info("Updated extractor definition for " + info.name) + + extractorsService.getExtractorInfo(info.name, info.unique_key) match { + case Some(infoFromDB) => { + Logger.info("...found existing extractor...") + if (info.unique_key == infoFromDB.unique_key) { + info.unique_key match { + case None => { + // TODO only update if new semantic version is greater than old semantic version + if (infoFromDB.version != info.version) { + // TODO keep older versions of extractor info instead of just the latest one + extractorsService.updateExtractorInfo(info) + Logger.info("Updated extractor definition for " + info.name) + } + } + case Some(ek) => { + Logger.info("Checking permissions.") + val reqUser = new ResourceRef('user, user.get.id) + if (infoFromDB.permissions.contains(reqUser)) { + Logger.info("pass") + extractorsService.updateExtractorInfo(info) + } else { + Logger.info(reqUser.toString) + Logger.info(infoFromDB.permissions.toString) + Logger.error("User doesn't have permission.") + } + } } + } else { + Logger.error("Mismatch between given extractor unique key and existing registration.") } - case None => { - extractorsService.updateExtractorInfo(info) match { - case None => {} - case Some(eInfo) => { - Logger.info("updated info, with key") - // Create (if needed) and assign default labels - eInfo.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") + + + } + case None => { + Logger.info("...no existing extractor found") + extractorsService.updateExtractorInfo(updatedInfo) match { + case None => {} + case Some(eInfo) => { + Logger.info("registered") + // Create (if needed) and assign default labels + eInfo.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") + } + extractorsService.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) } - extractorsService.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) - } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(eInfo.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) - val updatedLabel = extractorsService.updateExtractorsLabel(label) - } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(eInfo.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) + val updatedLabel = extractorsService.updateExtractorsLabel(label) } } - }) - } + } + }) } - - Logger.info(s"New extractor ${info.name} registered from heartbeat") } + + Logger.info(s"New extractor ${info.name} registered from heartbeat with key "+info.unique_key.toString) } } - } + } } ) } diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 1764d673e..4df0d0847 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -209,7 +209,7 @@ class MongoDBExtractorService @Inject() ( def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] = { // TODO: Make this account for version as well e.unique_key match { - case "" => { + case None => { ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match { case Some(old) => { val updated = e.copy(id = old.id) @@ -222,8 +222,7 @@ class MongoDBExtractorService @Inject() ( } } } - case ek => { - Logger.info("using key lookup on "+ek) + case Some(ek) => { ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> ek)) match { case Some(old) => { val updated = e.copy(id = old.id) @@ -237,7 +236,6 @@ class MongoDBExtractorService @Inject() ( } } } - } def deleteExtractor(extractorName: String, extractorKey: Option[String]) { From 49d1d09ff083668423fddbdc22a88a0edf860203 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Fri, 22 Oct 2021 10:31:54 -0500 Subject: [PATCH 06/11] Fix up queries --- app/api/Extractions.scala | 2 - app/models/Extraction.scala | 2 +- app/services/MessageService.scala | 64 ++++++++----------- .../mongodb/MongoDBExtractorService.scala | 30 ++++++--- 4 files changed, 51 insertions(+), 47 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index f39430b3b..50ac08890 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -435,11 +435,9 @@ class Extractions @Inject()( // Update database extractionInfoResult.fold( errors => { - Logger.info("Couldn't parse info") BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors))) }, info => { - // TODO: Include version number in queue name as well // Check private extractor flags val submissionInfo: Option[ExtractorInfo] = extractor_key match { case Some(ek) => { diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index d8d4d1be1..d38fe3b34 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -174,7 +174,7 @@ object ExtractorInfo { (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and - (JsPath \ "unique_key").read[Option[String]] and + (JsPath \ "unique_key").read[Option[String]].orElse(Reads.pure(None)) and (JsPath \ "permissions").read[List[ResourceRef]].orElse(Reads.pure(List.empty)) )(ExtractorInfo.apply _) } diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala index 3f50f3384..d3509d73c 100644 --- a/app/services/MessageService.scala +++ b/app/services/MessageService.scala @@ -328,54 +328,46 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { errors => Logger.debug("Received extractor heartbeat with bad format: " + extractor_info), info => { if (info.unique_key.isDefined && user.isEmpty) { - Logger.error("Unique extractor keys must have a user associated with them.") - + Logger.error("Extractor keys must have a user associated with them.") } else { - val userRRef = ResourceRef('user, user.get.id) - // We just verified that if we have a key, we have a user, so we can add the user ID to permissions - val infoWithPerms = info.unique_key match { - case Some(ek) => if info.copy(permissions=List() :+ userRRef) - case None => info - } - extractorsService.getExtractorInfo(info.name, info.unique_key) match { case Some(infoFromDB) => { - Logger.info("...found existing extractor...") - if (info.unique_key == infoFromDB.unique_key) { - info.unique_key match { - case None => { - // TODO only update if new semantic version is greater than old semantic version - if (infoFromDB.version != info.version) { - // TODO keep older versions of extractor info instead of just the latest one - extractorsService.updateExtractorInfo(info) - Logger.info("Updated extractor definition for " + info.name) - } - } - case Some(ek) => { - Logger.info("Checking permissions.") - val reqUser = new ResourceRef('user, user.get.id) - if (infoFromDB.permissions.contains(reqUser)) { - Logger.info("pass") - extractorsService.updateExtractorInfo(info) - } else { - Logger.info(reqUser.toString) - Logger.info(infoFromDB.permissions.toString) - Logger.error("User doesn't have permission.") + Logger.debug("Found existing extractor, checking permissions...") + if (info.unique_key.isDefined) { + if (user.isEmpty) + Logger.error("User authentication required to modify this extractor.") + else { + val submittingUser = ResourceRef('user, user.get.id) + if (!infoFromDB.permissions.contains(submittingUser)) + Logger.error("User does not have permission to modify this extractor.") + else { + // Retain existing permissions + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=infoFromDB.permissions) + case None => info } + extractorsService.updateExtractorInfo(registrationInfo) + Logger.info(s"Updated private extractor definition for ${info.name} - ${info.unique_key}") } } } else { - Logger.error("Mismatch between given extractor unique key and existing registration.") + // TODO only update if new semantic version is greater than old semantic version + if (infoFromDB.version != info.version) { + // TODO keep older versions of extractor info instead of just the latest one + extractorsService.updateExtractorInfo(info) + Logger.info(s"Updated extractor definition for ${info.name}") + } } - - } case None => { - Logger.info("...no existing extractor found") - extractorsService.updateExtractorInfo(updatedInfo) match { + // Inject user into permissions list if a key is given + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=List(ResourceRef('user, user.get.id))) + case None => info + } + extractorsService.updateExtractorInfo(registrationInfo) match { case None => {} case Some(eInfo) => { - Logger.info("registered") // Create (if needed) and assign default labels eInfo.defaultLabels.foreach(labelStr => { val segments = labelStr.split("/") diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 4df0d0847..c76b88e40 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -202,7 +202,7 @@ class MongoDBExtractorService @Inject() ( def getExtractorInfo(extractorName: String, extractorKey: Option[String]): Option[ExtractorInfo] = { extractorKey match { case Some(ek) => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) - case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) + case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) } } @@ -210,10 +210,10 @@ class MongoDBExtractorService @Inject() ( // TODO: Make this account for version as well e.unique_key match { case None => { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false))) match { case Some(old) => { val updated = e.copy(id = old.id) - ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false)), updated, false, false, WriteConcern.Safe) Some(updated) } case None => { @@ -226,7 +226,7 @@ class MongoDBExtractorService @Inject() ( ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> ek)) match { case Some(old) => { val updated = e.copy(id = old.id) - ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> ek), updated, false, false, WriteConcern.Safe) Some(updated) } case None => { @@ -239,12 +239,26 @@ class MongoDBExtractorService @Inject() ( } def deleteExtractor(extractorName: String, extractorKey: Option[String]) { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) match { - case Some(extractor) => { - ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name)) + extractorKey match { + case Some(ek) => { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) match { + case Some(extractor) => { + ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> ek)) + } + case None => { + Logger.error(s"No extractor found with name ${extractorName} and key ${ek}") + } + } } case None => { - Logger.info("No extractor found with name: " + extractorName) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) match { + case Some(extractor) => { + ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> MongoDBObject("$exists" -> false))) + } + case None => { + Logger.error("No extractor found with name: " + extractorName) + } + } } } } From d08a3ae885043bf781d618350629a1db62215acf Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Wed, 17 Aug 2022 10:36:31 -0500 Subject: [PATCH 07/11] list user extractors if no process type specified --- app/controllers/Extractors.scala | 9 ++++++++- app/services/mongodb/MongoDBExtractorService.scala | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala index 6f3a52285..ad30eeafc 100644 --- a/app/controllers/Extractors.scala +++ b/app/controllers/Extractors.scala @@ -315,7 +315,14 @@ class Extractors @Inject() (extractions: ExtractionService, fileService.get(file_id) match { case Some(file) => { val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) - val extractors = all_extractors.filter(!_.process.file.isEmpty) + var extractors = all_extractors.filter(!_.process.file.isEmpty) + + val user_extra = userid match { + case Some(uid) => all_extractors.filter(_.permissions.contains(ResourceRef('user, uid))) + case None => List.empty + } + + extractors = (extractors ++ user_extra).distinct val foldersContainingFile = folders.findByFileId(file.id).sortBy(_.name) var folderHierarchy = new ListBuffer[Folder]() diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index c76b88e40..9d3fe4776 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -188,7 +188,7 @@ class MongoDBExtractorService @Inject() ( case None => doc.permissions.isEmpty // If no user filter in registered extractor, everyone can see } val upper_categories = categories.map(cat => cat.toUpperCase) - val category_match = categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 + val category_match = doc.categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 filter_match = (category_match && user_match) } From 93d0562ffeecade6e485a8df899c8707a6b5622a Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Fri, 19 Aug 2022 10:30:45 -0500 Subject: [PATCH 08/11] revert back category change --- app/services/mongodb/MongoDBExtractorService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 9d3fe4776..c76b88e40 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -188,7 +188,7 @@ class MongoDBExtractorService @Inject() ( case None => doc.permissions.isEmpty // If no user filter in registered extractor, everyone can see } val upper_categories = categories.map(cat => cat.toUpperCase) - val category_match = doc.categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 + val category_match = categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 filter_match = (category_match && user_match) } From a43db1e2ae6d5c0586ae7a19c194fd2f719540bb Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Fri, 19 Aug 2022 11:01:10 -0500 Subject: [PATCH 09/11] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eff74f999..c4ac18ce5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased ### Added - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266) - +- Extractors can now specify an extractor_key and an owner (email address) when sending a +registration or heartbeat to Clowder that will restrict use of that extractor to them. ## 1.18.1 - 2021-08-16 From 42ab1a9963e1acc3b8f36561b9f058916e8ad2d0 Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Tue, 30 Aug 2022 09:59:37 -0500 Subject: [PATCH 10/11] Permission check updates --- app/api/Extractions.scala | 35 +++---------------- app/services/ExtractorRoutingService.scala | 31 ++++++++++------ app/services/ExtractorService.scala | 2 +- app/services/MessageService.scala | 24 ++++--------- .../mongodb/MongoDBExtractorService.scala | 16 +++++++-- 5 files changed, 47 insertions(+), 61 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 50ac08890..fb0e03bb8 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -127,36 +127,6 @@ class Extractions @Inject()( } } - /** - * - * Given a file id (UUID), submit this file for extraction - */ - def submitExtraction(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id)))(parse.json) { implicit request => - if (UUID.isValid(id.stringify)) { - files.get(id) match { - case Some(file) => { - // FIXME dataset not available? - routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match { - case Some(jobId) => { - Ok(Json.obj("status" -> "OK", "job_id" -> jobId)) - } - case None => { - val message = "No jobId found for Extraction" - Logger.error(message) - InternalServerError(toJson(Map("status" -> "KO", "message" -> message))) - } - } - } - case None => { - Logger.error("Could not retrieve file that was just saved.") - InternalServerError("Error uploading file") - } - } //file match - } else { - BadRequest("Not valid id") - } - } - /** * For a given file id, checks for the status of all extractors processing that file. * REST endpoint GET /api/extractions/:id/status @@ -550,11 +520,14 @@ class Extractions @Inject()( } // if extractor_id is not specified default to execution of all extractors matching mime type (request.body \ "extractor").asOpt[String] match { - case Some(extractorId) => + case Some(extractorId) => { + // TODO: Check extractor permissions + extractors.getExtractorInfo() val job_id = routing.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra, datasetId, newFlags, request.apiKey, request.user) sink.logSubmitFileToExtractorEvent(file, extractorId, request.user) Ok(Json.obj("status" -> "OK", "job_id" -> job_id)) + } case None => { routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match { case Some(job_id) => { diff --git a/app/services/ExtractorRoutingService.scala b/app/services/ExtractorRoutingService.scala index 18b469773..a8bc104fc 100644 --- a/app/services/ExtractorRoutingService.scala +++ b/app/services/ExtractorRoutingService.scala @@ -72,12 +72,12 @@ class ExtractorRoutingService { * @param resourceType the type of resource to check * @return filtered list of extractors */ - private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value): List[String] = { + private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value, user: Option[User] = None): List[String] = { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) extractorIds.flatMap(exId => - extractorsService.getExtractorInfo(exId, None)).filter(exInfo => - resourceType match { + extractorsService.getExtractorInfo(exId, None)).filter(exInfo => { + val processMatch = resourceType match { case ResourceType.dataset => containsOperation(exInfo.process.dataset, operation) case ResourceType.file => @@ -87,7 +87,17 @@ class ExtractorRoutingService { case _ => false } - ).map(_.name) + val permissionMatch = exInfo.unique_key match { + case Some(key) => { + user match { + case None => false // User must be provided for a key-protected extractor + case Some(u) => exInfo.permissions.contains(new ResourceRef('user,u.id)) + } + } + case None => true + } + processMatch && permissionMatch + }).map(_.name) } /** @@ -96,15 +106,15 @@ class ExtractorRoutingService { * @param operation The dataset operation requested. * @return A list of extractors IDs. */ - private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value): (List[String], List[String]) = { + private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value, user: Option[User] = None): (List[String], List[String]) = { val spacesService = DI.injector.getInstance(classOf[SpaceService]) var enabledExtractors = new ListBuffer[String]() var disabledExtractors = new ListBuffer[String]() dataset.spaces.map(space => { spacesService.getAllExtractors(space).foreach { extractors => - enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType)) - disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType)) + enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType, user)) + disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType, user)) } }) (enabledExtractors.toList, disabledExtractors.toList) @@ -145,7 +155,7 @@ class ExtractorRoutingService { * @param contentType the content type of the file in the case of a file * @return a set of unique rabbitmq queues */ - private def getQueues(dataset: Dataset, routingKey: String, contentType: String): Set[String] = { + private def getQueues(dataset: Dataset, routingKey: String, contentType: String, user: Option[User] = None): Set[String] = { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) // drop the first fragment from the routing key and replace characters to create operation id @@ -160,9 +170,9 @@ class ExtractorRoutingService { else return Set.empty[String] // get extractors enabled at the global level - val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType) + val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType, user) // get extractors enabled/disabled at the space level - val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType) + val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType, user) // get queues based on RabbitMQ bindings (old method). val queuesFromBindings = getQueuesFromBindings(routingKey) // take the union of queues so that we publish to a specific queue only once @@ -229,6 +239,7 @@ class ExtractorRoutingService { var jobId: Option[UUID] = None dataset match { case Some(d) => { + // TODO: Check private extractor behavior getQueues(d, routingKey, file.contentType).foreach { queue => val source = Entity(ResourceRef(ResourceRef.file, file.id), Some(file.contentType), sourceExtra) diff --git a/app/services/ExtractorService.scala b/app/services/ExtractorService.scala index fb2defe0d..d750eaf58 100644 --- a/app/services/ExtractorService.scala +++ b/app/services/ExtractorService.scala @@ -37,7 +37,7 @@ trait ExtractorService { def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] - def getExtractorInfo(extractorName: String, extractorKey: Option[String]): Option[ExtractorInfo] + def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala index d3509d73c..c6eda512a 100644 --- a/app/services/MessageService.scala +++ b/app/services/MessageService.scala @@ -330,26 +330,16 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { if (info.unique_key.isDefined && user.isEmpty) { Logger.error("Extractor keys must have a user associated with them.") } else { - extractorsService.getExtractorInfo(info.name, info.unique_key) match { + extractorsService.getExtractorInfo(info.name, info.unique_key, user) match { case Some(infoFromDB) => { - Logger.debug("Found existing extractor, checking permissions...") if (info.unique_key.isDefined) { - if (user.isEmpty) - Logger.error("User authentication required to modify this extractor.") - else { - val submittingUser = ResourceRef('user, user.get.id) - if (!infoFromDB.permissions.contains(submittingUser)) - Logger.error("User does not have permission to modify this extractor.") - else { - // Retain existing permissions - val registrationInfo = info.unique_key match { - case Some(ek) => info.copy(permissions=infoFromDB.permissions) - case None => info - } - extractorsService.updateExtractorInfo(registrationInfo) - Logger.info(s"Updated private extractor definition for ${info.name} - ${info.unique_key}") - } + // Retain existing permissions + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=infoFromDB.permissions) + case None => info } + extractorsService.updateExtractorInfo(registrationInfo) + Logger.info(s"Updated private extractor definition for ${info.name} - ${info.unique_key}") } else { // TODO only update if new semantic version is greater than old semantic version if (infoFromDB.version != info.version) { diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index c76b88e40..514327b15 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -199,9 +199,21 @@ class MongoDBExtractorService @Inject() ( list_queue } - def getExtractorInfo(extractorName: String, extractorKey: Option[String]): Option[ExtractorInfo] = { + def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] = { extractorKey match { - case Some(ek) => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) + case Some(ek) => { + user match { + case None => { + Logger.error("User authentication required to view extractor info with a unique key.") + None + } + case Some(u) => { + val userRef = new ResourceRef('user, u.id) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek, "permissions" -> userRef)) + } + } + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) + } case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) } } From 8c5a5dd7acf04ceac17e9f3e16e2110a01e3f42e Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Thu, 8 Sep 2022 09:25:34 -0500 Subject: [PATCH 11/11] clean up user arg in several places --- app/api/Extractions.scala | 6 +++--- app/controllers/Extractors.scala | 2 +- app/services/ExtractorRoutingService.scala | 2 +- app/services/mongodb/MongoDBExtractorService.scala | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index fb0e03bb8..120b02472 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -380,7 +380,7 @@ class Extractions @Inject()( } def getExtractorInfo(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => - extractors.getExtractorInfo(extractorName, extractor_key) match { + extractors.getExtractorInfo(extractorName, extractor_key, request.user) match { case Some(info) => Ok(Json.toJson(info)) case None => NotFound(Json.obj("status" -> "KO", "message" -> "Extractor info not found")) } @@ -521,8 +521,8 @@ class Extractions @Inject()( // if extractor_id is not specified default to execution of all extractors matching mime type (request.body \ "extractor").asOpt[String] match { case Some(extractorId) => { - // TODO: Check extractor permissions - extractors.getExtractorInfo() + val extractorKey = (request.body \ "extractor").asOpt[String] + extractors.getExtractorInfo(extractorId, extractorKey, request.user) val job_id = routing.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra, datasetId, newFlags, request.apiKey, request.user) sink.logSubmitFileToExtractorEvent(file, extractorId, request.user) diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala index ad30eeafc..664fcbc3a 100644 --- a/app/controllers/Extractors.scala +++ b/app/controllers/Extractors.scala @@ -41,7 +41,7 @@ class Extractors @Inject() (extractions: ExtractionService, */ def showJobHistory(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => implicit val user = request.user - extractorService.getExtractorInfo(extractorName, extractor_key) match { + extractorService.getExtractorInfo(extractorName, extractor_key, user) match { case None => NotFound(s"No extractor found with name=${extractorName}") case Some(info) => { val allExtractions = extractions.findAll() diff --git a/app/services/ExtractorRoutingService.scala b/app/services/ExtractorRoutingService.scala index a8bc104fc..4915bb35a 100644 --- a/app/services/ExtractorRoutingService.scala +++ b/app/services/ExtractorRoutingService.scala @@ -76,7 +76,7 @@ class ExtractorRoutingService { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) extractorIds.flatMap(exId => - extractorsService.getExtractorInfo(exId, None)).filter(exInfo => { + extractorsService.getExtractorInfo(exId, None, None)).filter(exInfo => { val processMatch = resourceType match { case ResourceType.dataset => containsOperation(exInfo.process.dataset, operation) diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 514327b15..3938e34bb 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -212,7 +212,6 @@ class MongoDBExtractorService @Inject() ( ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek, "permissions" -> userRef)) } } - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) } case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) }