diff --git a/CHANGELOG.md b/CHANGELOG.md index c5e98ecde..747cf9e44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +### Added +- Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266) +- Extractors can now specify an extractor_key and an owner (email address) when sending a +registration or heartbeat to Clowder that will restrict use of that extractor to them. + ## Fixed - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI [386](https://github.com/clowder-framework/clowder/issues/386) - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 0ee8701c9..120b02472 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -127,36 +127,6 @@ class Extractions @Inject()( } } - /** - * - * Given a file id (UUID), submit this file for extraction - */ - def submitExtraction(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id)))(parse.json) { implicit request => - if (UUID.isValid(id.stringify)) { - files.get(id) match { - case Some(file) => { - // FIXME dataset not available? - routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match { - case Some(jobId) => { - Ok(Json.obj("status" -> "OK", "job_id" -> jobId)) - } - case None => { - val message = "No jobId found for Extraction" - Logger.error(message) - InternalServerError(toJson(Map("status" -> "KO", "message" -> message))) - } - } - } - case None => { - Logger.error("Could not retrieve file that was just saved.") - InternalServerError("Error uploading file") - } - } //file match - } else { - BadRequest("Not valid id") - } - } - /** * For a given file id, checks for the status of all extractors processing that file. * REST endpoint GET /api/extractions/:id/status @@ -404,24 +374,24 @@ class Extractions @Inject()( Ok(jarr) } - def listExtractors(categories: List[String]) = AuthenticatedAction { implicit request => - Ok(Json.toJson(extractors.listExtractorsInfo(categories))) + def listExtractors(categories: List[String], space: Option[UUID]) = AuthenticatedAction { implicit request => + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + Ok(Json.toJson(extractors.listExtractorsInfo(categories, userid))) } - def getExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => - extractors.getExtractorInfo(extractorName) match { + def getExtractorInfo(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => + extractors.getExtractorInfo(extractorName, extractor_key, request.user) match { case Some(info) => Ok(Json.toJson(info)) case None => NotFound(Json.obj("status" -> "KO", "message" -> "Extractor info not found")) } } - def deleteExtractor(extractorName: String) = ServerAdminAction { implicit request => - extractors.deleteExtractor(extractorName) + def deleteExtractor(extractorName: String, extractor_key: Option[String]) = ServerAdminAction { implicit request => + extractors.deleteExtractor(extractorName, extractor_key) Ok(toJson(Map("status" -> "success"))) } - def addExtractorInfo() = AuthenticatedAction(parse.json) { implicit request => - + def addExtractorInfo(extractor_key: Option[String], user: Option[String]) = AuthenticatedAction(parse.json) { implicit request => // If repository is of type object, change it into an array. // This is for backward compatibility with requests from existing extractors. var requestJson = request.body \ "repository" match { @@ -438,34 +408,66 @@ class Extractions @Inject()( BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors))) }, info => { - extractors.updateExtractorInfo(info) match { - case Some(u) => { - // Create/assign any default labels for this extractor - u.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") + // Check private extractor flags + val submissionInfo: Option[ExtractorInfo] = extractor_key match { + case Some(ek) => { + user match { + case None => { + Logger.error("Extractors with a private key must also specify a user email.") + None } - extractors.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name)) - } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(u.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name)) - val updatedLabel = extractors.updateExtractorsLabel(label) + case Some(userEmail) => { + userservice.findByEmail(userEmail) match { + case Some(u) => { + val perms = List(new ResourceRef('user, u.id)) + Some(info.copy(unique_key=Some(ek), permissions=perms)) + } + case None => { + Logger.error("No user found with email "+userEmail) + None } } } - }) + } + } + case None => Some(info) + } + + // TODO: Check user permissions if the extractor_key has already been registered + + submissionInfo match { + case None => BadRequest("Extractors with a private key must also specify a non-anonymous user.") + case Some(subInfo) => { + extractors.updateExtractorInfo(subInfo) match { + case Some(u) => { + // Create/assign any default labels for this extractor + u.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") + } + extractors.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(u.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name)) + val updatedLabel = extractors.updateExtractorsLabel(label) + } + } + } + }) - Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id))) + Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id))) + } + case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) + } } - case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) } } ) @@ -518,11 +520,14 @@ class Extractions @Inject()( } // if extractor_id is not specified default to execution of all extractors matching mime type (request.body \ "extractor").asOpt[String] match { - case Some(extractorId) => + case Some(extractorId) => { + val extractorKey = (request.body \ "extractor").asOpt[String] + extractors.getExtractorInfo(extractorId, extractorKey, request.user) val job_id = routing.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra, datasetId, newFlags, request.apiKey, request.user) sink.logSubmitFileToExtractorEvent(file, extractorId, request.user) Ok(Json.obj("status" -> "OK", "job_id" -> job_id)) + } case None => { routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match { case Some(job_id) => { diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala index b3e63e948..664fcbc3a 100644 --- a/app/controllers/Extractors.scala +++ b/app/controllers/Extractors.scala @@ -39,9 +39,9 @@ class Extractors @Inject() (extractions: ExtractionService, /** * Gets a map of all updates from all jobs given to this extractor. */ - def showJobHistory(extractorName: String) = AuthenticatedAction { implicit request => + def showJobHistory(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => implicit val user = request.user - extractorService.getExtractorInfo(extractorName) match { + extractorService.getExtractorInfo(extractorName, extractor_key, user) match { case None => NotFound(s"No extractor found with name=${extractorName}") case Some(info) => { val allExtractions = extractions.findAll() @@ -56,9 +56,10 @@ class Extractors @Inject() (extractions: ExtractionService, */ def selectExtractors() = AuthenticatedAction { implicit request => implicit val user = request.user - + val userid = request.user.map(u => Some(u.id)).getOrElse(None) // Filter extractors by user filters necessary - var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty) + // TODO: Filter by multiple spaces + var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty, userid) val selectedExtractors: List[String] = extractorService.getEnabledExtractors() val groups = extractions.groupByType(extractions.findAll()) val allLabels = extractorService.listExtractorsLabels() @@ -166,7 +167,7 @@ class Extractors @Inject() (extractions: ExtractionService, def manageLabels = ServerAdminAction { implicit request => implicit val user = request.user val categories = List[String]("EXTRACT") - val extractors = extractorService.listExtractorsInfo(categories) + val extractors = extractorService.listExtractorsInfo(categories, None) val labels = extractorService.listExtractorsLabels() Ok(views.html.extractorLabels(labels, extractors)) @@ -211,7 +212,8 @@ class Extractors @Inject() (extractions: ExtractionService, def showExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => implicit val user = request.user - val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName) targetExtractor match { case Some(extractor) => { val labels = extractorService.getLabelsForExtractor(extractor.name) @@ -223,6 +225,7 @@ class Extractors @Inject() (extractions: ExtractionService, def showExtractorMetrics(extractorName: String) = AuthenticatedAction { implicit request => implicit val user = request.user + val userid = request.user.map(u => Some(u.id)).getOrElse(None) val dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS") val todaydate = dateFormatter.format(new java.util.Date()) @@ -299,7 +302,7 @@ class Extractors @Inject() (extractions: ExtractionService, } Logger.warn("last 10 average: " + lastTenAverage) - val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName) + val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName) targetExtractor match { case Some(extractor) => Ok(views.html.extractorMetrics(extractorName, average.toString, lastTenAverage.toString, lastweeksubmitted, lastmonthsubmitted)) case None => InternalServerError("Extractor Info not found: " + extractorName) @@ -308,11 +311,19 @@ class Extractors @Inject() (extractions: ExtractionService, def submitFileExtraction(file_id: UUID) = PermissionAction(Permission.EditFile, Some(ResourceRef(ResourceRef.file, file_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) - val extractors = all_extractors.filter(!_.process.file.isEmpty) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) fileService.get(file_id) match { - case Some(file) => { + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) + var extractors = all_extractors.filter(!_.process.file.isEmpty) + + val user_extra = userid match { + case Some(uid) => all_extractors.filter(_.permissions.contains(ResourceRef('user, uid))) + case None => List.empty + } + + extractors = (extractors ++ user_extra).distinct + val foldersContainingFile = folders.findByFileId(file.id).sortBy(_.name) var folderHierarchy = new ListBuffer[Folder]() if(foldersContainingFile.length > 0) { @@ -352,7 +363,8 @@ class Extractors @Inject() (extractions: ExtractionService, def submitSelectedExtractions(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) val extractors = all_extractors.filter(!_.process.file.isEmpty) datasets.get(ds_id) match { case Some(dataset) => { @@ -372,10 +384,13 @@ class Extractors @Inject() (extractions: ExtractionService, def submitDatasetExtraction(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) - val extractors = all_extractors.filter(!_.process.dataset.isEmpty) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) datasetService.get(ds_id) match { - case Some(ds) => Ok(views.html.extractions.submitDatasetExtraction(extractors, ds)) + case Some(ds) => { + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) + val extractors = all_extractors.filter(!_.process.dataset.isEmpty) + Ok(views.html.extractions.submitDatasetExtraction(extractors, ds)) + } case None => InternalServerError("Dataset not found") } } diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index a8388812f..49e724aea 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -83,10 +83,11 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS def selectExtractors(id: UUID) = AuthenticatedAction { implicit request => implicit val user = request.user + val userid = request.user.map(u => Some(u.id)).getOrElse(None) spaces.get(id) match { case Some(s) => { // get list of registered extractors - val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty) + val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty, userid) // list of extractors enabled globally val globalSelections: List[String] = extractors.getEnabledExtractors() // get list of extractors registered with a specific space diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index e6b35b503..d38fe3b34 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -78,6 +78,7 @@ case class ExtractorDetail( * * @param id id internal to the system * @param name lower case, no spaces, can use dashes + * @param uniqueName name+suffix to uniquely identify extractor for private use e.g. clowder.extractor.v2.johndoe123 * @param version the version, for example 1.3.5 * @param updated date when this information was last updated * @param description short description of what the extractor does @@ -117,7 +118,9 @@ case class ExtractorInfo( defaultLabels: List[String] = List[String](), process: ExtractorProcessTriggers = new ExtractorProcessTriggers(), categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString), - parameters: JsValue = JsObject(Seq()) + parameters: JsValue = JsObject(Seq()), + unique_key: Option[String] = None, + permissions: List[ResourceRef] =List[ResourceRef]() ) /** what are the categories of the extractor? @@ -170,7 +173,9 @@ object ExtractorInfo { (JsPath \ "labels").read[List[String]].orElse(Reads.pure(List.empty)) and (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and - (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) + (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and + (JsPath \ "unique_key").read[Option[String]].orElse(Reads.pure(None)) and + (JsPath \ "permissions").read[List[ResourceRef]].orElse(Reads.pure(List.empty)) )(ExtractorInfo.apply _) } diff --git a/app/services/ExtractorRoutingService.scala b/app/services/ExtractorRoutingService.scala index 545eb4194..4915bb35a 100644 --- a/app/services/ExtractorRoutingService.scala +++ b/app/services/ExtractorRoutingService.scala @@ -72,12 +72,12 @@ class ExtractorRoutingService { * @param resourceType the type of resource to check * @return filtered list of extractors */ - private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value): List[String] = { + private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value, user: Option[User] = None): List[String] = { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) extractorIds.flatMap(exId => - extractorsService.getExtractorInfo(exId)).filter(exInfo => - resourceType match { + extractorsService.getExtractorInfo(exId, None, None)).filter(exInfo => { + val processMatch = resourceType match { case ResourceType.dataset => containsOperation(exInfo.process.dataset, operation) case ResourceType.file => @@ -87,7 +87,17 @@ class ExtractorRoutingService { case _ => false } - ).map(_.name) + val permissionMatch = exInfo.unique_key match { + case Some(key) => { + user match { + case None => false // User must be provided for a key-protected extractor + case Some(u) => exInfo.permissions.contains(new ResourceRef('user,u.id)) + } + } + case None => true + } + processMatch && permissionMatch + }).map(_.name) } /** @@ -96,15 +106,15 @@ class ExtractorRoutingService { * @param operation The dataset operation requested. * @return A list of extractors IDs. */ - private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value): (List[String], List[String]) = { + private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value, user: Option[User] = None): (List[String], List[String]) = { val spacesService = DI.injector.getInstance(classOf[SpaceService]) var enabledExtractors = new ListBuffer[String]() var disabledExtractors = new ListBuffer[String]() dataset.spaces.map(space => { spacesService.getAllExtractors(space).foreach { extractors => - enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType)) - disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType)) + enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType, user)) + disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType, user)) } }) (enabledExtractors.toList, disabledExtractors.toList) @@ -145,7 +155,7 @@ class ExtractorRoutingService { * @param contentType the content type of the file in the case of a file * @return a set of unique rabbitmq queues */ - private def getQueues(dataset: Dataset, routingKey: String, contentType: String): Set[String] = { + private def getQueues(dataset: Dataset, routingKey: String, contentType: String, user: Option[User] = None): Set[String] = { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) // drop the first fragment from the routing key and replace characters to create operation id @@ -160,9 +170,9 @@ class ExtractorRoutingService { else return Set.empty[String] // get extractors enabled at the global level - val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType) + val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType, user) // get extractors enabled/disabled at the space level - val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType) + val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType, user) // get queues based on RabbitMQ bindings (old method). val queuesFromBindings = getQueuesFromBindings(routingKey) // take the union of queues so that we publish to a specific queue only once @@ -229,6 +239,7 @@ class ExtractorRoutingService { var jobId: Option[UUID] = None dataset match { case Some(d) => { + // TODO: Check private extractor behavior getQueues(d, routingKey, file.contentType).foreach { queue => val source = Entity(ResourceRef(ResourceRef.file, file.id), Some(file.contentType), sourceExtra) diff --git a/app/services/ExtractorService.scala b/app/services/ExtractorService.scala index 75acf38e1..d750eaf58 100644 --- a/app/services/ExtractorService.scala +++ b/app/services/ExtractorService.scala @@ -35,13 +35,13 @@ trait ExtractorService { def dropAllExtractorStatusCollection() - def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] + def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] - def getExtractorInfo(extractorName: String): Option[ExtractorInfo] + def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] - def deleteExtractor(extractorName: String) + def deleteExtractor(extractorName: String, extractorKey: Option[String]) def listExtractorsLabels(): List[ExtractorsLabel] diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala index 5eb95d557..c6eda512a 100644 --- a/app/services/MessageService.scala +++ b/app/services/MessageService.scala @@ -298,6 +298,7 @@ class EventFilter(channel: Channel, queue: String) extends Actor { * @param queue */ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { + val users: UserService = DI.injector.getInstance(classOf[UserService]) val extractions: ExtractionService = DI.injector.getInstance(classOf[ExtractionService]) val extractorsService: ExtractorService = DI.injector.getInstance(classOf[ExtractorService]) @@ -305,6 +306,7 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { case statusBody: String => Logger.debug("Received extractor heartbeat: " + statusBody) val json = Json.parse(statusBody) + Logger.debug(json.toString) // TODO store running extractors ids val id = UUID((json \ "id").as[String]) val queue = (json \ "queue").as[String] @@ -313,52 +315,78 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { // Validate document val extractionInfoResult = extractor_info.validate[ExtractorInfo] + // Determine if there is a user associated with this request + val owner = (json \ "owner").as[String] + val user: Option[User] = if (owner.length > 0) { + users.findByEmail(owner) + } else { + None + } + // Update database extractionInfoResult.fold( - errors => { - Logger.debug("Received extractor heartbeat with bad format: " + extractor_info) - }, + errors => Logger.debug("Received extractor heartbeat with bad format: " + extractor_info), info => { - extractorsService.getExtractorInfo(info.name) match { - case Some(infoFromDB) => { - // TODO only update if new semantic version is greater than old semantic version - if (infoFromDB.version != info.version) { - // TODO keep older versions of extractor info instead of just the latest one - extractorsService.updateExtractorInfo(info) - Logger.info("Updated extractor definition for " + info.name) + if (info.unique_key.isDefined && user.isEmpty) { + Logger.error("Extractor keys must have a user associated with them.") + } else { + extractorsService.getExtractorInfo(info.name, info.unique_key, user) match { + case Some(infoFromDB) => { + if (info.unique_key.isDefined) { + // Retain existing permissions + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=infoFromDB.permissions) + case None => info + } + extractorsService.updateExtractorInfo(registrationInfo) + Logger.info(s"Updated private extractor definition for ${info.name} - ${info.unique_key}") + } else { + // TODO only update if new semantic version is greater than old semantic version + if (infoFromDB.version != info.version) { + // TODO keep older versions of extractor info instead of just the latest one + extractorsService.updateExtractorInfo(info) + Logger.info(s"Updated extractor definition for ${info.name}") + } + } } - } - case None => { - extractorsService.updateExtractorInfo(info) match { - case None => {} - case Some(eInfo) => { - // Create (if needed) and assign default labels - eInfo.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") - } - extractorsService.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) + case None => { + // Inject user into permissions list if a key is given + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=List(ResourceRef('user, user.get.id))) + case None => info + } + extractorsService.updateExtractorInfo(registrationInfo) match { + case None => {} + case Some(eInfo) => { + // Create (if needed) and assign default labels + eInfo.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(eInfo.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) - val updatedLabel = extractorsService.updateExtractorsLabel(label) + extractorsService.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(eInfo.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) + val updatedLabel = extractorsService.updateExtractorsLabel(label) + } } } - } - }) + }) + } } - } - Logger.info(s"New extractor ${info.name} registered from heartbeat") + Logger.info(s"New extractor ${info.name} registered from heartbeat with key "+info.unique_key.toString) + } } + } } ) diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 039d7df06..3938e34bb 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -1,6 +1,6 @@ package services.mongodb -import javax.inject.Singleton +import javax.inject.{Inject, Singleton} import com.mongodb.casbah.Imports._ import com.mongodb.casbah.WriteConcern import com.mongodb.casbah.commons.MongoDBObject @@ -12,11 +12,12 @@ import play.api.Play.current import play.api.libs.json.{JsArray, JsNumber, JsObject, JsString, JsValue, Json} import services._ import services.mongodb.MongoContext.context - import org.bson.types.ObjectId @Singleton -class MongoDBExtractorService extends ExtractorService { +class MongoDBExtractorService @Inject() ( + users: MongoDBUserService + ) extends ExtractorService { def getExtractorServerIPList() = { var listServersIPs = List[String]() @@ -169,51 +170,106 @@ class MongoDBExtractorService extends ExtractorService { } } - def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] = { + def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] = { + Logger.info("listing: "+categories.toString) var list_queue = List[ExtractorInfo]() val allDocs = ExtractorInfoDAO.findAll().sort(orderBy = MongoDBObject("name" -> -1)) for (doc <- allDocs) { - // If no categories are specified, return all extractor names - var category_match = categories.isEmpty - if (!category_match) { + // If no filters are specified, return all extractor names + var filter_match = (categories.isEmpty && doc.permissions.isEmpty) + if (!filter_match) { // Otherwise check if any extractor categories overlap requested categories (force uppercase) + val user_match = user match { + case Some(u) => { + val rr = new ResourceRef('user, u) + doc.permissions.contains(rr) || doc.permissions.isEmpty + } + case None => doc.permissions.isEmpty // If no user filter in registered extractor, everyone can see + } val upper_categories = categories.map(cat => cat.toUpperCase) - category_match = doc.categories.intersect(upper_categories).length > 0 + val category_match = categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 + filter_match = (category_match && user_match) } - if (category_match) + if (filter_match) list_queue = doc :: list_queue } list_queue } - def getExtractorInfo(extractorName: String): Option[ExtractorInfo] = { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) + def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] = { + extractorKey match { + case Some(ek) => { + user match { + case None => { + Logger.error("User authentication required to view extractor info with a unique key.") + None + } + case Some(u) => { + val userRef = new ResourceRef('user, u.id) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek, "permissions" -> userRef)) + } + } + } + case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) + } } def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] = { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match { - case Some(old) => { - val updated = e.copy(id = old.id) - ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) - Some(updated) - } + // TODO: Make this account for version as well + e.unique_key match { case None => { - ExtractorInfoDAO.save(e) - Some(e) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false))) match { + case Some(old) => { + val updated = e.copy(id = old.id) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false)), updated, false, false, WriteConcern.Safe) + Some(updated) + } + case None => { + ExtractorInfoDAO.save(e) + Some(e) + } + } + } + case Some(ek) => { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> ek)) match { + case Some(old) => { + val updated = e.copy(id = old.id) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> ek), updated, false, false, WriteConcern.Safe) + Some(updated) + } + case None => { + ExtractorInfoDAO.save(e) + Some(e) + } + } } } } - def deleteExtractor(extractorName: String) { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) match { - case Some(extractor) => { - ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name)) + def deleteExtractor(extractorName: String, extractorKey: Option[String]) { + extractorKey match { + case Some(ek) => { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) match { + case Some(extractor) => { + ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> ek)) + } + case None => { + Logger.error(s"No extractor found with name ${extractorName} and key ${ek}") + } + } } case None => { - Logger.info("No extractor found with name: " + extractorName) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) match { + case Some(extractor) => { + ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> MongoDBObject("$exists" -> false))) + } + case None => { + Logger.error("No extractor found with name: " + extractorName) + } + } } } } @@ -246,15 +302,11 @@ class MongoDBExtractorService extends ExtractorService { def getLabelsForExtractor(extractorName: String): List[ExtractorsLabel] = { var results = List[ExtractorsLabel]() - ExtractorInfoDAO.findOne(MongoDBObject("name"->extractorName)) match { - case Some(info) => { - ExtractorsLabelDAO.findAll().foreach(label => { - if (label.extractors.contains(extractorName)) { - results = results ++ List[ExtractorsLabel](label) - } - }) + ExtractorsLabelDAO.findAll().foreach(label => { + if (label.extractors.contains(extractorName) && !results.contains(label)) { + results = results ++ List[ExtractorsLabel](label) } - } + }) results } } diff --git a/app/views/updateExtractors.scala.html b/app/views/updateExtractors.scala.html index 198f709c3..932713ed1 100644 --- a/app/views/updateExtractors.scala.html +++ b/app/views/updateExtractors.scala.html @@ -152,7 +152,7 @@

Extractor Catalog

Name Authors Version - Maturity + Unique Key All Jobs @if(showOptional("ratings")) { Rating } @@ -187,7 +187,7 @@

Extractor Catalog

- @extractor.maturity + @extractor.unique_key diff --git a/conf/routes b/conf/routes index 0eb5f62b6..9ab4f1b56 100644 --- a/conf/routes +++ b/conf/routes @@ -183,7 +183,7 @@ GET /extractors/labels GET /extractors/:extractorName @controllers.Extractors.showExtractorInfo(extractorName) GET /extractors/:extractorName/metrics @controllers.Extractors.showExtractorMetrics(extractorName) GET /extractors/:extractorName/logs @controllers.Extractors.showExtractorLog(extractorName) -GET /extractors/:extractorName/history @controllers.Extractors.showJobHistory(extractorName) +GET /extractors/:extractorName/history @controllers.Extractors.showJobHistory(extractorName, extractor_key: Option[String] ?= None) # ---------------------------------------------------------------------- @@ -423,9 +423,9 @@ POST /api/files/:id/sendUnarchiveRequest # ---------------------------------------------------------------------- # EXTRACTORS ENDPOINTS # ---------------------------------------------------------------------- -GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty) -GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String) -POST /api/extractors @api.Extractions.addExtractorInfo() +GET /api/extractors @api.Extractions.listExtractors(categories: List[String] ?= List.empty, space: Option[UUID] ?= None) +GET /api/extractors/:name @api.Extractions.getExtractorInfo(name: String, extractor_key: Option[String] ?= None) +POST /api/extractors @api.Extractions.addExtractorInfo(extractor_key: Option[String] ?= None, user: Option[String] ?= None) POST /api/extractors/labels @api.Extractions.createExtractorsLabel() PUT /api/extractors/labels/:id @api.Extractions.updateExtractorsLabel(id: UUID) DELETE /api/extractors/labels/:id @api.Extractions.deleteExtractorsLabel(id: UUID) @@ -447,7 +447,7 @@ GET /api/extractions/:id/status GET /api/extractions/:id/metadata @api.Extractions.fetch(id:UUID) GET /api/extractions/:id/statuses @api.Extractions.checkExtractionsStatuses(id:UUID) -DELETE /api/extractions/:extractorName/delete @api.Extractions.deleteExtractor(extractorName : String) +DELETE /api/extractions/:extractorName/delete @api.Extractions.deleteExtractor(extractorName : String, extractor_key: Option[String] ?= None) DELETE /api/files/:file_id/extractions/:msg_id @api.Extractions.cancelFileExtractionSubmission(file_id:UUID, msg_id: UUID) DELETE /api/datasets/:ds_id/extractions/:msg_id @api.Extractions.cancelDatasetExtractionSubmission(ds_id:UUID, msg_id: UUID)