From e40798427a1b5c0079eb9013d35c235c9a9294eb Mon Sep 17 00:00:00 2001 From: Dipannita Dey Date: Thu, 7 Apr 2022 15:42:24 -0500 Subject: [PATCH 1/8] Added logic for primary Clowder instance which handles multiple extractors problem and multiple indexing issue --- CHANGELOG.md | 3 ++ .../mongodb/MongoDBQueueService.scala | 3 +- .../rabbitmq/RabbitMQMessageService.scala | 39 ++++++++++--------- conf/application.conf | 1 + docker/custom.conf | 4 ++ 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b23d0aae..f6916f495 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased +- Fixes bug where same extractor shows up multiple times and all Clowder instances index db on reindex [#327](https://github.com/clowder-framework/clowder/issues/327) + ## 1.20.1 - 2022-04-04 ### Fixed diff --git a/app/services/mongodb/MongoDBQueueService.scala b/app/services/mongodb/MongoDBQueueService.scala index b52030465..6fab9539f 100644 --- a/app/services/mongodb/MongoDBQueueService.scala +++ b/app/services/mongodb/MongoDBQueueService.scala @@ -93,7 +93,8 @@ trait MongoDBQueueService { // start pool to being processing queue actions def listen() = { - if (queueTimer == null) { + //only if this is the primary clowder instance + if (queueTimer == null && configuration.getBoolean("clowder.primary").getOrElse(true)) { // TODO: Need to make these in a separate pool queueTimer = Akka.system().scheduler.schedule(0 seconds, 5 millis) { getNextQueuedAction match { diff --git a/app/services/rabbitmq/RabbitMQMessageService.scala b/app/services/rabbitmq/RabbitMQMessageService.scala index a5d5699a6..47ed9da6b 100644 --- a/app/services/rabbitmq/RabbitMQMessageService.scala +++ b/app/services/rabbitmq/RabbitMQMessageService.scala @@ -128,24 +128,27 @@ class RabbitMQMessageService extends MessageService { new MsgConsumer(channel.get, event_filter.get) ) - // Start actor to listen to extractor heartbeats - Logger.info("Starting extractor heartbeat listener") - // create fanout exchange if it doesn't already exist - channel.get.exchangeDeclare("extractors", "fanout", true) - // anonymous queue - val heartbeatsQueue = channel.get.queueDeclare().getQueue - // bind queue to exchange - channel.get.queueBind(heartbeatsQueue, "extractors", "*") - extractorsHeartbeats = Some(Akka.system.actorOf( - Props(new ExtractorsHeartbeats(channel.get, heartbeatsQueue)), name = "ExtractorsHeartbeats" - )) - Logger.debug("Initializing a MsgConsumer for the ExtractorsHeartbeats") - channel.get.basicConsume( - heartbeatsQueue, - false, // do not auto ack - "ExtractorsHeartbeats", // tagging the consumer is important if you want to stop it later - new MsgConsumer(channel.get, extractorsHeartbeats.get) - ) + //register new extractor only if this is the primary clowder instance + if (configuration.getBoolean("clowder.primary").getOrElse(true)) { + // Start actor to listen to extractor heartbeats + Logger.info("Starting extractor heartbeat listener") + // create fanout exchange if it doesn't already exist + channel.get.exchangeDeclare("extractors", "fanout", true) + // anonymous queue + val heartbeatsQueue = channel.get.queueDeclare().getQueue + // bind queue to exchange + channel.get.queueBind(heartbeatsQueue, "extractors", "*") + extractorsHeartbeats = Some(Akka.system.actorOf( + Props(new ExtractorsHeartbeats(channel.get, heartbeatsQueue)), name = "ExtractorsHeartbeats" + )) + Logger.debug("Initializing a MsgConsumer for the ExtractorsHeartbeats") + channel.get.basicConsume( + heartbeatsQueue, + false, // do not auto ack + "ExtractorsHeartbeats", // tagging the consumer is important if you want to stop it later + new MsgConsumer(channel.get, extractorsHeartbeats.get) + ) + } // Setup Actor to submit new extractions to broker extractQueue = Some(Akka.system.actorOf(Props(new PublishDirectActor(channel = channel.get, diff --git a/conf/application.conf b/conf/application.conf index c05870d3d..e08f4896a 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -190,6 +190,7 @@ mongodbURI = "mongodb://127.0.0.1:27017/clowder" # All requests to extractors and any other external process is send # using rabbitmq, this will setup the connection information. clowder.rabbitmq.uri="amqp://guest:guest@localhost:5672/%2f" +clowder.primary=true #clowder.rabbitmq.managmentPort=15672 #clowder.rabbitmq.exchange=clowder # Following variable will use the url when sending messages over rabbitmq. This can be used in a docker diff --git a/docker/custom.conf b/docker/custom.conf index be917b637..29fedbe9c 100644 --- a/docker/custom.conf +++ b/docker/custom.conf @@ -37,6 +37,10 @@ smtp.host=${?SMTP_SERVER} service.byteStorage=services.filesystem.DiskByteStorageService service.byteStorage=${?CLOWDER_STORAGE} +#primary Clowder instance +clowder.primary=true +clowder.primary=${?CLOWDER_PRIMARY} + # location in case of services.filesystem.DiskByteStorageService clowder.diskStorage.path="/home/clowder/data" clowder.diskStorage.path=${?CLOWDER_DISKPATH} From 160c5a12a405b2856e9b79584b946172a0932af7 Mon Sep 17 00:00:00 2001 From: Dipannita Dey Date: Tue, 12 Apr 2022 08:50:30 -0500 Subject: [PATCH 2/8] Added the review suggestions --- CHANGELOG.md | 2 ++ conf/application.conf | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6916f495..25ae56a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased + +### Fixed - Fixes bug where same extractor shows up multiple times and all Clowder instances index db on reindex [#327](https://github.com/clowder-framework/clowder/issues/327) ## 1.20.1 - 2022-04-04 diff --git a/conf/application.conf b/conf/application.conf index e08f4896a..2aad1f5df 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -183,6 +183,13 @@ api.version="beta" # mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] mongodbURI = "mongodb://127.0.0.1:27017/clowder" +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Clowder Primary Instance +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# This is required if there are multiple clowder instances. This config variable indicates +# the primary clowder instance who takes care of special actions like registering new extractors, +# indexing db on a reindex etc. The default value is true which means only one instance of clowder running. +clowder.primary=true # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # RabbitMQ @@ -190,7 +197,6 @@ mongodbURI = "mongodb://127.0.0.1:27017/clowder" # All requests to extractors and any other external process is send # using rabbitmq, this will setup the connection information. clowder.rabbitmq.uri="amqp://guest:guest@localhost:5672/%2f" -clowder.primary=true #clowder.rabbitmq.managmentPort=15672 #clowder.rabbitmq.exchange=clowder # Following variable will use the url when sending messages over rabbitmq. This can be used in a docker From be0902d428f7bba2429aef0ce931a10c8cfaa90d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 19 Apr 2022 14:39:17 -0500 Subject: [PATCH 3/8] fix masonry --- CHANGELOG.md | 5 +++++ app/views/spaces/space.scala.html | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b23d0aae..b5006da45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased + +### Fixed +- Fix for a rare race condition with masonry where tiles could end up overlapping in space page. + ## 1.20.1 - 2022-04-04 ### Fixed diff --git a/app/views/spaces/space.scala.html b/app/views/spaces/space.scala.html index 696b81112..363b67a3a 100644 --- a/app/views/spaces/space.scala.html +++ b/app/views/spaces/space.scala.html @@ -148,7 +148,7 @@

@Messages("collections.title")

match.masonry(); // layout Masonry again after all images have loaded - imagesLoaded( '.tiled-image', function() { + imagesLoaded( match, function() { match.masonry({ itemSelector: '.tiled-image', columnWidth: '.post-box', From 76cbcc8b96b1a3ddc8ac3e89e90b3bc6b0d3071d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Fri, 22 Apr 2022 14:39:20 -0500 Subject: [PATCH 4/8] fix/cleanup filename --- CHANGELOG.md | 5 +++++ app/util/FileUtils.scala | 39 +++++++++++---------------------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b23d0aae..4b0bc4a0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased + +### Fixed +- When downloading a file with a `'` in the name it would save the file as blob + ## 1.20.1 - 2022-04-04 ### Fixed diff --git a/app/util/FileUtils.scala b/app/util/FileUtils.scala index 1d66d2f93..fd5790f1d 100644 --- a/app/util/FileUtils.scala +++ b/app/util/FileUtils.scala @@ -856,37 +856,20 @@ object FileUtils { //Download CONTENT-DISPOSITION encoding // def encodeAttachment(filename: String, userAgent: String) : String = { - val filenameStar = if (userAgent.indexOf("MSIE") > -1) { - URLEncoder.encode(filename, "UTF-8") - } else if (userAgent.indexOf("Edge") > -1){ - MimeUtility.encodeText(filename - .replaceAll(",","%2C") - .replaceAll("\"","%22") - .replaceAll("/","%2F") - .replaceAll("=","%3D") - .replaceAll("&","%26") - .replaceAll(":","%3A") - .replaceAll(";","%3B") - .replaceAll("\\?","%3F") - .replaceAll("\\*","%2A") + val filenameStar = MimeUtility.encodeText(filename + .replaceAll("%","%25") + .replaceAll(" ","%20") + .replaceAll("\"","%22") + .replaceAll("'","%27") + .replaceAll(",","%2C") + .replaceAll("/","%2F") + .replaceAll("=","%3D") + .replaceAll(":","%3A") + .replaceAll(";","%3B") + .replaceAll("\\*","%2A") ,"utf-8","Q") - } else { - MimeUtility.encodeText(filename - .replaceAll("%","%25") - .replaceAll(" ","%20") - .replaceAll("\"","%22") - .replaceAll(",","%2C") - .replaceAll("/","%2F") - .replaceAll("=","%3D") - .replaceAll(":","%3A") - .replaceAll(";","%3B") - .replaceAll("\\*","%2A") - ,"utf-8","Q") - } - Logger.debug(userAgent + ": " + filenameStar) //Return the complete attachment header info "attachment; filename*=UTF-8''" + filenameStar } - } From e6f51e49adc9704bdeeddc5bef780381244c949d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 30 Apr 2022 11:06:28 -0500 Subject: [PATCH 5/8] version bump --- CHANGELOG.md | 2 ++ doc/src/sphinx/conf.py | 2 +- project/Build.scala | 2 +- public/swagger.yml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b23d0aae..2489e58b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## 1.20.2 - 2022-04-30 + ## 1.20.1 - 2022-04-04 ### Fixed diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py index c103902ec..cf120a58b 100644 --- a/doc/src/sphinx/conf.py +++ b/doc/src/sphinx/conf.py @@ -22,7 +22,7 @@ author = 'Luigi Marini' # The full version, including alpha/beta/rc tags -release = '1.20.1' +release = '1.20.2' # -- General configuration --------------------------------------------------- diff --git a/project/Build.scala b/project/Build.scala index e12fa7db2..196e7df4f 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -13,7 +13,7 @@ import NativePackagerKeys._ object ApplicationBuild extends Build { val appName = "clowder" - val version = "1.20.1" + val version = "1.20.2" val jvm = "1.7" def appVersion: String = { diff --git a/public/swagger.yml b/public/swagger.yml index 78ade0465..548169eae 100644 --- a/public/swagger.yml +++ b/public/swagger.yml @@ -9,7 +9,7 @@ info: Clowder is a customizable and scalable data management system to support any data format and multiple research domains. It is under active development and deployed for a variety of research projects. - version: 1.20.1 + version: 1.20.2 termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos contact: name: Clowder From 1503af5d0b2403f406ef75496ef3d0bc953f0fb5 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 30 Apr 2022 11:10:52 -0500 Subject: [PATCH 6/8] enabled by server admin (fixes #344) --- CHANGELOG.md | 3 +++ app/views/spaces/updateExtractors.scala.html | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2489e58b6..52532121f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## 1.20.2 - 2022-04-30 +### CHanged +- Changed `Enabled By SuperAdmin` to read `Enabled by Server Admin` [#344](https://github.com/clowder-framework/clowder/issues/344) + ## 1.20.1 - 2022-04-04 ### Fixed diff --git a/app/views/spaces/updateExtractors.scala.html b/app/views/spaces/updateExtractors.scala.html index 0e53789a7..6b4147435 100644 --- a/app/views/spaces/updateExtractors.scala.html +++ b/app/views/spaces/updateExtractors.scala.html @@ -35,8 +35,8 @@

Space Extractors

Name Description Process Triggers - Enabled By SuperAdmin - Use Default + Enabled by Server Admin + Use Default Enable in Space Disable in Space From c6e604ed8653d292783700404fc9166a43761847 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 30 Apr 2022 11:20:06 -0500 Subject: [PATCH 7/8] fix swagger lint --- .github/workflows/swagger.yml | 5 +++-- CHANGELOG.md | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index 9db7bbfca..e67d293df 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -22,7 +22,8 @@ jobs: steps: - uses: actions/checkout@v2 + example.openapi.yaml - name: openapi-lint - uses: mhiew/redoc-lint-github-action@v2 + uses: mbowman100/swagger-validator-action@master with: - args: 'public/swagger.yml --skip-rule operation-operationId' + files: public/swagger.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 52532121f..8c6c47351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## 1.20.2 - 2022-04-30 -### CHanged +### Fixed +- swagger lint action + +### Changed - Changed `Enabled By SuperAdmin` to read `Enabled by Server Admin` [#344](https://github.com/clowder-framework/clowder/issues/344) ## 1.20.1 - 2022-04-04 From 9b6e7625a8fb90276708fe2a6253fdff8c055159 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 30 Apr 2022 11:21:12 -0500 Subject: [PATCH 8/8] remove typo --- .github/workflows/swagger.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index e67d293df..b0f328a67 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -22,7 +22,6 @@ jobs: steps: - uses: actions/checkout@v2 - example.openapi.yaml - name: openapi-lint uses: mbowman100/swagger-validator-action@master with: