From b19e0cb56b38872dbd30164d30f8040b0eb9cd52 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 13 Aug 2020 22:42:59 -0700 Subject: [PATCH 01/12] circle filter --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bbcfcda5d..905caab33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,12 +68,15 @@ workflows: build-test-deploy: jobs: - build_and_test + filters: + tags: + only: /[0-9.]+(-(a|b|rc)[0-9]+)?/ - deploy_snapshot: requires: - build_and_test filters: branches: - only: /((release|hotfix|support)/[0-9.]+(-(a|b|rc)[0-9]+)?|master|develop)/ + only: /((release|hotfix|support)/[0-9.]+(-(a|b|rc)[0-9]+)?|develop)/ - deploy_release: requires: - build_and_test From 738001826694228f0dae21b156cb2e9416cad2a1 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 13 Aug 2020 23:02:51 -0700 Subject: [PATCH 02/12] go back to snapshot --- .circleci/config.yml | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 905caab33..eabe38789 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,7 +67,7 @@ workflows: version: 2 build-test-deploy: jobs: - - build_and_test + - build_and_test: filters: tags: only: /[0-9.]+(-(a|b|rc)[0-9]+)?/ diff --git a/gradle.properties b/gradle.properties index b9bcc01e9..8eb2def0d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0-alpha-1 +version=4.0.0-SNAPSHOT group=org.openmbee.mms springBootVersion=2.2.6.RELEASE From 115c17354c20489df981dea9ed38dfa406c225cd Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 3 Sep 2020 14:30:33 -0700 Subject: [PATCH 03/12] Adding new bulk processor for elasticsearch implementation --- Dockerfile | 4 +- .../sdvc/elastic/BaseElasticDAOImpl.java | 63 ++++++++++++++++--- example/example.gradle | 1 - .../example/config/ExampleSecurityConfig.java | 5 -- gradle.properties | 2 +- .../TwcRevisionMmsCommitMapService.java | 2 +- 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index b1e42e79b..70d3db842 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM openjdk:11.0.8-jdk COPY . /mms WORKDIR /mms -RUN ./gradlew --no-daemon build -x test +RUN ./gradlew --no-daemon bootJar RUN cp /mms/example/build/libs/example*.jar /app.jar -ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED","-jar", "/app.jar"] +ENTRYPOINT ["java", "-Djdk.tls.client.protocols=TLSv1", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"] EXPOSE 8080 diff --git a/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java index 179070f03..49c9e6b1d 100644 --- a/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java @@ -7,8 +7,14 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.BackoffPolicy; +import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -21,8 +27,12 @@ import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.openmbee.sdvc.core.exceptions.SdvcException; import org.openmbee.sdvc.elastic.utils.Index; import org.openmbee.sdvc.json.BaseJson; import org.springframework.beans.factory.annotation.Autowired; @@ -31,6 +41,8 @@ public abstract class BaseElasticDAOImpl> { + private final Logger logger = LogManager.getLogger(getClass()); + @Value("${elasticsearch.limit.result}") protected int resultLimit; @Value("${elasticsearch.limit.term}") @@ -39,10 +51,10 @@ public abstract class BaseElasticDAOImpl> { protected RestHighLevelClient client; private static final RequestOptions REQUEST_OPTIONS; static { - RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + RequestOptions.Builder requestBuilder = RequestOptions.DEFAULT.toBuilder(); // TODO: Should be configureable - builder.setHttpAsyncResponseConsumerFactory(new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(1024 * 1024 * 1024)); - REQUEST_OPTIONS = builder.build(); + requestBuilder.setHttpAsyncResponseConsumerFactory(new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(1024 * 1024 * 1024)); + REQUEST_OPTIONS = requestBuilder.build(); } @Autowired @@ -133,18 +145,18 @@ public List findAllById(String index, Set docIds) { } public void indexAll(String index, Collection jsons) { + BulkProcessor bulkProcessor = getBulkProcessor(client); try { - BulkRequest bulkIndex = new BulkRequest(); - for (BaseJson json : jsons) { - bulkIndex.add(new IndexRequest(index).id(json.getDocId()).source(json)); + if(!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) { + logger.error("Timed out in bulk processing"); } - client.bulk(bulkIndex, REQUEST_OPTIONS); - } catch (IOException e) { - throw new RuntimeException(e); + } catch (InterruptedException e) { + logger.error(e); } + } - public void index(String index, BaseJson json) { + public void index(String index, BaseJson json) { try { client.index(new IndexRequest(index).id(json.getDocId()).source(json), REQUEST_OPTIONS); @@ -174,4 +186,35 @@ public E update(String index, BaseJson json) { } return response; } + + private static BulkProcessor getBulkProcessor(RestHighLevelClient client) { + return getBulkProcessor(client, null); + } + + private static BulkProcessor getBulkProcessor(RestHighLevelClient client, BulkProcessor.Listener listener) { + if (listener == null) { + listener = new BulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + } + }; + } + BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> client + .bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); + bpBuilder.setBulkActions(5000); + bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)); + bpBuilder.setConcurrentRequests(1); + bpBuilder.setFlushInterval(TimeValue.timeValueSeconds(5)); + bpBuilder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(100), 3)); + + return bpBuilder.build(); + } } diff --git a/example/example.gradle b/example/example.gradle index 8aefd1c54..946eebb4f 100644 --- a/example/example.gradle +++ b/example/example.gradle @@ -24,7 +24,6 @@ dependencies { project(':jupyter'), project(':permissions'), project(':webhooks'), - project(':twc'), project(':search'), project(':artifacts'), 'org.springframework.boot:spring-boot-starter-web', diff --git a/example/src/main/java/org/openmbee/sdvc/example/config/ExampleSecurityConfig.java b/example/src/main/java/org/openmbee/sdvc/example/config/ExampleSecurityConfig.java index 9aea0ce22..f42a5657c 100644 --- a/example/src/main/java/org/openmbee/sdvc/example/config/ExampleSecurityConfig.java +++ b/example/src/main/java/org/openmbee/sdvc/example/config/ExampleSecurityConfig.java @@ -1,7 +1,6 @@ package org.openmbee.sdvc.example.config; import org.openmbee.sdvc.authenticator.config.AuthSecurityConfig; -import org.openmbee.sdvc.twc.config.TwcAuthSecurityConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -36,15 +35,11 @@ public class ExampleSecurityConfig extends WebSecurityConfigurerAdapter implemen @Autowired AuthSecurityConfig authSecurityConfig; - @Autowired - TwcAuthSecurityConfig twcAuthSecurityConfig; - @Override public void configure(HttpSecurity http) throws Exception { http.csrf().disable().authorizeRequests().anyRequest().permitAll().and().httpBasic(); http.headers().cacheControl(); http.addFilterAfter(corsFilter(), ExceptionTranslationFilter.class); - twcAuthSecurityConfig.setAuthConfig(http); authSecurityConfig.setAuthConfig(http); } diff --git a/gradle.properties b/gradle.properties index 8eb2def0d..9906d3482 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,4 +7,4 @@ springSecurityVersion=5.3.1.RELEASE springDataVersion=2.2.6.RELEASE jacksonVersion=2.10.3 log4jVersion=2.13.1 -elasticVersion=7.1.1 +elasticVersion=7.9.0 diff --git a/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java b/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java index de07f4a79..bd7f6ca30 100644 --- a/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java +++ b/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java @@ -36,7 +36,7 @@ public void setBranchRepository(BranchDAO branchRepository) { */ public CommitsResponse updateTwcRevisionID(String projectId, String commitId, String revisionId) { CommitsResponse commitsResponse = new CommitsResponse(); - if (revisionId.isEmpty() || revisionId.isBlank()) { + if (revisionId.isEmpty()) { return commitsResponse.addMessage("Revision id can not be empty"); } ContextHolder.setContext(projectId); From 381230aafba6d9e4fc0894ce368f5884c879d17e Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 15 Sep 2020 17:58:06 -0700 Subject: [PATCH 04/12] Add back IndexRequest adding --- .../java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java index 49c9e6b1d..cc3152c51 100644 --- a/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java @@ -146,6 +146,9 @@ public List findAllById(String index, Set docIds) { public void indexAll(String index, Collection jsons) { BulkProcessor bulkProcessor = getBulkProcessor(client); + for (BaseJson json : jsons) { + bulkProcessor.add(new IndexRequest(index).id(json.getDocId()).source(json)); + } try { if(!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) { logger.error("Timed out in bulk processing"); From e33b453b4385aebab9eedc508b0758b26cc4f7a5 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 17 Sep 2020 17:54:14 -0700 Subject: [PATCH 05/12] release to bintray --- .circleci/config.yml | 7 +++++-- build.gradle | 35 +++++++++++++++++++++++++++++------ gradle.properties | 2 +- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eabe38789..de870b02b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: at: ~/ - run: name: Deploy snapshot to Artifactory - command: ./gradlew -PbuildAccess=opensource -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$SNAPSHOT_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish + command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$SNAPSHOT_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish deploy_release: executor: openjdk_executor working_directory: /home/circleci/mms @@ -61,7 +61,10 @@ jobs: at: ~/ - run: name: Deploy release to Artifactory - command: ./gradlew -PbuildAccess=opensource -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$RELEASE_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish + command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$RELEASE_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish + - run: + name: Deploy release to Bintray + command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PbintrayUser=$BINTRAY_USER -PbintrayKey=$BINTRAY_KEY --info --stacktrace bintrayUpload workflows: version: 2 diff --git a/build.gradle b/build.gradle index 65a379e73..5a5b1e225 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,7 @@ buildscript { dependencies { classpath "org.springframework.boot:spring-boot-gradle-plugin:$springBootVersion" classpath "org.jfrog.buildinfo:build-info-extractor-gradle:latest.release" + classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4' } repositories { maven { url 'https://repo.spring.io/plugins-release' } @@ -12,10 +13,6 @@ buildscript { description = 'Structured Data Version Control' -ext.snapshotBuild = version.contains("SNAPSHOT") -ext.releaseBuild = !version.contains("SNAPSHOT") -ext.milestoneBuild = !(snapshotBuild || releaseBuild) - ext { commonDependencies = [ 'swagger-annotations' : 'io.swagger.core.v3:swagger-annotations:2.1.2', @@ -42,6 +39,17 @@ ext { ] } +String buildTag = project.getProperties().get('buildTag') +if (buildTag != null && !buildTag.isEmpty() && buildTag != version) { + throw new GradleException('Version mismatch: '+buildTag+' vs '+version) +} +if (buildTag == null || buildTag.isEmpty() || project.getProperties().getOrDefault('artifactoryRepository', '').toLowerCase().contains('snapshot')) { + version += '-SNAPSHOT' +} +ext.snapshotBuild = version.contains("SNAPSHOT") +ext.releaseBuild = !version.contains("SNAPSHOT") +ext.milestoneBuild = !(snapshotBuild || releaseBuild) + subprojects { repositories { maven { url 'https://repo.spring.io/plugins-release' } @@ -53,13 +61,13 @@ subprojects { project.sourceCompatibility = '10' project.targetCompatibility = '10' } -} + version = rootProject.version -allprojects { apply plugin: 'java-library' apply plugin: 'maven-publish' apply plugin: 'io.spring.dependency-management' apply plugin: "com.jfrog.artifactory" + apply plugin: 'com.jfrog.bintray' Map commonDependencies = rootProject.ext.commonDependencies @@ -111,4 +119,19 @@ allprojects { } } } + + bintray { + user = project.getProperties().get('bintrayUser') + key = project.getProperties().get('bintrayKey') + publications = ['mavenJava'] + publish = true + dryRun = true + pkg { + repo = 'maven' + name = project.name + userOrg = 'openmbee' + licenses = ['Apache-2.0'] + vcsUrl = 'https://github.com/Open-MBEE/mms.git' + } + } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 8eb2def0d..1f2de8cf8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0-SNAPSHOT +version=4.0.0 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE From db619ad66d60e8a0d8983863af2cbde249012e68 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Fri, 18 Sep 2020 11:11:39 -0700 Subject: [PATCH 06/12] add mms prefix to bintray package name --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5a5b1e225..ba709622a 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ subprojects { dryRun = true pkg { repo = 'maven' - name = project.name + name = 'mms-' + project.name userOrg = 'openmbee' licenses = ['Apache-2.0'] vcsUrl = 'https://github.com/Open-MBEE/mms.git' From c1a25a07348ffc64f0b3dda8c2b6497741989d20 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Fri, 18 Sep 2020 12:32:56 -0700 Subject: [PATCH 07/12] add optional db prefiix config --- example/src/main/resources/application-test.properties | 2 ++ permissions/permissions.gradle | 4 +--- .../openmbee/sdvc/rdb/config/DatabaseDefinitionService.java | 4 ++-- .../org/openmbee/sdvc/rdb/datasources/CrudDataSources.java | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/example/src/main/resources/application-test.properties b/example/src/main/resources/application-test.properties index f32632cfb..652c8cf5b 100644 --- a/example/src/main/resources/application-test.properties +++ b/example/src/main/resources/application-test.properties @@ -6,6 +6,8 @@ jwt.secret=12345678901234567890123456789012 jwt.expiration=86400 jwt.header=Authorization +rdb.project.prefix=mms + # See ldap module for example configuration ldap.provider.base=ou=something,dc=openmbee,dc=org ldap.provider.url=ldaps://ldap.openmbee.org/${ldap.provider.base} diff --git a/permissions/permissions.gradle b/permissions/permissions.gradle index 1ada45812..22c620275 100644 --- a/permissions/permissions.gradle +++ b/permissions/permissions.gradle @@ -1,7 +1,5 @@ -apply plugin: 'java-library' -apply plugin: 'io.spring.dependency-management' dependencies { implementation project(':rdb') implementation 'org.apache.commons:commons-lang3:3.10' - implementation 'org.springframework.boot:spring-boot:2.2.6.RELEASE' + implementation commonDependencies.'spring-boot' } diff --git a/rdb/src/main/java/org/openmbee/sdvc/rdb/config/DatabaseDefinitionService.java b/rdb/src/main/java/org/openmbee/sdvc/rdb/config/DatabaseDefinitionService.java index 198548f1c..d5e4292dd 100644 --- a/rdb/src/main/java/org/openmbee/sdvc/rdb/config/DatabaseDefinitionService.java +++ b/rdb/src/main/java/org/openmbee/sdvc/rdb/config/DatabaseDefinitionService.java @@ -55,8 +55,8 @@ public void setCrudDataSources(CrudDataSources crudDataSources) { public boolean createProjectDatabase(Project project) throws SQLException { ContextHolder.setContext(null); - //TODO sanitize projectId? or reject if not valid id - String queryString = String.format("CREATE DATABASE \"_%s\"", project.getProjectId()); + String prefix = env.getProperty("rdb.project.prefix", ""); + String queryString = String.format("CREATE DATABASE \"%s_%s\"", prefix, project.getProjectId()); JdbcTemplate jdbcTemplate = new JdbcTemplate( crudDataSources.getDataSource(ContextHolder.getContext().getKey())); List created = new ArrayList<>(); diff --git a/rdb/src/main/java/org/openmbee/sdvc/rdb/datasources/CrudDataSources.java b/rdb/src/main/java/org/openmbee/sdvc/rdb/datasources/CrudDataSources.java index 6f74c1c44..577c2d465 100644 --- a/rdb/src/main/java/org/openmbee/sdvc/rdb/datasources/CrudDataSources.java +++ b/rdb/src/main/java/org/openmbee/sdvc/rdb/datasources/CrudDataSources.java @@ -72,8 +72,9 @@ public boolean removeDataSource(String projectId) { public DataSource addDataSource(Project project) { String url = project.getConnectionString(); + String prefix = env.getProperty("rdb.project.prefix", ""); if (url == null || url.isEmpty()) { - url = env.getProperty("spring.datasource.url") + "/_" + project.getProjectId(); + url = env.getProperty("spring.datasource.url") + "/" + prefix + "_" + project.getProjectId(); } DataSource dataSource = buildDataSource(url); targetDataSources.put(project.getProjectId(), dataSource); From 7607bbafe30622efdbec2d61b795185cea564bac Mon Sep 17 00:00:00 2001 From: Jason Han Date: Fri, 18 Sep 2020 13:10:09 -0700 Subject: [PATCH 08/12] Adding null check and logging. Remove ldap from example --- .../org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java | 11 ++++++----- example/example.gradle | 1 - gradle.properties | 2 +- .../twc/services/TwcRevisionMmsCommitMapService.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java index cc3152c51..77864b63a 100644 --- a/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java @@ -8,8 +8,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkProcessor; @@ -32,16 +30,17 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.openmbee.sdvc.core.exceptions.SdvcException; import org.openmbee.sdvc.elastic.utils.Index; import org.openmbee.sdvc.json.BaseJson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; public abstract class BaseElasticDAOImpl> { - private final Logger logger = LogManager.getLogger(getClass()); + private final Logger logger = LoggerFactory.getLogger(getClass()); @Value("${elasticsearch.limit.result}") protected int resultLimit; @@ -154,7 +153,7 @@ public void indexAll(String index, Collection jsons) { logger.error("Timed out in bulk processing"); } } catch (InterruptedException e) { - logger.error(e); + logger.error("Index all interrupted: ", e); } } @@ -197,6 +196,7 @@ private static BulkProcessor getBulkProcessor(RestHighLevelClient client) { private static BulkProcessor getBulkProcessor(RestHighLevelClient client, BulkProcessor.Listener listener) { if (listener == null) { listener = new BulkProcessor.Listener() { + private final Logger logger = LoggerFactory.getLogger(getClass()); @Override public void beforeBulk(long executionId, BulkRequest request) { } @@ -207,6 +207,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Error in bulk processing: ", failure); } }; } diff --git a/example/example.gradle b/example/example.gradle index 946eebb4f..bd0bf45ef 100644 --- a/example/example.gradle +++ b/example/example.gradle @@ -18,7 +18,6 @@ dependencies { implementation( project(':authenticator'), project(':localuser'), - project(':ldap'), project(':cameo'), project(':elastic'), project(':jupyter'), diff --git a/gradle.properties b/gradle.properties index 9906d3482..268a966a9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,4 +7,4 @@ springSecurityVersion=5.3.1.RELEASE springDataVersion=2.2.6.RELEASE jacksonVersion=2.10.3 log4jVersion=2.13.1 -elasticVersion=7.9.0 +elasticVersion=7.7.1 diff --git a/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java b/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java index bd7f6ca30..04ae42101 100644 --- a/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java +++ b/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java @@ -36,7 +36,7 @@ public void setBranchRepository(BranchDAO branchRepository) { */ public CommitsResponse updateTwcRevisionID(String projectId, String commitId, String revisionId) { CommitsResponse commitsResponse = new CommitsResponse(); - if (revisionId.isEmpty()) { + if (revisionId != null && revisionId.isEmpty()) { return commitsResponse.addMessage("Revision id can not be empty"); } ContextHolder.setContext(projectId); From c7be98a5c6172573c912271241992d74a408dfbf Mon Sep 17 00:00:00 2001 From: Jason Han Date: Fri, 18 Sep 2020 14:45:05 -0700 Subject: [PATCH 09/12] Fix comp --- .../sdvc/twc/services/TwcRevisionMmsCommitMapService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java b/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java index 04ae42101..9548e32dc 100644 --- a/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java +++ b/twc/src/main/java/org/openmbee/sdvc/twc/services/TwcRevisionMmsCommitMapService.java @@ -36,7 +36,7 @@ public void setBranchRepository(BranchDAO branchRepository) { */ public CommitsResponse updateTwcRevisionID(String projectId, String commitId, String revisionId) { CommitsResponse commitsResponse = new CommitsResponse(); - if (revisionId != null && revisionId.isEmpty()) { + if (revisionId == null || revisionId.isEmpty()) { return commitsResponse.addMessage("Revision id can not be empty"); } ContextHolder.setContext(projectId); From b1b216fa833cda28691649da6427edb221b6f5c1 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 23 Sep 2020 15:13:41 -0700 Subject: [PATCH 10/12] separate configs --- build.gradle | 1 - .../localuser/config/AuthProviderConfig.java | 51 +++++++++++++++++++ .../config/LocalUserSecurityConfig.java | 49 ++---------------- .../config/PasswordEncoderConfig.java | 24 +++++++++ 4 files changed, 80 insertions(+), 45 deletions(-) create mode 100644 localuser/src/main/java/org/openmbee/sdvc/localuser/config/AuthProviderConfig.java create mode 100644 localuser/src/main/java/org/openmbee/sdvc/localuser/config/PasswordEncoderConfig.java diff --git a/build.gradle b/build.gradle index ba709622a..b03e98369 100644 --- a/build.gradle +++ b/build.gradle @@ -125,7 +125,6 @@ subprojects { key = project.getProperties().get('bintrayKey') publications = ['mavenJava'] publish = true - dryRun = true pkg { repo = 'maven' name = 'mms-' + project.name diff --git a/localuser/src/main/java/org/openmbee/sdvc/localuser/config/AuthProviderConfig.java b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/AuthProviderConfig.java new file mode 100644 index 000000000..fa40ae41e --- /dev/null +++ b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/AuthProviderConfig.java @@ -0,0 +1,51 @@ +package org.openmbee.sdvc.localuser.config; + +import org.openmbee.sdvc.localuser.security.UserDetailsServiceImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.authentication.dao.DaoAuthenticationProvider; +import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +public class AuthProviderConfig { + + private static Logger logger = LoggerFactory.getLogger(LocalUserSecurityConfig.class); + + private UserDetailsServiceImpl userDetailsService; + private PasswordEncoder passwordEncoder; + + @Value("${sdvc.admin.username}") + private String adminUsername; + @Value("${sdvc.admin.password}") + private String adminPassword; + + @Autowired + public void setUserDetailsService(UserDetailsServiceImpl userDetailsService) { + this.userDetailsService = userDetailsService; + } + + @Autowired + public void setPasswordEncoder(PasswordEncoder passwordEncoder) { + this.passwordEncoder = passwordEncoder; + } + + @Bean + public DaoAuthenticationProvider daoAuthenticationProvider() { + try { + userDetailsService.loadUserByUsername(adminUsername); + } catch (UsernameNotFoundException e) { + userDetailsService.register(adminUsername, adminPassword, true); + logger.info(String.format("Creating root user: %s with specified password.", + adminUsername)); + } + DaoAuthenticationProvider authProvider = new DaoAuthenticationProvider(); + authProvider.setUserDetailsService(userDetailsService); + authProvider.setPasswordEncoder(passwordEncoder); + return authProvider; + } +} diff --git a/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java index 998d835a7..5c728bc4a 100644 --- a/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java +++ b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java @@ -1,67 +1,28 @@ package org.openmbee.sdvc.localuser.config; ; -import org.openmbee.sdvc.localuser.security.UserDetailsServiceImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.security.authentication.dao.DaoAuthenticationProvider; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; -import org.springframework.security.core.userdetails.UsernameNotFoundException; -import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; -import org.springframework.security.crypto.password.PasswordEncoder; @Configuration @EnableGlobalMethodSecurity(prePostEnabled = true) public class LocalUserSecurityConfig { private static Logger logger = LoggerFactory.getLogger(LocalUserSecurityConfig.class); - @Autowired - public UserDetailsServiceImpl userDetailsService; - @Value("${sdvc.admin.username}") - private String adminUsername; - @Value("${sdvc.admin.password}") - private String adminPassword; - public LocalUserSecurityConfig() { - } + private DaoAuthenticationProvider daoAuthenticationProvider; - @Bean - public PasswordEncoder passwordEncoder() { - return new BCryptPasswordEncoder(){ - //Turn off warnings for null/empty passwords - @Override - public boolean matches(CharSequence rawPassword, String encodedPassword) { - if (encodedPassword == null || encodedPassword.length() == 0) { - return false; - } - return super.matches(rawPassword, encodedPassword); - } - }; + @Autowired + public void setDaoAuthenticationProvider(DaoAuthenticationProvider daoAuthenticationProvider) { + this.daoAuthenticationProvider = daoAuthenticationProvider; } @Autowired public void configureDaoAuth(AuthenticationManagerBuilder auth) { - auth.authenticationProvider(authenticationProvider()); + auth.authenticationProvider(daoAuthenticationProvider); } - - @Bean - public DaoAuthenticationProvider authenticationProvider() { - try { - userDetailsService.loadUserByUsername(adminUsername); - } catch (UsernameNotFoundException e) { - userDetailsService.register(adminUsername, adminPassword, true); - logger.info(String.format("Creating root user: %s with specified password.", - adminUsername)); - } - DaoAuthenticationProvider authProvider = new DaoAuthenticationProvider(); - authProvider.setUserDetailsService(userDetailsService); - authProvider.setPasswordEncoder(passwordEncoder()); - return authProvider; - } - } diff --git a/localuser/src/main/java/org/openmbee/sdvc/localuser/config/PasswordEncoderConfig.java b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/PasswordEncoderConfig.java new file mode 100644 index 000000000..1c9655215 --- /dev/null +++ b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/PasswordEncoderConfig.java @@ -0,0 +1,24 @@ +package org.openmbee.sdvc.localuser.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +public class PasswordEncoderConfig { + + @Bean + public PasswordEncoder passwordEncoder() { + return new BCryptPasswordEncoder(){ + //Turn off warnings for null/empty passwords + @Override + public boolean matches(CharSequence rawPassword, String encodedPassword) { + if (encodedPassword == null || encodedPassword.length() == 0) { + return false; + } + return super.matches(rawPassword, encodedPassword); + } + }; + } +} From cab6b6f737758ed3915c12ca3808fbfce12fea9c Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 23 Sep 2020 15:39:25 -0700 Subject: [PATCH 11/12] fix --- .../sdvc/localuser/config/LocalUserSecurityConfig.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java index 5c728bc4a..be13fb86b 100644 --- a/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java +++ b/localuser/src/main/java/org/openmbee/sdvc/localuser/config/LocalUserSecurityConfig.java @@ -14,15 +14,9 @@ public class LocalUserSecurityConfig { private static Logger logger = LoggerFactory.getLogger(LocalUserSecurityConfig.class); - private DaoAuthenticationProvider daoAuthenticationProvider; - - @Autowired - public void setDaoAuthenticationProvider(DaoAuthenticationProvider daoAuthenticationProvider) { - this.daoAuthenticationProvider = daoAuthenticationProvider; - } - @Autowired - public void configureDaoAuth(AuthenticationManagerBuilder auth) { + public void configureDaoAuth(AuthenticationManagerBuilder auth, + DaoAuthenticationProvider daoAuthenticationProvider) { auth.authenticationProvider(daoAuthenticationProvider); } } From 32d5fa825a4f4219acb33b2386891d16145fb18a Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 23 Sep 2020 16:56:21 -0700 Subject: [PATCH 12/12] Bump version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 6e528c733..7aa18d96a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0 +version=4.0.0-a2 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE