From 2264dd6bfe22b68a1bcad4a146436c0b8319e488 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 09:01:10 -0800 Subject: [PATCH 01/17] Add configuration for MavenCentral sonatype repository upload. --- build.gradle | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 779f1045a..c24824613 100644 --- a/build.gradle +++ b/build.gradle @@ -59,19 +59,32 @@ subprojects { project.sourceCompatibility = '10' project.targetCompatibility = '10' } - version = rootProject.version apply plugin: 'java-library' apply plugin: 'maven-publish' apply plugin: 'io.spring.dependency-management' apply plugin: "com.jfrog.artifactory" apply plugin: 'com.jfrog.bintray' + apply plugin: 'maven' + apply plugin: 'signing' + + group = "org.openmbee.mms" + archivesBaseName = "mms-" + project.name + version = rootProject.version Map commonDependencies = rootProject.ext.commonDependencies + task javadocJar(type: Jar) { + from javadoc + } + + task sourcesJar(type: Jar) { + from sourceSets.main.allSource + } + java { - //withJavadocJar() - //withSourcesJar() + withJavadocJar() + withSourcesJar() } javadoc { @@ -79,6 +92,15 @@ subprojects { options.addBooleanOption('html5', true) } } + + artifacts { + archives javadocJar, sourcesJar + } + + signing { + sign configurations.archives + } + //https://docs.gradle.org/current/userguide/publishing_maven.html publishing { publications { @@ -131,4 +153,56 @@ subprojects { vcsUrl = 'https://github.com/Open-MBEE/mms.git' } } + + uploadArchives { + repositories { + mavenDeployer { + beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } + + repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + + snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + + pom.project { + name 'mms-' + project.name + packaging 'jar' + // optionally artifactId can be defined here + description project.description + url 'http://github.com/OpenMBEE/mms' + + scm { + connection 'scm:git:https://github.com/Open-MBEE/mms.git' + developerConnection 'scm:git:https://github.com/Open-MBEE/mms.git' + url 'https://github.com/Open-MBEE/mms.git' + } + + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + + developers { + developer { + id 'huijun' + name 'Jason Han' + } + developer { + id 'dlam' + name 'Doris Lam' + } + developer { + id 'ward' + name 'Ward Schweitzer' + } + } + } + } + } + } } \ No newline at end of file From c75bd72a82ae90ef5363a6db1d10a6accceb691a Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 09:44:24 -0800 Subject: [PATCH 02/17] Add classifier --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index c24824613..367d92545 100644 --- a/build.gradle +++ b/build.gradle @@ -75,10 +75,12 @@ subprojects { Map commonDependencies = rootProject.ext.commonDependencies task javadocJar(type: Jar) { + archiveClassifier = "javadoc" from javadoc } task sourcesJar(type: Jar) { + archiveClassifier = "sources" from sourceSets.main.allSource } From 30f1daf01c5ce322c4ac97b5d365d9856e14f49e Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Tue, 16 Feb 2021 12:38:45 -0800 Subject: [PATCH 03/17] copy over creator and created keys on element overwrite --- .../java/org/openmbee/mms/crud/services/NodePostHelper.java | 2 ++ gradle.properties | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java index 082b0db21..e5cf967c5 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java @@ -91,6 +91,8 @@ public NodeChangeInfo processPostJson(List elements, boolean overwr } } else { updated = true; + element.setCreated(indexElement.getCreated()); + element.setCreator(indexElement.getCreator()); } } diff --git a/gradle.properties b/gradle.properties index 54c95dcf2..88dfda6dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0 +version=4.0.1 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE From 59e49b152446efbc54f07731172f133830c109ba Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 13:26:17 -0800 Subject: [PATCH 04/17] Conditional sonatype deployment --- build.gradle | 78 +++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/build.gradle b/build.gradle index 367d92545..c509ba834 100644 --- a/build.gradle +++ b/build.gradle @@ -100,7 +100,9 @@ subprojects { } signing { - sign configurations.archives + if (project.hasProperty('signing.keyId') && project.hasProperty('signing.password') && project.hasProperty('signing.secretKeyRingFile')) { + sign configurations.archives + } } //https://docs.gradle.org/current/userguide/publishing_maven.html @@ -158,49 +160,51 @@ subprojects { uploadArchives { repositories { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } + if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) { + mavenDeployer { + beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - pom.project { - name 'mms-' + project.name - packaging 'jar' - // optionally artifactId can be defined here - description project.description - url 'http://github.com/OpenMBEE/mms' - - scm { - connection 'scm:git:https://github.com/Open-MBEE/mms.git' - developerConnection 'scm:git:https://github.com/Open-MBEE/mms.git' - url 'https://github.com/Open-MBEE/mms.git' + repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { + authentication(userName: ossrhUsername, password: ossrhPassword) } - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } + snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { + authentication(userName: ossrhUsername, password: ossrhPassword) } - developers { - developer { - id 'huijun' - name 'Jason Han' + pom.project { + name 'mms-' + project.name + packaging 'jar' + // optionally artifactId can be defined here + description project.description + url 'http://github.com/OpenMBEE/mms' + + scm { + connection 'scm:git:https://github.com/Open-MBEE/mms.git' + developerConnection 'scm:git:https://github.com/Open-MBEE/mms.git' + url 'https://github.com/Open-MBEE/mms.git' } - developer { - id 'dlam' - name 'Doris Lam' + + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } } - developer { - id 'ward' - name 'Ward Schweitzer' + + developers { + developer { + id 'huijun' + name 'Jason Han' + } + developer { + id 'dlam' + name 'Doris Lam' + } + developer { + id 'ward' + name 'Ward Schweitzer' + } } } } From bce560d461092a13e048b1e2b2b2dec384788423 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 13:36:27 -0800 Subject: [PATCH 05/17] Fix new package name in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9d3830627..0cfff2c5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ COPY . /mms WORKDIR /mms RUN ./gradlew --no-daemon bootJar -RUN cp /mms/example/build/libs/example*.jar /app.jar +RUN cp /mms/example/build/libs/mms-example*.jar /app.jar ENV JDK_JAVA_OPTIONS "-XX:MaxRAMPercentage=90.0" ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"] EXPOSE 8080 From 856cbf54aafb81eb16bd21fe419da8ddd18ec543 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 13:44:09 -0800 Subject: [PATCH 06/17] Find single file in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0cfff2c5c..cd7eaf6cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ COPY . /mms WORKDIR /mms RUN ./gradlew --no-daemon bootJar -RUN cp /mms/example/build/libs/mms-example*.jar /app.jar +RUN find . -type f -name mms-example-*.jar -not -iname '*javadoc*' -not -iname '*sources*' -exec cp '{}' '/app.jar' ';' ENV JDK_JAVA_OPTIONS "-XX:MaxRAMPercentage=90.0" ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"] EXPOSE 8080 From bc75833a52527e9354f350083cf75f70fc766048 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 14:18:09 -0800 Subject: [PATCH 07/17] Remove unneeded namespace in package names --- Dockerfile | 2 +- build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd7eaf6cc..c1ee92baa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ COPY . /mms WORKDIR /mms RUN ./gradlew --no-daemon bootJar -RUN find . -type f -name mms-example-*.jar -not -iname '*javadoc*' -not -iname '*sources*' -exec cp '{}' '/app.jar' ';' +RUN find . -type f -name example-*.jar -not -iname '*javadoc*' -not -iname '*sources*' -exec cp '{}' '/app.jar' ';' ENV JDK_JAVA_OPTIONS "-XX:MaxRAMPercentage=90.0" ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"] EXPOSE 8080 diff --git a/build.gradle b/build.gradle index c509ba834..f834ada2c 100644 --- a/build.gradle +++ b/build.gradle @@ -69,7 +69,7 @@ subprojects { apply plugin: 'signing' group = "org.openmbee.mms" - archivesBaseName = "mms-" + project.name + archivesBaseName = project.name version = rootProject.version Map commonDependencies = rootProject.ext.commonDependencies @@ -173,7 +173,7 @@ subprojects { } pom.project { - name 'mms-' + project.name + name project.name packaging 'jar' // optionally artifactId can be defined here description project.description From 6aeffe29c9fe34115af15b5c34e9d34385123a9c Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 14:41:58 -0800 Subject: [PATCH 08/17] Disable upload of example module --- build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index f834ada2c..580eba809 100644 --- a/build.gradle +++ b/build.gradle @@ -211,4 +211,7 @@ subprojects { } } } +} +project('example') { + uploadArchives.enabled = false } \ No newline at end of file From 751f5bfdac4fdf9d0d7a21a97da54103a1e8dbff Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 16:14:17 -0800 Subject: [PATCH 09/17] Remove upload to artifactory --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ffb261255..60e2dd8c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,18 +52,12 @@ jobs: steps: - attach_workspace: at: ~/ - - run: - name: Deploy snapshot to Artifactory - command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$SNAPSHOT_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish - deploy_release: + deploy_release: executor: openjdk_executor working_directory: /home/circleci/mms steps: - attach_workspace: at: ~/ - - run: - name: Deploy release to Artifactory - command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$RELEASE_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish - run: name: Deploy release to Bintray command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PbintrayUser=$BINTRAY_USER -PbintrayKey=$BINTRAY_KEY --info --stacktrace bintrayUpload From fe886aaaa992e01486e088d12630a132237fd0c0 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 16:16:54 -0800 Subject: [PATCH 10/17] Add message for step --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60e2dd8c6..e59c8331f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,6 +52,9 @@ jobs: steps: - attach_workspace: at: ~/ + - run: + name: Deploy snapshot to sonatype + command: echo "CircleCI publishing not implemented yet." deploy_release: executor: openjdk_executor working_directory: /home/circleci/mms From 873dcc4bf5cf0324cae6a3437729e79d367c1219 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 16:36:44 -0800 Subject: [PATCH 11/17] Fix yaml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e59c8331f..58425046b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ jobs: - run: name: Deploy snapshot to sonatype command: echo "CircleCI publishing not implemented yet." - deploy_release: + deploy_release: executor: openjdk_executor working_directory: /home/circleci/mms steps: From 019b83b4bb7f778950b0b99274b20b3055ae49ee Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 16 Feb 2021 21:03:51 -0800 Subject: [PATCH 12/17] Update to use maven-publish --- build.gradle | 116 +++++++++++++++++++++------------------------------ 1 file changed, 48 insertions(+), 68 deletions(-) diff --git a/build.gradle b/build.gradle index 580eba809..f42fbfa20 100644 --- a/build.gradle +++ b/build.gradle @@ -65,18 +65,16 @@ subprojects { apply plugin: 'io.spring.dependency-management' apply plugin: "com.jfrog.artifactory" apply plugin: 'com.jfrog.bintray' - apply plugin: 'maven' apply plugin: 'signing' group = "org.openmbee.mms" - archivesBaseName = project.name version = rootProject.version Map commonDependencies = rootProject.ext.commonDependencies task javadocJar(type: Jar) { archiveClassifier = "javadoc" - from javadoc + from javadoc.destinationDir } task sourcesJar(type: Jar) { @@ -96,32 +94,68 @@ subprojects { } artifacts { - archives javadocJar, sourcesJar - } - - signing { - if (project.hasProperty('signing.keyId') && project.hasProperty('signing.password') && project.hasProperty('signing.secretKeyRingFile')) { - sign configurations.archives - } + archives jar, javadocJar, sourcesJar } //https://docs.gradle.org/current/userguide/publishing_maven.html publishing { publications { mavenJava(MavenPublication) { - from components.java + from project.components.java + pom { + name = project.name + description = project.description + url = 'http://github.com/OpenMBEE/mms' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + id = 'huijun' + name = 'Jason Han' + } + developer { + id = 'dlam' + name = 'Doris Lam' + } + developer { + id = 'ward' + name = 'Ward Schweitzer' + } + } + scm { + connection = 'scm:git:https://github.com/Open-MBEE/mms.git' + developerConnection = 'scm:git:https://github.com/Open-MBEE/mms.git' + url = 'https://github.com/Open-MBEE/mms.git' + } + } } } repositories { maven { - def releasesRepoUrl = "$buildDir/repos/releases" - def snapshotsRepoUrl = "$buildDir/repos/snapshots" + def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/" + def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/" url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) { + credentials { + username = ossrhUsername + password = ossrhPassword + } + } } } } + signing { + if (project.hasProperty('signing.keyId') && project.hasProperty('signing.password') && project.hasProperty('signing.secretKeyRingFile')) { + sign publishing.publications.mavenJava + } + } + //https://www.jfrog.com/confluence/display/JFROG/Gradle+Artifactory+Plugin //can use this or set artifactory info directly under publishing/repositories? artifactory { @@ -157,61 +191,7 @@ subprojects { vcsUrl = 'https://github.com/Open-MBEE/mms.git' } } - - uploadArchives { - repositories { - if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - pom.project { - name project.name - packaging 'jar' - // optionally artifactId can be defined here - description project.description - url 'http://github.com/OpenMBEE/mms' - - scm { - connection 'scm:git:https://github.com/Open-MBEE/mms.git' - developerConnection 'scm:git:https://github.com/Open-MBEE/mms.git' - url 'https://github.com/Open-MBEE/mms.git' - } - - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - - developers { - developer { - id 'huijun' - name 'Jason Han' - } - developer { - id 'dlam' - name 'Doris Lam' - } - developer { - id 'ward' - name 'Ward Schweitzer' - } - } - } - } - } - } - } } project('example') { - uploadArchives.enabled = false + publishing.publications.remove(publishing.publications.mavenJava) } \ No newline at end of file From 49908160e37b3e6368471086017e8c9a93bf0f40 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 17 Feb 2021 07:48:55 -0800 Subject: [PATCH 13/17] Fix description and move signing config --- build.gradle | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index f42fbfa20..2742a3896 100644 --- a/build.gradle +++ b/build.gradle @@ -104,7 +104,7 @@ subprojects { from project.components.java pom { name = project.name - description = project.description + description = project.name + ', a module of the MMS' url = 'http://github.com/OpenMBEE/mms' licenses { license { @@ -191,7 +191,4 @@ subprojects { vcsUrl = 'https://github.com/Open-MBEE/mms.git' } } -} -project('example') { - publishing.publications.remove(publishing.publications.mavenJava) } \ No newline at end of file From 5d5f2ad961d9a2c842fb12712f24ea140977630d Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 17 Feb 2021 10:27:04 -0800 Subject: [PATCH 14/17] Bump version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 88dfda6dc..efead23bc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.1 +version=4.0.2 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE From 550ce361d29dd5c18748ff26731f00edb9725688 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 17 Feb 2021 16:38:58 -0800 Subject: [PATCH 15/17] Add custom bulk processor --- .../mms/elastic/BaseElasticDAOImpl.java | 47 +------------- .../mms/elastic/NodeElasticDAOImpl.java | 2 +- .../mms/elastic/utils/BulkProcessor.java | 65 +++++++++++++++++++ 3 files changed, 69 insertions(+), 45 deletions(-) create mode 100644 elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index 1ef83b76c..9d90260e0 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -8,10 +8,8 @@ import java.util.Optional; import java.util.Set; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.bulk.BackoffPolicy; -import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; + import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -24,12 +22,10 @@ import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.openmbee.mms.core.exceptions.InternalErrorException; +import org.openmbee.mms.elastic.utils.BulkProcessor; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.slf4j.Logger; @@ -209,43 +205,6 @@ public E update(String index, BaseJson json) { } protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { - BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> { - try { - BulkResponse response = client.bulk(request, RequestOptions.DEFAULT); - if (response.hasFailures()) { - String failure = response.buildFailureMessage(); - logger.error("Bulk response error: {}", failure); - throw new InternalErrorException(failure); - } - } catch (IOException ioe) { - logger.error(ioe.getMessage(), ioe); - throw new InternalErrorException(ioe); - } - }, getListener()); - bpBuilder.setBulkActions(bulkLimit); - bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)); - bpBuilder.setConcurrentRequests(1); - bpBuilder.setFlushInterval(TimeValue.timeValueSeconds(5)); - bpBuilder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(100), 3)); - - return bpBuilder.build(); - } - - private BulkProcessor.Listener getListener() { - return new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - throw new InternalErrorException(failure); - } - }; + return new BulkProcessor(client, REQUEST_OPTIONS, bulkLimit); } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index 8f1f31aad..f47a42efe 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -23,6 +22,7 @@ import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.dao.NodeIndexDAO; import org.openmbee.mms.core.exceptions.InternalErrorException; +import org.openmbee.mms.elastic.utils.BulkProcessor; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.ElementJson; diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java new file mode 100644 index 000000000..849d5bdff --- /dev/null +++ b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java @@ -0,0 +1,65 @@ +package org.openmbee.mms.elastic.utils; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.openmbee.mms.core.exceptions.InternalErrorException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +public class BulkProcessor { + + protected int bulkLimit; + RestHighLevelClient client; + RequestOptions options; + List> allRequests = new ArrayList<>(); + + public BulkProcessor(RestHighLevelClient client, RequestOptions options, int bulkLimit) { + this.client = client; + this.options = options; + this.bulkLimit = bulkLimit; + } + + public void add(IndexRequest action) { + allRequests.add(action); + clear(); + } + + public void add(UpdateRequest action) { + allRequests.add(action); + clear(); + } + + public void clear() { + if (allRequests.size() > bulkLimit) { + bulkBatchRequests(allRequests); + allRequests = new ArrayList<>(); + } + } + + public void close() { + batches(allRequests, bulkLimit).forEach(this::bulkBatchRequests); + } + + protected static Stream> batches(List source, int length) { + return IntStream.iterate(0, i -> i < source.size(), i -> i + length) + .mapToObj(i -> source.subList(i, Math.min(i + length, source.size()))); + } + + protected void bulkBatchRequests(List> actionRequest) { + BulkRequest bulkRequest = new BulkRequest(); + actionRequest.forEach(bulkRequest::add); + try { + client.bulk(bulkRequest, options); + } catch (IOException ioe) { + throw new InternalErrorException(ioe); + } + } +} From 52d20e177c9592213013e81b72df4429dbf60ab6 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 17 Feb 2021 17:01:04 -0800 Subject: [PATCH 16/17] Optimize --- .../org/openmbee/mms/elastic/utils/BulkProcessor.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java index 849d5bdff..d5c5da870 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java @@ -11,8 +11,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.stream.IntStream; -import java.util.stream.Stream; public class BulkProcessor { @@ -38,19 +36,14 @@ public void add(UpdateRequest action) { } public void clear() { - if (allRequests.size() > bulkLimit) { + if (allRequests.size() >= bulkLimit) { bulkBatchRequests(allRequests); allRequests = new ArrayList<>(); } } public void close() { - batches(allRequests, bulkLimit).forEach(this::bulkBatchRequests); - } - - protected static Stream> batches(List source, int length) { - return IntStream.iterate(0, i -> i < source.size(), i -> i + length) - .mapToObj(i -> source.subList(i, Math.min(i + length, source.size()))); + bulkBatchRequests(allRequests); } protected void bulkBatchRequests(List> actionRequest) { From d81f716d2143e129cf0fd932f9a78238e2c07c1c Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 17 Feb 2021 17:19:18 -0800 Subject: [PATCH 17/17] Remove redundant list and simplify --- .../mms/elastic/utils/BulkProcessor.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java index d5c5da870..8b8f0fc5c 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java @@ -1,7 +1,7 @@ package org.openmbee.mms.elastic.utils; -import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; @@ -9,15 +9,13 @@ import org.openmbee.mms.core.exceptions.InternalErrorException; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; public class BulkProcessor { protected int bulkLimit; RestHighLevelClient client; RequestOptions options; - List> allRequests = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(); public BulkProcessor(RestHighLevelClient client, RequestOptions options, int bulkLimit) { this.client = client; @@ -26,31 +24,35 @@ public BulkProcessor(RestHighLevelClient client, RequestOptions options, int bul } public void add(IndexRequest action) { - allRequests.add(action); + bulkRequest.add(action); clear(); } public void add(UpdateRequest action) { - allRequests.add(action); + bulkRequest.add(action); clear(); } public void clear() { - if (allRequests.size() >= bulkLimit) { - bulkBatchRequests(allRequests); - allRequests = new ArrayList<>(); + if (bulkRequest.numberOfActions() >= bulkLimit) { + bulkBatchRequests(); } } public void close() { - bulkBatchRequests(allRequests); + if (bulkRequest.numberOfActions() > 0) { + bulkBatchRequests(); + } } - protected void bulkBatchRequests(List> actionRequest) { - BulkRequest bulkRequest = new BulkRequest(); - actionRequest.forEach(bulkRequest::add); + protected void bulkBatchRequests() { try { - client.bulk(bulkRequest, options); + BulkResponse response = client.bulk(bulkRequest, options); + if (response.hasFailures()) { + String failure = response.buildFailureMessage(); + throw new InternalErrorException(failure); + } + bulkRequest = new BulkRequest(); } catch (IOException ioe) { throw new InternalErrorException(ioe); }