diff --git a/.circleci/config.yml b/.circleci/config.yml index ffb261255..58425046b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,17 +53,14 @@ jobs: - attach_workspace: at: ~/ - run: - name: Deploy snapshot to Artifactory - command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$SNAPSHOT_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish + name: Deploy snapshot to sonatype + command: echo "CircleCI publishing not implemented yet." deploy_release: executor: openjdk_executor working_directory: /home/circleci/mms steps: - attach_workspace: at: ~/ - - run: - name: Deploy release to Artifactory - command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$RELEASE_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish - run: name: Deploy release to Bintray command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PbintrayUser=$BINTRAY_USER -PbintrayKey=$BINTRAY_KEY --info --stacktrace bintrayUpload diff --git a/Dockerfile b/Dockerfile index 9d3830627..c1ee92baa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ COPY . /mms WORKDIR /mms RUN ./gradlew --no-daemon bootJar -RUN cp /mms/example/build/libs/example*.jar /app.jar +RUN find . -type f -name example-*.jar -not -iname '*javadoc*' -not -iname '*sources*' -exec cp '{}' '/app.jar' ';' ENV JDK_JAVA_OPTIONS "-XX:MaxRAMPercentage=90.0" ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"] EXPOSE 8080 diff --git a/build.gradle b/build.gradle index 779f1045a..2742a3896 100644 --- a/build.gradle +++ b/build.gradle @@ -59,19 +59,32 @@ subprojects { project.sourceCompatibility = '10' project.targetCompatibility = '10' } - version = rootProject.version apply plugin: 'java-library' apply plugin: 'maven-publish' apply plugin: 'io.spring.dependency-management' apply plugin: "com.jfrog.artifactory" apply plugin: 'com.jfrog.bintray' + apply plugin: 'signing' + + group = "org.openmbee.mms" + version = rootProject.version Map commonDependencies = rootProject.ext.commonDependencies + task javadocJar(type: Jar) { + archiveClassifier = "javadoc" + from javadoc.destinationDir + } + + task sourcesJar(type: Jar) { + archiveClassifier = "sources" + from sourceSets.main.allSource + } + java { - //withJavadocJar() - //withSourcesJar() + withJavadocJar() + withSourcesJar() } javadoc { @@ -79,23 +92,70 @@ subprojects { options.addBooleanOption('html5', true) } } + + artifacts { + archives jar, javadocJar, sourcesJar + } + //https://docs.gradle.org/current/userguide/publishing_maven.html publishing { publications { mavenJava(MavenPublication) { - from components.java + from project.components.java + pom { + name = project.name + description = project.name + ', a module of the MMS' + url = 'http://github.com/OpenMBEE/mms' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + id = 'huijun' + name = 'Jason Han' + } + developer { + id = 'dlam' + name = 'Doris Lam' + } + developer { + id = 'ward' + name = 'Ward Schweitzer' + } + } + scm { + connection = 'scm:git:https://github.com/Open-MBEE/mms.git' + developerConnection = 'scm:git:https://github.com/Open-MBEE/mms.git' + url = 'https://github.com/Open-MBEE/mms.git' + } + } } } repositories { maven { - def releasesRepoUrl = "$buildDir/repos/releases" - def snapshotsRepoUrl = "$buildDir/repos/snapshots" + def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/" + def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/" url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) { + credentials { + username = ossrhUsername + password = ossrhPassword + } + } } } } + signing { + if (project.hasProperty('signing.keyId') && project.hasProperty('signing.password') && project.hasProperty('signing.secretKeyRingFile')) { + sign publishing.publications.mavenJava + } + } + //https://www.jfrog.com/confluence/display/JFROG/Gradle+Artifactory+Plugin //can use this or set artifactory info directly under publishing/repositories? artifactory { diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java index 082b0db21..e5cf967c5 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodePostHelper.java @@ -91,6 +91,8 @@ public NodeChangeInfo processPostJson(List elements, boolean overwr } } else { updated = true; + element.setCreated(indexElement.getCreated()); + element.setCreator(indexElement.getCreator()); } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index 1ef83b76c..9d90260e0 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -8,10 +8,8 @@ import java.util.Optional; import java.util.Set; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.bulk.BackoffPolicy; -import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; + import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -24,12 +22,10 @@ import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.openmbee.mms.core.exceptions.InternalErrorException; +import org.openmbee.mms.elastic.utils.BulkProcessor; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.slf4j.Logger; @@ -209,43 +205,6 @@ public E update(String index, BaseJson json) { } protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { - BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> { - try { - BulkResponse response = client.bulk(request, RequestOptions.DEFAULT); - if (response.hasFailures()) { - String failure = response.buildFailureMessage(); - logger.error("Bulk response error: {}", failure); - throw new InternalErrorException(failure); - } - } catch (IOException ioe) { - logger.error(ioe.getMessage(), ioe); - throw new InternalErrorException(ioe); - } - }, getListener()); - bpBuilder.setBulkActions(bulkLimit); - bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)); - bpBuilder.setConcurrentRequests(1); - bpBuilder.setFlushInterval(TimeValue.timeValueSeconds(5)); - bpBuilder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(100), 3)); - - return bpBuilder.build(); - } - - private BulkProcessor.Listener getListener() { - return new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - throw new InternalErrorException(failure); - } - }; + return new BulkProcessor(client, REQUEST_OPTIONS, bulkLimit); } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index 8f1f31aad..f47a42efe 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -23,6 +22,7 @@ import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.dao.NodeIndexDAO; import org.openmbee.mms.core.exceptions.InternalErrorException; +import org.openmbee.mms.elastic.utils.BulkProcessor; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.ElementJson; diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java new file mode 100644 index 000000000..8b8f0fc5c --- /dev/null +++ b/elastic/src/main/java/org/openmbee/mms/elastic/utils/BulkProcessor.java @@ -0,0 +1,60 @@ +package org.openmbee.mms.elastic.utils; + +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.openmbee.mms.core.exceptions.InternalErrorException; + +import java.io.IOException; + +public class BulkProcessor { + + protected int bulkLimit; + RestHighLevelClient client; + RequestOptions options; + BulkRequest bulkRequest = new BulkRequest(); + + public BulkProcessor(RestHighLevelClient client, RequestOptions options, int bulkLimit) { + this.client = client; + this.options = options; + this.bulkLimit = bulkLimit; + } + + public void add(IndexRequest action) { + bulkRequest.add(action); + clear(); + } + + public void add(UpdateRequest action) { + bulkRequest.add(action); + clear(); + } + + public void clear() { + if (bulkRequest.numberOfActions() >= bulkLimit) { + bulkBatchRequests(); + } + } + + public void close() { + if (bulkRequest.numberOfActions() > 0) { + bulkBatchRequests(); + } + } + + protected void bulkBatchRequests() { + try { + BulkResponse response = client.bulk(bulkRequest, options); + if (response.hasFailures()) { + String failure = response.buildFailureMessage(); + throw new InternalErrorException(failure); + } + bulkRequest = new BulkRequest(); + } catch (IOException ioe) { + throw new InternalErrorException(ioe); + } + } +} diff --git a/gradle.properties b/gradle.properties index 54c95dcf2..efead23bc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0 +version=4.0.2 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE