From acdf2e77a44721380519d5988e5cf82ca937f216 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 3 Mar 2021 11:37:21 -0800 Subject: [PATCH 01/13] trying ci publish again --- .circleci/config.yml | 6 +++--- gradle.properties | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 58425046b..8b66228c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: at: ~/ - run: name: Deploy snapshot to sonatype - command: echo "CircleCI publishing not implemented yet." + command: ./gradlew -PossrhUsername=$OSSRH_USERNAME -PossrhPassword=$OSSRH_PASSWORD -PsigningKey="$SIGNING_KEY" -PsigningPassword=$SIGNING_PASSWORD publish deploy_release: executor: openjdk_executor working_directory: /home/circleci/mms @@ -62,8 +62,8 @@ jobs: - attach_workspace: at: ~/ - run: - name: Deploy release to Bintray - command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PbintrayUser=$BINTRAY_USER -PbintrayKey=$BINTRAY_KEY --info --stacktrace bintrayUpload + name: Deploy release to sonatype + command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PossrhUsername=$OSSRH_USERNAME -PossrhPassword=$OSSRH_PASSWORD -PsigningKey="$SIGNING_KEY" -PsigningPassword=$SIGNING_PASSWORD --info --stacktrace publish workflows: version: 2 diff --git a/gradle.properties b/gradle.properties index b145c5e77..4224876f0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.3 +version=4.0.4 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE From 9db7dac44770b5fa25e43a051da8857095b4a857 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 3 Mar 2021 13:19:32 -0800 Subject: [PATCH 02/13] again --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 2e472dd6a..7046ebd8e 100644 --- a/build.gradle +++ b/build.gradle @@ -154,8 +154,8 @@ subprojects { if (project.hasProperty('signing.keyId') && project.hasProperty('signing.password') && project.hasProperty('signing.secretKeyRingFile')) { sign publishing.publications.mavenJava } else if (project.hasProperty('signingKey') && project.hasProperty('signingPassword')) { - def signingKey = findProperty("signingKey") - def signingPassword = findProperty("signingPassword") + //def signingKey = project.getProperties().get("signingKey") + //def signingPassword = project.getProperties().get("signingPassword") useInMemoryPgpKeys(signingKey, signingPassword) sign publishing.publications.mavenJava } From b242462cb14c186933c43b3bcdd5bb1aeea59b2f Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 3 Mar 2021 14:43:14 -0800 Subject: [PATCH 03/13] again... --- build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 7046ebd8e..de527b153 100644 --- a/build.gradle +++ b/build.gradle @@ -154,8 +154,7 @@ subprojects { if (project.hasProperty('signing.keyId') && project.hasProperty('signing.password') && project.hasProperty('signing.secretKeyRingFile')) { sign publishing.publications.mavenJava } else if (project.hasProperty('signingKey') && project.hasProperty('signingPassword')) { - //def signingKey = project.getProperties().get("signingKey") - //def signingPassword = project.getProperties().get("signingPassword") + def signingKey = signingKey.replaceAll(/\\n/, "\n") useInMemoryPgpKeys(signingKey, signingPassword) sign publishing.publications.mavenJava } From 40feb0eb4075adc5c3f914e3b9007748cdfabf69 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Fri, 16 Apr 2021 19:56:16 -0700 Subject: [PATCH 04/13] Create FUNDING.yml --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..cbfd17432 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: ['https://numfocus.salsalabs.org/supportopenmbee'] From abcb18a7916fb9fc0e085c938db80178eb7c0b85 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Fri, 16 Apr 2021 19:57:37 -0700 Subject: [PATCH 05/13] Update FUNDING.yml --- .github/FUNDING.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index cbfd17432..ef698e8e2 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,12 +1 @@ -# These are supported funding model platforms - -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username custom: ['https://numfocus.salsalabs.org/supportopenmbee'] From 4645325f334612779ca4d496c7c8f8e7e2a61c76 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 4 May 2021 08:49:47 -0700 Subject: [PATCH 06/13] Set column length for description on scoped branch object --- .../main/java/org/openmbee/mms/data/domains/scoped/Branch.java | 1 + 1 file changed, 1 insertion(+) diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java index b1f609f83..899df1801 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java @@ -17,6 +17,7 @@ public class Branch { @Column(name = "id", updatable = false, nullable = false) Long id; + @Column(length = 512) private String description; @Column(unique = true, length = 512) From 7a5bb017e9560dd1c14125193077d753d60f28cd Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 5 May 2021 18:44:50 -0700 Subject: [PATCH 07/13] Add more logging when commit error happens --- .../openmbee/mms/crud/services/DefaultNodeService.java | 9 ++++++++- .../openmbee/mms/elastic/config/ElasticsearchConfig.java | 8 +++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 482e052fa..2336f1382 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.*; @@ -234,7 +236,12 @@ public void commitChanges(NodeChangeInfo info) { }); this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); } catch (Exception e) { - logger.error("commitChanges error: {}", e.getMessage()); + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + String sStackTrace = sw.toString(); + + logger.error("Error in commitChanges: {}", sStackTrace); throw new InternalErrorException("Error committing changes: " + e.getMessage()); } eventPublisher.forEach((pub) -> pub.publish( diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/config/ElasticsearchConfig.java b/elastic/src/main/java/org/openmbee/mms/elastic/config/ElasticsearchConfig.java index 89bd9ff9f..7a8866c50 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/config/ElasticsearchConfig.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/config/ElasticsearchConfig.java @@ -1,12 +1,12 @@ package org.openmbee.mms.elastic.config; import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.beans.factory.annotation.Value; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.RestClient; @Configuration public class ElasticsearchConfig { @@ -20,7 +20,9 @@ public class ElasticsearchConfig { @Bean(name = "clientElastic", destroyMethod = "close") public RestHighLevelClient restClient() { + RestClientBuilder builder = RestClient.builder(new HttpHost(elasticsearchHost, elasticsearchPort, elasticsearchHttp)); + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(10000).setSocketTimeout(1000000)); RestHighLevelClient client = new RestHighLevelClient(builder); return client; } From 49ebcb1476b7c5e5e4d727e3dc59f7c919f9180e Mon Sep 17 00:00:00 2001 From: Jason Han Date: Mon, 10 May 2021 13:44:16 -0700 Subject: [PATCH 08/13] Index each action --- .../java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index 2f13e8a2d..ce0b357df 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -42,7 +42,6 @@ public void index(BaseJson json) { public void index(CommitJson json) { int commitCount = getCommitSize(json); - List broken = new ArrayList<>(); if (commitCount > commitLimit) { List> allActions = new ArrayList<>(); allActions.addAll(json.getAdded().stream().peek(toAdd -> toAdd.put("action", "added")).collect(Collectors.toList())); @@ -71,10 +70,8 @@ public void index(CommitJson json) { break; } } while(getCommitSize(currentCommitCopy) < commitLimit && !allActions.isEmpty()); - broken.add(currentCommitCopy); - + this.index(getIndex(), currentCommitCopy); } - this.indexAll(broken); } else { this.index(getIndex(), json); From 0c2855971905c2bd229b25eb3ea98626aea6547e Mon Sep 17 00:00:00 2001 From: Jason Han Date: Mon, 10 May 2021 16:23:18 -0700 Subject: [PATCH 09/13] Use REQUEST_OPTIONS for search requests that require larger max_content_length --- .../java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java | 2 +- .../java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java | 2 +- .../java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index 9d90260e0..f83e20224 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -52,7 +52,7 @@ public abstract class BaseElasticDAOImpl> { protected static int readTimeout = 1000000000; protected RestHighLevelClient client; - private static final RequestOptions REQUEST_OPTIONS; + protected static final RequestOptions REQUEST_OPTIONS; static { RequestOptions.Builder requestBuilder = RequestOptions.DEFAULT.toBuilder(); // TODO: Should be configureable diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index ce0b357df..9cb2ce422 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -210,7 +210,7 @@ private SearchHits getCommitResults(QueryBuilder query) throws IOException { sourceBuilder.size(this.resultLimit); // TODO handle paging requests sourceBuilder.sort(new FieldSortBuilder(CommitJson.CREATED).order(SortOrder.DESC)); searchRequest.source(sourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = client.search(searchRequest, REQUEST_OPTIONS); return searchResponse.getHits(); } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index f47a42efe..15d82b963 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -74,7 +74,7 @@ public Optional getByCommitId(String commitId, String nodeId) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(query); searchRequest.source(sourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = client.search(searchRequest, REQUEST_OPTIONS); if (searchResponse.getHits().getTotalHits().value == 0) { return Optional.empty(); } @@ -136,7 +136,7 @@ public Optional getElementLessThanOrEqualTimestamp(String nodeId, searchSourceBuilder.size(1); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = client.search(searchRequest, REQUEST_OPTIONS); SearchHit[] searchHits = searchResponse.getHits().getHits(); if (searchHits != null && searchHits.length > 0) { ElementJson elementJson = newInstance(); From b129603d2f47ddaf0f3209042ae81c03355055d8 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 13 May 2021 15:25:26 -0700 Subject: [PATCH 10/13] Remove string extraction --- .../org/openmbee/mms/crud/services/DefaultNodeService.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 2336f1382..b55dac043 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -236,12 +236,7 @@ public void commitChanges(NodeChangeInfo info) { }); this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); } catch (Exception e) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e.printStackTrace(pw); - String sStackTrace = sw.toString(); - - logger.error("Error in commitChanges: {}", sStackTrace); + logger.error("Error in commitChanges: ", e); throw new InternalErrorException("Error committing changes: " + e.getMessage()); } eventPublisher.forEach((pub) -> pub.publish( From d1b78e872ee52f4502dbc4d6237bcc6d9fff323f Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 13 May 2021 15:43:05 -0700 Subject: [PATCH 11/13] Throw exception for ioe while writing to stream --- .../java/org/openmbee/mms/crud/services/DefaultNodeService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index b55dac043..48f28e4c1 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -135,6 +135,7 @@ public void readAsStream(String projectId, String refId, .getBytes(StandardCharsets.UTF_8)); } catch (IOException ioe) { logger.error("Error writing to stream", ioe); + throw new InternalErrorException("Error writing to stream."); } }); if (!"application/x-ndjson".equals(accept)) { From c6c5d608a3269017e6922c3fec4f1407f1d5d588 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 13 May 2021 19:52:40 -0700 Subject: [PATCH 12/13] Remove unused imports --- .../java/org/openmbee/mms/crud/services/DefaultNodeService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 48f28e4c1..59b860992 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.*; From 7719ab2a9393dfb57aab89061fd7caa70c077d34 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Fri, 14 May 2021 14:23:08 -0700 Subject: [PATCH 13/13] remove old stuff from build file --- build.gradle | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/build.gradle b/build.gradle index de527b153..8e2930869 100644 --- a/build.gradle +++ b/build.gradle @@ -63,8 +63,6 @@ subprojects { apply plugin: 'java-library' apply plugin: 'maven-publish' apply plugin: 'io.spring.dependency-management' - apply plugin: "com.jfrog.artifactory" - apply plugin: 'com.jfrog.bintray' apply plugin: 'signing' group = "org.openmbee.mms" @@ -159,40 +157,4 @@ subprojects { sign publishing.publications.mavenJava } } - - //https://www.jfrog.com/confluence/display/JFROG/Gradle+Artifactory+Plugin - //can use this or set artifactory info directly under publishing/repositories? - artifactory { - contextUrl = 'https://mms.jfrog.io/mms' - publish { - repository { - repoKey = project.getProperties().get('artifactoryRepository') - username = project.getProperties().get('artifactoryUsername') - password = project.getProperties().get('artifactoryPassword') - maven = true - } - defaults { - publications('mavenJava') - publishArtifacts = true - // Properties to be attached to the published artifacts. - //properties = ['qa.level': 'basic', 'dev.team' : 'core'] - // Publish generated POM files to Artifactory (true by default) - publishPom = true - } - } - } - - bintray { - user = project.getProperties().get('bintrayUser') - key = project.getProperties().get('bintrayKey') - publications = ['mavenJava'] - publish = true - pkg { - repo = 'maven' - name = 'mms-' + project.name - userOrg = 'openmbee' - licenses = ['Apache-2.0'] - vcsUrl = 'https://github.com/Open-MBEE/mms.git' - } - } } \ No newline at end of file