Prepare release 4.0.0 a2

.circleci/config.yml

@@ -52,7 +52,7 @@ jobs:
                 at: ~/
           - run:
                 name: Deploy snapshot to Artifactory
-                command: ./gradlew -PbuildAccess=opensource -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$SNAPSHOT_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish
+                command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$SNAPSHOT_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish
   deploy_release:
       executor: openjdk_executor
       working_directory: /home/circleci/mms
@@ -61,19 +61,25 @@ jobs:
                 at: ~/
           - run:
                 name: Deploy release to Artifactory
-                command: ./gradlew -PbuildAccess=opensource -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$RELEASE_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish
+                command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PartifactoryUrl=$ARTIFACTORY_URL -PartifactoryRepository=$RELEASE_ARTIFACTORY_REPOSITORY -PartifactoryUsername=$ARTIFACTORY_USERNAME -PartifactoryPassword=$ARTIFACTORY_PASSWORD --info --stacktrace artifactoryPublish
+          - run:
+                name: Deploy release to Bintray
+                command: ./gradlew -PbuildNumber=$CIRCLE_BUILD_NUM -PbuildTag=$CIRCLE_TAG -PbintrayUser=$BINTRAY_USER -PbintrayKey=$BINTRAY_KEY --info --stacktrace bintrayUpload
 
 workflows:
     version: 2
     build-test-deploy:
       jobs:
-        - build_and_test
+        - build_and_test:
+              filters:
+                  tags:
+                      only: /[0-9.]+(-(a|b|rc)[0-9]+)?/
         - deploy_snapshot:
               requires:
                   - build_and_test
               filters:
                   branches:
-                      only: /((release|hotfix|support)/[0-9.]+(-(a|b|rc)[0-9]+)?|master|develop)/
+                      only: /((release|hotfix|support)/[0-9.]+(-(a|b|rc)[0-9]+)?|develop)/
         - deploy_release:
               requires:
                   - build_and_test

Dockerfile

@@ -1,8 +1,8 @@
 FROM openjdk:11.0.8-jdk
 COPY . /mms
 WORKDIR /mms
-RUN ./gradlew --no-daemon build -x test
+RUN ./gradlew --no-daemon bootJar
 
 RUN cp /mms/example/build/libs/example*.jar /app.jar
-ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED","-jar", "/app.jar"] 
+ENTRYPOINT ["java", "-Djdk.tls.client.protocols=TLSv1", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"]
 EXPOSE 8080

build.gradle

@@ -2,6 +2,7 @@ buildscript {
     dependencies {
         classpath "org.springframework.boot:spring-boot-gradle-plugin:$springBootVersion"
         classpath "org.jfrog.buildinfo:build-info-extractor-gradle:latest.release"
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4'
     }
     repositories {
         maven { url 'https://repo.spring.io/plugins-release' }
@@ -12,10 +13,6 @@ buildscript {
 
 description = 'Structured Data Version Control'
 
-ext.snapshotBuild = version.contains("SNAPSHOT")
-ext.releaseBuild = !version.contains("SNAPSHOT")
-ext.milestoneBuild = !(snapshotBuild || releaseBuild)
-
 ext {
     commonDependencies = [
         'swagger-annotations'           : 'io.swagger.core.v3:swagger-annotations:2.1.2',
@@ -42,6 +39,17 @@ ext {
     ]
 }
 
+String buildTag = project.getProperties().get('buildTag')
+if (buildTag != null && !buildTag.isEmpty() && buildTag != version) {
+    throw new GradleException('Version mismatch: '+buildTag+' vs '+version)
+}
+if (buildTag == null || buildTag.isEmpty() || project.getProperties().getOrDefault('artifactoryRepository', '').toLowerCase().contains('snapshot')) {
+    version += '-SNAPSHOT'
+}
+ext.snapshotBuild = version.contains("SNAPSHOT")
+ext.releaseBuild = !version.contains("SNAPSHOT")
+ext.milestoneBuild = !(snapshotBuild || releaseBuild)
+
 subprojects {
     repositories {
         maven { url 'https://repo.spring.io/plugins-release' }
@@ -53,13 +61,13 @@ subprojects {
         project.sourceCompatibility = '10'
         project.targetCompatibility = '10'
     }
-}
+    version = rootProject.version
 
-allprojects {
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'
     apply plugin: 'io.spring.dependency-management'
     apply plugin: "com.jfrog.artifactory"
+    apply plugin: 'com.jfrog.bintray'
 
     Map<String, String> commonDependencies = rootProject.ext.commonDependencies
 
@@ -111,4 +119,18 @@ allprojects {
             }
         }
     }
+
+    bintray {
+        user = project.getProperties().get('bintrayUser')
+        key = project.getProperties().get('bintrayKey')
+        publications = ['mavenJava']
+        publish = true
+        pkg {
+            repo = 'maven'
+            name = 'mms-' + project.name
+            userOrg = 'openmbee'
+            licenses = ['Apache-2.0']
+            vcsUrl = 'https://github.com/Open-MBEE/mms.git'
+        }
+    }
 }

elastic/src/main/java/org/openmbee/sdvc/elastic/BaseElasticDAOImpl.java

@@ -7,8 +7,12 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.BulkProcessor;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
@@ -21,16 +25,23 @@
 import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.openmbee.sdvc.elastic.utils.Index;
 import org.openmbee.sdvc.json.BaseJson;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 
 public abstract class BaseElasticDAOImpl<E extends Map<String, Object>> {
 
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
     @Value("${elasticsearch.limit.result}")
     protected int resultLimit;
     @Value("${elasticsearch.limit.term}")
@@ -39,10 +50,10 @@ public abstract class BaseElasticDAOImpl<E extends Map<String, Object>> {
     protected RestHighLevelClient client;
     private static final RequestOptions REQUEST_OPTIONS;
     static {
-        RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+        RequestOptions.Builder requestBuilder = RequestOptions.DEFAULT.toBuilder();
         // TODO: Should be configureable
-        builder.setHttpAsyncResponseConsumerFactory(new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(1024 * 1024 * 1024));
-        REQUEST_OPTIONS = builder.build();
+        requestBuilder.setHttpAsyncResponseConsumerFactory(new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(1024 * 1024 * 1024));
+        REQUEST_OPTIONS = requestBuilder.build();
     }
 
     @Autowired
@@ -133,18 +144,21 @@ public List<E> findAllById(String index, Set<String> docIds) {
     }
 
     public void indexAll(String index, Collection<? extends BaseJson> jsons) {
+        BulkProcessor bulkProcessor = getBulkProcessor(client);
+        for (BaseJson json : jsons) {
+            bulkProcessor.add(new IndexRequest(index).id(json.getDocId()).source(json));
+        }
         try {
-            BulkRequest bulkIndex = new BulkRequest();
-            for (BaseJson json : jsons) {
-                bulkIndex.add(new IndexRequest(index).id(json.getDocId()).source(json));
+            if(!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) {
+                logger.error("Timed out in bulk processing");
             }
-            client.bulk(bulkIndex, REQUEST_OPTIONS);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
+        } catch (InterruptedException e) {
+            logger.error("Index all interrupted: ", e);
         }
+
     }
 
-    public void index(String index, BaseJson json) {
+    public void index(String index, BaseJson<?> json) {
         try {
             client.index(new IndexRequest(index).id(json.getDocId()).source(json),
                 REQUEST_OPTIONS);
@@ -174,4 +188,37 @@ public E update(String index, BaseJson json) {
         }
         return response;
     }
+
+    private static BulkProcessor getBulkProcessor(RestHighLevelClient client) {
+        return getBulkProcessor(client, null);
+    }
+
+    private static BulkProcessor getBulkProcessor(RestHighLevelClient client,  BulkProcessor.Listener listener) {
+        if (listener == null) {
+            listener = new BulkProcessor.Listener() {
+                private final Logger logger = LoggerFactory.getLogger(getClass());
+                @Override
+                public void beforeBulk(long executionId, BulkRequest request) {
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                }
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+                    logger.error("Error in bulk processing: ", failure);
+                }
+            };
+        }
+        BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> client
+            .bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener);
+        bpBuilder.setBulkActions(5000);
+        bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB));
+        bpBuilder.setConcurrentRequests(1);
+        bpBuilder.setFlushInterval(TimeValue.timeValueSeconds(5));
+        bpBuilder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(100), 3));
+
+        return bpBuilder.build();
+    }
 }

example/example.gradle

@@ -18,13 +18,11 @@ dependencies {
     implementation(
         project(':authenticator'),
         project(':localuser'),
-        project(':ldap'),
         project(':cameo'),
         project(':elastic'),
         project(':jupyter'),
         project(':permissions'),
         project(':webhooks'),
-        project(':twc'),
         project(':search'),
         project(':artifacts'),
         'org.springframework.boot:spring-boot-starter-web',

example/src/main/java/org/openmbee/sdvc/example/config/ExampleSecurityConfig.java

@@ -1,7 +1,6 @@
 package org.openmbee.sdvc.example.config;
 
 import org.openmbee.sdvc.authenticator.config.AuthSecurityConfig;
-import org.openmbee.sdvc.twc.config.TwcAuthSecurityConfig;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -36,15 +35,11 @@ public class ExampleSecurityConfig extends WebSecurityConfigurerAdapter implemen
     @Autowired
     AuthSecurityConfig authSecurityConfig;
 
-    @Autowired
-    TwcAuthSecurityConfig twcAuthSecurityConfig;
-
     @Override
     public void configure(HttpSecurity http) throws Exception {
         http.csrf().disable().authorizeRequests().anyRequest().permitAll().and().httpBasic();
         http.headers().cacheControl();
         http.addFilterAfter(corsFilter(), ExceptionTranslationFilter.class);
-        twcAuthSecurityConfig.setAuthConfig(http);
         authSecurityConfig.setAuthConfig(http);
     }
 

example/src/main/resources/application-test.properties

@@ -6,6 +6,8 @@ jwt.secret=12345678901234567890123456789012
 jwt.expiration=86400
 jwt.header=Authorization
 
+rdb.project.prefix=mms
+
 # See ldap module for example configuration
 ldap.provider.base=ou=something,dc=openmbee,dc=org
 ldap.provider.url=ldaps://ldap.openmbee.org/${ldap.provider.base}

gradle.properties

@@ -1,4 +1,4 @@
-version=4.0.0-alpha-1
+version=4.0.0-a2
 group=org.openmbee.mms
 
 springBootVersion=2.2.6.RELEASE
@@ -7,4 +7,4 @@ springSecurityVersion=5.3.1.RELEASE
 springDataVersion=2.2.6.RELEASE
 jacksonVersion=2.10.3
 log4jVersion=2.13.1
-elasticVersion=7.1.1
+elasticVersion=7.7.1

localuser/src/main/java/org/openmbee/sdvc/localuser/config/AuthProviderConfig.java

@@ -0,0 +1,51 @@
+package org.openmbee.sdvc.localuser.config;
+
+import org.openmbee.sdvc.localuser.security.UserDetailsServiceImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.authentication.dao.DaoAuthenticationProvider;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
+import org.springframework.security.crypto.password.PasswordEncoder;
+
+@Configuration
+public class AuthProviderConfig {
+
+    private static Logger logger = LoggerFactory.getLogger(LocalUserSecurityConfig.class);
+
+    private UserDetailsServiceImpl userDetailsService;
+    private PasswordEncoder passwordEncoder;
+
+    @Value("${sdvc.admin.username}")
+    private String adminUsername;
+    @Value("${sdvc.admin.password}")
+    private String adminPassword;
+
+    @Autowired
+    public void setUserDetailsService(UserDetailsServiceImpl userDetailsService) {
+        this.userDetailsService = userDetailsService;
+    }
+
+    @Autowired
+    public void setPasswordEncoder(PasswordEncoder passwordEncoder) {
+        this.passwordEncoder = passwordEncoder;
+    }
+
+    @Bean
+    public DaoAuthenticationProvider daoAuthenticationProvider() {
+        try {
+            userDetailsService.loadUserByUsername(adminUsername);
+        } catch (UsernameNotFoundException e) {
+            userDetailsService.register(adminUsername, adminPassword, true);
+            logger.info(String.format("Creating root user: %s with specified password.",
+                adminUsername));
+        }
+        DaoAuthenticationProvider authProvider = new DaoAuthenticationProvider();
+        authProvider.setUserDetailsService(userDetailsService);
+        authProvider.setPasswordEncoder(passwordEncoder);
+        return authProvider;
+    }
+}