BI-1650 - Store and fetch genotype data

.env → .env.template

@@ -32,11 +32,21 @@ BRAPI_DEFAULT_URL=http://localhost:8083
 BRAPI_CORE_URL=http://brapiserver:8080/
 BRAPI_PHENO_URL=http://brapiserver:8080/
 BRAPI_GENO_URL=http://brapiserver:8080/
-BRAPI_REFERENCE_SOURCE=breeding-insight.org
+BRAPI_REFERENCE_SOURCE=breedinginsight.org
 
 WEB_BASE_URL=http://localhost:8080
 
 # Email server
 EMAIL_RELAY_HOST=mailhog
 EMAIL_RELAY_PORT=1025
-EMAIL_FROM=bidevteam@cornell.edu
+EMAIL_FROM=bidevteam@cornell.edu
+
+GIGWA_HOST=<gigwa host (including port)>
+GIGWA_USER=<username>
+GIGWA_PASSWORD=<password>
+
+AWS_REGION=<aws region>
+AWS_ACCESS_KEY_ID=<access key>
+AWS_SECRET_KEY=<secret>
+AWS_GENO_BUCKET=<s3 bucket for genotypic data uploads>
+AWS_S3_ENDPOINT=<s3 endpoint, default https://s3.us-east-1.amazonaws.com>

.github/workflows/build.yml

@@ -59,3 +59,4 @@ jobs:
           JWT_SECRET: ${{ secrets.JWT_SECRET }}
           OAUTH_CLIENT_ID: 123abc
           OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf
+          BRAPI_REFERENCE_SOURCE: breedinginsight.org

.github/workflows/release.yml

@@ -66,6 +66,7 @@ jobs:
           JWT_SECRET: ${{ secrets.JWT_SECRET }}
           OAUTH_CLIENT_ID: 123abc
           OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf
+          BRAPI_REFERENCE_SOURCE: breedinginsight.org
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:

.github/workflows/versioner-develop.yml

@@ -80,6 +80,7 @@ jobs:
           JWT_SECRET: ${{ secrets.JWT_SECRET }}
           OAUTH_CLIENT_ID: 123abc
           OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf
+          BRAPI_REFERENCE_SOURCE: breedinginsight.org
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:

.gitignore

@@ -17,5 +17,4 @@ src/build/*.dev.*
 src/build/*.test.*
 application-dev.yml
 application-test.yml
-.env
-.env.local
+.env.*

README.md

@@ -79,6 +79,46 @@ to run the brapi test server, update the proper variables in the .env file and t
 
 ```docker-compose up -d brapi-server```
 
+#### Redis
+
+Run this docker command in terminal to start up a redis docker container
+
+```docker-compose up -d redis```
+
+Then update the following environment variables:
+
+```
+REDIS_URL=redis://localhost:6379
+```
+
+#### Gigwa
+
+Run this docker command in terminal to start up a Gigwa container
+
+```docker-compose up -d gigwa```
+
+Then update the following environment variable:
+
+```
+GIGWA_HOST=http://localhost:5080/
+```
+
+#### Localstack (AWS service emulator)
+
+If you do not want to configure an S3 bucket and IAM roles in S3, you can use Localstack (https://localstack.cloud/) to emulate the S3 service on your machine
+
+Run this docker command in terminal to start up a Localstack container
+
+```docker-compose up -d localstack```
+
+Then update the following environment variables:
+
+```
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_KEY=test
+AWS_S3_ENDPOINT=http://localhost:4566
+```
+
 ### Building the project
 
 Once you have the project pulled down and your database running, follow these steps to build the project.

docker-compose.yml

@@ -32,6 +32,7 @@ services:
       - brapi-server
       - redis
       - mailhog
+      - gigwa
     volumes:
       - /usr/bin/docker:/usr/bin/docker
       - /var/run/docker.sock:/var/run/docker.sock
@@ -56,6 +57,14 @@ services:
       - REDIS_URL=${REDIS_URL:-redis://redis:6379}
       - REDIS_TIMEOUT=${REDIS_TIMEOUT:-30s}
       - REDIS_SSL=${REDIS_SSL:-false}
+      - GIGWA_HOST=${GIGWA_HOST:-http://gigwa:8080}
+      - GIGWA_USER=${GIGWA_USER}
+      - GIGWA_PASSWORD=${GIGWA_PASSWORD}
+      - AWS_REGION=${AWS_REGION:-us-east-1}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_KEY=${AWS_SECRET_KEY}
+      - AWS_GENO_BUCKET=${AWS_GENO_BUCKET}
+      - AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT:-https://s3.us-east-1.amazonaws.com}
     ports:
       - ${API_INTERNAL_PORT}:${API_INTERNAL_PORT}
     networks:
@@ -108,10 +117,62 @@ services:
       - "6379:6379"
     networks:
       backend:
+  gigwa:
+    depends_on:
+      - mongo
+    image: breedinginsight/gigwa:develop
+    container_name: ${GIGWA_CONTAINER_NAME:-gigwa}
+    ports:
+      - ${GIGWA_PORT:-5080}:8080
+    restart: always
+    environment:
+      MONGO_IP: ${GIGWA_CONTAINER_NAME:-gigwa}_db
+      MONGO_PORT: 27017
+      MONGO_INITDB_ROOT_USERNAME: ${GIGWA_MONGO_USER:-mongo}
+      MONGO_INITDB_ROOT_PASSWORD: ${GIGWA_MONGO_PASSWORD:-mongo}
+      HOST_LOCALE: "${LANG}"
+      GIGWA.serversAllowedToImport: ${BRAPI_REFERENCE_SOURCE}
+    networks:
+      backend:
+    volumes:
+      - type: volume
+        source: gigwa_data
+        target: /usr/local/tomcat/config
+  mongo:
+    image: mongo:4.2.21
+    container_name: ${GIGWA_CONTAINER_NAME:-gigwa}_db
+    restart: always
+    command: --profile 0 --slowms 60000 --storageEngine wiredTiger --wiredTigerCollectionBlockCompressor=zstd --directoryperdb --quiet
+    ports:
+      - ${GIGWA_MONGO_PORT:-27017}:27017
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${GIGWA_MONGO_USER:-mongo}
+      MONGO_INITDB_ROOT_PASSWORD: ${GIGWA_MONGO_PASSWORD:-mongo}
+    volumes:
+      - type: volume
+        source: gigwa_mongo_data
+        target: /data/db
+    networks:
+      backend:
+  localstack:
+    container_name: "localstack"
+    image: localstack/localstack
+    ports:
+      - "4566:4566"
+    networks:
+      backend:
+        aliases:
+          - localstack
+    environment:
+      - HOSTNAME_EXTERNAL=localstack
 
 networks:
   backend:
 
 volumes:
   biapi_data:
     name: biapi_data
+  gigwa_data:
+    name: ${GIGWA_CONTAINER_NAME:-gigwa}_data
+  gigwa_mongo_data:
+    name: ${GIGWA_CONTAINER_NAME:-gigwa}_mongo_data

pom.xml

@@ -49,6 +49,14 @@
             <name>Chris Tucker</name>
             <email>ct447@cornell.edu</email>
         </developer>
+        <developer>
+            <name>Heather Sweeney</name>
+            <email>hms243@cornell.edu</email>
+        </developer>
+        <developer>
+            <name>David Phillips</name>
+            <email>drp227@cornell.edu</email>
+        </developer>
     </developers>
 
     <properties>
@@ -77,7 +85,7 @@
         <guava.version>31.0.1-jre</guava.version>
         <okhttp.version>4.9.3</okhttp.version>
         <mockito.version>4.3.1</mockito.version>
-        <brapi-java-client.version>2.0.2</brapi-java-client.version>
+        <brapi-java-client.version>2.1-SNAPSHOT</brapi-java-client.version>
         <commons-io.version>2.11.0</commons-io.version>
         <tika-app.version>2.2.1</tika-app.version>
         <!-- Version of apache-poi depends on version of tika. Tika uses these. -->
@@ -319,6 +327,24 @@
             <version>${testcontainers.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
+            <version>${testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-core</artifactId>
+            <version>1.12.342</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>io.kowalski</groupId>
             <artifactId>fannypack</artifactId>
@@ -400,6 +426,16 @@
             <artifactId>redisson-micronaut-20</artifactId>
             <version>3.17.5</version>
         </dependency>
+
+        <dependency>
+            <groupId>io.micronaut.aws</groupId>
+            <artifactId>micronaut-aws-sdk-v2</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.agorapulse</groupId>
+            <artifactId>micronaut-amazon-awssdk-s3</artifactId>
+            <version>2.0.5-micronaut-2.0</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -609,7 +645,7 @@
                             <brapi.server.pheno-url>${BRAPI_PHENO_URL}</brapi.server.pheno-url>
                             <brapi.server.geno-url>${BRAPI_GENO_URL}</brapi.server.geno-url>
                             <brapi.server.default-url>${BRAPI_DEFAULT_URL}</brapi.server.default-url>
-                            <brapi.server.reference-source>breeding-insight.org</brapi.server.reference-source>
+                            <brapi.server.reference-source>${BRAPI_REFERENCE_SOURCE}</brapi.server.reference-source>
                             <email.relay-server.host>test</email.relay-server.host>
                             <email.relay-server.port>25</email.relay-server.port>
                             <email.from>test@test.com</email.from>

src/build/build.properties

@@ -28,4 +28,5 @@ BRAPI_GENO_URL=${env.BRAPI_GENO_URL}
 BRAPI_DEFAULT_URL=${env.BRAPI_DEFAULT_URL}
 JWT_SECRET=${env.JWT_SECRET}
 WEB_BASE_URL=${env.WEB_BASE_URL}
-FLYWAY_SCHEMA=flyway_build
+FLYWAY_SCHEMA=flyway_build
+BRAPI_REFERENCE_SOURCE=${env.BRAPI_REFERENCE_SOURCE}

src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java

@@ -0,0 +1,57 @@
+package org.breedinginsight.api.v1.controller.geno;
+
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.annotation.*;
+import io.micronaut.http.multipart.CompletedFileUpload;
+import lombok.extern.slf4j.Slf4j;
+import org.brapi.client.v2.model.exceptions.ApiException;
+import org.breedinginsight.api.auth.AuthenticatedUser;
+import org.breedinginsight.api.auth.ProgramSecured;
+import org.breedinginsight.api.auth.ProgramSecuredRole;
+import org.breedinginsight.api.auth.SecurityService;
+import org.breedinginsight.api.model.v1.response.Response;
+import org.breedinginsight.api.v1.controller.metadata.AddMetadata;
+import org.breedinginsight.brapps.importer.model.response.ImportResponse;
+import org.breedinginsight.services.exceptions.AuthorizationException;
+import org.breedinginsight.services.exceptions.DoesNotExistException;
+import org.breedinginsight.services.geno.GenotypeService;
+
+import javax.inject.Inject;
+import java.util.UUID;
+
+@Slf4j
+@Controller("/${micronaut.bi.api.version}")
+public class GenotypeDataUploadController {
+    private final GenotypeService genoService;
+    private final SecurityService securityService;
+
+    @Inject
+    public GenotypeDataUploadController(GenotypeService genoService, SecurityService securityService) {
+        this.genoService = genoService;
+        this.securityService = securityService;
+    }
+
+    @Post("programs/{programId}/experiments/{experimentId}/geno/import")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    @Produces(MediaType.APPLICATION_JSON)
+    @AddMetadata
+    @ProgramSecured(roles = {ProgramSecuredRole.BREEDER})
+    public HttpResponse<Response<ImportResponse>> uploadData(@PathVariable UUID programId, @PathVariable UUID experimentId, @Part("file") CompletedFileUpload upload) {
+        AuthenticatedUser actingUser = securityService.getUser();
+        try {
+            ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, experimentId, upload);
+            Response<ImportResponse> response = new Response<>(result);
+            return HttpResponse.ok(response);
+        } catch (DoesNotExistException e) {
+            log.error("Missing data", e);
+            return HttpResponse.notFound();
+        } catch (AuthorizationException e) {
+            log.error("Error authorizing to backing service", e);
+            return HttpResponse.unauthorized();
+        } catch (ApiException e) {
+            log.error("Error importing geno data", e);
+            return HttpResponse.serverError();
+        }
+    }
+}