From 3c2a7a36b49b82df4b3be9a51e06a8abe4533414 Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Wed, 20 Mar 2024 11:51:04 -0400
Subject: [PATCH 01/11] Begin work on bucket interface

---
 packages/server/src/app.module.ts             |  4 ++-
 .../src/bucket/bucket-factory.service.ts      | 21 ++++++++++++
 .../server/src/bucket/bucket-info.model.ts    | 33 +++++++++++++++++++
 packages/server/src/bucket/bucket.module.ts   | 11 +++++++
 packages/server/src/bucket/bucket.ts          | 19 +++++++++++
 5 files changed, 87 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/bucket/bucket-factory.service.ts
 create mode 100644 packages/server/src/bucket/bucket-info.model.ts
 create mode 100644 packages/server/src/bucket/bucket.module.ts
 create mode 100644 packages/server/src/bucket/bucket.ts

diff --git a/packages/server/src/app.module.ts b/packages/server/src/app.module.ts
index 49a797f0..3dd95afc 100644
--- a/packages/server/src/app.module.ts
+++ b/packages/server/src/app.module.ts
@@ -14,6 +14,7 @@ import { SharedModule } from './shared/shared.module';
 import { JwtModule } from './jwt/jwt.module';
 import { PermissionModule } from './permission/permission.module';
 import { UserModule } from './user/user.module';
+import { BucketModule } from './bucket/bucket.module';
 
 @Module({
   imports: [
@@ -44,7 +45,8 @@ import { UserModule } from './user/user.module';
     SharedModule,
     JwtModule,
     PermissionModule,
-    UserModule
+    UserModule,
+    BucketModule
   ]
 })
 export class AppModule {}
diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
new file mode 100644
index 00000000..ac3a0154
--- /dev/null
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -0,0 +1,21 @@
+import { Injectable } from '@nestjs/common';
+import { InjectModel } from '@nestjs/mongoose';
+import { Model } from 'mongoose';
+import { BucketInfo } from './bucket-info.model';
+
+@Injectable()
+export class BucketFactory {
+  constructor(@InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>) {}
+
+  /**
+   * Factory method which gets the correct storage bucket inforamtion for the
+   * given organization.
+   */
+  async getBucket(organization: string): Promise<Bucket | null> {
+    // Get the information on the bucket for the given organization
+    const bucketInfo = await this.bucketInfoModel.findOne({ organization });
+    if (!bucketInfo) {
+      return bucketInfo;
+    }
+  }
+}
diff --git a/packages/server/src/bucket/bucket-info.model.ts b/packages/server/src/bucket/bucket-info.model.ts
new file mode 100644
index 00000000..3aa70c0a
--- /dev/null
+++ b/packages/server/src/bucket/bucket-info.model.ts
@@ -0,0 +1,33 @@
+import { Schema, Prop, SchemaFactory } from '@nestjs/mongoose';
+import { Document } from 'mongoose';
+
+export enum BucketType {
+  GCP,
+  S3
+}
+
+/**
+ * Represents information on a cloud storage endpoint used
+ * for an organization
+ */
+@Schema()
+export class BucketInfo {
+  _id: string;
+
+  @Prop()
+  bucketName: string;
+
+  @Prop({ type: String, enum: BucketType })
+  bucketType: BucketType;
+
+  /** The GCP Secret Manager name for the bucket credentials */
+  @Prop()
+  secretName: string;
+
+  /** Organization the bucket is associated with */
+  @Prop()
+  organization: string;
+}
+
+export type BucketInfoDocument = BucketInfo & Document;
+export const BucketInfoSchema = SchemaFactory.createForClass(BucketInfo);
diff --git a/packages/server/src/bucket/bucket.module.ts b/packages/server/src/bucket/bucket.module.ts
new file mode 100644
index 00000000..cab5077b
--- /dev/null
+++ b/packages/server/src/bucket/bucket.module.ts
@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { MongooseModule } from '@nestjs/mongoose';
+import { BucketInfo, BucketInfoSchema } from './bucket-info.model';
+import { BucketFactory } from './bucket-factory.service';
+
+@Module({
+  imports: [MongooseModule.forFeature([{ name: BucketInfo.name, schema: BucketInfoSchema }])],
+  providers: [BucketFactory],
+  exports: [BucketFactory]
+})
+export class BucketModule {}
diff --git a/packages/server/src/bucket/bucket.ts b/packages/server/src/bucket/bucket.ts
new file mode 100644
index 00000000..dceb3723
--- /dev/null
+++ b/packages/server/src/bucket/bucket.ts
@@ -0,0 +1,19 @@
+export enum BucketObjectAction {
+  READ,
+  WRITE,
+  DELETE
+}
+
+export interface Bucket {
+  /** Get a signed URL for the given bucket location */
+  getSignedUrl(location: string, action: BucketObjectAction, expiration: Date, contentType?: string): Promise<string>;
+
+  /** Delete the object at the given location */
+  delete(location: string): Promise<void>;
+
+  /** Move an object between two locations */
+  move(originalLocation: string, finalLocation: string): Promise<void>;
+
+  /** Check if an object exists */
+  exists(location: string): Promise<boolean>;
+}

From 615da34c2304b57b73ddf7cf1084cc4ae115132a Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Wed, 20 Mar 2024 12:07:54 -0400
Subject: [PATCH 02/11] Add another layer of redirection for injecting into GCP
 bucket maker

---
 .../src/bucket/bucket-factory.service.ts      | 13 ++++++--
 packages/server/src/bucket/bucket.module.ts   |  3 +-
 packages/server/src/bucket/gcp-bucket.ts      | 31 +++++++++++++++++++
 3 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 packages/server/src/bucket/gcp-bucket.ts

diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
index ac3a0154..aaa5cd9b 100644
--- a/packages/server/src/bucket/bucket-factory.service.ts
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -1,11 +1,13 @@
 import { Injectable } from '@nestjs/common';
 import { InjectModel } from '@nestjs/mongoose';
 import { Model } from 'mongoose';
-import { BucketInfo } from './bucket-info.model';
+import { BucketInfo, BucketType } from './bucket-info.model';
+import { GcpBucketMaker } from './gcp-bucket';
+import { Bucket } from './bucket';
 
 @Injectable()
 export class BucketFactory {
-  constructor(@InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>) {}
+  constructor(@InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>, private readonly gcpBucketMaker: GcpBucketMaker) {}
 
   /**
    * Factory method which gets the correct storage bucket inforamtion for the
@@ -17,5 +19,12 @@ export class BucketFactory {
     if (!bucketInfo) {
       return bucketInfo;
     }
+
+    switch(bucketInfo.bucketType) {
+      case BucketType.GCP:
+        return this.gcpBucketMaker.getGcpBucket(bucketInfo);
+      default:
+        throw new Error(`Unsupported bucket type ${bucketInfo.bucketType}`);
+    }
   }
 }
diff --git a/packages/server/src/bucket/bucket.module.ts b/packages/server/src/bucket/bucket.module.ts
index cab5077b..d7e96b62 100644
--- a/packages/server/src/bucket/bucket.module.ts
+++ b/packages/server/src/bucket/bucket.module.ts
@@ -2,10 +2,11 @@ import { Module } from '@nestjs/common';
 import { MongooseModule } from '@nestjs/mongoose';
 import { BucketInfo, BucketInfoSchema } from './bucket-info.model';
 import { BucketFactory } from './bucket-factory.service';
+import { GcpBucketMaker } from './gcp-bucket';
 
 @Module({
   imports: [MongooseModule.forFeature([{ name: BucketInfo.name, schema: BucketInfoSchema }])],
-  providers: [BucketFactory],
+  providers: [BucketFactory, GcpBucketMaker],
   exports: [BucketFactory]
 })
 export class BucketModule {}
diff --git a/packages/server/src/bucket/gcp-bucket.ts b/packages/server/src/bucket/gcp-bucket.ts
new file mode 100644
index 00000000..a1a51759
--- /dev/null
+++ b/packages/server/src/bucket/gcp-bucket.ts
@@ -0,0 +1,31 @@
+import { Bucket, BucketObjectAction } from './bucket';
+import { BucketInfo } from './bucket-info.model';
+import { Injectable } from '@nestjs/common';
+
+/** Wrapper maker for the bucket so that dependencies can be injected */
+@Injectable()
+export class GcpBucketMaker {
+  async getGcpBucket(bucketInfo: BucketInfo): Promise<GcpBucket> {
+    return new GcpBucket(bucketInfo);
+  }
+}
+
+class GcpBucket implements Bucket {
+  constructor(bucketInfo: BucketInfo) {}
+
+  async getSignedUrl(location: string, action: BucketObjectAction, expiration: Date, contentType?: string | undefined): Promise<string> {
+      return '';
+  }
+
+  async delete(location: string): Promise<void> {
+      return;
+  }
+
+  async move(originalLocation: string, finalLocation: string): Promise<void> {
+      return;
+  }
+
+  async exists(location: string): Promise<boolean> {
+      return true;
+  }
+}

From 305f3d5373dbb7eecc3fe18aed8fcdffc999238b Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Wed, 20 Mar 2024 15:24:24 -0400
Subject: [PATCH 03/11] Make secret provider and wrap bucket creation

---
 package-lock.json                             | 37 ++++++++++++-------
 packages/server/package.json                  |  1 +
 .../src/bucket/bucket-factory.service.ts      | 18 ++++++++-
 packages/server/src/bucket/gcp-bucket.ts      | 22 +++++++++--
 packages/server/src/gcp/gcp.module.ts         |  5 ++-
 .../src/gcp/providers/secret.provider.ts      | 20 ++++++++++
 6 files changed, 82 insertions(+), 21 deletions(-)
 create mode 100644 packages/server/src/gcp/providers/secret.provider.ts

diff --git a/package-lock.json b/package-lock.json
index d2c6affb..55904f86 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -4206,6 +4206,17 @@
         "node": ">=14"
       }
     },
+    "node_modules/@google-cloud/secret-manager": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/secret-manager/-/secret-manager-5.2.0.tgz",
+      "integrity": "sha512-sGIbXCq20CAI7w8Mh5Rl6yASwkq1K350s+//P/DIdNLBHbTdVcfDYcsS+bSOQwt0JEUXTgn/imIL69VqR8+6gw==",
+      "dependencies": {
+        "google-gax": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
     "node_modules/@google-cloud/storage": {
       "version": "7.7.0",
       "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.7.0.tgz",
@@ -17846,7 +17857,6 @@
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.3.1.tgz",
       "integrity": "sha512-qpSfslpwqToIgQ+Tf3MjWIDjYK4UFIZ0uz6nLtttlW9N1NQA4PhGf9tlGo6KDYJ4rgL2w4CjXVd0z5yeNpN/Iw==",
-      "optional": true,
       "dependencies": {
         "@grpc/grpc-js": "~1.10.0",
         "@grpc/proto-loader": "^0.7.0",
@@ -17869,7 +17879,6 @@
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.0.tgz",
       "integrity": "sha512-tx+eoEsqkMkLCHR4OOplwNIaJ7SVZWzeVKzEMBz8VR+TbssgBYOP4a0P+KQiQ6LaTG4SGaIEu7YTS8xOmkOWLA==",
-      "optional": true,
       "dependencies": {
         "@grpc/proto-loader": "^0.7.8",
         "@types/node": ">=12.12.47"
@@ -17882,7 +17891,6 @@
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz",
       "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==",
-      "optional": true,
       "dependencies": {
         "end-of-stream": "^1.4.1",
         "inherits": "^2.0.3",
@@ -17894,7 +17902,6 @@
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
       "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "optional": true,
       "dependencies": {
         "inherits": "^2.0.3",
         "string_decoder": "^1.1.1",
@@ -22341,7 +22348,6 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
       "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
-      "optional": true,
       "engines": {
         "node": ">= 6"
       }
@@ -23095,7 +23101,6 @@
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.1.tgz",
       "integrity": "sha512-8awBvjO+FwkMd6gNoGFZyqkHZXCFd54CIYTb6De7dPaufGJ2XNW+QUNqbMr8MaAocMdb+KpsD4rxEOaTBDCffA==",
-      "optional": true,
       "dependencies": {
         "protobufjs": "^7.2.5"
       },
@@ -28169,6 +28174,7 @@
       "license": "UNLICENSED",
       "dependencies": {
         "@apollo/subgraph": "^2.4.12",
+        "@google-cloud/secret-manager": "^5.2.0",
         "@google-cloud/storage": "^7.7.0",
         "@jsonforms/core": "^3.2.1",
         "@nestjs/apollo": "^12.0.7",
@@ -31197,6 +31203,14 @@
       "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz",
       "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g=="
     },
+    "@google-cloud/secret-manager": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/secret-manager/-/secret-manager-5.2.0.tgz",
+      "integrity": "sha512-sGIbXCq20CAI7w8Mh5Rl6yASwkq1K350s+//P/DIdNLBHbTdVcfDYcsS+bSOQwt0JEUXTgn/imIL69VqR8+6gw==",
+      "requires": {
+        "google-gax": "^4.0.3"
+      }
+    },
     "@google-cloud/storage": {
       "version": "7.7.0",
       "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.7.0.tgz",
@@ -41282,7 +41296,6 @@
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.3.1.tgz",
       "integrity": "sha512-qpSfslpwqToIgQ+Tf3MjWIDjYK4UFIZ0uz6nLtttlW9N1NQA4PhGf9tlGo6KDYJ4rgL2w4CjXVd0z5yeNpN/Iw==",
-      "optional": true,
       "requires": {
         "@grpc/grpc-js": "~1.10.0",
         "@grpc/proto-loader": "^0.7.0",
@@ -41302,7 +41315,6 @@
           "version": "1.10.0",
           "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.0.tgz",
           "integrity": "sha512-tx+eoEsqkMkLCHR4OOplwNIaJ7SVZWzeVKzEMBz8VR+TbssgBYOP4a0P+KQiQ6LaTG4SGaIEu7YTS8xOmkOWLA==",
-          "optional": true,
           "requires": {
             "@grpc/proto-loader": "^0.7.8",
             "@types/node": ">=12.12.47"
@@ -41312,7 +41324,6 @@
           "version": "4.1.2",
           "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz",
           "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==",
-          "optional": true,
           "requires": {
             "end-of-stream": "^1.4.1",
             "inherits": "^2.0.3",
@@ -41324,7 +41335,6 @@
           "version": "3.6.2",
           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
           "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-          "optional": true,
           "requires": {
             "inherits": "^2.0.3",
             "string_decoder": "^1.1.1",
@@ -44683,8 +44693,7 @@
     "object-hash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
-      "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
-      "optional": true
+      "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="
     },
     "object-inspect": {
       "version": "1.12.3",
@@ -45233,7 +45242,6 @@
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.1.tgz",
       "integrity": "sha512-8awBvjO+FwkMd6gNoGFZyqkHZXCFd54CIYTb6De7dPaufGJ2XNW+QUNqbMr8MaAocMdb+KpsD4rxEOaTBDCffA==",
-      "optional": true,
       "requires": {
         "protobufjs": "^7.2.5"
       }
@@ -46230,6 +46238,7 @@
       "version": "file:packages/server",
       "requires": {
         "@apollo/subgraph": "^2.4.12",
+        "@google-cloud/secret-manager": "*",
         "@google-cloud/storage": "^7.7.0",
         "@graphql-codegen/typescript-graphql-request": "^6.1.0",
         "@jsonforms/core": "^3.2.1",
@@ -46260,7 +46269,7 @@
         "eslint": "^8.0.1",
         "eslint-config-prettier": "^8.3.0",
         "eslint-plugin-prettier": "^4.0.0",
-        "firebase-admin": "*",
+        "firebase-admin": "^12.0.0",
         "graphql-request": "^6.1.0",
         "graphql-type-json": "^0.3.2",
         "jest": "28.1.3",
diff --git a/packages/server/package.json b/packages/server/package.json
index 1a2f2111..052d752b 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@apollo/subgraph": "^2.4.12",
+    "@google-cloud/secret-manager": "^5.2.0",
     "@google-cloud/storage": "^7.7.0",
     "@jsonforms/core": "^3.2.1",
     "@nestjs/apollo": "^12.0.7",
diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
index aaa5cd9b..07377c34 100644
--- a/packages/server/src/bucket/bucket-factory.service.ts
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -1,13 +1,19 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Inject } from '@nestjs/common';
 import { InjectModel } from '@nestjs/mongoose';
 import { Model } from 'mongoose';
 import { BucketInfo, BucketType } from './bucket-info.model';
 import { GcpBucketMaker } from './gcp-bucket';
 import { Bucket } from './bucket';
+import { SECRET_MANAGER_PROVIDER } from 'src/gcp/providers/secret.provider';
+import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
 
 @Injectable()
 export class BucketFactory {
-  constructor(@InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>, private readonly gcpBucketMaker: GcpBucketMaker) {}
+  constructor(
+    @InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>,
+    private readonly gcpBucketMaker: GcpBucketMaker,
+    @Inject(SECRET_MANAGER_PROVIDER) private readonly secretClient: SecretManagerServiceClient
+  ) {}
 
   /**
    * Factory method which gets the correct storage bucket inforamtion for the
@@ -20,6 +26,14 @@ export class BucketFactory {
       return bucketInfo;
     }
 
+    // Get the secret associated with the bucket
+    const [version] = await this.secretClient.accessSecretVersion({ name: bucketInfo.secretName });
+    if (!version) {
+      throw new Error('Could not get credentials for bucket');
+    }
+
+    const credentials = version.payload.data.toString();
+
     switch(bucketInfo.bucketType) {
       case BucketType.GCP:
         return this.gcpBucketMaker.getGcpBucket(bucketInfo);
diff --git a/packages/server/src/bucket/gcp-bucket.ts b/packages/server/src/bucket/gcp-bucket.ts
index a1a51759..e9f6266f 100644
--- a/packages/server/src/bucket/gcp-bucket.ts
+++ b/packages/server/src/bucket/gcp-bucket.ts
@@ -1,17 +1,33 @@
 import { Bucket, BucketObjectAction } from './bucket';
 import { BucketInfo } from './bucket-info.model';
 import { Injectable } from '@nestjs/common';
+import { Storage, Bucket as StorageBucket } from '@google-cloud/storage';
 
 /** Wrapper maker for the bucket so that dependencies can be injected */
 @Injectable()
 export class GcpBucketMaker {
-  async getGcpBucket(bucketInfo: BucketInfo): Promise<GcpBucket> {
-    return new GcpBucket(bucketInfo);
+  async getGcpBucket(bucketInfo: BucketInfo, credentials: string): Promise<GcpBucket> {
+    // Expects the key to be a GCP service account key
+    let key: any = null;
+    try {
+      key = JSON.parse(credentials);
+    } catch(e) {
+      throw new Error('Failed to parse credentails');
+    }
+
+    // Make the GCP storage client
+    const storage = new Storage({ credentials: key });
+
+    // Get the bucket
+    const bucket = storage.bucket(bucketInfo.bucketName);
+
+    // Return the wrapper around the GCP bucket implementation
+    return new GcpBucket(bucket);
   }
 }
 
 class GcpBucket implements Bucket {
-  constructor(bucketInfo: BucketInfo) {}
+  constructor(private readonly storageBucket: StorageBucket) {}
 
   async getSignedUrl(location: string, action: BucketObjectAction, expiration: Date, contentType?: string | undefined): Promise<string> {
       return '';
diff --git a/packages/server/src/gcp/gcp.module.ts b/packages/server/src/gcp/gcp.module.ts
index 0d20dcd6..97deb406 100644
--- a/packages/server/src/gcp/gcp.module.ts
+++ b/packages/server/src/gcp/gcp.module.ts
@@ -1,9 +1,10 @@
 import { Module } from '@nestjs/common';
 import { storageProvider } from './providers/storage.provider';
 import { firebaseProvider } from './providers/firebase.provider';
+import { secretManagerProvider } from './providers/secret.provider';
 
 @Module({
-  providers: [storageProvider, firebaseProvider],
-  exports: [storageProvider, firebaseProvider]
+  providers: [storageProvider, firebaseProvider, secretManagerProvider],
+  exports: [storageProvider, firebaseProvider, secretManagerProvider]
 })
 export class GcpModule {}
diff --git a/packages/server/src/gcp/providers/secret.provider.ts b/packages/server/src/gcp/providers/secret.provider.ts
new file mode 100644
index 00000000..ad2b4039
--- /dev/null
+++ b/packages/server/src/gcp/providers/secret.provider.ts
@@ -0,0 +1,20 @@
+import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
+import { Provider } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+
+export const SECRET_MANAGER_PROVIDER = 'GCP_SECRET_MANAGER_PROVIDER';
+
+export const secretManagerProvider: Provider<SecretManagerServiceClient> = {
+  provide: SECRET_MANAGER_PROVIDER,
+  useFactory: (configService: ConfigService) => {
+    const keyFilename: string | undefined = configService.get<string>('gcp.storage.keyFilename');
+
+    // If no key file is provided, use the default credentials
+    if (!keyFilename) {
+      return new SecretManagerServiceClient();
+    }
+
+    return new SecretManagerServiceClient({ keyFilename });
+  },
+  inject: [ConfigService]
+};

From 161349313b8c02357ff4dd50f75bb18d7244b200 Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Wed, 20 Mar 2024 16:01:27 -0400
Subject: [PATCH 04/11] Begin process of moving over to using bucket wrapper

---
 .../src/bucket/bucket-factory.service.ts      |  8 ++-
 packages/server/src/bucket/bucket.ts          |  9 +++
 packages/server/src/bucket/gcp-bucket.ts      | 39 ++++++++++-
 packages/server/src/entry/entry.module.ts     |  4 +-
 .../src/entry/models/upload-session.model.ts  |  3 +
 .../src/entry/services/entry.service.ts       | 21 +++---
 .../entry/services/upload-session.service.ts  | 69 +++++++++++--------
 7 files changed, 107 insertions(+), 46 deletions(-)

diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
index 07377c34..304db7aa 100644
--- a/packages/server/src/bucket/bucket-factory.service.ts
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -32,11 +32,15 @@ export class BucketFactory {
       throw new Error('Could not get credentials for bucket');
     }
 
-    const credentials = version.payload.data.toString();
+    // Get the payload of the secret
+    const credentials = version.payload?.data?.toString();
+    if (!credentials) {
+      throw new Error('Unable to parse credentials');
+    }
 
     switch(bucketInfo.bucketType) {
       case BucketType.GCP:
-        return this.gcpBucketMaker.getGcpBucket(bucketInfo);
+        return this.gcpBucketMaker.getGcpBucket(bucketInfo, credentials);
       default:
         throw new Error(`Unsupported bucket type ${bucketInfo.bucketType}`);
     }
diff --git a/packages/server/src/bucket/bucket.ts b/packages/server/src/bucket/bucket.ts
index dceb3723..02b0434f 100644
--- a/packages/server/src/bucket/bucket.ts
+++ b/packages/server/src/bucket/bucket.ts
@@ -16,4 +16,13 @@ export interface Bucket {
 
   /** Check if an object exists */
   exists(location: string): Promise<boolean>;
+
+  /** Get the content type for a file */
+  getContentType(location: string): Promise<string | null>;
+
+  /** Get the contents of an object */
+  download(location: string): Promise<Buffer | null>;
+
+  /** Delete many files */
+  deleteFiles(location: string): Promise<void>;
 }
diff --git a/packages/server/src/bucket/gcp-bucket.ts b/packages/server/src/bucket/gcp-bucket.ts
index e9f6266f..fb41d849 100644
--- a/packages/server/src/bucket/gcp-bucket.ts
+++ b/packages/server/src/bucket/gcp-bucket.ts
@@ -30,7 +30,14 @@ class GcpBucket implements Bucket {
   constructor(private readonly storageBucket: StorageBucket) {}
 
   async getSignedUrl(location: string, action: BucketObjectAction, expiration: Date, contentType?: string | undefined): Promise<string> {
-      return '';
+    const file = this.storageBucket.file(location);
+
+    const [url] = await file.getSignedUrl({
+      action: this.actionToString(action),
+      expires: expiration,
+      contentType
+    });
+    return url;
   }
 
   async delete(location: string): Promise<void> {
@@ -38,10 +45,38 @@ class GcpBucket implements Bucket {
   }
 
   async move(originalLocation: string, finalLocation: string): Promise<void> {
-      return;
+    const file = this.storageBucket.file(originalLocation);
+    await file.move(finalLocation);
   }
 
   async exists(location: string): Promise<boolean> {
       return true;
   }
+
+  async getContentType(location: string): Promise<string | null> {
+    const file = this.storageBucket.file(location);
+    return file.metadata.contentType || null;
+  }
+
+  async download(location: string): Promise<Buffer | null> {
+    const file = this.storageBucket.file(location);
+    return (await file.download())[0];
+  }
+
+  async deleteFiles(location: string): Promise<void> {
+   this.storageBucket.deleteFiles({ prefix: location });
+  }
+
+  private actionToString(action: BucketObjectAction): 'read' | 'write' | 'delete' {
+    switch (action) {
+      case BucketObjectAction.READ:
+        return 'read';
+      case BucketObjectAction.WRITE:
+        return 'write';
+      case BucketObjectAction.DELETE:
+        return 'delete';
+      default:
+        throw new Error(`Unsupported action ${action}`);
+    }
+  }
 }
diff --git a/packages/server/src/entry/entry.module.ts b/packages/server/src/entry/entry.module.ts
index 01f291cd..5a8d6148 100644
--- a/packages/server/src/entry/entry.module.ts
+++ b/packages/server/src/entry/entry.module.ts
@@ -18,6 +18,7 @@ import { JwtModule } from '../jwt/jwt.module';
 import { MongooseMiddlewareService } from '../shared/service/mongoose-callback.service';
 import { SharedModule } from '../shared/shared.module';
 import { OrganizationModule } from '../organization/organization.module';
+import { BucketModule } from 'src/bucket/bucket.module';
 
 @Module({
   imports: [
@@ -44,7 +45,8 @@ import { OrganizationModule } from '../organization/organization.module';
     GcpModule,
     PermissionModule,
     JwtModule,
-    OrganizationModule
+    OrganizationModule,
+    BucketModule
   ],
   providers: [
     EntryResolver,
diff --git a/packages/server/src/entry/models/upload-session.model.ts b/packages/server/src/entry/models/upload-session.model.ts
index e07a9a41..bb7c73ca 100644
--- a/packages/server/src/entry/models/upload-session.model.ts
+++ b/packages/server/src/entry/models/upload-session.model.ts
@@ -32,6 +32,9 @@ export class UploadSession {
 
   @Prop({ required: false, type: String })
   entryPrefix: string | null;
+
+  @Prop()
+  organization: string;
 }
 
 export type UploadSessionDocument = UploadSession & Document;
diff --git a/packages/server/src/entry/services/entry.service.ts b/packages/server/src/entry/services/entry.service.ts
index 343369b5..f03a5ea9 100644
--- a/packages/server/src/entry/services/entry.service.ts
+++ b/packages/server/src/entry/services/entry.service.ts
@@ -4,21 +4,19 @@ import { Entry } from '../models/entry.model';
 import { Model } from 'mongoose';
 import { EntryCreate } from '../dtos/create.dto';
 import { Dataset } from '../../dataset/dataset.model';
-import { GCP_STORAGE_PROVIDER } from '../../gcp/providers/storage.provider';
-import { Bucket, Storage } from '@google-cloud/storage';
 import { ConfigService } from '@nestjs/config';
 import { TokenPayload } from '../../jwt/token.dto';
+import { BucketFactory } from 'src/bucket/bucket-factory.service';
+import { BucketObjectAction } from 'src/bucket/bucket';
 
 @Injectable()
 export class EntryService {
-  private readonly bucketName = this.configService.getOrThrow<string>('gcp.storage.bucket');
-  private readonly bucket: Bucket = this.storage.bucket(this.bucketName);
   private readonly expiration = this.configService.getOrThrow<number>('entry.signedURLExpiration');
 
   constructor(
     @InjectModel(Entry.name) private readonly entryModel: Model<Entry>,
-    @Inject(GCP_STORAGE_PROVIDER) private readonly storage: Storage,
-    private readonly configService: ConfigService
+    private readonly configService: ConfigService,
+    private readonly bucketFactory: BucketFactory
   ) {}
 
   async find(entryID: string): Promise<Entry | null> {
@@ -56,12 +54,11 @@ export class EntryService {
   }
 
   async getSignedUrl(entry: Entry): Promise<string> {
-    const file = this.bucket.file(entry.bucketLocation);
-    const [url] = await file.getSignedUrl({
-      action: 'read',
-      expires: Date.now() + this.expiration
-    });
-    return url;
+    const bucket = await this.bucketFactory.getBucket(entry.organization);
+    if (!bucket) {
+      throw new Error('Missing bucket for entry');
+    }
+    return bucket.getSignedUrl(entry.bucketLocation, BucketObjectAction.READ, new Date(Date.now() + this.expiration));
   }
 
   /**
diff --git a/packages/server/src/entry/services/upload-session.service.ts b/packages/server/src/entry/services/upload-session.service.ts
index ae726bde..8256f258 100644
--- a/packages/server/src/entry/services/upload-session.service.ts
+++ b/packages/server/src/entry/services/upload-session.service.ts
@@ -1,10 +1,8 @@
-import { Injectable, Inject, BadRequestException } from '@nestjs/common';
+import { Injectable, BadRequestException } from '@nestjs/common';
 import { UploadSession } from '../models/upload-session.model';
 import { InjectModel } from '@nestjs/mongoose';
 import { Model } from 'mongoose';
 import { Dataset } from '../../dataset/dataset.model';
-import { GCP_STORAGE_PROVIDER } from '../../gcp/providers/storage.provider';
-import { Bucket, Storage } from '@google-cloud/storage';
 import { ConfigService } from '@nestjs/config';
 import { CsvValidationService } from './csv-validation.service';
 import { DatasetService } from '../../dataset/dataset.service';
@@ -12,23 +10,23 @@ import { EntryUploadService } from '../services/entry-upload.service';
 import { UploadStatus, UploadResult } from '../dtos/upload-result.dto';
 import { EntryService } from './entry.service';
 import { TokenPayload } from '../../jwt/token.dto';
+import { BucketFactory } from 'src/bucket/bucket-factory.service';
+import { BucketObjectAction } from 'src/bucket/bucket';
 
 @Injectable()
 export class UploadSessionService {
-  private readonly uploadBucket = this.configService.getOrThrow<string>('gcp.storage.bucket');
   private readonly uploadPrefix = this.configService.getOrThrow<string>('upload.prefix');
   private readonly csvFileName = this.configService.getOrThrow<string>('upload.csvFileName');
   private readonly entryFolder = this.configService.getOrThrow<string>('upload.entryFolder');
-  private readonly bucket: Bucket = this.storage.bucket(this.uploadBucket);
 
   constructor(
     @InjectModel(UploadSession.name) private readonly uploadSessionModel: Model<UploadSession>,
-    @Inject(GCP_STORAGE_PROVIDER) private readonly storage: Storage,
     private readonly configService: ConfigService,
     private readonly csvValidation: CsvValidationService,
     private readonly datasetService: DatasetService,
     private readonly entryUploadService: EntryUploadService,
-    private readonly entryService: EntryService
+    private readonly entryService: EntryService,
+    private readonly bucketFactory: BucketFactory
   ) {}
 
   async find(id: string): Promise<UploadSession | null> {
@@ -73,15 +71,17 @@ export class UploadSessionService {
 
     const missingEntries: string[] = [];
 
+    const bucket = await this.bucketFactory.getBucket(uploadSession.organization);
+    if (!bucket) {
+      throw new Error('Could not find bucket for organization');
+    }
+
     // Go over each entry and move it to the dataset
     for (const entryUpload of entryUploads) {
       const entryURL = `${uploadSession.entryPrefix}/${entryUpload.filename}`;
 
-      const entryFile = this.bucket.file(entryURL);
-
       // Verify the entry is in the bucket
-      const exists = await entryFile.exists();
-      if (!exists[0]) {
+      if (await bucket.exists(entryURL)) {
         missingEntries.push(`Entry ${entryUpload.filename} not found`);
         continue;
       }
@@ -90,7 +90,7 @@ export class UploadSessionService {
       const entry = await this.entryService.create(
         {
           entryID: entryUpload.entryID,
-          contentType: entryFile.metadata.contentType,
+          contentType: bucket.getContentType(entryURL)!,
           meta: entryUpload.metadata
         },
         dataset,
@@ -102,7 +102,7 @@ export class UploadSessionService {
       const fileExtension = entryUpload.filename.split('.').pop();
       const filename = `${entry._id}.${fileExtension}`;
       const newName = `${dataset.bucketPrefix}/${filename}`;
-      await entryFile.move(newName);
+      await bucket.move(entryURL, newName);
 
       // Add the bucket URL to the entry
       await this.entryService.setBucketLocation(entry, newName);
@@ -128,12 +128,13 @@ export class UploadSessionService {
   async getCSVUploadURL(uploadSession: UploadSession): Promise<string> {
     const csvURL = `${this.uploadPrefix}/${uploadSession.bucketPrefix}/${this.csvFileName}`;
     const entryPrefix = `${this.uploadPrefix}/${uploadSession.bucketPrefix}/${this.entryFolder}`;
+    const bucket = await this.bucketFactory.getBucket(uploadSession.organization);
+    if (!bucket) {
+      throw new Error('Bucket not found for organization');
+    }
 
-    const [url] = await this.bucket.file(csvURL).getSignedUrl({
-      action: 'write',
-      expires: Date.now() + 2 * 60 * 1000, // 2 minutes
-      contentType: 'text/csv'
-    });
+    // Make the URL
+    const url = await bucket.getSignedUrl(csvURL, BucketObjectAction.WRITE, new Date(Date.now() + 2 * 60 * 1000), 'text/csv');
 
     // Add the url to the upload session to signify the upload is ready
     await this.uploadSessionModel.updateOne({ _id: uploadSession._id }, { $set: { csvURL, entryPrefix } });
@@ -145,15 +146,14 @@ export class UploadSessionService {
     if (!uploadSession.entryPrefix) {
       throw new BadRequestException('CSV must be uploaded before entries');
     }
+    const bucket = await this.bucketFactory.getBucket(uploadSession.organization);
+    if (!bucket) {
+      throw new Error('Bucket not found for organization');
+    }
 
     const entryURL = `${uploadSession.entryPrefix}/${filename}`;
 
-    const [url] = await this.bucket.file(entryURL).getSignedUrl({
-      action: 'write',
-      expires: Date.now() + 2 * 60 * 1000, // 2 minutes
-      contentType: filetype
-    });
-
+    const url = await bucket.getSignedUrl(entryURL, BucketObjectAction.WRITE, new Date(Date.now() + 2 * 60 * 1000), filetype);
     return url;
   }
 
@@ -162,14 +162,21 @@ export class UploadSessionService {
     if (!uploadSession.csvURL) {
       throw new BadRequestException('CSV URL not found');
     }
-    const exists = await this.bucket.file(uploadSession.csvURL).exists();
+    const bucket = await this.bucketFactory.getBucket(uploadSession.organization);
+    if (!bucket) {
+      throw new Error('Bucket not found for organization');
+    }
+
+    const exists = await bucket.exists(uploadSession.csvURL);
     if (!exists) {
       throw new BadRequestException('CSV not found');
     }
 
     // Download the CSV to /tmp location
-    const csvFile = this.bucket.file(uploadSession.csvURL);
-    const csvFileContents = await csvFile.download();
+    const csvFileContents = await bucket.download(uploadSession.csvURL);
+    if (!csvFileContents) {
+      throw new Error('Failed to read file');
+    }
 
     // Get the cooresponding dataset
     const dataset = await this.datasetService.findById(uploadSession.dataset);
@@ -178,7 +185,7 @@ export class UploadSessionService {
     }
 
     // Validate the CSV contents against the target dataset
-    const csvValidationResults = await this.csvValidation.validate(csvFileContents[0], dataset, uploadSession);
+    const csvValidationResults = await this.csvValidation.validate(csvFileContents, dataset, uploadSession);
 
     if (!csvValidationResults.success) {
       // TODO: Add object type return here
@@ -197,11 +204,15 @@ export class UploadSessionService {
   // TODO: Provide user information
   private async deleteOldSession(dataset: Dataset): Promise<void> {
     const existing = await this.uploadSessionModel.findOne({ dataset: dataset._id }).exec();
+    const bucket = await this.bucketFactory.getBucket(dataset.organization);
+    if (!bucket) {
+      throw new Error('Bucket not found for organization');
+    }
     if (existing) {
       // Delete the in progress entry uploads
       await this.entryUploadService.deleteForSession(existing);
       // Remove cooresponding upload files
-      await this.bucket.deleteFiles({ prefix: `${this.uploadPrefix}/${existing.bucketPrefix}` });
+      await bucket.deleteFiles(`${this.uploadPrefix}/${existing.bucketPrefix}`)
       // Remove the upload session itself
       await this.uploadSessionModel.deleteOne({ _id: existing._id }).exec();
     }

From c3fb4c81d92d5e7dfca1126cb90df5828dc86a8b Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Wed, 20 Mar 2024 16:11:21 -0400
Subject: [PATCH 05/11] Removed usage of GCP storage directly

---
 .../entry/services/upload-session.service.ts  |  3 +-
 .../src/tag/models/video-field.model.ts       |  3 ++
 .../src/tag/services/video-field.service.ts   | 36 ++++++++++---------
 3 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/entry/services/upload-session.service.ts b/packages/server/src/entry/services/upload-session.service.ts
index 8256f258..0f6750a9 100644
--- a/packages/server/src/entry/services/upload-session.service.ts
+++ b/packages/server/src/entry/services/upload-session.service.ts
@@ -39,7 +39,8 @@ export class UploadSessionService {
     // Make the session
     const uploadSession = await this.uploadSessionModel.create({
       dataset: dataset._id,
-      created: new Date()
+      created: new Date(),
+      organization: dataset.organization
     });
 
     // Add in the bucket prefix for the session
diff --git a/packages/server/src/tag/models/video-field.model.ts b/packages/server/src/tag/models/video-field.model.ts
index 71550632..8fb1f084 100644
--- a/packages/server/src/tag/models/video-field.model.ts
+++ b/packages/server/src/tag/models/video-field.model.ts
@@ -28,6 +28,9 @@ export class VideoField {
   /** Where within the bucket the video is stored */
   @Prop()
   bucketLocation: string;
+
+  @Prop()
+  organization: string;
 }
 
 export type VideoFieldDocument = VideoField & Document;
diff --git a/packages/server/src/tag/services/video-field.service.ts b/packages/server/src/tag/services/video-field.service.ts
index c6c4b7ae..92d3f12d 100644
--- a/packages/server/src/tag/services/video-field.service.ts
+++ b/packages/server/src/tag/services/video-field.service.ts
@@ -1,24 +1,22 @@
-import { BadRequestException, Injectable, Inject } from '@nestjs/common';
+import { BadRequestException, Injectable } from '@nestjs/common';
 import { InjectModel } from '@nestjs/mongoose';
 import { VideoField, VideoFieldDocument } from '../models/video-field.model';
 import { Model } from 'mongoose';
 import { Tag } from '../models/tag.model';
 import { StudyService } from '../../study/study.service';
 import { ConfigService } from '@nestjs/config';
-import { GCP_STORAGE_PROVIDER } from '../../gcp/providers/storage.provider';
-import { Storage, Bucket } from '@google-cloud/storage';
 import { Entry } from '../../entry/models/entry.model';
 import { EntryService } from '../../entry/services/entry.service';
 import { DatasetPipe } from '../../dataset/pipes/dataset.pipe';
 import { TokenPayload } from '../../jwt/token.dto';
 import { Dataset } from '../../dataset/dataset.model';
+import { BucketFactory } from 'src/bucket/bucket-factory.service';
+import { BucketObjectAction } from 'src/bucket/bucket';
 
 @Injectable()
 export class VideoFieldService {
   private readonly bucketPrefix = this.configService.getOrThrow<string>('tag.videoFieldFolder');
   private readonly videoRecordFileType = this.configService.getOrThrow<string>('tag.videoRecordFileType');
-  private readonly bucketName = this.configService.getOrThrow<string>('gcp.storage.bucket');
-  private readonly bucket: Bucket = this.storage.bucket(this.bucketName);
   private readonly expiration = this.configService.getOrThrow<number>('tag.videoUploadExpiration');
   private readonly trainingPrefix = this.configService.getOrThrow<string>('tag.trainingPrefix');
 
@@ -26,9 +24,9 @@ export class VideoFieldService {
     @InjectModel(VideoField.name) private readonly videoFieldModel: Model<VideoFieldDocument>,
     private readonly studyService: StudyService,
     private readonly configService: ConfigService,
-    @Inject(GCP_STORAGE_PROVIDER) private readonly storage: Storage,
     private readonly entryService: EntryService,
-    private readonly datasetPipe: DatasetPipe
+    private readonly datasetPipe: DatasetPipe,
+    private readonly bucketFactory: BucketFactory
   ) {}
 
   async saveVideoField(tag: Tag, field: string, index: number): Promise<VideoField> {
@@ -56,17 +54,19 @@ export class VideoFieldService {
       tag: tag._id,
       field,
       index,
-      bucketLocation: this.getVideoFieldBucketLocation(tag._id, field, index)
+      bucketLocation: this.getVideoFieldBucketLocation(tag._id, field, index),
+      organization: study.organization
     });
   }
 
   async getUploadURL(videoField: VideoField): Promise<string> {
-    const file = this.bucket.file(this.getVideoFieldBucketLocation(videoField.tag, videoField.field, videoField.index));
-    const [url] = await file.getSignedUrl({
-      action: 'write',
-      expires: Date.now() + this.expiration,
-      contentType: 'video/webm'
-    });
+    const bucket = await this.bucketFactory.getBucket(videoField.organization);
+    if (!bucket) {
+      throw new Error('Could not find bucket for video field');
+    }
+
+    const file = this.getVideoFieldBucketLocation(videoField.tag, videoField.field, videoField.index);
+    const url = await bucket.getSignedUrl(file, BucketObjectAction.WRITE, new Date(Date.now() + this.expiration), 'video/webm');
     return url;
   }
 
@@ -79,6 +79,10 @@ export class VideoFieldService {
     if (!videoField) {
       throw new BadRequestException(`Video field ${videoFieldID} not found`);
     }
+    const bucket = await this.bucketFactory.getBucket(videoField.organization);
+    if (!bucket) {
+      throw new Error('Could not find bucket for video field');
+    }
 
     // The dataset that the entry would be associated with
     const dataset: Dataset = await this.datasetPipe.transform(datasetID);
@@ -102,9 +106,7 @@ export class VideoFieldService {
     }
 
     // Move the video to the permanent location
-    const source = this.bucket.file(videoField.bucketLocation);
-    await source.move(newLocation);
-    await this.entryService.setBucketLocation(entry, newLocation);
+    await bucket.move(videoField.bucketLocation, newLocation);
     entry.bucketLocation = newLocation;
 
     // Remove the video field

From 98b031c7a9d0749346ee2fe1761a32c5d932de74 Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Wed, 20 Mar 2024 16:16:06 -0400
Subject: [PATCH 06/11] Add missing import into module

---
 packages/server/src/bucket/bucket.module.ts | 3 ++-
 packages/server/src/tag/tag.module.ts       | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/bucket/bucket.module.ts b/packages/server/src/bucket/bucket.module.ts
index d7e96b62..5a579b91 100644
--- a/packages/server/src/bucket/bucket.module.ts
+++ b/packages/server/src/bucket/bucket.module.ts
@@ -3,9 +3,10 @@ import { MongooseModule } from '@nestjs/mongoose';
 import { BucketInfo, BucketInfoSchema } from './bucket-info.model';
 import { BucketFactory } from './bucket-factory.service';
 import { GcpBucketMaker } from './gcp-bucket';
+import { GcpModule } from 'src/gcp/gcp.module';
 
 @Module({
-  imports: [MongooseModule.forFeature([{ name: BucketInfo.name, schema: BucketInfoSchema }])],
+  imports: [MongooseModule.forFeature([{ name: BucketInfo.name, schema: BucketInfoSchema }]), GcpModule],
   providers: [BucketFactory, GcpBucketMaker],
   exports: [BucketFactory]
 })
diff --git a/packages/server/src/tag/tag.module.ts b/packages/server/src/tag/tag.module.ts
index 0d5e81a7..09b06125 100644
--- a/packages/server/src/tag/tag.module.ts
+++ b/packages/server/src/tag/tag.module.ts
@@ -19,6 +19,7 @@ import { DatasetModule } from '../dataset/dataset.module';
 import { TrainingSet, TrainingSetSchema } from './models/training-set';
 import { TrainingSetResolver } from './resolvers/training-set.resolver';
 import { TrainingSetService } from './services/training-set.service';
+import { BucketModule } from 'src/bucket/bucket.module';
 
 @Module({
   imports: [
@@ -32,7 +33,8 @@ import { TrainingSetService } from './services/training-set.service';
     SharedModule,
     PermissionModule,
     GcpModule,
-    DatasetModule
+    DatasetModule,
+    BucketModule
   ],
   providers: [
     TagService,

From d90c0af5f765220cf5ef56a416dfedd78d611c59 Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Fri, 22 Mar 2024 11:49:23 -0400
Subject: [PATCH 07/11] Working loading of bucket info

---
 packages/server/src/bucket/bucket-factory.service.ts | 4 +++-
 packages/server/src/bucket/bucket-info.model.ts      | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
index 304db7aa..daeab0e7 100644
--- a/packages/server/src/bucket/bucket-factory.service.ts
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -13,7 +13,9 @@ export class BucketFactory {
     @InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>,
     private readonly gcpBucketMaker: GcpBucketMaker,
     @Inject(SECRET_MANAGER_PROVIDER) private readonly secretClient: SecretManagerServiceClient
-  ) {}
+  ) {
+    // this.bucketInfoModel.create({ bucketName: 'asl-lex', bucketType: BucketType.GCP, secretName: 'projects/294876375307/secrets/bucket-service-account/versions/latest', organization: '659591685de45d852d4202bd' })
+  }
 
   /**
    * Factory method which gets the correct storage bucket inforamtion for the
diff --git a/packages/server/src/bucket/bucket-info.model.ts b/packages/server/src/bucket/bucket-info.model.ts
index 3aa70c0a..697b912d 100644
--- a/packages/server/src/bucket/bucket-info.model.ts
+++ b/packages/server/src/bucket/bucket-info.model.ts
@@ -2,8 +2,8 @@ import { Schema, Prop, SchemaFactory } from '@nestjs/mongoose';
 import { Document } from 'mongoose';
 
 export enum BucketType {
-  GCP,
-  S3
+  GCP = 'GCP',
+  S3 = 'S3'
 }
 
 /**

From 51ff2a39307e9362e7b154a7a1841a4adee23ea2 Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Fri, 22 Mar 2024 12:47:05 -0400
Subject: [PATCH 08/11] Fix file upload process

---
 packages/server/src/bucket/gcp-bucket.ts                   | 7 ++++---
 .../server/src/entry/services/upload-session.service.ts    | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/bucket/gcp-bucket.ts b/packages/server/src/bucket/gcp-bucket.ts
index fb41d849..f6e53c78 100644
--- a/packages/server/src/bucket/gcp-bucket.ts
+++ b/packages/server/src/bucket/gcp-bucket.ts
@@ -41,7 +41,6 @@ class GcpBucket implements Bucket {
   }
 
   async delete(location: string): Promise<void> {
-      return;
   }
 
   async move(originalLocation: string, finalLocation: string): Promise<void> {
@@ -50,11 +49,13 @@ class GcpBucket implements Bucket {
   }
 
   async exists(location: string): Promise<boolean> {
-      return true;
+    const exists = await this.storageBucket.file(location).exists();
+    return exists[0];
   }
 
   async getContentType(location: string): Promise<string | null> {
     const file = this.storageBucket.file(location);
+    await file.getMetadata();
     return file.metadata.contentType || null;
   }
 
@@ -64,7 +65,7 @@ class GcpBucket implements Bucket {
   }
 
   async deleteFiles(location: string): Promise<void> {
-   this.storageBucket.deleteFiles({ prefix: location });
+    await this.storageBucket.deleteFiles({ prefix: location });
   }
 
   private actionToString(action: BucketObjectAction): 'read' | 'write' | 'delete' {
diff --git a/packages/server/src/entry/services/upload-session.service.ts b/packages/server/src/entry/services/upload-session.service.ts
index 0f6750a9..a12a609c 100644
--- a/packages/server/src/entry/services/upload-session.service.ts
+++ b/packages/server/src/entry/services/upload-session.service.ts
@@ -82,7 +82,7 @@ export class UploadSessionService {
       const entryURL = `${uploadSession.entryPrefix}/${entryUpload.filename}`;
 
       // Verify the entry is in the bucket
-      if (await bucket.exists(entryURL)) {
+      if (!await bucket.exists(entryURL)) {
         missingEntries.push(`Entry ${entryUpload.filename} not found`);
         continue;
       }
@@ -91,7 +91,7 @@ export class UploadSessionService {
       const entry = await this.entryService.create(
         {
           entryID: entryUpload.entryID,
-          contentType: bucket.getContentType(entryURL)!,
+          contentType: await bucket.getContentType(entryURL),
           meta: entryUpload.metadata
         },
         dataset,

From 7600c4f52254ba1119ccf1eaffc8b0dbb89e66ab Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Fri, 22 Mar 2024 13:08:04 -0400
Subject: [PATCH 09/11] Fix issue with no organization on study object

---
 packages/server/src/bucket/bucket-factory.service.ts |  4 +---
 packages/server/src/bucket/gcp-bucket.ts             |  1 +
 packages/server/src/study/study.module.ts            |  4 +++-
 packages/server/src/study/study.resolver.ts          | 10 +++++++---
 packages/server/src/study/study.service.ts           |  4 ++--
 5 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
index daeab0e7..304db7aa 100644
--- a/packages/server/src/bucket/bucket-factory.service.ts
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -13,9 +13,7 @@ export class BucketFactory {
     @InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>,
     private readonly gcpBucketMaker: GcpBucketMaker,
     @Inject(SECRET_MANAGER_PROVIDER) private readonly secretClient: SecretManagerServiceClient
-  ) {
-    // this.bucketInfoModel.create({ bucketName: 'asl-lex', bucketType: BucketType.GCP, secretName: 'projects/294876375307/secrets/bucket-service-account/versions/latest', organization: '659591685de45d852d4202bd' })
-  }
+  ) {}
 
   /**
    * Factory method which gets the correct storage bucket inforamtion for the
diff --git a/packages/server/src/bucket/gcp-bucket.ts b/packages/server/src/bucket/gcp-bucket.ts
index f6e53c78..45c314d2 100644
--- a/packages/server/src/bucket/gcp-bucket.ts
+++ b/packages/server/src/bucket/gcp-bucket.ts
@@ -41,6 +41,7 @@ class GcpBucket implements Bucket {
   }
 
   async delete(location: string): Promise<void> {
+    await this.deleteFiles(location);
   }
 
   async move(originalLocation: string, finalLocation: string): Promise<void> {
diff --git a/packages/server/src/study/study.module.ts b/packages/server/src/study/study.module.ts
index 4d38bf84..93bc015c 100644
--- a/packages/server/src/study/study.module.ts
+++ b/packages/server/src/study/study.module.ts
@@ -10,6 +10,7 @@ import { MongooseMiddlewareService } from '../shared/service/mongoose-callback.s
 import { SharedModule } from '../shared/shared.module';
 import { JwtModule } from '../jwt/jwt.module';
 import { PermissionModule } from '../permission/permission.module';
+import { OrganizationModule } from '../organization/organization.module';
 
 @Module({
   imports: [
@@ -33,7 +34,8 @@ import { PermissionModule } from '../permission/permission.module';
     ProjectModule,
     SharedModule,
     JwtModule,
-    forwardRef(() => PermissionModule)
+    forwardRef(() => PermissionModule),
+    OrganizationModule
   ],
   providers: [StudyService, StudyResolver, StudyPipe, StudyCreatePipe],
   exports: [StudyService, StudyPipe]
diff --git a/packages/server/src/study/study.resolver.ts b/packages/server/src/study/study.resolver.ts
index 1fad952f..cfac4afd 100644
--- a/packages/server/src/study/study.resolver.ts
+++ b/packages/server/src/study/study.resolver.ts
@@ -13,8 +13,11 @@ import * as casbin from 'casbin';
 import { StudyPermissions } from '../permission/permissions/study';
 import { TokenContext } from '../jwt/token.context';
 import { TokenPayload } from '../jwt/token.dto';
+import { OrganizationContext } from '../organization/organization.context';
+import { OrganizationGuard } from '../organization/organization.guard';
+import { Organization } from '../organization/organization.model';
 
-@UseGuards(JwtAuthGuard)
+@UseGuards(JwtAuthGuard, OrganizationGuard)
 @Resolver(() => Study)
 export class StudyResolver {
   constructor(
@@ -25,13 +28,14 @@ export class StudyResolver {
   @Mutation(() => Study)
   async createStudy(
     @Args('study', { type: () => StudyCreate }, StudyCreatePipe) study: StudyCreate,
-    @TokenContext() user: TokenPayload
+    @TokenContext() user: TokenPayload,
+    @OrganizationContext() organization: Organization
   ): Promise<Study> {
     if (!(await this.enforcer.enforce(user.user_id, StudyPermissions.CREATE, study.project))) {
       throw new UnauthorizedException('User cannot create studies on this project');
     }
 
-    return this.studyService.create(study);
+    return this.studyService.create(study, organization._id);
   }
 
   @Query(() => Boolean)
diff --git a/packages/server/src/study/study.service.ts b/packages/server/src/study/study.service.ts
index a831e83a..48631998 100644
--- a/packages/server/src/study/study.service.ts
+++ b/packages/server/src/study/study.service.ts
@@ -23,8 +23,8 @@ export class StudyService {
     });
   }
 
-  async create(study: StudyCreate): Promise<Study> {
-    const newStudy = await this.studyModel.create(study);
+  async create(study: StudyCreate, organization: string): Promise<Study> {
+    const newStudy = await this.studyModel.create({...study, organization });
 
     // Make the study - project relation in the enforcer model
     await this.enforcer.addNamedGroupingPolicy('g2', study.project, newStudy._id.toString());

From a7c6be7b31e907d7ade43fbb9b4a18b4a397257a Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Fri, 22 Mar 2024 14:13:24 -0400
Subject: [PATCH 10/11] Fix missing upload on entry file location

---
 packages/server/src/tag/services/video-field.service.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/tag/services/video-field.service.ts b/packages/server/src/tag/services/video-field.service.ts
index 92d3f12d..02547d45 100644
--- a/packages/server/src/tag/services/video-field.service.ts
+++ b/packages/server/src/tag/services/video-field.service.ts
@@ -107,6 +107,7 @@ export class VideoFieldService {
 
     // Move the video to the permanent location
     await bucket.move(videoField.bucketLocation, newLocation);
+    await this.entryService.setBucketLocation(entry, newLocation);
     entry.bucketLocation = newLocation;
 
     // Remove the video field

From a59c3269be91ed2c5bc12bc273e031ce78a23ca1 Mon Sep 17 00:00:00 2001
From: cbolles <collinbolles@gmail.com>
Date: Mon, 25 Mar 2024 11:47:50 -0400
Subject: [PATCH 11/11] Fix formatting

---
 .../src/bucket/bucket-factory.service.ts       |  2 +-
 packages/server/src/bucket/gcp-bucket.ts       |  9 +++++++--
 .../entry/services/upload-session.service.ts   | 18 ++++++++++++++----
 packages/server/src/study/study.service.ts     |  2 +-
 .../src/tag/services/video-field.service.ts    |  7 ++++++-
 5 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/bucket/bucket-factory.service.ts b/packages/server/src/bucket/bucket-factory.service.ts
index 304db7aa..292cce8c 100644
--- a/packages/server/src/bucket/bucket-factory.service.ts
+++ b/packages/server/src/bucket/bucket-factory.service.ts
@@ -38,7 +38,7 @@ export class BucketFactory {
       throw new Error('Unable to parse credentials');
     }
 
-    switch(bucketInfo.bucketType) {
+    switch (bucketInfo.bucketType) {
       case BucketType.GCP:
         return this.gcpBucketMaker.getGcpBucket(bucketInfo, credentials);
       default:
diff --git a/packages/server/src/bucket/gcp-bucket.ts b/packages/server/src/bucket/gcp-bucket.ts
index 45c314d2..dbcaf911 100644
--- a/packages/server/src/bucket/gcp-bucket.ts
+++ b/packages/server/src/bucket/gcp-bucket.ts
@@ -11,7 +11,7 @@ export class GcpBucketMaker {
     let key: any = null;
     try {
       key = JSON.parse(credentials);
-    } catch(e) {
+    } catch (e) {
       throw new Error('Failed to parse credentails');
     }
 
@@ -29,7 +29,12 @@ export class GcpBucketMaker {
 class GcpBucket implements Bucket {
   constructor(private readonly storageBucket: StorageBucket) {}
 
-  async getSignedUrl(location: string, action: BucketObjectAction, expiration: Date, contentType?: string | undefined): Promise<string> {
+  async getSignedUrl(
+    location: string,
+    action: BucketObjectAction,
+    expiration: Date,
+    contentType?: string | undefined
+  ): Promise<string> {
     const file = this.storageBucket.file(location);
 
     const [url] = await file.getSignedUrl({
diff --git a/packages/server/src/entry/services/upload-session.service.ts b/packages/server/src/entry/services/upload-session.service.ts
index a12a609c..fb8e98b2 100644
--- a/packages/server/src/entry/services/upload-session.service.ts
+++ b/packages/server/src/entry/services/upload-session.service.ts
@@ -82,7 +82,7 @@ export class UploadSessionService {
       const entryURL = `${uploadSession.entryPrefix}/${entryUpload.filename}`;
 
       // Verify the entry is in the bucket
-      if (!await bucket.exists(entryURL)) {
+      if (!(await bucket.exists(entryURL))) {
         missingEntries.push(`Entry ${entryUpload.filename} not found`);
         continue;
       }
@@ -135,7 +135,12 @@ export class UploadSessionService {
     }
 
     // Make the URL
-    const url = await bucket.getSignedUrl(csvURL, BucketObjectAction.WRITE, new Date(Date.now() + 2 * 60 * 1000), 'text/csv');
+    const url = await bucket.getSignedUrl(
+      csvURL,
+      BucketObjectAction.WRITE,
+      new Date(Date.now() + 2 * 60 * 1000),
+      'text/csv'
+    );
 
     // Add the url to the upload session to signify the upload is ready
     await this.uploadSessionModel.updateOne({ _id: uploadSession._id }, { $set: { csvURL, entryPrefix } });
@@ -154,7 +159,12 @@ export class UploadSessionService {
 
     const entryURL = `${uploadSession.entryPrefix}/${filename}`;
 
-    const url = await bucket.getSignedUrl(entryURL, BucketObjectAction.WRITE, new Date(Date.now() + 2 * 60 * 1000), filetype);
+    const url = await bucket.getSignedUrl(
+      entryURL,
+      BucketObjectAction.WRITE,
+      new Date(Date.now() + 2 * 60 * 1000),
+      filetype
+    );
     return url;
   }
 
@@ -213,7 +223,7 @@ export class UploadSessionService {
       // Delete the in progress entry uploads
       await this.entryUploadService.deleteForSession(existing);
       // Remove cooresponding upload files
-      await bucket.deleteFiles(`${this.uploadPrefix}/${existing.bucketPrefix}`)
+      await bucket.deleteFiles(`${this.uploadPrefix}/${existing.bucketPrefix}`);
       // Remove the upload session itself
       await this.uploadSessionModel.deleteOne({ _id: existing._id }).exec();
     }
diff --git a/packages/server/src/study/study.service.ts b/packages/server/src/study/study.service.ts
index 48631998..8671b7de 100644
--- a/packages/server/src/study/study.service.ts
+++ b/packages/server/src/study/study.service.ts
@@ -24,7 +24,7 @@ export class StudyService {
   }
 
   async create(study: StudyCreate, organization: string): Promise<Study> {
-    const newStudy = await this.studyModel.create({...study, organization });
+    const newStudy = await this.studyModel.create({ ...study, organization });
 
     // Make the study - project relation in the enforcer model
     await this.enforcer.addNamedGroupingPolicy('g2', study.project, newStudy._id.toString());
diff --git a/packages/server/src/tag/services/video-field.service.ts b/packages/server/src/tag/services/video-field.service.ts
index 02547d45..8e658681 100644
--- a/packages/server/src/tag/services/video-field.service.ts
+++ b/packages/server/src/tag/services/video-field.service.ts
@@ -66,7 +66,12 @@ export class VideoFieldService {
     }
 
     const file = this.getVideoFieldBucketLocation(videoField.tag, videoField.field, videoField.index);
-    const url = await bucket.getSignedUrl(file, BucketObjectAction.WRITE, new Date(Date.now() + this.expiration), 'video/webm');
+    const url = await bucket.getSignedUrl(
+      file,
+      BucketObjectAction.WRITE,
+      new Date(Date.now() + this.expiration),
+      'video/webm'
+    );
     return url;
   }