feat: Organization Bucket

packages/server/package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@apollo/subgraph": "^2.4.12",
+    "@google-cloud/secret-manager": "^5.2.0",
     "@google-cloud/storage": "^7.7.0",
     "@jsonforms/core": "^3.2.1",
     "@nestjs/apollo": "^12.0.7",

packages/server/src/app.module.ts

@@ -14,6 +14,7 @@ import { SharedModule } from './shared/shared.module';
 import { JwtModule } from './jwt/jwt.module';
 import { PermissionModule } from './permission/permission.module';
 import { UserModule } from './user/user.module';
+import { BucketModule } from './bucket/bucket.module';
 
 @Module({
   imports: [
@@ -44,7 +45,8 @@ import { UserModule } from './user/user.module';
     SharedModule,
     JwtModule,
     PermissionModule,
-    UserModule
+    UserModule,
+    BucketModule
   ]
 })
 export class AppModule {}

packages/server/src/bucket/bucket-factory.service.ts

@@ -0,0 +1,48 @@
+import { Injectable, Inject } from '@nestjs/common';
+import { InjectModel } from '@nestjs/mongoose';
+import { Model } from 'mongoose';
+import { BucketInfo, BucketType } from './bucket-info.model';
+import { GcpBucketMaker } from './gcp-bucket';
+import { Bucket } from './bucket';
+import { SECRET_MANAGER_PROVIDER } from 'src/gcp/providers/secret.provider';
+import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
+
+@Injectable()
+export class BucketFactory {
+  constructor(
+    @InjectModel(BucketInfo.name) private readonly bucketInfoModel: Model<BucketInfo>,
+    private readonly gcpBucketMaker: GcpBucketMaker,
+    @Inject(SECRET_MANAGER_PROVIDER) private readonly secretClient: SecretManagerServiceClient
+  ) {}
+
+  /**
+   * Factory method which gets the correct storage bucket inforamtion for the
+   * given organization.
+   */
+  async getBucket(organization: string): Promise<Bucket | null> {
+    // Get the information on the bucket for the given organization
+    const bucketInfo = await this.bucketInfoModel.findOne({ organization });
+    if (!bucketInfo) {
+      return bucketInfo;
+    }
+
+    // Get the secret associated with the bucket
+    const [version] = await this.secretClient.accessSecretVersion({ name: bucketInfo.secretName });
+    if (!version) {
+      throw new Error('Could not get credentials for bucket');
+    }
+
+    // Get the payload of the secret
+    const credentials = version.payload?.data?.toString();
+    if (!credentials) {
+      throw new Error('Unable to parse credentials');
+    }
+
+    switch (bucketInfo.bucketType) {
+      case BucketType.GCP:
+        return this.gcpBucketMaker.getGcpBucket(bucketInfo, credentials);
+      default:
+        throw new Error(`Unsupported bucket type ${bucketInfo.bucketType}`);
+    }
+  }
+}

packages/server/src/bucket/bucket-info.model.ts

@@ -0,0 +1,33 @@
+import { Schema, Prop, SchemaFactory } from '@nestjs/mongoose';
+import { Document } from 'mongoose';
+
+export enum BucketType {
+  GCP = 'GCP',
+  S3 = 'S3'
+}
+
+/**
+ * Represents information on a cloud storage endpoint used
+ * for an organization
+ */
+@Schema()
+export class BucketInfo {
+  _id: string;
+
+  @Prop()
+  bucketName: string;
+
+  @Prop({ type: String, enum: BucketType })
+  bucketType: BucketType;
+
+  /** The GCP Secret Manager name for the bucket credentials */
+  @Prop()
+  secretName: string;
+
+  /** Organization the bucket is associated with */
+  @Prop()
+  organization: string;
+}
+
+export type BucketInfoDocument = BucketInfo & Document;
+export const BucketInfoSchema = SchemaFactory.createForClass(BucketInfo);

packages/server/src/bucket/bucket.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+import { MongooseModule } from '@nestjs/mongoose';
+import { BucketInfo, BucketInfoSchema } from './bucket-info.model';
+import { BucketFactory } from './bucket-factory.service';
+import { GcpBucketMaker } from './gcp-bucket';
+import { GcpModule } from 'src/gcp/gcp.module';
+
+@Module({
+  imports: [MongooseModule.forFeature([{ name: BucketInfo.name, schema: BucketInfoSchema }]), GcpModule],
+  providers: [BucketFactory, GcpBucketMaker],
+  exports: [BucketFactory]
+})
+export class BucketModule {}

packages/server/src/bucket/bucket.ts

@@ -0,0 +1,28 @@
+export enum BucketObjectAction {
+  READ,
+  WRITE,
+  DELETE
+}
+
+export interface Bucket {
+  /** Get a signed URL for the given bucket location */
+  getSignedUrl(location: string, action: BucketObjectAction, expiration: Date, contentType?: string): Promise<string>;
+
+  /** Delete the object at the given location */
+  delete(location: string): Promise<void>;
+
+  /** Move an object between two locations */
+  move(originalLocation: string, finalLocation: string): Promise<void>;
+
+  /** Check if an object exists */
+  exists(location: string): Promise<boolean>;
+
+  /** Get the content type for a file */
+  getContentType(location: string): Promise<string | null>;
+
+  /** Get the contents of an object */
+  download(location: string): Promise<Buffer | null>;
+
+  /** Delete many files */
+  deleteFiles(location: string): Promise<void>;
+}

packages/server/src/bucket/gcp-bucket.ts

@@ -0,0 +1,89 @@
+import { Bucket, BucketObjectAction } from './bucket';
+import { BucketInfo } from './bucket-info.model';
+import { Injectable } from '@nestjs/common';
+import { Storage, Bucket as StorageBucket } from '@google-cloud/storage';
+
+/** Wrapper maker for the bucket so that dependencies can be injected */
+@Injectable()
+export class GcpBucketMaker {
+  async getGcpBucket(bucketInfo: BucketInfo, credentials: string): Promise<GcpBucket> {
+    // Expects the key to be a GCP service account key
+    let key: any = null;
+    try {
+      key = JSON.parse(credentials);
+    } catch (e) {
+      throw new Error('Failed to parse credentails');
+    }
+
+    // Make the GCP storage client
+    const storage = new Storage({ credentials: key });
+
+    // Get the bucket
+    const bucket = storage.bucket(bucketInfo.bucketName);
+
+    // Return the wrapper around the GCP bucket implementation
+    return new GcpBucket(bucket);
+  }
+}
+
+class GcpBucket implements Bucket {
+  constructor(private readonly storageBucket: StorageBucket) {}
+
+  async getSignedUrl(
+    location: string,
+    action: BucketObjectAction,
+    expiration: Date,
+    contentType?: string | undefined
+  ): Promise<string> {
+    const file = this.storageBucket.file(location);
+
+    const [url] = await file.getSignedUrl({
+      action: this.actionToString(action),
+      expires: expiration,
+      contentType
+    });
+    return url;
+  }
+
+  async delete(location: string): Promise<void> {
+    await this.deleteFiles(location);
+  }
+
+  async move(originalLocation: string, finalLocation: string): Promise<void> {
+    const file = this.storageBucket.file(originalLocation);
+    await file.move(finalLocation);
+  }
+
+  async exists(location: string): Promise<boolean> {
+    const exists = await this.storageBucket.file(location).exists();
+    return exists[0];
+  }
+
+  async getContentType(location: string): Promise<string | null> {
+    const file = this.storageBucket.file(location);
+    await file.getMetadata();
+    return file.metadata.contentType || null;
+  }
+
+  async download(location: string): Promise<Buffer | null> {
+    const file = this.storageBucket.file(location);
+    return (await file.download())[0];
+  }
+
+  async deleteFiles(location: string): Promise<void> {
+    await this.storageBucket.deleteFiles({ prefix: location });
+  }
+
+  private actionToString(action: BucketObjectAction): 'read' | 'write' | 'delete' {
+    switch (action) {
+      case BucketObjectAction.READ:
+        return 'read';
+      case BucketObjectAction.WRITE:
+        return 'write';
+      case BucketObjectAction.DELETE:
+        return 'delete';
+      default:
+        throw new Error(`Unsupported action ${action}`);
+    }
+  }
+}