feat: Training Video Storage

packages/client/src/graphql/tag/tag.graphql

@@ -53,7 +53,6 @@ query getTags($study: ID!) {
       organization
       entryID
       contentType
-      dataset
       creator
       dateCreated
       meta
@@ -73,7 +72,6 @@ query getTrainingTags($study: ID!, $user: String!) {
       organization
       entryID
       contentType
-      dataset
       creator
       dateCreated
       meta

packages/client/src/graphql/tag/tag.ts

@@ -67,15 +67,15 @@ export type GetTagsQueryVariables = Types.Exact<{
 }>;
 
 
-export type GetTagsQuery = { __typename?: 'Query', getTags: Array<{ __typename?: 'Tag', _id: string, data?: any | null, complete: boolean, entry: { __typename?: 'Entry', _id: string, organization: string, entryID: string, contentType: string, dataset: string, creator: string, dateCreated: any, meta?: any | null, signedUrl: string, signedUrlExpiration: number } }> };
+export type GetTagsQuery = { __typename?: 'Query', getTags: Array<{ __typename?: 'Tag', _id: string, data?: any | null, complete: boolean, entry: { __typename?: 'Entry', _id: string, organization: string, entryID: string, contentType: string, creator: string, dateCreated: any, meta?: any | null, signedUrl: string, signedUrlExpiration: number } }> };
 
 export type GetTrainingTagsQueryVariables = Types.Exact<{
   study: Types.Scalars['ID']['input'];
   user: Types.Scalars['String']['input'];
 }>;
 
 
-export type GetTrainingTagsQuery = { __typename?: 'Query', getTrainingTags: Array<{ __typename?: 'Tag', _id: string, data?: any | null, complete: boolean, entry: { __typename?: 'Entry', _id: string, organization: string, entryID: string, contentType: string, dataset: string, creator: string, dateCreated: any, meta?: any | null, signedUrl: string, signedUrlExpiration: number } }> };
+export type GetTrainingTagsQuery = { __typename?: 'Query', getTrainingTags: Array<{ __typename?: 'Tag', _id: string, data?: any | null, complete: boolean, entry: { __typename?: 'Entry', _id: string, organization: string, entryID: string, contentType: string, creator: string, dateCreated: any, meta?: any | null, signedUrl: string, signedUrlExpiration: number } }> };
 
 
 export const CreateTagsDocument = gql`
@@ -333,7 +333,6 @@ export const GetTagsDocument = gql`
       organization
       entryID
       contentType
-      dataset
       creator
       dateCreated
       meta
@@ -382,7 +381,6 @@ export const GetTrainingTagsDocument = gql`
       organization
       entryID
       contentType
-      dataset
       creator
       dateCreated
       meta

packages/server/src/config/configuration.ts

@@ -34,6 +34,7 @@ export default () => ({
   tag: {
     videoFieldFolder: process.env.TAG_VIDEO_FIELD_FOLDER || 'video-fields',
     videoRecordFileType: 'webm',
-    videoUploadExpiration: process.env.TAG_VIDEO_UPLOAD_EXPIRATION || 15 * 60 * 1000 // 15 minutes
+    videoUploadExpiration: process.env.TAG_VIDEO_UPLOAD_EXPIRATION || 15 * 60 * 1000, // 15 minutes
+    trainingPrefix: process.env.TAG_TRAINING_PREFIX || 'training'
   }
 });

packages/server/src/entry/models/entry.model.ts

@@ -51,6 +51,10 @@ export class Entry {
   @Prop({ required: false })
   signedURLExpiration: Date;
 
+  @Prop()
+  @Field()
+  isTraining: boolean;
+
   // TODO: Add creator field
 }
 

packages/server/src/entry/services/entry.service.ts

@@ -25,14 +25,16 @@ export class EntryService {
     return this.entryModel.findOne({ _id: entryID });
   }
 
-  async create(entryCreate: EntryCreate, dataset: Dataset, user: TokenPayload): Promise<Entry> {
+  async create(entryCreate: EntryCreate, dataset: Dataset, user: TokenPayload, isTraining: boolean): Promise<Entry> {
+    // Make the entry, note that training entries are not associated with a dataset
     return this.entryModel.create({
       ...entryCreate,
       dataset: dataset._id,
       organization: dataset.organization,
       recordedInSignLab: false,
       dateCreated: new Date(),
-      creator: user.user_id
+      creator: user.user_id,
+      isTraining
     });
   }
 
@@ -41,7 +43,7 @@ export class EntryService {
   }
 
   async findForDataset(dataset: Dataset): Promise<Entry[]> {
-    return this.entryModel.find({ dataset: dataset._id.toString() });
+    return this.entryModel.find({ dataset: dataset._id.toString(), isTraining: false });
   }
 
   async exists(entryID: string, dataset: Dataset): Promise<boolean> {

packages/server/src/entry/services/upload-session.service.ts

@@ -94,7 +94,8 @@ export class UploadSessionService {
           meta: entryUpload.metadata
         },
         dataset,
-        user
+        user,
+        false
       );
 
       // Move the entry to the dataset

packages/server/src/tag/resolvers/tag.resolver.ts

@@ -63,6 +63,7 @@ export class TagResolver {
   ): Promise<boolean> {
     // TODO: Add user context and verify the correct user has completed the tag
     const study = await this.studyPipe.transform(tag.study);
+    tag.data = data;
     await this.tagService.complete(tag, data, study, user);
     return true;
   }

packages/server/src/tag/services/tag-transformer.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { Study } from '../../study/study.model';
 import { FieldTransformerFactory } from '../transformers/field-transformer-factory';
 import { TokenPayload } from '../../jwt/token.dto';
+import { Tag } from '../../tag/models/tag.model';
 
 @Injectable()
 export class TagTransformer {
@@ -11,7 +12,7 @@ export class TagTransformer {
    * Transforms the tag data. Takes in the whole tag and produces the modified
    * tag data.
    */
-  async transformTagData(data: any, study: Study, user: TokenPayload): Promise<any> {
+  async transformTagData(tag: Tag, data: any, study: Study, user: TokenPayload): Promise<any> {
     const transformedData: { [property: string]: any } = {};
 
     const schema = study.tagSchema.dataSchema;
@@ -36,7 +37,7 @@ export class TagTransformer {
 
       // Apply the transformation if present, otherwise just return the data
       const transformed = transformer
-        ? await transformer.transformField(data[field], fieldUiSchema, fieldSchema, user)
+        ? await transformer.transformField(tag, data[field], fieldUiSchema, fieldSchema, user)
         : data[field];
       transformedData[field] = transformed;
     }

packages/server/src/tag/services/tag.service.ts

@@ -146,7 +146,7 @@ export class TagService {
     await this.tagModel.db.transaction(async (): Promise<void> => {
       const searchResult = await this.tagModel.aggregate([
         // Only search on tags that are enabled for the current study
-        { $match: { enabled: true, study: study._id.toString() } },
+        { $match: { enabled: true, study: study._id.toString(), training: false } },
         // Grab tags that are unassigned (user field doesn't exist) or have been completed by the user
         { $match: { $or: [{ user: { $exists: false } }, { user: { $eq: user } }] } },
         // Group by the entrys and expand tags
@@ -188,7 +188,7 @@ export class TagService {
     }
 
     // Handle any transformations
-    const transformed = await this.tagTransformService.transformTagData(data, study, user);
+    const transformed = await this.tagTransformService.transformTagData(tag, data, study, user);
 
     // Save the tag information and mark the tag as complete
     await this.tagModel.findOneAndUpdate({ _id: tag._id }, { $set: { data: transformed, complete: true } });
@@ -218,7 +218,7 @@ export class TagService {
   }
 
   async getTags(study: Study): Promise<Tag[]> {
-    return this.tagModel.find({ study: study._id });
+    return this.tagModel.find({ study: study._id, training: false });
   }
 
   private async getIncomplete(study: Study, user: string): Promise<Tag | null> {

packages/server/src/tag/services/video-field.service.ts

@@ -11,6 +11,7 @@ import { Entry } from '../../entry/models/entry.model';
 import { EntryService } from '../../entry/services/entry.service';
 import { DatasetPipe } from '../../dataset/pipes/dataset.pipe';
 import { TokenPayload } from '../../jwt/token.dto';
+import { Dataset } from '../../dataset/dataset.model';
 
 @Injectable()
 export class VideoFieldService {
@@ -19,6 +20,7 @@ export class VideoFieldService {
   private readonly bucketName = this.configService.getOrThrow<string>('gcp.storage.bucket');
   private readonly bucket: Bucket = this.storage.bucket(this.bucketName);
   private readonly expiration = this.configService.getOrThrow<number>('tag.videoUploadExpiration');
+  private readonly trainingPrefix = this.configService.getOrThrow<string>('tag.trainingPrefix');
 
   constructor(
     @InjectModel(VideoField.name) private readonly videoFieldModel: Model<VideoFieldDocument>,
@@ -72,13 +74,14 @@ export class VideoFieldService {
    * Move the video itself to the permanent storage location and create the
    * cooresponding entry.
    */
-  async markComplete(videoFieldID: string, datasetID: string, user: TokenPayload): Promise<Entry> {
+  async markComplete(videoFieldID: string, datasetID: string, user: TokenPayload, tag: Tag): Promise<Entry> {
     const videoField = await this.videoFieldModel.findById(videoFieldID);
     if (!videoField) {
       throw new BadRequestException(`Video field ${videoFieldID} not found`);
     }
 
-    const dataset = await this.datasetPipe.transform(datasetID);
+    // The dataset that the entry would be associated with
+    const dataset: Dataset = await this.datasetPipe.transform(datasetID);
 
     // Make the entry
     const entry = await this.entryService.create(
@@ -88,12 +91,18 @@ export class VideoFieldService {
         meta: {}
       },
       dataset,
-      user
+      user,
+      tag.training
     );
 
+    // Where to move the entry video
+    let newLocation = `${dataset.bucketPrefix}/${entry._id}.webm`;
+    if (tag.training) {
+      newLocation = `${this.trainingPrefix}/${dataset.organization}/${tag.study}/${entry._id}.webm`;
+    }
+
     // Move the video to the permanent location
     const source = this.bucket.file(videoField.bucketLocation);
-    const newLocation = `${dataset.bucketPrefix}/${entry._id}.webm`;
     await source.move(newLocation);
     await this.entryService.setBucketLocation(entry, newLocation);
     entry.bucketLocation = newLocation;

packages/server/src/tag/transformers/field-transformer.ts

@@ -1,13 +1,14 @@
 import { JsonSchema, UISchemaElement } from '@jsonforms/core';
 import { TokenPayload } from '../../jwt/token.dto';
+import { Tag } from '../models/tag.model';
 
 /**
  * A field transformer handles converting and operating on fields of a tag.
  * It handles adding in additional information, update intermediate data,
  * and ensuring that the data meets any additional formatting requirements.
  */
 export interface FieldTransformer {
-  transformField(field: any, uischema: UISchemaElement, schema: JsonSchema, user: TokenPayload): Promise<any>;
+  transformField(tag: Tag, data: any, uischema: UISchemaElement, schema: JsonSchema, user: TokenPayload): Promise<any>;
 }
 
 /**

packages/server/src/tag/transformers/video-field-transformer.ts

@@ -3,12 +3,14 @@ import { FieldTransformer } from './field-transformer';
 import { JsonSchema, UISchemaElement } from '@jsonforms/core';
 import { VideoFieldService } from '../services/video-field.service';
 import { TokenPayload } from '../../jwt/token.dto';
+import { Tag } from '../models/tag.model';
 
 @Injectable()
 export class VideoFieldTransformer implements FieldTransformer {
   constructor(private readonly videoFieldService: VideoFieldService) {}
 
   async transformField(
+    tag: Tag,
     data: string[],
     uischema: UISchemaElement,
     _schema: JsonSchema,
@@ -21,7 +23,7 @@ export class VideoFieldTransformer implements FieldTransformer {
 
     const videoFields = await Promise.all(
       data.map(async (videoFieldId) => {
-        const entry = await this.videoFieldService.markComplete(videoFieldId, datasetID, user);
+        const entry = await this.videoFieldService.markComplete(videoFieldId, datasetID, user, tag);
         return entry._id;
       })
     );