From 606b041f73665bc61790b0a74c3e0d582b71a8a8 Mon Sep 17 00:00:00 2001 From: cbolles Date: Thu, 25 Jan 2024 14:08:48 -0500 Subject: [PATCH 1/9] Make dedicated services, resolvers, and models folder for tags --- .../server/src/tag/{ => models}/tag.model.ts | 4 ++-- packages/server/src/tag/pipes/tag.pipe.ts | 4 ++-- .../src/tag/{ => resolvers}/tag.resolver.ts | 24 +++++++++---------- .../src/tag/{ => services}/tag.service.ts | 10 ++++---- packages/server/src/tag/tag.module.ts | 6 ++--- 5 files changed, 24 insertions(+), 24 deletions(-) rename packages/server/src/tag/{ => models}/tag.model.ts (91%) rename packages/server/src/tag/{ => resolvers}/tag.resolver.ts (81%) rename packages/server/src/tag/{ => services}/tag.service.ts (94%) diff --git a/packages/server/src/tag/tag.model.ts b/packages/server/src/tag/models/tag.model.ts similarity index 91% rename from packages/server/src/tag/tag.model.ts rename to packages/server/src/tag/models/tag.model.ts index d2876634..d5da7bfb 100644 --- a/packages/server/src/tag/tag.model.ts +++ b/packages/server/src/tag/models/tag.model.ts @@ -2,8 +2,8 @@ import { Schema, Prop, SchemaFactory } from '@nestjs/mongoose'; import { Field, ObjectType } from '@nestjs/graphql'; import JSON from 'graphql-type-json'; import mongoose, { Document } from 'mongoose'; -import { Study } from '../study/study.model'; -import { Entry } from '../entry/models/entry.model'; +import { Study } from '../../study/study.model'; +import { Entry } from '../../entry/models/entry.model'; @Schema() @ObjectType() diff --git a/packages/server/src/tag/pipes/tag.pipe.ts b/packages/server/src/tag/pipes/tag.pipe.ts index 9b53456a..e3d7febd 100644 --- a/packages/server/src/tag/pipes/tag.pipe.ts +++ b/packages/server/src/tag/pipes/tag.pipe.ts @@ -1,6 +1,6 @@ import { BadRequestException, Injectable, PipeTransform } from '@nestjs/common'; -import { Tag } from '../tag.model'; -import { TagService } from '../tag.service'; +import { Tag } from '../models/tag.model'; +import { TagService } from '../services/tag.service'; @Injectable() export class TagPipe implements PipeTransform> { diff --git a/packages/server/src/tag/tag.resolver.ts b/packages/server/src/tag/resolvers/tag.resolver.ts similarity index 81% rename from packages/server/src/tag/tag.resolver.ts rename to packages/server/src/tag/resolvers/tag.resolver.ts index 02519613..41fda85f 100644 --- a/packages/server/src/tag/tag.resolver.ts +++ b/packages/server/src/tag/resolvers/tag.resolver.ts @@ -1,19 +1,19 @@ import { Resolver, Mutation, Query, Args, ID, ResolveField, Parent } from '@nestjs/graphql'; -import { TagService } from './tag.service'; -import { Tag } from './tag.model'; -import { StudyPipe } from '../study/pipes/study.pipe'; -import { Study } from '../study/study.model'; -import { EntriesPipe, EntryPipe } from '../entry/pipes/entry.pipe'; -import { Entry } from '../entry/models/entry.model'; -import { TagPipe } from './pipes/tag.pipe'; +import { TagService } from '../services/tag.service'; +import { Tag } from '../models/tag.model'; +import { StudyPipe } from '../../study/pipes/study.pipe'; +import { Study } from '../../study/study.model'; +import { EntriesPipe, EntryPipe } from '../../entry/pipes/entry.pipe'; +import { Entry } from '../../entry/models/entry.model'; +import { TagPipe } from '../pipes/tag.pipe'; import JSON from 'graphql-type-json'; import { Inject, UseGuards, UnauthorizedException } from '@nestjs/common'; -import { JwtAuthGuard } from '../jwt/jwt.guard'; -import { CASBIN_PROVIDER } from 'src/permission/casbin.provider'; +import { JwtAuthGuard } from '../../jwt/jwt.guard'; +import { CASBIN_PROVIDER } from '../../permission/casbin.provider'; import * as casbin from 'casbin'; -import { TokenContext } from '../jwt/token.context'; -import { TokenPayload } from '../jwt/token.dto'; -import { StudyPermissions } from '../permission/permissions/study'; +import { TokenContext } from '../../jwt/token.context'; +import { TokenPayload } from '../../jwt/token.dto'; +import { StudyPermissions } from '../../permission/permissions/study'; import { TagPermissions } from 'src/permission/permissions/tag'; // TODO: Add permissioning diff --git a/packages/server/src/tag/tag.service.ts b/packages/server/src/tag/services/tag.service.ts similarity index 94% rename from packages/server/src/tag/tag.service.ts rename to packages/server/src/tag/services/tag.service.ts index 5efe6d4d..91762fdb 100644 --- a/packages/server/src/tag/tag.service.ts +++ b/packages/server/src/tag/services/tag.service.ts @@ -1,11 +1,11 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { Tag } from './tag.model'; +import { Tag } from '../models/tag.model'; import { Model } from 'mongoose'; -import { Study } from '../study/study.model'; -import { Entry } from '../entry/models/entry.model'; -import { StudyService } from '../study/study.service'; -import { MongooseMiddlewareService } from '../shared/service/mongoose-callback.service'; +import { Study } from '../../study/study.model'; +import { Entry } from '../../entry/models/entry.model'; +import { StudyService } from '../../study/study.service'; +import { MongooseMiddlewareService } from '../../shared/service/mongoose-callback.service'; @Injectable() export class TagService { diff --git a/packages/server/src/tag/tag.module.ts b/packages/server/src/tag/tag.module.ts index 4c020e52..934637cf 100644 --- a/packages/server/src/tag/tag.module.ts +++ b/packages/server/src/tag/tag.module.ts @@ -1,8 +1,8 @@ import { Module } from '@nestjs/common'; -import { TagService } from './tag.service'; -import { TagResolver } from './tag.resolver'; +import { TagService } from './services/tag.service'; +import { TagResolver } from './resolvers/tag.resolver'; import { MongooseModule } from '@nestjs/mongoose'; -import { Tag, TagSchema } from './tag.model'; +import { Tag, TagSchema } from './models/tag.model'; import { StudyModule } from '../study/study.module'; import { EntryModule } from '../entry/entry.module'; import { TagPipe } from './pipes/tag.pipe'; From 592b0fa487b26307af124e7c3c00622630d63cce Mon Sep 17 00:00:00 2001 From: cbolles Date: Thu, 25 Jan 2024 15:05:34 -0500 Subject: [PATCH 2/9] Untested ability to save a video field --- packages/server/schema.gql | 1 + packages/server/src/config/configuration.ts | 5 ++ .../src/tag/models/video-field.model.ts | 34 ++++++++++ .../src/tag/resolvers/video-field.resolver.ts | 45 ++++++++++++ .../src/tag/services/video-field.service.ts | 68 +++++++++++++++++++ packages/server/src/tag/tag.module.ts | 11 ++- 6 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 packages/server/src/tag/models/video-field.model.ts create mode 100644 packages/server/src/tag/resolvers/video-field.resolver.ts create mode 100644 packages/server/src/tag/services/video-field.service.ts diff --git a/packages/server/schema.gql b/packages/server/schema.gql index 00e8bab4..440453ed 100644 --- a/packages/server/schema.gql +++ b/packages/server/schema.gql @@ -184,6 +184,7 @@ type Mutation { assignTag(study: ID!): Tag completeTag(tag: ID!, data: JSON!): Boolean! setEntryEnabled(study: ID!, entry: ID!, enabled: Boolean!): Boolean! + saveVideoField(tag: ID!, field: ID!, index: Float!): String! } input OrganizationCreate { diff --git a/packages/server/src/config/configuration.ts b/packages/server/src/config/configuration.ts index 59598b66..b7a06c44 100644 --- a/packages/server/src/config/configuration.ts +++ b/packages/server/src/config/configuration.ts @@ -28,5 +28,10 @@ export default () => ({ mongo: { uri: process.env.CASBIN_MONGO_URI || 'mongodb://127.0.0.1:27017/casbin' } + }, + tag: { + videoFieldFolder: process.env.TAG_VIDEO_FIELD_FOLDER || 'video-fields', + videoRecordFileType: 'webm', + videoUploadExpiration: process.env.TAG_VIDEO_UPLOAD_EXPIRATION || 15 * 60 * 1000 // 15 minutes } }); diff --git a/packages/server/src/tag/models/video-field.model.ts b/packages/server/src/tag/models/video-field.model.ts new file mode 100644 index 00000000..71550632 --- /dev/null +++ b/packages/server/src/tag/models/video-field.model.ts @@ -0,0 +1,34 @@ +import { Schema, Prop, SchemaFactory } from '@nestjs/mongoose'; +import { Field, ObjectType } from '@nestjs/graphql'; +import { Document } from 'mongoose'; + +@Schema() +@ObjectType() +/** + * Represents a single video field in a study. This is used for temporarily + * storing the recording video data before the tag is submitted and the video + * is turned into an Entry. + */ +export class VideoField { + @Field() + _id: string; + + /** The tag the video field is a part of */ + @Prop() + tag: string; + + /** The field of the tag the video field is a part of */ + @Prop() + field: string; + + /** The index of the video field in the tag */ + @Prop() + index: number; + + /** Where within the bucket the video is stored */ + @Prop() + bucketLocation: string; +} + +export type VideoFieldDocument = VideoField & Document; +export const VideoFieldSchema = SchemaFactory.createForClass(VideoField); diff --git a/packages/server/src/tag/resolvers/video-field.resolver.ts b/packages/server/src/tag/resolvers/video-field.resolver.ts new file mode 100644 index 00000000..775c0d86 --- /dev/null +++ b/packages/server/src/tag/resolvers/video-field.resolver.ts @@ -0,0 +1,45 @@ +import { Resolver, Query, Args, Mutation, ID, ResolveField, Parent } from '@nestjs/graphql'; +import { VideoFieldService } from '../services/video-field.service'; +import { TagPipe } from '../pipes/tag.pipe'; +import { Tag } from '../models/tag.model'; +import { VideoField } from '../models/video-field.model'; +import { TokenContext } from '../../jwt/token.context'; +import { TokenPayload } from '../../jwt/token.dto'; +import { Inject, UnauthorizedException } from '@nestjs/common'; +import { CASBIN_PROVIDER } from '../../permission/casbin.provider'; +import * as casbin from 'casbin'; +import { TagPermissions } from '../../permission/permissions/tag'; + +@Resolver(() => VideoField) +export class VideoFieldResolver { + constructor(private readonly videoFieldService: VideoFieldService, @Inject(CASBIN_PROVIDER) private readonly enforcer: casbin.Enforcer) {} + + @Mutation(() => String) + async saveVideoField( + @Args('tag', { type: () => ID }, TagPipe) tag: Tag, + @Args('field', { type: () => ID }) field: string, + @Args('index', { type: () => Number }) index: number, + @TokenContext() user: TokenPayload + ): Promise { + // Make sure the user first has permission to create video fields for this tag + if (!(await this.enforcer.enforce(user.id, TagPermissions.CREATE, tag.study.toString()))) { + throw new UnauthorizedException('User does not have permission to create video fields for this tag'); + } + + // Make sure its the user assigned to the tag + if (user.id !== tag.user?.toString()) { + throw new UnauthorizedException('User does not have permission to create video fields for this tag'); + } + + return this.videoFieldService.saveVideoField(tag, field, index); + } + + @ResolveField(() => String) + async uploadURL(@Parent() videoField: VideoField, @TokenContext() user: TokenPayload): Promise { + if (!(await this.enforcer.enforce(user.id, TagPermissions.CREATE, videoField.tag.toString()))) { + throw new UnauthorizedException('User does not have permission to create video fields for this tag'); + } + + return this.videoFieldService.getUploadURL(videoField); + } +} diff --git a/packages/server/src/tag/services/video-field.service.ts b/packages/server/src/tag/services/video-field.service.ts new file mode 100644 index 00000000..2dc4d03f --- /dev/null +++ b/packages/server/src/tag/services/video-field.service.ts @@ -0,0 +1,68 @@ +import { BadRequestException, Injectable, Inject } from '@nestjs/common'; +import { InjectModel} from '@nestjs/mongoose'; +import { VideoField, VideoFieldDocument } from '../models/video-field.model'; +import { Model } from 'mongoose'; +import { Tag } from '../models/tag.model'; +import { StudyService } from '../../study/study.service'; +import { ConfigService } from '@nestjs/config'; +import { GCP_STORAGE_PROVIDER } from '../../gcp/providers/storage.provider'; +import { Storage, Bucket } from '@google-cloud/storage'; + +@Injectable() +export class VideoFieldService { + private readonly bucketPrefix = this.configService.getOrThrow('tag.videoFieldFolder'); + private readonly videoRecordFileType = this.configService.getOrThrow('tag.videoRecordFileType'); + private readonly bucketName = this.configService.getOrThrow('gcp.storage.bucket'); + private readonly bucket: Bucket = this.storage.bucket(this.bucketName); + private readonly expiration = this.configService.getOrThrow('tag.videoUploadExpiration'); + + constructor( + @InjectModel(VideoField.name) private readonly videoFieldModel: Model, + private readonly studyService: StudyService, + private readonly configService: ConfigService, + @Inject(GCP_STORAGE_PROVIDER) private readonly storage: Storage + ) {} + + async saveVideoField(tag: Tag, field: string, index: number): Promise { + // First do a correctness check to make sure the field shows up in the tag + // TODO: Can do a correctness check on the index and using the UI schema as well + const study = await this.studyService.findById(tag.study); + if (!study) { + // Unexpected error, got a tag with an invalid study + throw new Error(`Study ${tag.study} not found on tag ${tag._id}`); + } + const dataSchema = study.tagSchema.dataSchema; + if (!dataSchema.properties || !dataSchema.properties[field]) { + // User is trying to save a video field that doesn't exist in the tag + throw new BadRequestException(`Field ${field} not found in tag ${tag._id}`); + } + + // Check if the video field already exists, if so return it + const existingVideoField = await this.getVideoField(tag, field, index); + if (existingVideoField) { + return existingVideoField; + } + + // Otherwise make a new one and return it + return this.videoFieldModel.create({ + tag: tag._id, field, index, bucketLocation: this.getVideoFieldBucketLocation(tag._id, field, index) + }); + } + + async getUploadURL(videoField: VideoField): Promise { + const file = this.bucket.file(this.getVideoFieldBucketLocation(videoField.tag, videoField.field, videoField.index)); + const [url] = await file.getSignedUrl({ + action: 'write', + expires: Date.now() + this.expiration + }); + return url; + } + + private getVideoFieldBucketLocation(tagID: string, field: string, index: number): string { + return `${this.bucketPrefix}/${tagID}/${field}/${index}.${this.videoRecordFileType}`; + } + + private async getVideoField(tag: Tag, field: string, index: number): Promise { + return this.videoFieldModel.findOne({ tag: tag._id, field, index }).exec(); + } +} diff --git a/packages/server/src/tag/tag.module.ts b/packages/server/src/tag/tag.module.ts index 934637cf..a0aec128 100644 --- a/packages/server/src/tag/tag.module.ts +++ b/packages/server/src/tag/tag.module.ts @@ -8,15 +8,20 @@ import { EntryModule } from '../entry/entry.module'; import { TagPipe } from './pipes/tag.pipe'; import { SharedModule } from '../shared/shared.module'; import { PermissionModule } from '../permission/permission.module'; +import { VideoField, VideoFieldSchema } from './models/video-field.model'; +import { VideoFieldService } from './services/video-field.service'; +import { VideoFieldResolver } from './resolvers/video-field.resolver'; +import { GcpModule } from '../gcp/gcp.module'; @Module({ imports: [ - MongooseModule.forFeature([{ name: Tag.name, schema: TagSchema }]), + MongooseModule.forFeature([{ name: Tag.name, schema: TagSchema }, { name: VideoField.name, schema: VideoFieldSchema }]), StudyModule, EntryModule, SharedModule, - PermissionModule + PermissionModule, + GcpModule ], - providers: [TagService, TagResolver, TagPipe] + providers: [TagService, TagResolver, TagPipe, VideoFieldService, VideoFieldResolver] }) export class TagModule {} From f63b47ac5afb1222a6d8f8aaa549174a1ceeb64d Mon Sep 17 00:00:00 2001 From: cbolles Date: Thu, 25 Jan 2024 15:17:00 -0500 Subject: [PATCH 3/9] Working ability to save tag field --- packages/server/schema.gql | 7 ++++++- .../src/tag/resolvers/video-field.resolver.ts | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/server/schema.gql b/packages/server/schema.gql index 440453ed..8a5fe801 100644 --- a/packages/server/schema.gql +++ b/packages/server/schema.gql @@ -137,6 +137,11 @@ type Tag { enabled: Boolean! } +type VideoField { + _id: String! + uploadURL: String! +} + type Query { getOrganizations: [Organization!]! exists(name: String!): Boolean! @@ -184,7 +189,7 @@ type Mutation { assignTag(study: ID!): Tag completeTag(tag: ID!, data: JSON!): Boolean! setEntryEnabled(study: ID!, entry: ID!, enabled: Boolean!): Boolean! - saveVideoField(tag: ID!, field: ID!, index: Float!): String! + saveVideoField(tag: ID!, field: ID!, index: Float!): VideoField! } input OrganizationCreate { diff --git a/packages/server/src/tag/resolvers/video-field.resolver.ts b/packages/server/src/tag/resolvers/video-field.resolver.ts index 775c0d86..3d7d484d 100644 --- a/packages/server/src/tag/resolvers/video-field.resolver.ts +++ b/packages/server/src/tag/resolvers/video-field.resolver.ts @@ -5,16 +5,22 @@ import { Tag } from '../models/tag.model'; import { VideoField } from '../models/video-field.model'; import { TokenContext } from '../../jwt/token.context'; import { TokenPayload } from '../../jwt/token.dto'; -import { Inject, UnauthorizedException } from '@nestjs/common'; +import { Inject, UnauthorizedException, UseGuards } from '@nestjs/common'; import { CASBIN_PROVIDER } from '../../permission/casbin.provider'; import * as casbin from 'casbin'; import { TagPermissions } from '../../permission/permissions/tag'; +import { JwtAuthGuard } from '../../jwt/jwt.guard'; +@UseGuards(JwtAuthGuard) @Resolver(() => VideoField) export class VideoFieldResolver { - constructor(private readonly videoFieldService: VideoFieldService, @Inject(CASBIN_PROVIDER) private readonly enforcer: casbin.Enforcer) {} + constructor( + private readonly videoFieldService: VideoFieldService, + @Inject(CASBIN_PROVIDER) private readonly enforcer: casbin.Enforcer, + private readonly tagPipe: TagPipe + ) {} - @Mutation(() => String) + @Mutation(() => VideoField) async saveVideoField( @Args('tag', { type: () => ID }, TagPipe) tag: Tag, @Args('field', { type: () => ID }) field: string, @@ -36,7 +42,11 @@ export class VideoFieldResolver { @ResolveField(() => String) async uploadURL(@Parent() videoField: VideoField, @TokenContext() user: TokenPayload): Promise { - if (!(await this.enforcer.enforce(user.id, TagPermissions.CREATE, videoField.tag.toString()))) { + const tag = await this.tagPipe.transform(videoField.tag); + if (!tag) { + throw new Error(`Tag ${videoField.tag} not found`); + } + if (!(await this.enforcer.enforce(user.id, TagPermissions.CREATE, tag.study.toString()))) { throw new UnauthorizedException('User does not have permission to create video fields for this tag'); } From 8724c556b2b92f22244faa2828036aab3456067a Mon Sep 17 00:00:00 2001 From: cbolles Date: Thu, 25 Jan 2024 15:33:51 -0500 Subject: [PATCH 4/9] Generate GraphQL query for saving the video field --- packages/client/src/graphql/graphql.ts | 14 ++++++ packages/client/src/graphql/tag/tag.graphql | 7 +++ packages/client/src/graphql/tag/tag.ts | 47 ++++++++++++++++++- packages/server/schema.gql | 2 +- .../src/tag/resolvers/video-field.resolver.ts | 6 +-- 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/packages/client/src/graphql/graphql.ts b/packages/client/src/graphql/graphql.ts index b3eb8eef..b699c81b 100644 --- a/packages/client/src/graphql/graphql.ts +++ b/packages/client/src/graphql/graphql.ts @@ -212,6 +212,7 @@ export type Mutation = { refresh: AccessToken; resendInvite: InviteModel; resetPassword: Scalars['Boolean']['output']; + saveVideoField: VideoField; setEntryEnabled: Scalars['Boolean']['output']; signLabCreateProject: Project; signup: AccessToken; @@ -414,6 +415,13 @@ export type MutationResetPasswordArgs = { }; +export type MutationSaveVideoFieldArgs = { + field: Scalars['String']['input']; + index: Scalars['Int']['input']; + tag: Scalars['ID']['input']; +}; + + export type MutationSetEntryEnabledArgs = { enabled: Scalars['Boolean']['input']; entry: Scalars['ID']['input']; @@ -813,3 +821,9 @@ export type UsernameLoginDto = { projectId: Scalars['String']['input']; username: Scalars['String']['input']; }; + +export type VideoField = { + __typename?: 'VideoField'; + _id: Scalars['String']['output']; + uploadURL: Scalars['String']['output']; +}; diff --git a/packages/client/src/graphql/tag/tag.graphql b/packages/client/src/graphql/tag/tag.graphql index aa6cc4f6..bcd89b97 100644 --- a/packages/client/src/graphql/tag/tag.graphql +++ b/packages/client/src/graphql/tag/tag.graphql @@ -33,3 +33,10 @@ mutation assignTag($study: ID!) { mutation completeTag($tag: ID!, $data: JSON!) { completeTag(tag: $tag, data: $data) } + +mutation saveVideoField($tag: ID!, $field: String!, $index: Int!) { + saveVideoField(tag: $tag, field: $field, index: $index) { + _id, + uploadURL + } +} diff --git a/packages/client/src/graphql/tag/tag.ts b/packages/client/src/graphql/tag/tag.ts index a299e24c..52a61f7f 100644 --- a/packages/client/src/graphql/tag/tag.ts +++ b/packages/client/src/graphql/tag/tag.ts @@ -45,6 +45,15 @@ export type CompleteTagMutationVariables = Types.Exact<{ export type CompleteTagMutation = { __typename?: 'Mutation', completeTag: boolean }; +export type SaveVideoFieldMutationVariables = Types.Exact<{ + tag: Types.Scalars['ID']['input']; + field: Types.Scalars['String']['input']; + index: Types.Scalars['Int']['input']; +}>; + + +export type SaveVideoFieldMutation = { __typename?: 'Mutation', saveVideoField: { __typename?: 'VideoField', _id: string, uploadURL: string } }; + export const CreateTagsDocument = gql` mutation createTags($study: ID!, $entries: [ID!]!) { @@ -223,4 +232,40 @@ export function useCompleteTagMutation(baseOptions?: Apollo.MutationHookOptions< } export type CompleteTagMutationHookResult = ReturnType; export type CompleteTagMutationResult = Apollo.MutationResult; -export type CompleteTagMutationOptions = Apollo.BaseMutationOptions; \ No newline at end of file +export type CompleteTagMutationOptions = Apollo.BaseMutationOptions; +export const SaveVideoFieldDocument = gql` + mutation saveVideoField($tag: ID!, $field: String!, $index: Int!) { + saveVideoField(tag: $tag, field: $field, index: $index) { + _id + uploadURL + } +} + `; +export type SaveVideoFieldMutationFn = Apollo.MutationFunction; + +/** + * __useSaveVideoFieldMutation__ + * + * To run a mutation, you first call `useSaveVideoFieldMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useSaveVideoFieldMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [saveVideoFieldMutation, { data, loading, error }] = useSaveVideoFieldMutation({ + * variables: { + * tag: // value for 'tag' + * field: // value for 'field' + * index: // value for 'index' + * }, + * }); + */ +export function useSaveVideoFieldMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(SaveVideoFieldDocument, options); + } +export type SaveVideoFieldMutationHookResult = ReturnType; +export type SaveVideoFieldMutationResult = Apollo.MutationResult; +export type SaveVideoFieldMutationOptions = Apollo.BaseMutationOptions; \ No newline at end of file diff --git a/packages/server/schema.gql b/packages/server/schema.gql index 8a5fe801..56eac3f0 100644 --- a/packages/server/schema.gql +++ b/packages/server/schema.gql @@ -189,7 +189,7 @@ type Mutation { assignTag(study: ID!): Tag completeTag(tag: ID!, data: JSON!): Boolean! setEntryEnabled(study: ID!, entry: ID!, enabled: Boolean!): Boolean! - saveVideoField(tag: ID!, field: ID!, index: Float!): VideoField! + saveVideoField(tag: ID!, field: String!, index: Int!): VideoField! } input OrganizationCreate { diff --git a/packages/server/src/tag/resolvers/video-field.resolver.ts b/packages/server/src/tag/resolvers/video-field.resolver.ts index 3d7d484d..62211252 100644 --- a/packages/server/src/tag/resolvers/video-field.resolver.ts +++ b/packages/server/src/tag/resolvers/video-field.resolver.ts @@ -1,4 +1,4 @@ -import { Resolver, Query, Args, Mutation, ID, ResolveField, Parent } from '@nestjs/graphql'; +import { Resolver, Args, Mutation, ID, ResolveField, Parent, Int } from '@nestjs/graphql'; import { VideoFieldService } from '../services/video-field.service'; import { TagPipe } from '../pipes/tag.pipe'; import { Tag } from '../models/tag.model'; @@ -23,8 +23,8 @@ export class VideoFieldResolver { @Mutation(() => VideoField) async saveVideoField( @Args('tag', { type: () => ID }, TagPipe) tag: Tag, - @Args('field', { type: () => ID }) field: string, - @Args('index', { type: () => Number }) index: number, + @Args('field') field: string, + @Args('index', { type: () => Int }) index: number, @TokenContext() user: TokenPayload ): Promise { // Make sure the user first has permission to create video fields for this tag From b373e6f2b611097b02489fcecd1dfe622e189a6a Mon Sep 17 00:00:00 2001 From: cbolles Date: Fri, 26 Jan 2024 14:22:12 -0500 Subject: [PATCH 5/9] Create tag context --- .../VideoRecordField.component.tsx | 15 +++ packages/client/src/context/Tag.context.tsx | 45 +++++++++ .../src/pages/contribute/TaggingInterface.tsx | 98 ++++++++----------- 3 files changed, 99 insertions(+), 59 deletions(-) create mode 100644 packages/client/src/context/Tag.context.tsx diff --git a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx index 95bab254..6a54cfa5 100644 --- a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx +++ b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx @@ -5,6 +5,8 @@ import { withJsonFormsControlProps } from '@jsonforms/react'; import { StatusProcessCircles } from './StatusCircles.component'; import { VideoRecordInterface } from './VideoRecordInterface.component'; import { useEffect, useState, useRef } from 'react'; +import { useApolloClient } from '@apollo/client'; +import { SaveVideoFieldDocument, SaveVideoFieldMutationResult, SaveVideoFieldMutationVariables } from '../../../graphql/tag/tag'; const VideoRecordField: React.FC = (props) => { const [maxVideos, setMaxVideos] = useState(0); @@ -15,6 +17,7 @@ const VideoRecordField: React.FC = (props) => { const [recording, setRecording] = useState(false); const stateRef = useRef<{ validVideos: boolean[]; blobs: (Blob | null)[]; activeIndex: number }>(); stateRef.current = { validVideos, blobs, activeIndex }; + const client = useApolloClient(); useEffect(() => { if (!props.uischema.options?.minimumRequired) { @@ -34,6 +37,18 @@ const VideoRecordField: React.FC = (props) => { setBlobs(Array.from({ length: maxVideos }, (_, _i) => null)); }, [props.uischema]); + /** Handles saving the video fragment to the database and updating the JSON Forms representation of the data */ + /*const saveVideoFragment = async (blob: Blob) => { + const result = await client.mutate({ + mutation: SaveVideoFieldDocument, + variables: { + tag: props.data.id, + field: props. + }, + }); + }; */ + + /** Store the blob and check if the video needs to be saved */ const handleVideoRecord = (video: Blob | null) => { const updatedBlobs = stateRef.current!.blobs.map((blob, index) => { if (index === stateRef.current!.activeIndex) { diff --git a/packages/client/src/context/Tag.context.tsx b/packages/client/src/context/Tag.context.tsx new file mode 100644 index 00000000..c8ea0bd5 --- /dev/null +++ b/packages/client/src/context/Tag.context.tsx @@ -0,0 +1,45 @@ +import { ReactNode, FC, createContext, useContext, useEffect, useState } from 'react'; +import { useStudy } from './Study.context'; +import { AssignTagMutation, useAssignTagMutation } from '../graphql/tag/tag'; + +export interface TagContextProps { + tag: AssignTagMutation['assignTag'] | null; + requestTag: () => void; +} + +const TagContext = createContext({} as TagContextProps); + + +export interface TagProviderProps { + children: ReactNode; +} + +export const TagProvider: FC = ({ children }) => { + const { study } = useStudy(); + const [tag, setTag] = useState(null); + const [assignTag, assignTagResult] = useAssignTagMutation(); + + useEffect(() => { + requestTag(); + }, [study]); + + useEffect(() => { + if (!assignTagResult.data?.assignTag) { + return; + } + setTag(assignTagResult.data.assignTag); + }, [assignTagResult.data]); + + const requestTag = () => { + if (!study) { + setTag(null); + return; + } + + assignTag({ variables: { study: study._id } }); + }; + + return {children}; +}; + +export const useTag = () => useContext(TagContext); diff --git a/packages/client/src/pages/contribute/TaggingInterface.tsx b/packages/client/src/pages/contribute/TaggingInterface.tsx index 3eebe837..bff72654 100644 --- a/packages/client/src/pages/contribute/TaggingInterface.tsx +++ b/packages/client/src/pages/contribute/TaggingInterface.tsx @@ -2,40 +2,37 @@ import { Box } from '@mui/material'; import { EntryView } from '../../components/EntryView.component'; import { TagForm } from '../../components/contribute/TagForm.component'; import { useStudy } from '../../context/Study.context'; -import { Dispatch, SetStateAction, useEffect, useState } from 'react'; -import { AssignTagMutation, useAssignTagMutation } from '../../graphql/tag/tag'; +import { useEffect, useState } from 'react'; import { useCompleteTagMutation } from '../../graphql/tag/tag'; import { NoTagNotification } from '../../components/contribute/NoTagNotification.component'; import { Study } from '../../graphql/graphql'; +import { TagProvider, useTag } from '../../context/Tag.context'; export const TaggingInterface: React.FC = () => { const { study } = useStudy(); - const [tag, setTag] = useState(null); - const [assignTag, assignTagResult] = useAssignTagMutation(); - const [tagData, setTagData] = useState({}); - const [completeTag, completeTagResult] = useCompleteTagMutation(); - - // Changes to study will trigger a new tag assignment - useEffect(() => { - // No study, then no tag - if (!study) { - setTag(null); - return; - } - // Assign a tag - assignTag({ variables: { study: study._id } }); - }, [study]); + // TODO: View for when there is no study vs when there is no tag + return ( + <> + + {study && ( + <> + + + )} + + + ); +}; - // Update to the assigned tag - useEffect(() => { - if (!assignTagResult.data) { - setTag(null); - return; - } +interface MainViewProps { + study: Study; +} - setTag(assignTagResult.data.assignTag); - }, [assignTagResult.data]); +const MainView: React.FC = (props) => { + const { tag, requestTag } = useTag(); + const [completeTag, completeTagResult] = useCompleteTagMutation(); + const [tagData, setTagData] = useState({}); // Changes made to the tag data useEffect(() => { @@ -48,46 +45,29 @@ export const TaggingInterface: React.FC = () => { // Tag submission result // TODO: Handle errors useEffect(() => { - if (completeTagResult.data && study) { + if (completeTagResult.data) { // Assign a new tag - assignTag({ variables: { study: study._id } }); + requestTag(); } }, [completeTagResult.data]); - // TODO: View for when there is no study vs when there is no tag return ( - <> - {study && ( - <> - {tag ? ( - - ) : ( - - )} - + <> + {tag ? ( + + + + + ) : ( + )} ); }; - -interface MainViewProps { - tag: NonNullable; - setTagData: Dispatch>; - study: Study; -} - -const MainView: React.FC = (props) => { - return ( - - - - - ); -}; From 528d89f984cdc3f8668231c12594745044d2e4d0 Mon Sep 17 00:00:00 2001 From: cbolles Date: Fri, 26 Jan 2024 15:34:59 -0500 Subject: [PATCH 6/9] Working ability to save video field --- .../VideoRecordField.component.tsx | 57 ++++++++++++++++--- .../src/tag/services/video-field.service.ts | 3 +- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx index 6a54cfa5..21e51a99 100644 --- a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx +++ b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx @@ -6,7 +6,9 @@ import { StatusProcessCircles } from './StatusCircles.component'; import { VideoRecordInterface } from './VideoRecordInterface.component'; import { useEffect, useState, useRef } from 'react'; import { useApolloClient } from '@apollo/client'; -import { SaveVideoFieldDocument, SaveVideoFieldMutationResult, SaveVideoFieldMutationVariables } from '../../../graphql/tag/tag'; +import { SaveVideoFieldDocument, SaveVideoFieldMutation, SaveVideoFieldMutationVariables } from '../../../graphql/tag/tag'; +import { useTag } from '../../../context/Tag.context'; +import axios from 'axios'; const VideoRecordField: React.FC = (props) => { const [maxVideos, setMaxVideos] = useState(0); @@ -15,9 +17,16 @@ const VideoRecordField: React.FC = (props) => { const [activeIndex, setActiveIndex] = useState(0); const [blobs, setBlobs] = useState<(Blob | null)[]>([]); const [recording, setRecording] = useState(false); - const stateRef = useRef<{ validVideos: boolean[]; blobs: (Blob | null)[]; activeIndex: number }>(); - stateRef.current = { validVideos, blobs, activeIndex }; + const [videoFragmentID, setVideoFragmentID] = useState<(string | null)[]>([]); + const stateRef = useRef<{ + validVideos: boolean[]; + blobs: (Blob | null)[]; + activeIndex: number; + videoFragmentID: (string | null)[]; + }>(); + stateRef.current = { validVideos, blobs, activeIndex, videoFragmentID }; const client = useApolloClient(); + const { tag } = useTag(); useEffect(() => { if (!props.uischema.options?.minimumRequired) { @@ -35,18 +44,47 @@ const VideoRecordField: React.FC = (props) => { setMinimumVideos(minimumVideos); setMaxVideos(maxVideos); setBlobs(Array.from({ length: maxVideos }, (_, _i) => null)); + setVideoFragmentID(Array.from({ length: maxVideos }, (_, _i) => null)); }, [props.uischema]); + console.log(props); + /** Handles saving the video fragment to the database and updating the JSON Forms representation of the data */ - /*const saveVideoFragment = async (blob: Blob) => { - const result = await client.mutate({ + const saveVideoFragment = async (blob: Blob) => { + // Save the video fragment + const result = await client.mutate({ mutation: SaveVideoFieldDocument, variables: { - tag: props.data.id, - field: props. + tag: tag!._id, + field: props.path, + index: stateRef.current!.activeIndex }, }); - }; */ + + console.log(result); + + if (!result.data?.saveVideoField) { + console.error('Failed to save video fragment'); + return; + } + + // Upload the video to the provided URL + await axios.put(result.data.saveVideoField.uploadURL, blob, { + headers: { + 'Content-Type': 'video/webm' + } + }); + + // Update the JSON Forms representation of the data to be the ID of the video fragment + const updatedVideoFragmentID = stateRef.current!.videoFragmentID.map((id, index) => { + if (index === stateRef.current!.activeIndex) { + return result.data!.saveVideoField._id; + } + return id; + }); + setVideoFragmentID(updatedVideoFragmentID); + props.handleChange(props.path, updatedVideoFragmentID); + }; /** Store the blob and check if the video needs to be saved */ const handleVideoRecord = (video: Blob | null) => { @@ -63,6 +101,9 @@ const VideoRecordField: React.FC = (props) => { return valid; }); + if (video !== null) { + saveVideoFragment(video); + } setBlobs(updatedBlobs); setValidVideos(updateValidVideos); }; diff --git a/packages/server/src/tag/services/video-field.service.ts b/packages/server/src/tag/services/video-field.service.ts index 2dc4d03f..c62b5dcc 100644 --- a/packages/server/src/tag/services/video-field.service.ts +++ b/packages/server/src/tag/services/video-field.service.ts @@ -53,7 +53,8 @@ export class VideoFieldService { const file = this.bucket.file(this.getVideoFieldBucketLocation(videoField.tag, videoField.field, videoField.index)); const [url] = await file.getSignedUrl({ action: 'write', - expires: Date.now() + this.expiration + expires: Date.now() + this.expiration, + contentType: 'video/webm' }); return url; } From 6c2fcd9bb70df5afc8869bd440c8e64ef530f7a9 Mon Sep 17 00:00:00 2001 From: cbolles Date: Wed, 31 Jan 2024 10:12:22 -0500 Subject: [PATCH 7/9] Have the video IDs only be a list of strings --- .../contribute/TagForm.component.tsx | 2 ++ .../VideoRecordField.component.tsx | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/packages/client/src/components/contribute/TagForm.component.tsx b/packages/client/src/components/contribute/TagForm.component.tsx index f4c62151..3e57cfe3 100644 --- a/packages/client/src/components/contribute/TagForm.component.tsx +++ b/packages/client/src/components/contribute/TagForm.component.tsx @@ -21,6 +21,8 @@ export const TagForm: React.FC = (props) => { const handleFormChange = (data: any, errors: ErrorObject[] | undefined) => { setData(data); + console.log(data, errors); + // No errors, data could be submitted if (!errors || errors.length === 0) { setDataValid(true); diff --git a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx index 21e51a99..6b15a89a 100644 --- a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx +++ b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx @@ -17,12 +17,12 @@ const VideoRecordField: React.FC = (props) => { const [activeIndex, setActiveIndex] = useState(0); const [blobs, setBlobs] = useState<(Blob | null)[]>([]); const [recording, setRecording] = useState(false); - const [videoFragmentID, setVideoFragmentID] = useState<(string | null)[]>([]); + const [videoFragmentID, setVideoFragmentID] = useState([]); const stateRef = useRef<{ validVideos: boolean[]; blobs: (Blob | null)[]; activeIndex: number; - videoFragmentID: (string | null)[]; + videoFragmentID: string[]; }>(); stateRef.current = { validVideos, blobs, activeIndex, videoFragmentID }; const client = useApolloClient(); @@ -44,11 +44,8 @@ const VideoRecordField: React.FC = (props) => { setMinimumVideos(minimumVideos); setMaxVideos(maxVideos); setBlobs(Array.from({ length: maxVideos }, (_, _i) => null)); - setVideoFragmentID(Array.from({ length: maxVideos }, (_, _i) => null)); }, [props.uischema]); - console.log(props); - /** Handles saving the video fragment to the database and updating the JSON Forms representation of the data */ const saveVideoFragment = async (blob: Blob) => { // Save the video fragment @@ -61,8 +58,6 @@ const VideoRecordField: React.FC = (props) => { }, }); - console.log(result); - if (!result.data?.saveVideoField) { console.error('Failed to save video fragment'); return; @@ -76,14 +71,22 @@ const VideoRecordField: React.FC = (props) => { }); // Update the JSON Forms representation of the data to be the ID of the video fragment - const updatedVideoFragmentID = stateRef.current!.videoFragmentID.map((id, index) => { - if (index === stateRef.current!.activeIndex) { - return result.data!.saveVideoField._id; - } - return id; - }); - setVideoFragmentID(updatedVideoFragmentID); - props.handleChange(props.path, updatedVideoFragmentID); + + // If the index is longer than the current videoFragmentID array, then add the new ID to the end + if (stateRef.current!.activeIndex >= stateRef.current!.videoFragmentID.length) { + const updatedVideoFragmentID = [...stateRef.current!.videoFragmentID, result.data!.saveVideoField._id]; + setVideoFragmentID(updatedVideoFragmentID); + props.handleChange(props.path, updatedVideoFragmentID); + } else { + const updatedVideoFragmentID = stateRef.current!.videoFragmentID.map((id, index) => { + if (index === stateRef.current!.activeIndex) { + return result.data!.saveVideoField._id; + } + return id; + }); + setVideoFragmentID(updatedVideoFragmentID); + props.handleChange(props.path, updatedVideoFragmentID); + } }; /** Store the blob and check if the video needs to be saved */ From 30b4a42185c28717789ca391f56dd64f7dadd658 Mon Sep 17 00:00:00 2001 From: cbolles Date: Wed, 31 Jan 2024 10:14:19 -0500 Subject: [PATCH 8/9] Fix formatting --- .../src/components/contribute/TagForm.component.tsx | 2 -- .../tag/videorecord/VideoRecordField.component.tsx | 8 ++++++-- packages/client/src/context/Tag.context.tsx | 1 - packages/client/src/pages/contribute/TaggingInterface.tsx | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/client/src/components/contribute/TagForm.component.tsx b/packages/client/src/components/contribute/TagForm.component.tsx index 3e57cfe3..f4c62151 100644 --- a/packages/client/src/components/contribute/TagForm.component.tsx +++ b/packages/client/src/components/contribute/TagForm.component.tsx @@ -21,8 +21,6 @@ export const TagForm: React.FC = (props) => { const handleFormChange = (data: any, errors: ErrorObject[] | undefined) => { setData(data); - console.log(data, errors); - // No errors, data could be submitted if (!errors || errors.length === 0) { setDataValid(true); diff --git a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx index 6b15a89a..b5ba7caa 100644 --- a/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx +++ b/packages/client/src/components/tag/videorecord/VideoRecordField.component.tsx @@ -6,7 +6,11 @@ import { StatusProcessCircles } from './StatusCircles.component'; import { VideoRecordInterface } from './VideoRecordInterface.component'; import { useEffect, useState, useRef } from 'react'; import { useApolloClient } from '@apollo/client'; -import { SaveVideoFieldDocument, SaveVideoFieldMutation, SaveVideoFieldMutationVariables } from '../../../graphql/tag/tag'; +import { + SaveVideoFieldDocument, + SaveVideoFieldMutation, + SaveVideoFieldMutationVariables +} from '../../../graphql/tag/tag'; import { useTag } from '../../../context/Tag.context'; import axios from 'axios'; @@ -55,7 +59,7 @@ const VideoRecordField: React.FC = (props) => { tag: tag!._id, field: props.path, index: stateRef.current!.activeIndex - }, + } }); if (!result.data?.saveVideoField) { diff --git a/packages/client/src/context/Tag.context.tsx b/packages/client/src/context/Tag.context.tsx index c8ea0bd5..4ec1b8cd 100644 --- a/packages/client/src/context/Tag.context.tsx +++ b/packages/client/src/context/Tag.context.tsx @@ -9,7 +9,6 @@ export interface TagContextProps { const TagContext = createContext({} as TagContextProps); - export interface TagProviderProps { children: ReactNode; } diff --git a/packages/client/src/pages/contribute/TaggingInterface.tsx b/packages/client/src/pages/contribute/TaggingInterface.tsx index bff72654..84b8dc26 100644 --- a/packages/client/src/pages/contribute/TaggingInterface.tsx +++ b/packages/client/src/pages/contribute/TaggingInterface.tsx @@ -52,7 +52,7 @@ const MainView: React.FC = (props) => { }, [completeTagResult.data]); return ( - <> + <> {tag ? ( Date: Wed, 31 Jan 2024 10:16:32 -0500 Subject: [PATCH 9/9] Fix formatting --- packages/server/src/tag/services/video-field.service.ts | 7 +++++-- packages/server/src/tag/tag.module.ts | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/server/src/tag/services/video-field.service.ts b/packages/server/src/tag/services/video-field.service.ts index c62b5dcc..92df5398 100644 --- a/packages/server/src/tag/services/video-field.service.ts +++ b/packages/server/src/tag/services/video-field.service.ts @@ -1,5 +1,5 @@ import { BadRequestException, Injectable, Inject } from '@nestjs/common'; -import { InjectModel} from '@nestjs/mongoose'; +import { InjectModel } from '@nestjs/mongoose'; import { VideoField, VideoFieldDocument } from '../models/video-field.model'; import { Model } from 'mongoose'; import { Tag } from '../models/tag.model'; @@ -45,7 +45,10 @@ export class VideoFieldService { // Otherwise make a new one and return it return this.videoFieldModel.create({ - tag: tag._id, field, index, bucketLocation: this.getVideoFieldBucketLocation(tag._id, field, index) + tag: tag._id, + field, + index, + bucketLocation: this.getVideoFieldBucketLocation(tag._id, field, index) }); } diff --git a/packages/server/src/tag/tag.module.ts b/packages/server/src/tag/tag.module.ts index a0aec128..06f1a3f7 100644 --- a/packages/server/src/tag/tag.module.ts +++ b/packages/server/src/tag/tag.module.ts @@ -15,7 +15,10 @@ import { GcpModule } from '../gcp/gcp.module'; @Module({ imports: [ - MongooseModule.forFeature([{ name: Tag.name, schema: TagSchema }, { name: VideoField.name, schema: VideoFieldSchema }]), + MongooseModule.forFeature([ + { name: Tag.name, schema: TagSchema }, + { name: VideoField.name, schema: VideoFieldSchema } + ]), StudyModule, EntryModule, SharedModule,