diff --git a/packages/client/public/locales/en/translation.json b/packages/client/public/locales/en/translation.json index 25841393..26be8c0d 100644 --- a/packages/client/public/locales/en/translation.json +++ b/packages/client/public/locales/en/translation.json @@ -28,7 +28,8 @@ "redo": "Redo", "dataset": "Dataset", "status": "Status", - "dateFormat": "{{date, datetime}}" + "dateFormat": "{{date, datetime}}", + "download": "Download" }, "languages": { "en": "English", @@ -55,7 +56,8 @@ "contribute": "Contribute", "tagInStudy": "Tag in Study", "logout": "Logout", - "datasetDownloads": "Dataset Downloads" + "datasetDownloads": "Dataset Downloads", + "studyDownloads": "Study Downloads" }, "components": { "environment": { @@ -143,6 +145,14 @@ "tagView": { "originalEntry": "Original Entry", "export": "Export" + }, + "studyDownload": { + "csv": "Tag CSV", + "taggedEntries": "Entries Tagged", + "downloadStartedSuccess": "Download has started, the download will be available under the study download page", + "downloadFailed": "Could not download study data, please reach out to your administrator", + "downloadTitle": "Study Download Request", + "downloadDescription": "Would you like to download this study? The tag data, any recorded videos, and the original entries will be download, this may take a while, downloads will appear in the study download page when complete" } }, "errors": { diff --git a/packages/client/src/App.tsx b/packages/client/src/App.tsx index fc089741..1d5a7ba2 100644 --- a/packages/client/src/App.tsx +++ b/packages/client/src/App.tsx @@ -31,6 +31,7 @@ import { PermissionProvider } from './context/Permission.context'; import { TagTrainingView } from './pages/studies/TagTrainingView'; import { SnackbarProvider } from './context/Snackbar.context'; import { DatasetDownloads } from './pages/datasets/DatasetDownloads'; +import { StudyDownloads } from './pages/studies/StudyDownloads'; const drawerWidth = 256; const Main = styled('main', { shouldForwardProp: (prop) => prop !== 'open' })<{ @@ -135,6 +136,7 @@ const MyRoutes: FC = () => { } /> } /> } /> + } /> } /> } /> } /> diff --git a/packages/client/src/components/SideBar.component.tsx b/packages/client/src/components/SideBar.component.tsx index 00b7ee5a..e37073ae 100644 --- a/packages/client/src/components/SideBar.component.tsx +++ b/packages/client/src/components/SideBar.component.tsx @@ -52,7 +52,8 @@ export const SideBar: FC = ({ open, drawerWidth }) => { visible: (p) => p!.studyAdmin }, { name: t('menu.entryControl'), action: () => navigate('/study/entries'), visible: (p) => p!.studyAdmin }, - { name: t('menu.viewTags'), action: () => navigate('/study/tags'), visible: (p) => p!.studyAdmin } + { name: t('menu.viewTags'), action: () => navigate('/study/tags'), visible: (p) => p!.studyAdmin }, + { name: t('menu.studyDownloads'), action: () => navigate('/study/downloads'), visible: (p) => p!.studyAdmin } ] }, { diff --git a/packages/client/src/graphql/graphql.ts b/packages/client/src/graphql/graphql.ts index 431b4420..76f50a71 100644 --- a/packages/client/src/graphql/graphql.ts +++ b/packages/client/src/graphql/graphql.ts @@ -32,6 +32,10 @@ export type CreateDatasetDownloadRequest = { dataset: Scalars['ID']['input']; }; +export type CreateStudyDownloadRequest = { + study: Scalars['ID']['input']; +}; + export type Dataset = { __typename?: 'Dataset'; _id: Scalars['ID']['output']; @@ -78,6 +82,7 @@ export type Entry = { signedUrl: Scalars['String']['output']; /** Get the number of milliseconds the signed URL is valid for. */ signedUrlExpiration: Scalars['Float']['output']; + signlabRecording?: Maybe; }; export type FreeTextField = { @@ -146,6 +151,7 @@ export type Mutation = { createDatasetDownload: DatasetDownloadRequest; createOrganization: Organization; createStudy: Study; + createStudyDownload: StudyDownloadRequest; createTags: Array; createTrainingSet: Scalars['Boolean']['output']; createUploadSession: UploadSession; @@ -229,6 +235,11 @@ export type MutationCreateStudyArgs = { }; +export type MutationCreateStudyDownloadArgs = { + downloadRequest: CreateStudyDownloadRequest; +}; + + export type MutationCreateTagsArgs = { entries: Array; study: Scalars['ID']['input']; @@ -410,6 +421,7 @@ export type Query = { getProjectPermissions: Array; getProjects: Array; getRoles: Permission; + getStudyDownloads: Array; getStudyPermissions: Array; getTags: Array; getTrainingTags: Array; @@ -486,6 +498,11 @@ export type QueryGetRolesArgs = { }; +export type QueryGetStudyDownloadsArgs = { + study: Scalars['ID']['input']; +}; + + export type QueryGetStudyPermissionsArgs = { study: Scalars['ID']['input']; }; @@ -535,6 +552,11 @@ export type QueryValidateCsvArgs = { session: Scalars['ID']['input']; }; +export type SignLabRecorded = { + __typename?: 'SignLabRecorded'; + fieldName: Scalars['String']['output']; +}; + export type SliderField = { __typename?: 'SliderField'; value: Scalars['Float']['output']; @@ -560,6 +582,17 @@ export type StudyCreate = { tagsPerEntry: Scalars['Float']['input']; }; +export type StudyDownloadRequest = { + __typename?: 'StudyDownloadRequest'; + _id: Scalars['String']['output']; + date: Scalars['DateTime']['output']; + entryZip: Scalars['String']['output']; + status: Scalars['String']['output']; + study: Study; + tagCSV: Scalars['String']['output']; + taggedEntries: Scalars['String']['output']; +}; + export type StudyPermissionModel = { __typename?: 'StudyPermissionModel'; isContributor: Scalars['Boolean']['output']; diff --git a/packages/client/src/graphql/study/study.graphql b/packages/client/src/graphql/study/study.graphql index 2d880667..5a4f3c6c 100644 --- a/packages/client/src/graphql/study/study.graphql +++ b/packages/client/src/graphql/study/study.graphql @@ -35,3 +35,34 @@ mutation createStudy($study: StudyCreate!) { query studyExists($name: String!, $project: ID!) { studyExists(name: $name, project: $project) } + +query getStudyDownloads($study: ID!) { + getStudyDownloads(study: $study) { + _id, + date, + status, + entryZip, + tagCSV, + taggedEntries, + study { + _id + name + description + instructions + project + tagsPerEntry + tagSchema { + dataSchema + uiSchema + } + } + } +} + +mutation createStudyDownload($downloadRequest: CreateStudyDownloadRequest!) { + createStudyDownload(downloadRequest: $downloadRequest) { + _id, + status, + date + } +} diff --git a/packages/client/src/graphql/study/study.ts b/packages/client/src/graphql/study/study.ts index 7d85de42..e52e29fe 100644 --- a/packages/client/src/graphql/study/study.ts +++ b/packages/client/src/graphql/study/study.ts @@ -34,6 +34,20 @@ export type StudyExistsQueryVariables = Types.Exact<{ export type StudyExistsQuery = { __typename?: 'Query', studyExists: boolean }; +export type GetStudyDownloadsQueryVariables = Types.Exact<{ + study: Types.Scalars['ID']['input']; +}>; + + +export type GetStudyDownloadsQuery = { __typename?: 'Query', getStudyDownloads: Array<{ __typename?: 'StudyDownloadRequest', _id: string, date: any, status: string, entryZip: string, tagCSV: string, taggedEntries: string, study: { __typename?: 'Study', _id: string, name: string, description: string, instructions: string, project: string, tagsPerEntry: number, tagSchema: { __typename?: 'TagSchema', dataSchema: any, uiSchema: any } } }> }; + +export type CreateStudyDownloadMutationVariables = Types.Exact<{ + downloadRequest: Types.CreateStudyDownloadRequest; +}>; + + +export type CreateStudyDownloadMutation = { __typename?: 'Mutation', createStudyDownload: { __typename?: 'StudyDownloadRequest', _id: string, status: string, date: any } }; + export const FindStudiesDocument = gql` query findStudies($project: ID!) { @@ -185,4 +199,91 @@ export function useStudyExistsLazyQuery(baseOptions?: Apollo.LazyQueryHookOption } export type StudyExistsQueryHookResult = ReturnType; export type StudyExistsLazyQueryHookResult = ReturnType; -export type StudyExistsQueryResult = Apollo.QueryResult; \ No newline at end of file +export type StudyExistsQueryResult = Apollo.QueryResult; +export const GetStudyDownloadsDocument = gql` + query getStudyDownloads($study: ID!) { + getStudyDownloads(study: $study) { + _id + date + status + entryZip + tagCSV + taggedEntries + study { + _id + name + description + instructions + project + tagsPerEntry + tagSchema { + dataSchema + uiSchema + } + } + } +} + `; + +/** + * __useGetStudyDownloadsQuery__ + * + * To run a query within a React component, call `useGetStudyDownloadsQuery` and pass it any options that fit your needs. + * When your component renders, `useGetStudyDownloadsQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useGetStudyDownloadsQuery({ + * variables: { + * study: // value for 'study' + * }, + * }); + */ +export function useGetStudyDownloadsQuery(baseOptions: Apollo.QueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(GetStudyDownloadsDocument, options); + } +export function useGetStudyDownloadsLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(GetStudyDownloadsDocument, options); + } +export type GetStudyDownloadsQueryHookResult = ReturnType; +export type GetStudyDownloadsLazyQueryHookResult = ReturnType; +export type GetStudyDownloadsQueryResult = Apollo.QueryResult; +export const CreateStudyDownloadDocument = gql` + mutation createStudyDownload($downloadRequest: CreateStudyDownloadRequest!) { + createStudyDownload(downloadRequest: $downloadRequest) { + _id + status + date + } +} + `; +export type CreateStudyDownloadMutationFn = Apollo.MutationFunction; + +/** + * __useCreateStudyDownloadMutation__ + * + * To run a mutation, you first call `useCreateStudyDownloadMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useCreateStudyDownloadMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [createStudyDownloadMutation, { data, loading, error }] = useCreateStudyDownloadMutation({ + * variables: { + * downloadRequest: // value for 'downloadRequest' + * }, + * }); + */ +export function useCreateStudyDownloadMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(CreateStudyDownloadDocument, options); + } +export type CreateStudyDownloadMutationHookResult = ReturnType; +export type CreateStudyDownloadMutationResult = Apollo.MutationResult; +export type CreateStudyDownloadMutationOptions = Apollo.BaseMutationOptions; \ No newline at end of file diff --git a/packages/client/src/pages/studies/StudyControl.tsx b/packages/client/src/pages/studies/StudyControl.tsx index 8daa93f3..8bb6c034 100644 --- a/packages/client/src/pages/studies/StudyControl.tsx +++ b/packages/client/src/pages/studies/StudyControl.tsx @@ -1,14 +1,15 @@ -import { Typography, Box } from '@mui/material'; +import { Typography, Box, IconButton } from '@mui/material'; import { useStudy } from '../../context/Study.context'; import { DataGrid, GridColDef, GridRowId } from '@mui/x-data-grid'; import DeleteIcon from '@mui/icons-material/DeleteOutlined'; import { GridActionsCellItem } from '@mui/x-data-grid-pro'; import { Study } from '../../graphql/graphql'; -import { useDeleteStudyMutation } from '../../graphql/study/study'; +import { useCreateStudyDownloadMutation, useDeleteStudyMutation } from '../../graphql/study/study'; import { useEffect } from 'react'; import { useConfirmation } from '../../context/Confirmation.context'; import { useTranslation } from 'react-i18next'; import { useSnackbar } from '../../context/Snackbar.context'; +import { Download } from '@mui/icons-material'; export const StudyControl: React.FC = () => { const { studies, updateStudies } = useStudy(); @@ -18,6 +19,8 @@ export const StudyControl: React.FC = () => { const { t } = useTranslation(); const { pushSnackbarMessage } = useSnackbar(); + const [createDownloadMutation, createDownloadResults] = useCreateStudyDownloadMutation(); + const handleDelete = async (id: GridRowId) => { // Execute delete mutation confirmation.pushConfirmationRequest({ @@ -40,6 +43,32 @@ export const StudyControl: React.FC = () => { } }, [deleteStudyResults.called, deleteStudyResults.data, deleteStudyResults.error]); + const handleDownloadRequest = (study: Study) => { + confirmation.pushConfirmationRequest({ + title: t('components.studyDownload.downloadTitle'), + message: t('components.studyDownload.downloadDescription'), + onConfirm: () => { + createDownloadMutation({ + variables: { + downloadRequest: { + study: study._id + } + } + }); + }, + onCancel: () => {} + }); + }; + + // Share the results with the user + useEffect(() => { + if (createDownloadResults.data) { + pushSnackbarMessage(t('components.studyDownload.downloadStartedSuccess'), 'success'); + } else if (createDownloadResults.error) { + pushSnackbarMessage(t('components.studyDownload.downloadFailed'), 'error'); + } + }, [createDownloadResults.data, createDownloadResults.error]); + const columns: GridColDef[] = [ { field: 'name', @@ -53,6 +82,16 @@ export const StudyControl: React.FC = () => { width: 500, editable: false }, + { + field: 'download', + headerName: t('common.download'), + width: 200, + renderCell: (params) => ( + handleDownloadRequest(params.row)}> + + + ) + }, { field: 'delete', type: 'actions', diff --git a/packages/client/src/pages/studies/StudyDownloads.tsx b/packages/client/src/pages/studies/StudyDownloads.tsx new file mode 100644 index 00000000..a78a66d7 --- /dev/null +++ b/packages/client/src/pages/studies/StudyDownloads.tsx @@ -0,0 +1,104 @@ +import { DataGrid, GridColDef } from '@mui/x-data-grid'; +import { useEffect, useState } from 'react'; +import { StudyDownloadRequest, DownloadStatus } from '../../graphql/graphql'; +import { useStudy } from '../../context/Study.context'; +import { useGetStudyDownloadsLazyQuery } from '../../graphql/study/study'; +import { useTranslation } from 'react-i18next'; +import { IconButton } from '@mui/material'; +import { Download, HourglassTop, DownloadDone } from '@mui/icons-material'; + +export const StudyDownloads: React.FC = () => { + const [studyDownloadRequests, setStudyDownloadRequest] = useState([]); + const { study } = useStudy(); + const { t } = useTranslation(); + + const [getDownloadsQuery, getDownloadsResults] = useGetStudyDownloadsLazyQuery(); + + useEffect(() => { + if (!study) { + setStudyDownloadRequest([]); + return; + } + + getDownloadsQuery({ + variables: { + study: study._id + } + }); + }, [study]); + + useEffect(() => { + if (getDownloadsResults.data) { + setStudyDownloadRequest(getDownloadsResults.data.getStudyDownloads); + } + }, [getDownloadsResults.data]); + + const columns: GridColDef[] = [ + { + field: 'studyName', + headerName: t('common.study'), + width: 200, + valueGetter: (params) => params.row.study.name + }, + { + field: 'date', + width: 200, + headerName: t('components.datasetDownload.requestDate'), + valueGetter: (params) => t('common.dateFormat', { date: Date.parse(params.row.date) }) + }, + { + field: 'status', + width: 200, + headerName: t('common.status'), + renderCell: (params) => params.value && + }, + { + field: 'entryZip', + width: 200, + headerName: t('components.datasetDownload.entryDownload'), + renderCell: (params) => + params.value && ( + + + + ) + }, + { + field: 'tagCSV', + width: 200, + headerName: t('components.studyDownload.csv'), + renderCell: (params) => + params.value && ( + + + + ) + }, + { + field: 'taggedEntries', + width: 200, + headerName: t('components.studyDownload.taggedEntries'), + renderCell: (params) => + params.value && ( + + + + ) + } + ]; + + return row._id} />; +}; + +interface StatusViewProps { + status: DownloadStatus; +} + +const StatusView: React.FC = ({ status }) => { + switch (status) { + case DownloadStatus.Ready: + return ; + case DownloadStatus.InProgress: + return ; + } +}; diff --git a/packages/server/src/download-request/download-request.module.ts b/packages/server/src/download-request/download-request.module.ts index f81ea973..acfe5d38 100644 --- a/packages/server/src/download-request/download-request.module.ts +++ b/packages/server/src/download-request/download-request.module.ts @@ -11,6 +11,11 @@ import { CreateDatasetDownloadPipe } from './pipes/dataset-download-request-crea import { DatasetDownloadRequestResolver } from './resolvers/dataset-download-request.resolver'; import { DatasetDownloadService } from './services/dataset-download-request.service'; import { DownloadRequestService } from './services/download-request.service'; +import { CreateStudyDownloadPipe } from './pipes/study-download-request-create.pipe'; +import { StudyModule } from 'src/study/study.module'; +import { StudyDownloadRequestResolver } from './resolvers/study-download-request.resolver'; +import { StudyDownloadService } from './services/study-download-request.service'; +import { TagModule } from '../tag/tag.module'; @Module({ imports: [ @@ -22,8 +27,18 @@ import { DownloadRequestService } from './services/download-request.service'; DatasetModule, EntryModule, BucketModule, - GcpModule + GcpModule, + StudyModule, + TagModule ], - providers: [DatasetDownloadRequestResolver, DatasetDownloadService, DownloadRequestService, CreateDatasetDownloadPipe] + providers: [ + DatasetDownloadRequestResolver, + DatasetDownloadService, + DownloadRequestService, + CreateDatasetDownloadPipe, + CreateStudyDownloadPipe, + StudyDownloadRequestResolver, + StudyDownloadService + ] }) export class DownloadRequestModule {} diff --git a/packages/server/src/download-request/dtos/dataset-download-request-create.dto.ts b/packages/server/src/download-request/dtos/dataset-download-request-create.dto.ts index e731ce66..0bc54e2b 100644 --- a/packages/server/src/download-request/dtos/dataset-download-request-create.dto.ts +++ b/packages/server/src/download-request/dtos/dataset-download-request-create.dto.ts @@ -4,7 +4,15 @@ import { DatasetDownloadRequest } from '../models/dataset-download-request.model @InputType() export class CreateDatasetDownloadRequest extends OmitType( DatasetDownloadRequest, - ['_id', 'date', 'status'] as const, + [ + '_id', + 'date', + 'status', + 'entryZIPLocation', + 'bucketLocation', + 'entryJSONLocation', + 'webhookPayloadLocation' + ] as const, InputType ) { @Field(() => ID) diff --git a/packages/server/src/download-request/dtos/study-download-request-create.dto.ts b/packages/server/src/download-request/dtos/study-download-request-create.dto.ts new file mode 100644 index 00000000..773fac6c --- /dev/null +++ b/packages/server/src/download-request/dtos/study-download-request-create.dto.ts @@ -0,0 +1,24 @@ +import { Field, ID, InputType, OmitType } from '@nestjs/graphql'; +import { StudyDownloadRequest } from '../models/study-download-request.model'; + +@InputType() +export class CreateStudyDownloadRequest extends OmitType( + StudyDownloadRequest, + [ + '_id', + 'date', + 'status', + 'tagCSVLocation', + 'entryZIPLocation', + 'bucketLocation', + 'entryZIPLocation', + 'webhookPayloadLocation', + 'taggedEntriesJSONLocation', + 'taggedEntriesZipLocation', + 'taggedEntryWebhookPayloadLocation' + ] as const, + InputType +) { + @Field(() => ID) + study: string; +} diff --git a/packages/server/src/download-request/models/study-download-request.model.ts b/packages/server/src/download-request/models/study-download-request.model.ts index c7af1bfb..b0aa0daa 100644 --- a/packages/server/src/download-request/models/study-download-request.model.ts +++ b/packages/server/src/download-request/models/study-download-request.model.ts @@ -1,35 +1,59 @@ import { Schema, Prop, SchemaFactory } from '@nestjs/mongoose'; import { Document } from 'mongoose'; import { DownloadRequest, DownloadStatus } from './download-request.model'; +import { Field, ObjectType } from '@nestjs/graphql'; @Schema() +@ObjectType() export class StudyDownloadRequest implements DownloadRequest { + @Field() + _id: string; + @Prop({ required: true }) organization: string; @Prop({ required: true }) + @Field() date: Date; @Prop({ required: true, enum: DownloadStatus }) + @Field() status: DownloadStatus; @Prop({ required: true }) study: string; + /** Location in a bucket where the tag data as a CSV should be stored */ @Prop({ requied: false }) tagCSVLocation?: string; - @Prop({ required: true }) - entryZIPLocation: string; + /** Location in a bucket where any entries recorded as part of a study will be */ + @Prop({ required: false }) + entryZIPLocation?: string; + /** The prefix for all bucket locations */ @Prop({ required: false }) bucketLocation?: string; + /** Where the JSON list of entries recorded as part of the study will be */ @Prop({ required: false }) entryJSONLocation?: string; + /** Webhook payload to be called when the zipping of entries recorded in the study is complete */ @Prop({ required: false }) webhookPayloadLocation?: string; + + /** Location in a bucket where the entries tagged will be stored */ + @Prop({ required: false }) + taggedEntriesZipLocation?: string; + + /** Location in a bucket where the JSON list of entries tagged as part of the study will be */ + @Prop({ required: false }) + taggedEntriesJSONLocation?: string; + + /** Webhook payload to be used when the zipping of tagged entries is complete */ + @Prop({ required: false }) + taggedEntryWebhookPayloadLocation?: string; } export type StudyDownloadRequestDocument = Document & StudyDownloadRequest; diff --git a/packages/server/src/download-request/pipes/study-download-request-create.pipe.ts b/packages/server/src/download-request/pipes/study-download-request-create.pipe.ts new file mode 100644 index 00000000..2fc44efa --- /dev/null +++ b/packages/server/src/download-request/pipes/study-download-request-create.pipe.ts @@ -0,0 +1,18 @@ +import { BadRequestException, Injectable, PipeTransform } from '@nestjs/common'; +import { StudyService } from '../../study/study.service'; +import { CreateStudyDownloadRequest } from '../dtos/study-download-request-create.dto'; + +@Injectable() +export class CreateStudyDownloadPipe + implements PipeTransform> +{ + constructor(private readonly studyService: StudyService) {} + + async transform(value: CreateStudyDownloadRequest): Promise { + const exists = await this.studyService.existsById(value.study); + if (!exists) { + throw new BadRequestException(`Study with id ${value.study} does not exist`); + } + return value; + } +} diff --git a/packages/server/src/download-request/resolvers/study-download-request.resolver.ts b/packages/server/src/download-request/resolvers/study-download-request.resolver.ts new file mode 100644 index 00000000..c2aa363f --- /dev/null +++ b/packages/server/src/download-request/resolvers/study-download-request.resolver.ts @@ -0,0 +1,51 @@ +import { Resolver, Mutation, Args, ResolveField, Parent, ID, Query } from '@nestjs/graphql'; +import { JwtAuthGuard } from '../../jwt/jwt.guard'; +import { OrganizationGuard } from '../../organization/organization.guard'; +import { UseGuards } from '@nestjs/common'; +import { StudyDownloadRequest } from '../models/study-download-request.model'; +import { StudyDownloadService } from '../services/study-download-request.service'; +import { CreateStudyDownloadPipe } from '../pipes/study-download-request-create.pipe'; +import { CreateStudyDownloadRequest } from '../dtos/study-download-request-create.dto'; +import { OrganizationContext } from '../../organization/organization.context'; +import { Organization } from '../../organization/organization.model'; +import { StudyPipe } from '../../study/pipes/study.pipe'; +import { Study } from '../../study/study.model'; + +@UseGuards(JwtAuthGuard, OrganizationGuard) +@Resolver(() => StudyDownloadRequest) +export class StudyDownloadRequestResolver { + constructor(private readonly studyDownloadService: StudyDownloadService, private readonly studyPipe: StudyPipe) {} + + @Mutation(() => StudyDownloadRequest) + async createStudyDownload( + @Args('downloadRequest', CreateStudyDownloadPipe) downloadRequest: CreateStudyDownloadRequest, + @OrganizationContext() organization: Organization + ): Promise { + return this.studyDownloadService.createDownloadRequest(downloadRequest, organization); + } + + @Query(() => [StudyDownloadRequest]) + async getStudyDownloads(@Args('study', { type: () => ID }, StudyPipe) study: Study): Promise { + return this.studyDownloadService.getStudyDownloads(study); + } + + @ResolveField(() => String) + async entryZip(@Parent() downloadRequest: StudyDownloadRequest): Promise { + return this.studyDownloadService.getEntryZipUrl(downloadRequest); + } + + @ResolveField(() => Study) + async study(@Parent() downloadRequest: StudyDownloadRequest): Promise { + return this.studyPipe.transform(downloadRequest.study); + } + + @ResolveField(() => String) + async tagCSV(@Parent() downloadRequest: StudyDownloadRequest): Promise { + return this.studyDownloadService.getTagCSVUrl(downloadRequest); + } + + @ResolveField(() => String) + async taggedEntries(@Parent() downloadRequest: StudyDownloadRequest): Promise { + return this.studyDownloadService.getTaggedEntriesUrl(downloadRequest); + } +} diff --git a/packages/server/src/download-request/services/dataset-download-request.service.ts b/packages/server/src/download-request/services/dataset-download-request.service.ts index bf24a1b6..36adc3e0 100644 --- a/packages/server/src/download-request/services/dataset-download-request.service.ts +++ b/packages/server/src/download-request/services/dataset-download-request.service.ts @@ -1,11 +1,9 @@ -import { JobsClient } from '@google-cloud/run'; -import { Inject, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { InjectModel } from '@nestjs/mongoose'; import { Model } from 'mongoose'; import { BucketObjectAction } from 'src/bucket/bucket'; import { Dataset } from 'src/dataset/dataset.model'; -import { JOB_PROVIDER } from 'src/gcp/providers/job.provider'; import { BucketFactory } from '../../bucket/bucket-factory.service'; import { EntryService } from '../../entry/services/entry.service'; import { Organization } from '../../organization/organization.model'; @@ -16,7 +14,6 @@ import { DownloadRequestService } from './download-request.service'; @Injectable() export class DatasetDownloadService { - private readonly zipJobName: string = this.configService.getOrThrow('downloads.jobName'); private readonly expiration = this.configService.getOrThrow('entry.signedURLExpiration'); constructor( @@ -25,7 +22,6 @@ export class DatasetDownloadService { private readonly downloadService: DownloadRequestService, private readonly entryService: EntryService, private readonly bucketFactory: BucketFactory, - @Inject(JOB_PROVIDER) private readonly jobsClient: JobsClient, private readonly configService: ConfigService ) {} @@ -61,7 +57,16 @@ export class DatasetDownloadService { request = (await this.downloadRequestModel.findById(request._id))!; // Start the process of zipping the entries - await this.startZipJob(request); + await this.downloadService.startZipJob({ + entryJSONLocation: request.entryJSONLocation!, + entryZIPLocation: request.entryZIPLocation!, + webhookPayloadLocation: request.webhookPayloadLocation!, + webhookPayload: JSON.stringify({ test: 'hello' }), + webhook: 'http://localhost:3000/', + entries: await this.entryService.findForDataset(request.dataset), + bucket: (await this.bucketFactory.getBucket(request.organization))!, + organization: request.organization + }); return request; } @@ -83,49 +88,4 @@ export class DatasetDownloadService { new Date(Date.now() + this.expiration) ); } - - private async startZipJob(downloadRequest: DatasetDownloadRequest): Promise { - // First, get the entries that need to be zipped - const entries = await this.entryService.findForDataset(downloadRequest.dataset); - const entryLocations = entries.map((entry) => `/buckets/${downloadRequest.organization}/${entry.bucketLocation}`); - - // Make the content of the entry request file - const entryContent: string = JSON.stringify({ entries: entryLocations }); - - // Get the bucket for uploading supporting files - const bucket = await this.bucketFactory.getBucket(downloadRequest.organization); - if (!bucket) { - throw Error(`Bucket not found for organization ${downloadRequest.organization}`); - } - - // Write in the entries file - await bucket.writeText(downloadRequest.entryJSONLocation!, entryContent); - - // Upload the webhook payload - // TODO: Update webhook - await bucket.writeText( - downloadRequest.webhookPayloadLocation!, - JSON.stringify({ - code: '1234', - downloadRequest: '12' - }) - ); - - // Trigger the cloud run job - await this.jobsClient.runJob({ - name: this.zipJobName, - overrides: { - containerOverrides: [ - { - args: [ - `--target_entries=/buckets/${downloadRequest.organization}/${downloadRequest.entryJSONLocation!}`, - `--output_zip=/buckets/${downloadRequest.organization}/${downloadRequest.entryZIPLocation!}`, - `--notification_webhook=http://localhost:3000`, - `--webhook_payload=/buckets/${downloadRequest.organization}/${downloadRequest.webhookPayloadLocation!}` - ] - } - ] - } - }); - } } diff --git a/packages/server/src/download-request/services/download-request.service.ts b/packages/server/src/download-request/services/download-request.service.ts index e07edb64..890be9a7 100644 --- a/packages/server/src/download-request/services/download-request.service.ts +++ b/packages/server/src/download-request/services/download-request.service.ts @@ -1,11 +1,48 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { Entry } from '../../entry/models/entry.model'; +import { Bucket } from '../../bucket/bucket'; +import { JOB_PROVIDER } from '../../gcp/providers/job.provider'; +import { JobsClient } from '@google-cloud/run'; + +export interface ZipJobRequest { + /** Where to put the entry JSON file in the bucket */ + entryJSONLocation: string; + + /** The location in the bucket to put the zip */ + entryZIPLocation: string; + + /** Where the webhook payload should be placed */ + webhookPayloadLocation: string; + + /** The webhook payload */ + webhookPayload: string; + + /** The webhook endpoint */ + webhook: string; + + /** The entries that need to be zipped */ + entries: Entry[]; + + /** The bucket to upload into */ + bucket: Bucket; + + /** The organization ID */ + organization: string; +} @Injectable() export class DownloadRequestService { + /** Where in the organization bucket all downloads are stored */ private readonly downloadPrefix: string = this.configService.getOrThrow('downloads.bucketPrefix'); + /** The name of the GCP Job */ + private readonly zipJobName: string = this.configService.getOrThrow('downloads.jobName'); - constructor(private readonly configService: ConfigService) {} + constructor( + private readonly configService: ConfigService, + @Inject(JOB_PROVIDER) + private readonly jobsClient: JobsClient + ) {} /** Get a bucket location for download requests given the filename */ getBucketLocation(fileName: string): string { @@ -15,4 +52,38 @@ export class DownloadRequestService { getPrefix(): string { return this.downloadPrefix; } + + async startZipJob(request: ZipJobRequest): Promise { + const mountPoint = `/buckets/${request.organization}`; + + // Get the location of each entry based on the prefix. This will be where the + // entry is located for the GCP Cloud Run Job + const entryLocations = request.entries.map((entry) => `${mountPoint}/${entry.bucketLocation}`); + + // Convert the list to a string for saving + const entryContent: string = JSON.stringify({ entries: entryLocations }); + + // Now upload the generated JSON file with the entry locations into the bucket + await request.bucket.writeText(request.entryJSONLocation, entryContent); + + // Upload the webhook payload + await request.bucket.writeText(request.webhookPayloadLocation, request.webhookPayload); + + // Trigger the cloud run job + await this.jobsClient.runJob({ + name: this.zipJobName, + overrides: { + containerOverrides: [ + { + args: [ + `--target_entries=${mountPoint}/${request.entryJSONLocation}`, + `--output_zip=${mountPoint}/${request.entryZIPLocation}`, + `--notification_webhook=http://localhost:3000`, + `--webhook_payload=${mountPoint}/${request.webhookPayloadLocation}` + ] + } + ] + } + }); + } } diff --git a/packages/server/src/download-request/services/study-download-request.service.ts b/packages/server/src/download-request/services/study-download-request.service.ts new file mode 100644 index 00000000..0e5c9fb7 --- /dev/null +++ b/packages/server/src/download-request/services/study-download-request.service.ts @@ -0,0 +1,216 @@ +import { Injectable } from '@nestjs/common'; +import { StudyDownloadRequest } from '../models/study-download-request.model'; +import { InjectModel } from '@nestjs/mongoose'; +import { Model } from 'mongoose'; +import { CreateStudyDownloadRequest } from '../dtos/study-download-request-create.dto'; +import { DownloadStatus } from '../models/download-request.model'; +import { Organization } from '../../organization/organization.model'; +import { DownloadRequestService } from './download-request.service'; +import { EntryService } from '../../entry/services/entry.service'; +import { BucketFactory } from '../../bucket/bucket-factory.service'; +import { ConfigService } from '@nestjs/config'; +import { TagService } from '../../tag/services/tag.service'; +import { TagFieldType } from '../../tag/models/tag-field.model'; +import { VideoFieldService } from '../../tag/services/video-field.service'; +import { BucketObjectAction } from 'src/bucket/bucket'; +import { Entry } from 'src/entry/models/entry.model'; +import { Study } from 'src/study/study.model'; + +@Injectable() +export class StudyDownloadService { + private readonly expiration = this.configService.getOrThrow('entry.signedURLExpiration'); + + constructor( + @InjectModel(StudyDownloadRequest.name) + private readonly downloadRequestModel: Model, + private readonly downloadService: DownloadRequestService, + private readonly entryService: EntryService, + private readonly bucketFactory: BucketFactory, + private readonly configService: ConfigService, + private readonly tagService: TagService, + private readonly videoFieldService: VideoFieldService + ) {} + + async createDownloadRequest( + downloadRequest: CreateStudyDownloadRequest, + organization: Organization + ): Promise { + let request = await this.downloadRequestModel.create({ + ...downloadRequest, + date: new Date(), + status: DownloadStatus.IN_PROGRESS, + organization: organization._id + }); + + const bucketLocation = `${this.downloadService.getPrefix()}/${request._id}`; + + // Create the locations for all the artifacts + const zipLocation = `${bucketLocation}/entries.zip`; + const entryJSONLocation = `${bucketLocation}/entries.json`; + const webhookPayloadLocation = `${bucketLocation}/webhook.json`; + const tagCSVLocation = `${bucketLocation}/tag.csv`; + const taggedEntriesZipLocation = `${bucketLocation}/tagged_entries.zip`; + const taggedEntriesJSONLocation = `${bucketLocation}/tagged_entries.json`; + const taggedEntryWebhookPayloadLocation = `${bucketLocation}/tagged_entries_webhook.json`; + + await this.downloadRequestModel.updateOne( + { _id: request._id }, + { + $set: { + bucketLocation: bucketLocation, + entryZIPLocation: zipLocation, + entryJSONLocation: entryJSONLocation, + webhookPayloadLocation: webhookPayloadLocation, + tagCSVLocation: tagCSVLocation, + taggedEntriesZipLocation: taggedEntriesZipLocation, + taggedEntriesJSONLocation: taggedEntriesJSONLocation, + taggedEntryWebhookPayloadLocation: taggedEntryWebhookPayloadLocation + } + } + ); + request = (await this.downloadRequestModel.findById(request._id))!; + + // Download the entries that were generated as part of this study + await this.downloadService.startZipJob({ + entryJSONLocation: request.entryJSONLocation!, + entryZIPLocation: request.entryZIPLocation!, + webhookPayloadLocation: request.webhookPayloadLocation!, + webhookPayload: JSON.stringify({ test: 'hello' }), + webhook: 'http://localhost:3000', + entries: await this.entryService.getEntriesForStudy(request.study), + bucket: (await this.bucketFactory.getBucket(request.organization))!, + organization: request.organization + }); + // Download the tag data as a CSV + await this.generateCSV(request); + // Download the entries that were tagged in this study + await this.downloadService.startZipJob({ + entryJSONLocation: request.taggedEntriesJSONLocation!, + entryZIPLocation: request.taggedEntriesZipLocation!, + webhookPayloadLocation: request.taggedEntryWebhookPayloadLocation!, + webhookPayload: JSON.stringify({ test: 'hello' }), + webhook: 'http://localhost:3000', + entries: await this.getLabeledEntries(request), + bucket: (await this.bucketFactory.getBucket(request.organization))!, + organization: request.organization + }); + + return request; + } + + async getStudyDownloads(study: Study): Promise { + return this.downloadRequestModel.find({ study: study._id }); + } + + /** + * Handles generating the CSV for the tag data. This approach is a sub-optimal one. + * + * The overall need is to convert the tag information into a flat CSV format where + * any external information (like videos that are downloaded as a zip) can be associated + * with the data. + * + * For example, video fields need to be linked to the videos that are downloaded, + * therefore the video fields show up as multiple columns, one for each video recorded. + * + * This approach is sub-optimal for a number of reasons + * 1. The code should be isolated into different handlers that each know how to make + * the CSV representation for that field. + * 2. Expansion of video fields can be time consuming. This may need to be a process + * that runs in the background. + * 3. ASL-LEX fields are not expanded. Currently only the ID of the field will be + * stored + */ + private async generateCSV(downloadRequest: StudyDownloadRequest): Promise { + const tags = await this.tagService.getCompleteTags(downloadRequest.study); + + // Turn the tag fields into their "CSV-friendly" format + const converted: any[] = []; + for (const tag of tags) { + const tagFields: any = {}; + + // Add basic meta-fields + tagFields['prompt'] = (await this.entryService.find(tag.entry))!.bucketLocation.split('/').pop(); + + for (const field of tag.data!) { + // For video fields, each entry is represented by the filename + if (field.type == TagFieldType.VIDEO_RECORD) { + const videoField = (await this.videoFieldService.find(field.data))!; + for (let index = 0; index < videoField.entries.length; index++) { + const entryID = videoField.entries[index]; + const entry = (await this.entryService.find(entryID))!; + tagFields[`${field.name}-${index}`] = entry.bucketLocation.split('/').pop(); + } + } else { + tagFields[`${field.name}`] = field.data; + } + } + converted.push(tagFields); + } + + // Convert the data into a CSV + const dataString = this.convertToCSV(converted); + + // Store the CSV in the expected location in the bucket + const bucket = await this.bucketFactory.getBucket(downloadRequest.organization); + if (!bucket) { + throw new Error(`No bucket found for organization ${downloadRequest.organization}`); + } + await bucket.writeText(downloadRequest.tagCSVLocation!, dataString); + } + + async getEntryZipUrl(downloadRequest: StudyDownloadRequest): Promise { + return this.getSignedURL(downloadRequest, downloadRequest.entryZIPLocation!); + } + + async getTagCSVUrl(downloadRequest: StudyDownloadRequest): Promise { + return this.getSignedURL(downloadRequest, downloadRequest.tagCSVLocation!); + } + + async getTaggedEntriesUrl(downloadRequest: StudyDownloadRequest): Promise { + return this.getSignedURL(downloadRequest, downloadRequest.taggedEntriesZipLocation!); + } + + private async getSignedURL(downloadRequest: StudyDownloadRequest, location: string): Promise { + const bucket = await this.bucketFactory.getBucket(downloadRequest.organization); + if (!bucket) { + throw new Error(`Bucket not found for organization ${downloadRequest.organization}`); + } + return bucket.getSignedUrl(location, BucketObjectAction.READ, new Date(Date.now() + this.expiration)); + } + + /** + * TODO: Improve the CSV process, need a better method to determine the headers and handle default values + */ + private convertToCSV(arr: any[]): string { + const array = [Object.keys(arr[0])].concat(arr); + + return array + .map((it) => { + return Object.values(it).toString(); + }) + .join('\n'); + } + + /** + * Get the entries taged as part of the study + */ + private async getLabeledEntries(downloadRequest: StudyDownloadRequest): Promise { + // Get the complete tags + const tags = await this.tagService.getCompleteTags(downloadRequest.study); + + // Get the entries, make sure they are unique + let entryIDs: string[] = tags.map((tag) => tag.entry); + entryIDs = Array.from(new Set(entryIDs)); + + // Get all the entries + return Promise.all( + entryIDs.map(async (id) => { + const entry = await this.entryService.find(id); + if (!entry) { + throw new Error(`Invalid id for entry: ${id}`); + } + return entry; + }) + ); + } +} diff --git a/packages/server/src/entry/models/entry.model.ts b/packages/server/src/entry/models/entry.model.ts index 4643e6dc..6f7c83eb 100644 --- a/packages/server/src/entry/models/entry.model.ts +++ b/packages/server/src/entry/models/entry.model.ts @@ -3,6 +3,28 @@ import { ObjectType, Field, ID } from '@nestjs/graphql'; import mongoose, { Document } from 'mongoose'; import JSON from 'graphql-type-json'; +@Schema() +@ObjectType() +export class SignLabRecorded { + /** The tag the recording is associated with */ + @Prop({ required: true }) + tag: string; + + /** The name of the field within the tag */ + @Prop({ requied: true }) + @Field() + fieldName: string; + + /** The study the entry was recorded as part of */ + @Prop() + study: string; + + @Prop({ required: true }) + videoNumber: number; +} + +export const SignLabRecordedSchema = SchemaFactory.createForClass(SignLabRecorded); + @Schema() @ObjectType() export class Entry { @@ -28,7 +50,9 @@ export class Entry { @Prop({ required: true }) recordedInSignLab: boolean; - // TODO: Add info on in-SignLab recording + @Prop({ type: SignLabRecorded }) + @Field(() => SignLabRecorded, { nullable: true }) + signlabRecording?: SignLabRecorded; // TODO: Add GraphQL reference back to dataset object @Prop() diff --git a/packages/server/src/entry/services/entry.service.ts b/packages/server/src/entry/services/entry.service.ts index fe9a4419..d1fe70b8 100644 --- a/packages/server/src/entry/services/entry.service.ts +++ b/packages/server/src/entry/services/entry.service.ts @@ -1,6 +1,6 @@ import { Injectable, Inject } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { Entry } from '../models/entry.model'; +import { Entry, SignLabRecorded } from '../models/entry.model'; import { Model } from 'mongoose'; import { EntryCreate } from '../dtos/create.dto'; import { Dataset } from '../../dataset/dataset.model'; @@ -8,6 +8,7 @@ import { ConfigService } from '@nestjs/config'; import { TokenPayload } from '../../jwt/token.dto'; import { BucketFactory } from 'src/bucket/bucket-factory.service'; import { BucketObjectAction } from 'src/bucket/bucket'; +import { Study } from 'src/study/study.model'; @Injectable() export class EntryService { @@ -23,16 +24,23 @@ export class EntryService { return this.entryModel.findOne({ _id: entryID }); } - async create(entryCreate: EntryCreate, dataset: Dataset, user: TokenPayload, isTraining: boolean): Promise { + async create( + entryCreate: EntryCreate, + dataset: Dataset, + user: TokenPayload, + isTraining: boolean, + signLabRecorded?: SignLabRecorded + ): Promise { // Make the entry, note that training entries are not associated with a dataset return this.entryModel.create({ ...entryCreate, dataset: dataset._id, organization: dataset.organization, - recordedInSignLab: false, + recordedInSignLab: !!signLabRecorded, dateCreated: new Date(), creator: user.user_id, - isTraining + isTraining, + signlabRecording: signLabRecorded }); } @@ -69,6 +77,20 @@ export class EntryService { return bucket.getSignedUrl(entry.bucketLocation, BucketObjectAction.READ, new Date(Date.now() + this.expiration)); } + /** Get all entries recorded as part of the given study */ + async getEntriesForStudy(study: Study | string): Promise { + let studyID = ''; + if (typeof study === 'string') { + studyID = study; + } else { + studyID = study._id; + } + + return await this.entryModel.find({ + 'signlabRecording.study': studyID + }); + } + /** * Get how long the signed URL is valid for in milliseconds. * diff --git a/packages/server/src/study/study.service.ts b/packages/server/src/study/study.service.ts index 8671b7de..b7c6ef6d 100644 --- a/packages/server/src/study/study.service.ts +++ b/packages/server/src/study/study.service.ts @@ -41,6 +41,11 @@ export class StudyService { return !!study; } + async existsById(id: string): Promise { + const study = await this.studyModel.findOne({ _id: id }); + return !!study; + } + async findById(id: string): Promise { return this.studyModel.findById(id); } diff --git a/packages/server/src/tag/services/tag.service.ts b/packages/server/src/tag/services/tag.service.ts index 1fcbc9a8..1f5dd0ac 100644 --- a/packages/server/src/tag/services/tag.service.ts +++ b/packages/server/src/tag/services/tag.service.ts @@ -229,8 +229,24 @@ export class TagService { return true; } - async getTags(study: Study): Promise { - return this.tagModel.find({ study: study._id, training: false }); + async getTags(study: Study | string): Promise { + let studyID = ''; + if (typeof study === 'string') { + studyID = study; + } else { + studyID = study._id; + } + return this.tagModel.find({ study: studyID, training: false }); + } + + async getCompleteTags(study: Study | string): Promise { + let studyID = ''; + if (typeof study === 'string') { + studyID = study; + } else { + studyID = study._id; + } + return this.tagModel.find({ study: studyID, training: false, complete: true }); } private async getIncomplete(study: Study, user: string): Promise { diff --git a/packages/server/src/tag/services/video-field-inter.service.ts b/packages/server/src/tag/services/video-field-inter.service.ts index 27c5ef36..32517290 100644 --- a/packages/server/src/tag/services/video-field-inter.service.ts +++ b/packages/server/src/tag/services/video-field-inter.service.ts @@ -101,7 +101,13 @@ export class VideoFieldIntermediateService { }, dataset, user, - tag.training + tag.training, + { + study: tag.study, + tag: tag._id, + fieldName: videoField.field, + videoNumber: videoField.index + } ); // Where to move the entry video diff --git a/packages/server/src/tag/tag.module.ts b/packages/server/src/tag/tag.module.ts index 8c4eae56..d971b325 100644 --- a/packages/server/src/tag/tag.module.ts +++ b/packages/server/src/tag/tag.module.ts @@ -67,6 +67,7 @@ import { VideoFieldResolver } from './resolvers/video-field.resolver'; AslLexFieldTransformer, VideoFieldService, VideoFieldResolver - ] + ], + exports: [TagService, VideoFieldService] }) export class TagModule {}