From 17849a66bd64b69a8ce2a7db7a5bcd401db77646 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 21:06:59 -0700 Subject: [PATCH 01/10] fix size fetching --- .../resource/dashboard/user/dataset/type/DatasetFileNode.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala index 8b73b293047..b701085208f 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala @@ -103,7 +103,7 @@ object DatasetFileNode { val fileType = if (Files.isDirectory(currentPhysicalNode.getAbsolutePath)) "directory" else "file" val fileSize = - if (fileType == "file") Some(Files.size(currentPhysicalNode.getAbsolutePath)) else None + if (fileType == "file") Some(currentPhysicalNode.getSize) else None val existingNode = currentParent.getChildren.find(child => child.getName == nodeName && child.getNodeType == fileType ) From 6e143d0d3a2d320933060e4fb6f22630841fec58 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 21:07:25 -0700 Subject: [PATCH 02/10] replace file removal --- .../common/storage/DatasetFileDocument.scala | 12 ++- .../user/dataset/DatasetResource.scala | 95 +++---------------- 2 files changed, 22 insertions(+), 85 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala b/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala index bb5ef6b8e2b..0f6846e155b 100644 --- a/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala +++ b/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala @@ -1,8 +1,10 @@ package edu.uci.ics.amber.engine.common.storage import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.utils.PathUtils -import java.io.{File, InputStream, FileOutputStream} +import java.io.{File, FileOutputStream, InputStream} import java.net.URI import java.nio.file.{Files, Path} @@ -57,5 +59,13 @@ class DatasetFileDocument(fileFullPath: Path) extends VirtualDocument[Nothing] { case Some(file) => Files.delete(file.toPath) case None => // Do nothing } + + fileRelativePath match { + case Some(path) => GitVersionControlLocalFileStorage.removeFileFromRepo( + PathUtils.getDatasetPath(dataset.getDid), + PathUtils.getDatasetPath(dataset.getDid).resolve(path) + ) + case None => // Do nothing + } } } diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala index fb81e62d011..614cb819df6 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala @@ -1,64 +1,19 @@ package edu.uci.ics.texera.web.resource.dashboard.user.dataset import edu.uci.ics.amber.engine.common.Utils.withTransaction +import edu.uci.ics.amber.engine.common.storage.DatasetFileDocument import edu.uci.ics.texera.web.SqlServer import edu.uci.ics.texera.web.auth.SessionUser import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege -import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{ - DatasetDao, - DatasetUserAccessDao, - DatasetVersionDao -} -import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{ - Dataset, - DatasetUserAccess, - DatasetVersion, - User -} +import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{DatasetDao, DatasetUserAccessDao, DatasetVersionDao} +import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{Dataset, DatasetUserAccess, DatasetVersion, User} import edu.uci.ics.texera.web.model.jooq.generated.tables.Dataset.DATASET import edu.uci.ics.texera.web.model.jooq.generated.tables.User.USER import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetVersion.DATASET_VERSION -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{ - getDatasetUserAccessPrivilege, - getOwner, - userHasReadAccess, - userHasWriteAccess, - userOwnDataset -} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{ - DATASET_IS_PRIVATE, - DATASET_IS_PUBLIC, - DashboardDataset, - DashboardDatasetVersion, - DatasetDescriptionModification, - DatasetIDs, - DatasetNameModification, - DatasetVersionRootFileNodes, - DatasetVersionRootFileNodesResponse, - DatasetVersions, - ERR_DATASET_CREATION_FAILED_MESSAGE, - ERR_DATASET_NAME_ALREADY_EXISTS, - ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, - ListDatasetsResponse, - calculateLatestDatasetVersionSize, - calculateDatasetVersionSize, - context, - createNewDatasetVersionFromFormData, - getDashboardDataset, - getDatasetByID, - getDatasetVersionByID, - getDatasetVersions, - getFileNodesOfCertainVersion, - getLatestDatasetVersionWithAccessCheck, - getUserDatasets, - resolvePath, - retrievePublicDatasets -} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{ - DatasetFileNode, - PhysicalFileNode -} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{getDatasetUserAccessPrivilege, getOwner, userHasReadAccess, userHasWriteAccess, userOwnDataset} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{DATASET_IS_PRIVATE, DATASET_IS_PUBLIC, DashboardDataset, DashboardDatasetVersion, DatasetDescriptionModification, DatasetIDs, DatasetNameModification, DatasetVersionRootFileNodes, DatasetVersionRootFileNodesResponse, DatasetVersions, ERR_DATASET_CREATION_FAILED_MESSAGE, ERR_DATASET_NAME_ALREADY_EXISTS, ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, ListDatasetsResponse, calculateDatasetVersionSize, calculateLatestDatasetVersionSize, context, createNewDatasetVersionFromFormData, getDashboardDataset, getDatasetByID, getDatasetVersionByID, getDatasetVersions, getFileNodesOfCertainVersion, getLatestDatasetVersionWithAccessCheck, getUserDatasets, resolvePath, retrievePublicDatasets} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{DatasetFileNode, PhysicalFileNode} import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage import edu.uci.ics.texera.web.resource.dashboard.user.dataset.utils.PathUtils import io.dropwizard.auth.Auth @@ -76,19 +31,7 @@ import java.util.zip.{ZipEntry, ZipOutputStream} import java.util import java.util.concurrent.locks.ReentrantLock import javax.annotation.security.RolesAllowed -import javax.ws.rs.{ - BadRequestException, - Consumes, - ForbiddenException, - GET, - NotFoundException, - POST, - Path, - PathParam, - Produces, - QueryParam, - WebApplicationException -} +import javax.ws.rs.{BadRequestException, Consumes, ForbiddenException, GET, NotFoundException, POST, Path, PathParam, Produces, QueryParam, WebApplicationException} import javax.ws.rs.core.{MediaType, Response, StreamingOutput} import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable @@ -333,7 +276,7 @@ object DatasetResource { // DatasetOperation defines the operations that will be applied when creating a new dataset version private case class DatasetOperation( filesToAdd: Map[java.nio.file.Path, InputStream], - filesToRemove: List[java.nio.file.Path] + filesToRemove: List[DatasetFileDocument] ) private def parseUserUploadedFormToDatasetOperations( @@ -344,7 +287,7 @@ object DatasetResource { // Mutable collections for constructing DatasetOperation val filesToAdd = mutable.Map[java.nio.file.Path, InputStream]() - val filesToRemove = mutable.ListBuffer[java.nio.file.Path]() + val filesToRemove = mutable.ListBuffer[DatasetFileDocument]() val fields = multiPart.getFields.keySet.iterator() // Get all field names @@ -369,17 +312,7 @@ object DatasetResource { .parse(filePathsValue) .as[List[String]] .foreach(pathStr => { - val (_, _, _, fileRelativePath) = - resolvePath(Paths.get(pathStr), shouldContainFile = true) - - fileRelativePath - .map { path => - filesToRemove += datasetPath - .resolve(path) // When path exists, resolve it and add to filesToRemove - } - .getOrElse { - throw new IllegalArgumentException("File relative path is missing") - } + filesToRemove += new DatasetFileDocument(Paths.get(pathStr)) }) } } @@ -482,13 +415,7 @@ object DatasetResource { case (filePath, fileStream) => GitVersionControlLocalFileStorage.writeFileToRepo(datasetPath, filePath, fileStream) } - - datasetOperation.filesToRemove.foreach { filePath => - GitVersionControlLocalFileStorage.removeFileFromRepo( - datasetPath, - filePath - ) - } + datasetOperation.filesToRemove.foreach(f => f.remove()) } ) From 663e094b6eb50349d1e28e9286884b6864d021ec Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 21:37:59 -0700 Subject: [PATCH 03/10] refine selected file path retrieval --- .../user/dataset/DatasetResource.scala | 107 +++++++----------- .../service/user/dataset/dataset.service.ts | 6 +- .../file-selection.component.ts | 34 +++--- 3 files changed, 56 insertions(+), 91 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala index 614cb819df6..9ee913e490e 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala @@ -772,81 +772,54 @@ class DatasetResource { @Auth user: SessionUser, @QueryParam("includeVersions") includeVersions: Boolean = false, @QueryParam("includeFileNodes") includeFileNodes: Boolean = false, - @QueryParam("path") filePathStr: String ): ListDatasetsResponse = { val uid = user.getUid withTransaction(context)(ctx => { var accessibleDatasets: ListBuffer[DashboardDataset] = ListBuffer() - - if (filePathStr != null && filePathStr.nonEmpty) { - // if the file path is given, then only fetch the dataset and version this file is belonging to - val decodedPathStr = URLDecoder.decode(filePathStr, StandardCharsets.UTF_8.name()) - val (ownerEmail, dataset, version, _) = - resolvePath(Paths.get(decodedPathStr), shouldContainFile = true) - val accessPrivilege = getDatasetUserAccessPrivilege(ctx, dataset.getDid, uid) - if ( - accessPrivilege == DatasetUserAccessPrivilege.NONE && dataset.getIsPublic == DATASET_IS_PRIVATE - ) { - throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) - } - accessibleDatasets = accessibleDatasets :+ DashboardDataset( - dataset = dataset, - ownerEmail = ownerEmail, - accessPrivilege = accessPrivilege, - isOwner = dataset.getOwnerUid == uid, - versions = List( - DashboardDatasetVersion( - datasetVersion = version, - fileNodes = List() - ) - ), - size = calculateLatestDatasetVersionSize(dataset.getDid) - ) - } else { - // first fetch all datasets user have explicit access to - accessibleDatasets = ListBuffer.from( - ctx - .select() - .from( - DATASET - .leftJoin(DATASET_USER_ACCESS) - .on(DATASET_USER_ACCESS.DID.eq(DATASET.DID)) - .leftJoin(USER) - .on(USER.UID.eq(DATASET.OWNER_UID)) - ) - .where(DATASET_USER_ACCESS.UID.eq(uid)) - .fetch() - .map(record => { - val dataset = record.into(DATASET).into(classOf[Dataset]) - val datasetAccess = record.into(DATASET_USER_ACCESS).into(classOf[DatasetUserAccess]) - val ownerEmail = record.into(USER).getEmail - DashboardDataset( - isOwner = dataset.getOwnerUid == uid, - dataset = dataset, - accessPrivilege = datasetAccess.getPrivilege, - versions = List(), - ownerEmail = ownerEmail, - size = calculateLatestDatasetVersionSize(dataset.getDid) - ) - }) - ) - - // then we fetch the public datasets and merge it as a part of the result if not exist - val publicDatasets = retrievePublicDatasets(context) - publicDatasets.forEach { publicDataset => - if (!accessibleDatasets.exists(_.dataset.getDid == publicDataset.dataset.getDid)) { - val dashboardDataset = DashboardDataset( - isOwner = false, - dataset = publicDataset.dataset, - ownerEmail = publicDataset.ownerEmail, - accessPrivilege = DatasetUserAccessPrivilege.READ, + // first fetch all datasets user have explicit access to + accessibleDatasets = ListBuffer.from( + ctx + .select() + .from( + DATASET + .leftJoin(DATASET_USER_ACCESS) + .on(DATASET_USER_ACCESS.DID.eq(DATASET.DID)) + .leftJoin(USER) + .on(USER.UID.eq(DATASET.OWNER_UID)) + ) + .where(DATASET_USER_ACCESS.UID.eq(uid)) + .fetch() + .map(record => { + val dataset = record.into(DATASET).into(classOf[Dataset]) + val datasetAccess = record.into(DATASET_USER_ACCESS).into(classOf[DatasetUserAccess]) + val ownerEmail = record.into(USER).getEmail + DashboardDataset( + isOwner = dataset.getOwnerUid == uid, + dataset = dataset, + accessPrivilege = datasetAccess.getPrivilege, versions = List(), - size = calculateLatestDatasetVersionSize(publicDataset.dataset.getDid) + ownerEmail = ownerEmail, + size = calculateLatestDatasetVersionSize(dataset.getDid) ) - accessibleDatasets = accessibleDatasets :+ dashboardDataset - } + }) + ) + + // then we fetch the public datasets and merge it as a part of the result if not exist + val publicDatasets = retrievePublicDatasets(context) + publicDatasets.forEach { publicDataset => + if (!accessibleDatasets.exists(_.dataset.getDid == publicDataset.dataset.getDid)) { + val dashboardDataset = DashboardDataset( + isOwner = false, + dataset = publicDataset.dataset, + ownerEmail = publicDataset.ownerEmail, + accessPrivilege = DatasetUserAccessPrivilege.READ, + versions = List(), + size = calculateLatestDatasetVersionSize(publicDataset.dataset.getDid) + ) + accessibleDatasets = accessibleDatasets :+ dashboardDataset } } + val fileNodesMap = mutable.Map[(String, String, String), List[PhysicalFileNode]]() // iterate over datasets and retrieve the version diff --git a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts index ffba54d1316..d97aefcaff6 100644 --- a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts +++ b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts @@ -87,8 +87,7 @@ export class DatasetService { public retrieveAccessibleDatasets( includeVersions: boolean = false, - includeFileNodes: boolean = false, - filePath: string = "" + includeFileNodes: boolean = false ): Observable<{ datasets: DashboardDataset[]; fileNodes: DatasetFileNode[] }> { let params = new HttpParams(); if (includeVersions) { @@ -97,9 +96,6 @@ export class DatasetService { if (includeFileNodes) { params = params.set("includeFileNodes", "true"); } - if (filePath && filePath != "") { - params = params.set("path", encodeURIComponent(filePath)); - } return this.http.get<{ datasets: DashboardDataset[]; fileNodes: DatasetFileNode[] }>( `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}`, { params: params } diff --git a/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts b/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts index 1704e4d8fba..bc6b39773d8 100644 --- a/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts +++ b/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts @@ -31,26 +31,22 @@ export class FileSelectionComponent implements OnInit { ngOnInit() { // if users already select some file, then show that selected dataset & related version if (this.selectedFilePath && this.selectedFilePath !== "") { - this.datasetService - .retrieveAccessibleDatasets(false, false, this.selectedFilePath) - .pipe(untilDestroyed(this)) - .subscribe(response => { - const prevDataset = response.datasets[0]; - this.selectedDataset = this.datasets.find(d => d.dataset.did === prevDataset.dataset.did); - this.isDatasetSelected = !!this.selectedDataset; + const selectedDatasetFile = parseFilePathToDatasetFile(this.selectedFilePath); + this.selectedDataset = this.datasets.find( + d => d.ownerEmail === selectedDatasetFile.ownerEmail && d.dataset.name === selectedDatasetFile.datasetName + ); + this.isDatasetSelected = !!this.selectedDataset; - if (this.selectedDataset && this.selectedDataset.dataset.did !== undefined) { - this.datasetService - .retrieveDatasetVersionList(this.selectedDataset.dataset.did) - .pipe(untilDestroyed(this)) - .subscribe(versions => { - this.datasetVersions = versions; - const versionDvid = prevDataset.versions[0].datasetVersion.dvid; - this.selectedVersion = this.datasetVersions.find(v => v.dvid === versionDvid); - this.onVersionChange(); - }); - } - }); + if (this.selectedDataset && this.selectedDataset.dataset.did !== undefined) { + this.datasetService + .retrieveDatasetVersionList(this.selectedDataset.dataset.did) + .pipe(untilDestroyed(this)) + .subscribe(versions => { + this.datasetVersions = versions; + this.selectedVersion = this.datasetVersions.find(v => v.name === selectedDatasetFile.versionName); + this.onVersionChange(); + }); + } } } From 15c5aeafd3988f892268b5eaa6145b3974ef8cfb Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 21:55:50 -0700 Subject: [PATCH 04/10] refine dataset downloading logic --- .../common/storage/DatasetFileDocument.scala | 9 +- .../user/dataset/DatasetResource.scala | 104 +++++++++++++----- .../user-dataset-explorer.component.ts | 9 +- .../service/user/dataset/dataset.service.ts | 12 +- .../service/user/download/download.service.ts | 9 +- 5 files changed, 94 insertions(+), 49 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala b/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala index 0f6846e155b..63949f059ab 100644 --- a/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala +++ b/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala @@ -61,11 +61,12 @@ class DatasetFileDocument(fileFullPath: Path) extends VirtualDocument[Nothing] { } fileRelativePath match { - case Some(path) => GitVersionControlLocalFileStorage.removeFileFromRepo( - PathUtils.getDatasetPath(dataset.getDid), + case Some(path) => + GitVersionControlLocalFileStorage.removeFileFromRepo( + PathUtils.getDatasetPath(dataset.getDid), PathUtils.getDatasetPath(dataset.getDid).resolve(path) - ) - case None => // Do nothing + ) + case None => // Do nothing } } } diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala index 9ee913e490e..74348507584 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala @@ -5,15 +5,61 @@ import edu.uci.ics.amber.engine.common.storage.DatasetFileDocument import edu.uci.ics.texera.web.SqlServer import edu.uci.ics.texera.web.auth.SessionUser import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege -import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{DatasetDao, DatasetUserAccessDao, DatasetVersionDao} -import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{Dataset, DatasetUserAccess, DatasetVersion, User} +import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{ + DatasetDao, + DatasetUserAccessDao, + DatasetVersionDao +} +import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{ + Dataset, + DatasetUserAccess, + DatasetVersion, + User +} import edu.uci.ics.texera.web.model.jooq.generated.tables.Dataset.DATASET import edu.uci.ics.texera.web.model.jooq.generated.tables.User.USER import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetVersion.DATASET_VERSION -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{getDatasetUserAccessPrivilege, getOwner, userHasReadAccess, userHasWriteAccess, userOwnDataset} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{DATASET_IS_PRIVATE, DATASET_IS_PUBLIC, DashboardDataset, DashboardDatasetVersion, DatasetDescriptionModification, DatasetIDs, DatasetNameModification, DatasetVersionRootFileNodes, DatasetVersionRootFileNodesResponse, DatasetVersions, ERR_DATASET_CREATION_FAILED_MESSAGE, ERR_DATASET_NAME_ALREADY_EXISTS, ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, ListDatasetsResponse, calculateDatasetVersionSize, calculateLatestDatasetVersionSize, context, createNewDatasetVersionFromFormData, getDashboardDataset, getDatasetByID, getDatasetVersionByID, getDatasetVersions, getFileNodesOfCertainVersion, getLatestDatasetVersionWithAccessCheck, getUserDatasets, resolvePath, retrievePublicDatasets} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{DatasetFileNode, PhysicalFileNode} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{ + getDatasetUserAccessPrivilege, + getOwner, + userHasReadAccess, + userHasWriteAccess, + userOwnDataset +} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{ + DATASET_IS_PRIVATE, + DATASET_IS_PUBLIC, + DashboardDataset, + DashboardDatasetVersion, + DatasetDescriptionModification, + DatasetIDs, + DatasetNameModification, + DatasetVersionRootFileNodes, + DatasetVersionRootFileNodesResponse, + DatasetVersions, + ERR_DATASET_CREATION_FAILED_MESSAGE, + ERR_DATASET_NAME_ALREADY_EXISTS, + ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, + ListDatasetsResponse, + calculateDatasetVersionSize, + calculateLatestDatasetVersionSize, + context, + createNewDatasetVersionFromFormData, + getDashboardDataset, + getDatasetByID, + getDatasetVersionByID, + getDatasetVersions, + getFileNodesOfCertainVersion, + getLatestDatasetVersionWithAccessCheck, + getUserDatasets, + resolvePath, + retrievePublicDatasets +} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{ + DatasetFileNode, + PhysicalFileNode +} import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage import edu.uci.ics.texera.web.resource.dashboard.user.dataset.utils.PathUtils import io.dropwizard.auth.Auth @@ -29,9 +75,22 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import java.util.zip.{ZipEntry, ZipOutputStream} import java.util +import java.util.Optional import java.util.concurrent.locks.ReentrantLock import javax.annotation.security.RolesAllowed -import javax.ws.rs.{BadRequestException, Consumes, ForbiddenException, GET, NotFoundException, POST, Path, PathParam, Produces, QueryParam, WebApplicationException} +import javax.ws.rs.{ + BadRequestException, + Consumes, + ForbiddenException, + GET, + NotFoundException, + POST, + Path, + PathParam, + Produces, + QueryParam, + WebApplicationException +} import javax.ws.rs.core.{MediaType, Response, StreamingOutput} import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable @@ -771,7 +830,7 @@ class DatasetResource { def listDatasets( @Auth user: SessionUser, @QueryParam("includeVersions") includeVersions: Boolean = false, - @QueryParam("includeFileNodes") includeFileNodes: Boolean = false, + @QueryParam("includeFileNodes") includeFileNodes: Boolean = false ): ListDatasetsResponse = { val uid = user.getUid withTransaction(context)(ctx => { @@ -1035,25 +1094,29 @@ class DatasetResource { /** * Retrieves a ZIP file for a specific dataset version or the latest version. * - * @param pathStr The dataset version path in the format: /ownerEmail/datasetName/versionName - * Example: /user@example.com/dataset/v1 - * @param getLatest When true, retrieves the latest version regardless of the provided path. * @param did The dataset ID (used when getLatest is true). + * @param dvid The dataset version ID, if given, retrieve this version; if not given, retrieve the latest version * @param user The session user. * @return A Response containing the dataset version as a ZIP file. */ @GET @Path("/version-zip") def retrieveDatasetVersionZip( - @QueryParam("path") pathStr: String, - @QueryParam("getLatest") getLatest: Boolean, @QueryParam("did") did: UInteger, + @QueryParam("dvid") dvid: Optional[Integer], @Auth user: SessionUser ): Response = { - val (dataset, version) = if (getLatest) { + if (!userHasReadAccess(context, did, user.getUid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) + } + val (dataset, version) = if (dvid.isEmpty) { + // dvid is not given, retrieve latest getLatestVersionInfo(did, user) } else { - resolveAndValidatePath(pathStr, user) + // dvid is given, retrieve certain version + withTransaction(context)(ctx => + (getDatasetByID(ctx, did), getDatasetVersionByID(ctx, UInteger.valueOf(dvid.get))) + ) } val targetDatasetPath = PathUtils.getDatasetPath(dataset.getDid) val fileNodes = GitVersionControlLocalFileStorage.retrieveRootFileNodesOfVersion( @@ -1110,19 +1173,6 @@ class DatasetResource { .`type`("application/zip") .build() } - - private def resolveAndValidatePath( - pathStr: String, - user: SessionUser - ): (Dataset, DatasetVersion) = { - val decodedPathStr = URLDecoder.decode(pathStr, StandardCharsets.UTF_8.name()) - val (_, dataset, dsVersion, _) = - resolvePath(Paths.get(decodedPathStr), shouldContainFile = false) - - validateUserAccess(dataset.getDid, user.getUid) - (dataset, dsVersion) - } - private def getLatestVersionInfo(did: UInteger, user: SessionUser): (Dataset, DatasetVersion) = { validateUserAccess(did, user.getUid) val dataset = getDatasetByID(context, did) diff --git a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-explorer.component.ts b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-explorer.component.ts index b8f1d6d19d0..90c5ae05db6 100644 --- a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-explorer.component.ts +++ b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-explorer.component.ts @@ -193,18 +193,11 @@ export class UserDatasetExplorerComponent implements OnInit { this.downloadService.downloadSingleFile(this.currentDisplayedFileName).pipe(untilDestroyed(this)).subscribe(); }; - extractVersionPath(currentDisplayedFileName: string): string { - const pathParts = currentDisplayedFileName.split("/"); - - return `/${pathParts[1]}/${pathParts[2]}/${pathParts[3]}`; - } - onClickDownloadVersionAsZip = (): void => { if (!this.did || !this.selectedVersion?.dvid) return; - const versionPath = this.extractVersionPath(this.currentDisplayedFileName); this.downloadService - .downloadDatasetVersion(versionPath, this.datasetName, this.selectedVersion.name) + .downloadDatasetVersion(this.did, this.selectedVersion.dvid, this.datasetName, this.selectedVersion.name) .pipe(untilDestroyed(this)) .subscribe(); }; diff --git a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts index d97aefcaff6..2917ebb84cc 100644 --- a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts +++ b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts @@ -68,15 +68,11 @@ export class DatasetService { * - did: A number representing the dataset ID * @returns An Observable that emits a Blob containing the zip file */ - public retrieveDatasetZip(options: { path?: string; did?: number }): Observable { + public retrieveDatasetZip(options: { did: number; dvid?: number }): Observable { let params = new HttpParams(); - - if (options.path) { - params = params.set("path", encodeURIComponent(options.path)); - } - if (options.did) { - params = params.set("did", options.did.toString()); - params = params.set("getLatest", "true"); + params = params.set("did", options.did.toString()); + if (options.dvid) { + params = params.set("dvid", options.dvid.toString()); } return this.http.get(`${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/version-zip`, { diff --git a/core/gui/src/app/dashboard/service/user/download/download.service.ts b/core/gui/src/app/dashboard/service/user/download/download.service.ts index 2ce5a309c72..d2682853735 100644 --- a/core/gui/src/app/dashboard/service/user/download/download.service.ts +++ b/core/gui/src/app/dashboard/service/user/download/download.service.ts @@ -46,9 +46,14 @@ export class DownloadService { ); } - downloadDatasetVersion(versionPath: string, datasetName: string, versionName: string): Observable { + downloadDatasetVersion( + datasetId: number, + datasetVersionId: number, + datasetName: string, + versionName: string + ): Observable { return this.downloadWithNotification( - () => this.datasetService.retrieveDatasetZip({ path: versionPath }), + () => this.datasetService.retrieveDatasetZip({ did: datasetId, dvid: datasetVersionId }), `${datasetName}-${versionName}.zip`, `Starting to download version ${versionName} as ZIP`, `Version ${versionName} has been downloaded as ZIP`, From 03794cfe0e1f41433f2d805b90d19e0fc3647a9f Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 22:43:13 -0700 Subject: [PATCH 05/10] Revert "fix size fetching" This reverts commit 17849a66bd64b69a8ce2a7db7a5bcd401db77646. --- .../resource/dashboard/user/dataset/type/DatasetFileNode.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala index b701085208f..8b73b293047 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileNode.scala @@ -103,7 +103,7 @@ object DatasetFileNode { val fileType = if (Files.isDirectory(currentPhysicalNode.getAbsolutePath)) "directory" else "file" val fileSize = - if (fileType == "file") Some(currentPhysicalNode.getSize) else None + if (fileType == "file") Some(Files.size(currentPhysicalNode.getAbsolutePath)) else None val existingNode = currentParent.getChildren.find(child => child.getName == nodeName && child.getNodeType == fileType ) From 4ccd1242f599e29f7db4a7fdd860f846ec3a0617 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 22:45:00 -0700 Subject: [PATCH 06/10] Revert "replace file removal" This reverts commit 6e143d0d --- .../common/storage/DatasetFileDocument.scala | 13 +- .../user/dataset/DatasetResource.scala | 168 ++++++++++++------ 2 files changed, 110 insertions(+), 71 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala b/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala index 63949f059ab..bb5ef6b8e2b 100644 --- a/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala +++ b/core/amber/src/main/scala/edu/uci/ics/amber/engine/common/storage/DatasetFileDocument.scala @@ -1,10 +1,8 @@ package edu.uci.ics.amber.engine.common.storage import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.utils.PathUtils -import java.io.{File, FileOutputStream, InputStream} +import java.io.{File, InputStream, FileOutputStream} import java.net.URI import java.nio.file.{Files, Path} @@ -59,14 +57,5 @@ class DatasetFileDocument(fileFullPath: Path) extends VirtualDocument[Nothing] { case Some(file) => Files.delete(file.toPath) case None => // Do nothing } - - fileRelativePath match { - case Some(path) => - GitVersionControlLocalFileStorage.removeFileFromRepo( - PathUtils.getDatasetPath(dataset.getDid), - PathUtils.getDatasetPath(dataset.getDid).resolve(path) - ) - case None => // Do nothing - } } } diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala index 74348507584..fb81e62d011 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala @@ -1,7 +1,6 @@ package edu.uci.ics.texera.web.resource.dashboard.user.dataset import edu.uci.ics.amber.engine.common.Utils.withTransaction -import edu.uci.ics.amber.engine.common.storage.DatasetFileDocument import edu.uci.ics.texera.web.SqlServer import edu.uci.ics.texera.web.auth.SessionUser import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege @@ -42,8 +41,8 @@ import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{ ERR_DATASET_NAME_ALREADY_EXISTS, ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, ListDatasetsResponse, - calculateDatasetVersionSize, calculateLatestDatasetVersionSize, + calculateDatasetVersionSize, context, createNewDatasetVersionFromFormData, getDashboardDataset, @@ -75,7 +74,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import java.util.zip.{ZipEntry, ZipOutputStream} import java.util -import java.util.Optional import java.util.concurrent.locks.ReentrantLock import javax.annotation.security.RolesAllowed import javax.ws.rs.{ @@ -335,7 +333,7 @@ object DatasetResource { // DatasetOperation defines the operations that will be applied when creating a new dataset version private case class DatasetOperation( filesToAdd: Map[java.nio.file.Path, InputStream], - filesToRemove: List[DatasetFileDocument] + filesToRemove: List[java.nio.file.Path] ) private def parseUserUploadedFormToDatasetOperations( @@ -346,7 +344,7 @@ object DatasetResource { // Mutable collections for constructing DatasetOperation val filesToAdd = mutable.Map[java.nio.file.Path, InputStream]() - val filesToRemove = mutable.ListBuffer[DatasetFileDocument]() + val filesToRemove = mutable.ListBuffer[java.nio.file.Path]() val fields = multiPart.getFields.keySet.iterator() // Get all field names @@ -371,7 +369,17 @@ object DatasetResource { .parse(filePathsValue) .as[List[String]] .foreach(pathStr => { - filesToRemove += new DatasetFileDocument(Paths.get(pathStr)) + val (_, _, _, fileRelativePath) = + resolvePath(Paths.get(pathStr), shouldContainFile = true) + + fileRelativePath + .map { path => + filesToRemove += datasetPath + .resolve(path) // When path exists, resolve it and add to filesToRemove + } + .getOrElse { + throw new IllegalArgumentException("File relative path is missing") + } }) } } @@ -474,7 +482,13 @@ object DatasetResource { case (filePath, fileStream) => GitVersionControlLocalFileStorage.writeFileToRepo(datasetPath, filePath, fileStream) } - datasetOperation.filesToRemove.foreach(f => f.remove()) + + datasetOperation.filesToRemove.foreach { filePath => + GitVersionControlLocalFileStorage.removeFileFromRepo( + datasetPath, + filePath + ) + } } ) @@ -830,55 +844,82 @@ class DatasetResource { def listDatasets( @Auth user: SessionUser, @QueryParam("includeVersions") includeVersions: Boolean = false, - @QueryParam("includeFileNodes") includeFileNodes: Boolean = false + @QueryParam("includeFileNodes") includeFileNodes: Boolean = false, + @QueryParam("path") filePathStr: String ): ListDatasetsResponse = { val uid = user.getUid withTransaction(context)(ctx => { var accessibleDatasets: ListBuffer[DashboardDataset] = ListBuffer() - // first fetch all datasets user have explicit access to - accessibleDatasets = ListBuffer.from( - ctx - .select() - .from( - DATASET - .leftJoin(DATASET_USER_ACCESS) - .on(DATASET_USER_ACCESS.DID.eq(DATASET.DID)) - .leftJoin(USER) - .on(USER.UID.eq(DATASET.OWNER_UID)) - ) - .where(DATASET_USER_ACCESS.UID.eq(uid)) - .fetch() - .map(record => { - val dataset = record.into(DATASET).into(classOf[Dataset]) - val datasetAccess = record.into(DATASET_USER_ACCESS).into(classOf[DatasetUserAccess]) - val ownerEmail = record.into(USER).getEmail - DashboardDataset( - isOwner = dataset.getOwnerUid == uid, - dataset = dataset, - accessPrivilege = datasetAccess.getPrivilege, - versions = List(), - ownerEmail = ownerEmail, - size = calculateLatestDatasetVersionSize(dataset.getDid) + + if (filePathStr != null && filePathStr.nonEmpty) { + // if the file path is given, then only fetch the dataset and version this file is belonging to + val decodedPathStr = URLDecoder.decode(filePathStr, StandardCharsets.UTF_8.name()) + val (ownerEmail, dataset, version, _) = + resolvePath(Paths.get(decodedPathStr), shouldContainFile = true) + val accessPrivilege = getDatasetUserAccessPrivilege(ctx, dataset.getDid, uid) + if ( + accessPrivilege == DatasetUserAccessPrivilege.NONE && dataset.getIsPublic == DATASET_IS_PRIVATE + ) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) + } + accessibleDatasets = accessibleDatasets :+ DashboardDataset( + dataset = dataset, + ownerEmail = ownerEmail, + accessPrivilege = accessPrivilege, + isOwner = dataset.getOwnerUid == uid, + versions = List( + DashboardDatasetVersion( + datasetVersion = version, + fileNodes = List() ) - }) - ) + ), + size = calculateLatestDatasetVersionSize(dataset.getDid) + ) + } else { + // first fetch all datasets user have explicit access to + accessibleDatasets = ListBuffer.from( + ctx + .select() + .from( + DATASET + .leftJoin(DATASET_USER_ACCESS) + .on(DATASET_USER_ACCESS.DID.eq(DATASET.DID)) + .leftJoin(USER) + .on(USER.UID.eq(DATASET.OWNER_UID)) + ) + .where(DATASET_USER_ACCESS.UID.eq(uid)) + .fetch() + .map(record => { + val dataset = record.into(DATASET).into(classOf[Dataset]) + val datasetAccess = record.into(DATASET_USER_ACCESS).into(classOf[DatasetUserAccess]) + val ownerEmail = record.into(USER).getEmail + DashboardDataset( + isOwner = dataset.getOwnerUid == uid, + dataset = dataset, + accessPrivilege = datasetAccess.getPrivilege, + versions = List(), + ownerEmail = ownerEmail, + size = calculateLatestDatasetVersionSize(dataset.getDid) + ) + }) + ) - // then we fetch the public datasets and merge it as a part of the result if not exist - val publicDatasets = retrievePublicDatasets(context) - publicDatasets.forEach { publicDataset => - if (!accessibleDatasets.exists(_.dataset.getDid == publicDataset.dataset.getDid)) { - val dashboardDataset = DashboardDataset( - isOwner = false, - dataset = publicDataset.dataset, - ownerEmail = publicDataset.ownerEmail, - accessPrivilege = DatasetUserAccessPrivilege.READ, - versions = List(), - size = calculateLatestDatasetVersionSize(publicDataset.dataset.getDid) - ) - accessibleDatasets = accessibleDatasets :+ dashboardDataset + // then we fetch the public datasets and merge it as a part of the result if not exist + val publicDatasets = retrievePublicDatasets(context) + publicDatasets.forEach { publicDataset => + if (!accessibleDatasets.exists(_.dataset.getDid == publicDataset.dataset.getDid)) { + val dashboardDataset = DashboardDataset( + isOwner = false, + dataset = publicDataset.dataset, + ownerEmail = publicDataset.ownerEmail, + accessPrivilege = DatasetUserAccessPrivilege.READ, + versions = List(), + size = calculateLatestDatasetVersionSize(publicDataset.dataset.getDid) + ) + accessibleDatasets = accessibleDatasets :+ dashboardDataset + } } } - val fileNodesMap = mutable.Map[(String, String, String), List[PhysicalFileNode]]() // iterate over datasets and retrieve the version @@ -1094,29 +1135,25 @@ class DatasetResource { /** * Retrieves a ZIP file for a specific dataset version or the latest version. * + * @param pathStr The dataset version path in the format: /ownerEmail/datasetName/versionName + * Example: /user@example.com/dataset/v1 + * @param getLatest When true, retrieves the latest version regardless of the provided path. * @param did The dataset ID (used when getLatest is true). - * @param dvid The dataset version ID, if given, retrieve this version; if not given, retrieve the latest version * @param user The session user. * @return A Response containing the dataset version as a ZIP file. */ @GET @Path("/version-zip") def retrieveDatasetVersionZip( + @QueryParam("path") pathStr: String, + @QueryParam("getLatest") getLatest: Boolean, @QueryParam("did") did: UInteger, - @QueryParam("dvid") dvid: Optional[Integer], @Auth user: SessionUser ): Response = { - if (!userHasReadAccess(context, did, user.getUid)) { - throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) - } - val (dataset, version) = if (dvid.isEmpty) { - // dvid is not given, retrieve latest + val (dataset, version) = if (getLatest) { getLatestVersionInfo(did, user) } else { - // dvid is given, retrieve certain version - withTransaction(context)(ctx => - (getDatasetByID(ctx, did), getDatasetVersionByID(ctx, UInteger.valueOf(dvid.get))) - ) + resolveAndValidatePath(pathStr, user) } val targetDatasetPath = PathUtils.getDatasetPath(dataset.getDid) val fileNodes = GitVersionControlLocalFileStorage.retrieveRootFileNodesOfVersion( @@ -1173,6 +1210,19 @@ class DatasetResource { .`type`("application/zip") .build() } + + private def resolveAndValidatePath( + pathStr: String, + user: SessionUser + ): (Dataset, DatasetVersion) = { + val decodedPathStr = URLDecoder.decode(pathStr, StandardCharsets.UTF_8.name()) + val (_, dataset, dsVersion, _) = + resolvePath(Paths.get(decodedPathStr), shouldContainFile = false) + + validateUserAccess(dataset.getDid, user.getUid) + (dataset, dsVersion) + } + private def getLatestVersionInfo(did: UInteger, user: SessionUser): (Dataset, DatasetVersion) = { validateUserAccess(did, user.getUid) val dataset = getDatasetByID(context, did) From 5305bbc3286a06fb1544c96dfffc2cc16059af5f Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 22:45:12 -0700 Subject: [PATCH 07/10] Revert "refine selected file path retrieval" This reverts commit 663e094b6eb50349d1e28e9286884b6864d021ec. --- .../service/user/dataset/dataset.service.ts | 6 +++- .../file-selection.component.ts | 34 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts index 2917ebb84cc..9f0a414b010 100644 --- a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts +++ b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts @@ -83,7 +83,8 @@ export class DatasetService { public retrieveAccessibleDatasets( includeVersions: boolean = false, - includeFileNodes: boolean = false + includeFileNodes: boolean = false, + filePath: string = "" ): Observable<{ datasets: DashboardDataset[]; fileNodes: DatasetFileNode[] }> { let params = new HttpParams(); if (includeVersions) { @@ -92,6 +93,9 @@ export class DatasetService { if (includeFileNodes) { params = params.set("includeFileNodes", "true"); } + if (filePath && filePath != "") { + params = params.set("path", encodeURIComponent(filePath)); + } return this.http.get<{ datasets: DashboardDataset[]; fileNodes: DatasetFileNode[] }>( `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}`, { params: params } diff --git a/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts b/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts index bc6b39773d8..1704e4d8fba 100644 --- a/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts +++ b/core/gui/src/app/workspace/component/file-selection/file-selection.component.ts @@ -31,22 +31,26 @@ export class FileSelectionComponent implements OnInit { ngOnInit() { // if users already select some file, then show that selected dataset & related version if (this.selectedFilePath && this.selectedFilePath !== "") { - const selectedDatasetFile = parseFilePathToDatasetFile(this.selectedFilePath); - this.selectedDataset = this.datasets.find( - d => d.ownerEmail === selectedDatasetFile.ownerEmail && d.dataset.name === selectedDatasetFile.datasetName - ); - this.isDatasetSelected = !!this.selectedDataset; + this.datasetService + .retrieveAccessibleDatasets(false, false, this.selectedFilePath) + .pipe(untilDestroyed(this)) + .subscribe(response => { + const prevDataset = response.datasets[0]; + this.selectedDataset = this.datasets.find(d => d.dataset.did === prevDataset.dataset.did); + this.isDatasetSelected = !!this.selectedDataset; - if (this.selectedDataset && this.selectedDataset.dataset.did !== undefined) { - this.datasetService - .retrieveDatasetVersionList(this.selectedDataset.dataset.did) - .pipe(untilDestroyed(this)) - .subscribe(versions => { - this.datasetVersions = versions; - this.selectedVersion = this.datasetVersions.find(v => v.name === selectedDatasetFile.versionName); - this.onVersionChange(); - }); - } + if (this.selectedDataset && this.selectedDataset.dataset.did !== undefined) { + this.datasetService + .retrieveDatasetVersionList(this.selectedDataset.dataset.did) + .pipe(untilDestroyed(this)) + .subscribe(versions => { + this.datasetVersions = versions; + const versionDvid = prevDataset.versions[0].datasetVersion.dvid; + this.selectedVersion = this.datasetVersions.find(v => v.dvid === versionDvid); + this.onVersionChange(); + }); + } + }); } } From cf527d5f0b89c60a2b047504ff63b9ba6c51f901 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Sun, 27 Oct 2024 22:50:31 -0700 Subject: [PATCH 08/10] add backend change --- .../user/dataset/DatasetResource.scala | 97 +++++-------------- 1 file changed, 22 insertions(+), 75 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala index fb81e62d011..6985ba8ae2b 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala @@ -4,61 +4,15 @@ import edu.uci.ics.amber.engine.common.Utils.withTransaction import edu.uci.ics.texera.web.SqlServer import edu.uci.ics.texera.web.auth.SessionUser import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege -import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{ - DatasetDao, - DatasetUserAccessDao, - DatasetVersionDao -} -import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{ - Dataset, - DatasetUserAccess, - DatasetVersion, - User -} +import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{DatasetDao, DatasetUserAccessDao, DatasetVersionDao} +import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{Dataset, DatasetUserAccess, DatasetVersion, User} import edu.uci.ics.texera.web.model.jooq.generated.tables.Dataset.DATASET import edu.uci.ics.texera.web.model.jooq.generated.tables.User.USER import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetVersion.DATASET_VERSION -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{ - getDatasetUserAccessPrivilege, - getOwner, - userHasReadAccess, - userHasWriteAccess, - userOwnDataset -} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{ - DATASET_IS_PRIVATE, - DATASET_IS_PUBLIC, - DashboardDataset, - DashboardDatasetVersion, - DatasetDescriptionModification, - DatasetIDs, - DatasetNameModification, - DatasetVersionRootFileNodes, - DatasetVersionRootFileNodesResponse, - DatasetVersions, - ERR_DATASET_CREATION_FAILED_MESSAGE, - ERR_DATASET_NAME_ALREADY_EXISTS, - ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, - ListDatasetsResponse, - calculateLatestDatasetVersionSize, - calculateDatasetVersionSize, - context, - createNewDatasetVersionFromFormData, - getDashboardDataset, - getDatasetByID, - getDatasetVersionByID, - getDatasetVersions, - getFileNodesOfCertainVersion, - getLatestDatasetVersionWithAccessCheck, - getUserDatasets, - resolvePath, - retrievePublicDatasets -} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{ - DatasetFileNode, - PhysicalFileNode -} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{getDatasetUserAccessPrivilege, getOwner, userHasReadAccess, userHasWriteAccess, userOwnDataset} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{DATASET_IS_PRIVATE, DATASET_IS_PUBLIC, DashboardDataset, DashboardDatasetVersion, DatasetDescriptionModification, DatasetIDs, DatasetNameModification, DatasetVersionRootFileNodes, DatasetVersionRootFileNodesResponse, DatasetVersions, ERR_DATASET_CREATION_FAILED_MESSAGE, ERR_DATASET_NAME_ALREADY_EXISTS, ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, ListDatasetsResponse, calculateDatasetVersionSize, calculateLatestDatasetVersionSize, context, createNewDatasetVersionFromFormData, getDashboardDataset, getDatasetByID, getDatasetVersionByID, getDatasetVersions, getFileNodesOfCertainVersion, getLatestDatasetVersionWithAccessCheck, getUserDatasets, resolvePath, retrievePublicDatasets} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{DatasetFileNode, PhysicalFileNode} import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage import edu.uci.ics.texera.web.resource.dashboard.user.dataset.utils.PathUtils import io.dropwizard.auth.Auth @@ -74,21 +28,10 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import java.util.zip.{ZipEntry, ZipOutputStream} import java.util +import java.util.Optional import java.util.concurrent.locks.ReentrantLock import javax.annotation.security.RolesAllowed -import javax.ws.rs.{ - BadRequestException, - Consumes, - ForbiddenException, - GET, - NotFoundException, - POST, - Path, - PathParam, - Produces, - QueryParam, - WebApplicationException -} +import javax.ws.rs.{BadRequestException, Consumes, ForbiddenException, GET, NotFoundException, POST, Path, PathParam, Produces, QueryParam, WebApplicationException} import javax.ws.rs.core.{MediaType, Response, StreamingOutput} import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable @@ -1135,25 +1078,29 @@ class DatasetResource { /** * Retrieves a ZIP file for a specific dataset version or the latest version. * - * @param pathStr The dataset version path in the format: /ownerEmail/datasetName/versionName - * Example: /user@example.com/dataset/v1 - * @param getLatest When true, retrieves the latest version regardless of the provided path. - * @param did The dataset ID (used when getLatest is true). + * @param did The dataset ID (used when getLatest is true). + * @param dvid The dataset version ID, if given, retrieve this version; if not given, retrieve the latest version * @param user The session user. * @return A Response containing the dataset version as a ZIP file. */ @GET @Path("/version-zip") def retrieveDatasetVersionZip( - @QueryParam("path") pathStr: String, - @QueryParam("getLatest") getLatest: Boolean, - @QueryParam("did") did: UInteger, - @Auth user: SessionUser - ): Response = { - val (dataset, version) = if (getLatest) { + @QueryParam("did") did: UInteger, + @QueryParam("dvid") dvid: Optional[Integer], + @Auth user: SessionUser + ): Response = { + if (!userHasReadAccess(context, did, user.getUid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) + } + val (dataset, version) = if (dvid.isEmpty) { + // dvid is not given, retrieve latest getLatestVersionInfo(did, user) } else { - resolveAndValidatePath(pathStr, user) + // dvid is given, retrieve certain version + withTransaction(context)(ctx => + (getDatasetByID(ctx, did), getDatasetVersionByID(ctx, UInteger.valueOf(dvid.get))) + ) } val targetDatasetPath = PathUtils.getDatasetPath(dataset.getDid) val fileNodes = GitVersionControlLocalFileStorage.retrieveRootFileNodesOfVersion( From 05f5fb2bd6544d8398838527a876c87b9d34b514 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Mon, 28 Oct 2024 08:04:47 -0700 Subject: [PATCH 09/10] fix unit tests --- .../service/user/download/download.service.spec.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts b/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts index 75bbf2e8f8f..47bdb764e12 100644 --- a/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts +++ b/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts @@ -134,18 +134,19 @@ describe("DownloadService", () => { }); it("should download a dataset version successfully", done => { - const versionPath = "path/to/version"; + const datasetId = 1; + const datasetVersionId = 1; const datasetName = "TestDataset"; const versionName = "v1.0"; const mockBlob = new Blob(["version content"], { type: "application/zip" }); datasetServiceSpy.retrieveDatasetZip.and.returnValue(of(mockBlob)); - downloadService.downloadDatasetVersion(versionPath, datasetName, versionName).subscribe({ + downloadService.downloadDatasetVersion(datasetId, datasetVersionId, datasetName, versionName).subscribe({ next: blob => { expect(blob).toBe(mockBlob); expect(notificationServiceSpy.info).toHaveBeenCalledWith("Starting to download version v1.0 as ZIP"); - expect(datasetServiceSpy.retrieveDatasetZip).toHaveBeenCalledWith({ path: versionPath }); + expect(datasetServiceSpy.retrieveDatasetZip).toHaveBeenCalledWith({ did: datasetId, dvid: datasetVersionId }); expect(fileSaverServiceSpy.saveAs).toHaveBeenCalledWith(mockBlob, "TestDataset-v1.0.zip"); expect(notificationServiceSpy.success).toHaveBeenCalledWith("Version v1.0 has been downloaded as ZIP"); done(); @@ -157,21 +158,22 @@ describe("DownloadService", () => { }); it("should handle dataset version download failure correctly", done => { - const versionPath = "path/to/version"; + const datasetId = 1; + const datasetVersionId = 1; const datasetName = "TestDataset"; const versionName = "v1.0"; const errorMessage = "Dataset version download failed"; datasetServiceSpy.retrieveDatasetZip.and.returnValue(throwError(() => new Error(errorMessage))); - downloadService.downloadDatasetVersion(versionPath, datasetName, versionName).subscribe({ + downloadService.downloadDatasetVersion(datasetId, datasetVersionId, datasetName, versionName).subscribe({ next: () => { fail("Should have thrown an error"); }, error: (error: unknown) => { expect(error).toBeTruthy(); expect(notificationServiceSpy.info).toHaveBeenCalledWith("Starting to download version v1.0 as ZIP"); - expect(datasetServiceSpy.retrieveDatasetZip).toHaveBeenCalledWith({ path: versionPath }); + expect(datasetServiceSpy.retrieveDatasetZip).toHaveBeenCalledWith({ did: datasetId, dvid: datasetVersionId }); expect(fileSaverServiceSpy.saveAs).not.toHaveBeenCalled(); expect(notificationServiceSpy.error).toHaveBeenCalledWith("Error downloading version 'v1.0' as ZIP"); done(); From b416405873086164e1d25ee7be845ff54f0e3c51 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Mon, 28 Oct 2024 08:40:17 -0700 Subject: [PATCH 10/10] fmt --- .../user/dataset/DatasetResource.scala | 78 ++++++++++++++++--- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala index 6985ba8ae2b..25a83b92326 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala @@ -4,15 +4,61 @@ import edu.uci.ics.amber.engine.common.Utils.withTransaction import edu.uci.ics.texera.web.SqlServer import edu.uci.ics.texera.web.auth.SessionUser import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege -import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{DatasetDao, DatasetUserAccessDao, DatasetVersionDao} -import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{Dataset, DatasetUserAccess, DatasetVersion, User} +import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{ + DatasetDao, + DatasetUserAccessDao, + DatasetVersionDao +} +import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{ + Dataset, + DatasetUserAccess, + DatasetVersion, + User +} import edu.uci.ics.texera.web.model.jooq.generated.tables.Dataset.DATASET import edu.uci.ics.texera.web.model.jooq.generated.tables.User.USER import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetVersion.DATASET_VERSION -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{getDatasetUserAccessPrivilege, getOwner, userHasReadAccess, userHasWriteAccess, userOwnDataset} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{DATASET_IS_PRIVATE, DATASET_IS_PUBLIC, DashboardDataset, DashboardDatasetVersion, DatasetDescriptionModification, DatasetIDs, DatasetNameModification, DatasetVersionRootFileNodes, DatasetVersionRootFileNodesResponse, DatasetVersions, ERR_DATASET_CREATION_FAILED_MESSAGE, ERR_DATASET_NAME_ALREADY_EXISTS, ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, ListDatasetsResponse, calculateDatasetVersionSize, calculateLatestDatasetVersionSize, context, createNewDatasetVersionFromFormData, getDashboardDataset, getDatasetByID, getDatasetVersionByID, getDatasetVersions, getFileNodesOfCertainVersion, getLatestDatasetVersionWithAccessCheck, getUserDatasets, resolvePath, retrievePublicDatasets} -import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{DatasetFileNode, PhysicalFileNode} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{ + getDatasetUserAccessPrivilege, + getOwner, + userHasReadAccess, + userHasWriteAccess, + userOwnDataset +} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{ + DATASET_IS_PRIVATE, + DATASET_IS_PUBLIC, + DashboardDataset, + DashboardDatasetVersion, + DatasetDescriptionModification, + DatasetIDs, + DatasetNameModification, + DatasetVersionRootFileNodes, + DatasetVersionRootFileNodesResponse, + DatasetVersions, + ERR_DATASET_CREATION_FAILED_MESSAGE, + ERR_DATASET_NAME_ALREADY_EXISTS, + ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE, + ListDatasetsResponse, + calculateDatasetVersionSize, + calculateLatestDatasetVersionSize, + context, + createNewDatasetVersionFromFormData, + getDashboardDataset, + getDatasetByID, + getDatasetVersionByID, + getDatasetVersions, + getFileNodesOfCertainVersion, + getLatestDatasetVersionWithAccessCheck, + getUserDatasets, + resolvePath, + retrievePublicDatasets +} +import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.{ + DatasetFileNode, + PhysicalFileNode +} import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage import edu.uci.ics.texera.web.resource.dashboard.user.dataset.utils.PathUtils import io.dropwizard.auth.Auth @@ -31,7 +77,19 @@ import java.util import java.util.Optional import java.util.concurrent.locks.ReentrantLock import javax.annotation.security.RolesAllowed -import javax.ws.rs.{BadRequestException, Consumes, ForbiddenException, GET, NotFoundException, POST, Path, PathParam, Produces, QueryParam, WebApplicationException} +import javax.ws.rs.{ + BadRequestException, + Consumes, + ForbiddenException, + GET, + NotFoundException, + POST, + Path, + PathParam, + Produces, + QueryParam, + WebApplicationException +} import javax.ws.rs.core.{MediaType, Response, StreamingOutput} import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable @@ -1086,10 +1144,10 @@ class DatasetResource { @GET @Path("/version-zip") def retrieveDatasetVersionZip( - @QueryParam("did") did: UInteger, - @QueryParam("dvid") dvid: Optional[Integer], - @Auth user: SessionUser - ): Response = { + @QueryParam("did") did: UInteger, + @QueryParam("dvid") dvid: Optional[Integer], + @Auth user: SessionUser + ): Response = { if (!userHasReadAccess(context, did, user.getUid)) { throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) }