Introduce Dataset GUI

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/DashboardResource.scala

@@ -4,6 +4,7 @@ import edu.uci.ics.texera.web.auth.SessionUser
 import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos._
 import edu.uci.ics.texera.web.resource.dashboard.DashboardResource._
 import edu.uci.ics.texera.web.resource.dashboard.SearchQueryBuilder.ALL_RESOURCE_TYPE
+import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.DashboardDataset
 import edu.uci.ics.texera.web.resource.dashboard.user.file.UserFileResource.DashboardFile
 import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowResource.DashboardWorkflow
 import io.dropwizard.auth.Auth
@@ -21,7 +22,8 @@ object DashboardResource {
       resourceType: String,
       workflow: Option[DashboardWorkflow] = None,
       project: Option[Project] = None,
-      file: Option[DashboardFile] = None
+      file: Option[DashboardFile] = None,
+      dataset: Option[DashboardDataset] = None
   )
 
   case class DashboardSearchResult(results: List[DashboardClickableFileEntry], more: Boolean)
@@ -54,6 +56,7 @@ object DashboardResource {
       @QueryParam("id") workflowIDs: java.util.List[UInteger] = new util.ArrayList(),
       @QueryParam("operator") operators: java.util.List[String] = new util.ArrayList(),
       @QueryParam("projectId") projectIds: java.util.List[UInteger] = new util.ArrayList(),
+      @QueryParam("datasetId") datasetIds: java.util.List[UInteger] = new util.ArrayList(),
       @QueryParam("start") @DefaultValue("0") offset: Int = 0,
       @QueryParam("count") @DefaultValue("20") count: Int = 20,
       @QueryParam("orderBy") @DefaultValue("EditTimeDesc") orderBy: String = "EditTimeDesc"
@@ -73,11 +76,14 @@ object DashboardResource {
         FileSearchQueryBuilder.constructQuery(uid, params)
       case SearchQueryBuilder.PROJECT_RESOURCE_TYPE =>
         ProjectSearchQueryBuilder.constructQuery(uid, params)
+      case SearchQueryBuilder.DATASET_RESOURCE_TYPE =>
+        DatasetSearchQueryBuilder.constructQuery(uid, params)
       case SearchQueryBuilder.ALL_RESOURCE_TYPE =>
         val q1 = WorkflowSearchQueryBuilder.constructQuery(uid, params)
         val q2 = FileSearchQueryBuilder.constructQuery(uid, params)
         val q3 = ProjectSearchQueryBuilder.constructQuery(uid, params)
-        q1.unionAll(q2).unionAll(q3)
+        val q4 = DatasetSearchQueryBuilder.constructQuery(uid, params)
+        q1.unionAll(q2).unionAll(q3).unionAll(q4)
       case _ => throw new IllegalArgumentException(s"Unknown resource type: ${params.resourceType}")
     }
 
@@ -96,6 +102,8 @@ object DashboardResource {
             FileSearchQueryBuilder.toEntry(uid, record)
           case SearchQueryBuilder.PROJECT_RESOURCE_TYPE =>
             ProjectSearchQueryBuilder.toEntry(uid, record)
+          case SearchQueryBuilder.DATASET_RESOURCE_TYPE =>
+            DatasetSearchQueryBuilder.toEntry(uid, record)
         }
       })
 

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/DatasetSearchQueryBuilder.scala

@@ -0,0 +1,93 @@
+package edu.uci.ics.texera.web.resource.dashboard
+import org.jooq.impl.DSL
+import org.jooq.{Condition, GroupField, Record, TableLike}
+import org.jooq.types.UInteger
+import edu.uci.ics.texera.web.model.jooq.generated.Tables.{DATASET, DATASET_USER_ACCESS}
+import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege
+import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.Dataset
+import edu.uci.ics.texera.web.resource.dashboard.DashboardResource.DashboardClickableFileEntry
+import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource
+import edu.uci.ics.texera.web.resource.dashboard.FulltextSearchQueryUtils.{
+  getContainsFilter,
+  getDateFilter,
+  getFullTextSearchFilter,
+  getSubstringSearchFilter
+}
+import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.DashboardDataset
+
+import scala.jdk.CollectionConverters.CollectionHasAsScala
+object DatasetSearchQueryBuilder extends SearchQueryBuilder {
+  override protected val mappedResourceSchema: UnifiedResourceSchema = UnifiedResourceSchema(
+    resourceType = DSL.inline(SearchQueryBuilder.DATASET_RESOURCE_TYPE),
+    name = DATASET.NAME,
+    description = DATASET.DESCRIPTION,
+    creationTime = DATASET.CREATION_TIME,
+    did = DATASET.DID,
+    ownerId = DATASET.OWNER_UID,
+    isDatasetPublic = DATASET.IS_PUBLIC,
+    datasetStoragePath = DATASET.STORAGE_PATH,
+    datasetUserAccess = DATASET_USER_ACCESS.PRIVILEGE
+  )
+
+  override protected def constructFromClause(
+      uid: UInteger,
+      params: DashboardResource.SearchQueryParams
+  ): TableLike[_] = {
+    DATASET
+      .leftJoin(DATASET_USER_ACCESS)
+      .on(DATASET_USER_ACCESS.DID.eq(DATASET.DID))
+      .where(
+        DATASET.IS_PUBLIC
+          .eq(DatasetResource.DATASET_IS_PUBLIC)
+          .or(DATASET_USER_ACCESS.UID.eq(uid))
+      )
+  }
+
+  override protected def constructWhereClause(
+      uid: UInteger,
+      params: DashboardResource.SearchQueryParams
+  ): Condition = {
+    val splitKeywords = params.keywords.asScala
+      .flatMap(_.split("[+\\-()<>~*@\"]"))
+      .filter(_.nonEmpty)
+      .toSeq
+
+    getDateFilter(
+      params.creationStartDate,
+      params.creationEndDate,
+      DATASET.CREATION_TIME
+    )
+      .and(getContainsFilter(params.datasetIds, DATASET.DID))
+      .and(
+        getFullTextSearchFilter(splitKeywords, List(DATASET.NAME, DATASET.DESCRIPTION))
+          .or(
+            getSubstringSearchFilter(
+              splitKeywords,
+              List(DATASET.NAME, DATASET.DESCRIPTION)
+            )
+          )
+      )
+  }
+  override protected def getGroupByFields: Seq[GroupField] = Seq.empty
+  override protected def toEntryImpl(
+      uid: UInteger,
+      record: Record
+  ): DashboardResource.DashboardClickableFileEntry = {
+    val dataset = record.into(DATASET).into(classOf[Dataset])
+
+    val dd = DashboardDataset(
+      dataset,
+      record
+        .get(
+          DATASET_USER_ACCESS.PRIVILEGE,
+          classOf[DatasetUserAccessPrivilege]
+        ),
+      dataset.getOwnerUid == uid
+    )
+    DashboardClickableFileEntry(
+      resourceType = SearchQueryBuilder.DATASET_RESOURCE_TYPE,
+      dataset = Some(dd)
+    )
+  }
+}
+class DatasetSearchQueryBuilder {}

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/SearchQueryBuilder.scala

@@ -14,6 +14,7 @@ object SearchQueryBuilder {
   val FILE_RESOURCE_TYPE = "file"
   val WORKFLOW_RESOURCE_TYPE = "workflow"
   val PROJECT_RESOURCE_TYPE = "project"
+  val DATASET_RESOURCE_TYPE = "dataset"
   val ALL_RESOURCE_TYPE = ""
 }
 

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/UnifiedResourceSchema.scala

@@ -2,6 +2,7 @@ package edu.uci.ics.texera.web.resource.dashboard
 
 import edu.uci.ics.texera.web.SqlServer
 import edu.uci.ics.texera.web.model.jooq.generated.enums.{
+  DatasetUserAccessPrivilege,
   UserFileAccessPrivilege,
   WorkflowUserAccessPrivilege
 }
@@ -11,6 +12,7 @@ import org.jooq.types.UInteger
 import org.jooq.{Field, Record}
 
 import java.sql.Timestamp
+import java.lang.Byte
 import scala.collection.mutable
 
 object UnifiedResourceSchema {
@@ -54,7 +56,12 @@ object UnifiedResourceSchema {
       filePath: Field[String] = DSL.inline(""),
       fileSize: Field[UInteger] = DSL.inline(null, classOf[UInteger]),
       fileUserAccess: Field[UserFileAccessPrivilege] =
-        DSL.inline(null, classOf[UserFileAccessPrivilege])
+        DSL.inline(null, classOf[UserFileAccessPrivilege]),
+      did: Field[UInteger] = DSL.inline(null, classOf[UInteger]),
+      datasetStoragePath: Field[String] = DSL.inline(null, classOf[String]),
+      isDatasetPublic: Field[Byte] = DSL.inline(null, classOf[Byte]),
+      datasetUserAccess: Field[DatasetUserAccessPrivilege] =
+        DSL.inline(null, classOf[DatasetUserAccessPrivilege])
   ): UnifiedResourceSchema = {
     new UnifiedResourceSchema(
       Seq(
@@ -77,7 +84,11 @@ object UnifiedResourceSchema {
         fileUploadTime -> fileUploadTime.as("upload_time"),
         filePath -> filePath.as("path"),
         fileSize -> fileSize.as("size"),
-        fileUserAccess -> fileUserAccess.as("user_file_access")
+        fileUserAccess -> fileUserAccess.as("user_file_access"),
+        did -> did.as("did"),
+        datasetStoragePath -> datasetStoragePath.as("dataset_storage_path"),
+        datasetUserAccess -> datasetUserAccess.as("user_dataset_access"),
+        isDatasetPublic -> isDatasetPublic.as("is_dataset_public")
       )
     )
   }
@@ -113,6 +124,10 @@ object UnifiedResourceSchema {
   * - `filePath`: Path of the file, as a `String`.
   * - `fileSize`: Size of the file, as a `UInteger`.
   * - `fileUserAccess`: Access privileges for the file, as a `UserFileAccessPrivilege`.
+  *
+  * Attributes specific to datasets:
+  * - `did`: Dataset ID, as a `UInteger`.
+  * - `datasetUserAccess`: Access privileges for the dataset, as a `DatasetUserAccessPrivilege`
   */
 class UnifiedResourceSchema private (
     fieldMappingSeq: Seq[(Field[_], Field[_])]

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala

@@ -16,6 +16,8 @@ import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.{
 }
 import edu.uci.ics.texera.web.model.jooq.generated.tables.Dataset.DATASET
 import edu.uci.ics.texera.web.model.jooq.generated.tables.DatasetVersion.DATASET_VERSION
+import edu.uci.ics.texera.web.resource.dashboard.DashboardResource
+import edu.uci.ics.texera.web.resource.dashboard.DashboardResource.SearchQueryParams
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResource.{
   getDatasetUserAccessPrivilege,
   userHasReadAccess,
@@ -33,11 +35,11 @@ import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{
   ERR_DATASET_CREATION_FAILED_MESSAGE,
   ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE,
   context,
+  createNewDatasetVersion,
   getDashboardDataset,
   getDatasetByID,
   getDatasetLatestVersion,
-  getDatasetVersionHashByID,
-  createNewDatasetVersion
+  getDatasetVersionHashByID
 }
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.FileNode
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage
@@ -212,9 +214,10 @@ object DatasetResource {
               val filePathsValue = bodyPart.getValueAs(classOf[String])
               val filePaths = filePathsValue.split(",")
               filePaths.foreach { filePath =>
+                val normalizedFilePath = filePath.stripPrefix("/")
                 GitVersionControlLocalFileStorage.removeFileFromRepo(
                   datasetPath,
-                  datasetPath.resolve(filePath)
+                  datasetPath.resolve(normalizedFilePath)
                 )
               }
               fileOperationHappens = true
@@ -458,6 +461,23 @@ class DatasetResource {
     })
   }
 
+  /**
+    * This method returns a list of DashboardDatasets objects that are accessible by current user.
+    * @param user the session user
+    * @return list of user accessible DashboardDataset objects
+    */
+  @GET
+  @Path("")
+  def listDatasets(
+      @Auth user: SessionUser
+  ): List[DashboardDataset] = {
+    val result = DashboardResource.searchAllResources(
+      user,
+      SearchQueryParams(resourceType = "dataset")
+    )
+    result.results.map(_.dataset.get)
+  }
+
   @GET
   @Path("/{did}/version/list")
   def getDatasetVersionList(

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/utils/JGitVersionControl.java

@@ -85,45 +85,51 @@ public static Set<FileNode> getRootFileNodeOfCommit(Path repoPath, String commit
       ObjectId commitId = repository.resolve(commitHash);
       RevCommit commit = revWalk.parseCommit(commitId);
 
-      // initialize the treeWalk to traverse the file tree
-      TreeWalk treeWalk = new TreeWalk(repository);
-      treeWalk.addTree(commit.getTree());
-      treeWalk.setRecursive(true);
-
-      while (treeWalk.next()) {
-        Path fullPath = repoPath.resolve(treeWalk.getPathString());
-        String pathStr = fullPath.toString();
-
-        // Determine if the current path is at the root level
-        if (treeWalk.getDepth() == 0) {
-          FileNode rootNode = new FileNode(repoPath, fullPath);
-          rootNodes.add(rootNode);
-          pathToFileNodeMap.put(pathStr, rootNode);
-        } else {
-          // For child nodes, find or create the parent node based on the directory structure
-          Path parentPath = fullPath.getParent();
-          String parentPathStr = parentPath.toString();
-          FileNode parentNode = pathToFileNodeMap.get(parentPathStr);
-
-          if (parentNode == null) {
-            parentNode = new FileNode(repoPath, parentPath);
-            pathToFileNodeMap.put(parentPathStr, parentNode);
-            // Determine if this parent should be added to rootNodes
-            if (parentPath.getParent().equals(repoPath)) {
-              rootNodes.add(parentNode);
-            }
+      try (TreeWalk treeWalk = new TreeWalk(repository)) {
+        treeWalk.addTree(commit.getTree());
+        treeWalk.setRecursive(false);
+
+        while (treeWalk.next()) {
+          Path fullPath = repoPath.resolve(treeWalk.getPathString());
+          FileNode currentNode = createOrGetNode(pathToFileNodeMap, repoPath, fullPath);
+
+          if (treeWalk.isSubtree()) {
+            treeWalk.enterSubtree();
+          } else {
+            // For files, we've already added them. Just ensure parent linkage is correct.
+            ensureParentChildLink(pathToFileNodeMap, repoPath, fullPath, currentNode);
           }
 
-          FileNode childNode = new FileNode(repoPath, fullPath);
-          parentNode.addChildNode(childNode);
-          // Map child node to its path for potential future children
-          pathToFileNodeMap.put(pathStr, childNode);
+          // For directories, also ensure they are correctly linked
+          if (currentNode.isDirectory()) {
+            ensureParentChildLink(pathToFileNodeMap, repoPath, fullPath, currentNode);
+          }
         }
       }
     }
+
+    // Extract root nodes
+    pathToFileNodeMap.values().forEach(node -> {
+      if (node.getAbsolutePath().getParent().equals(repoPath)) {
+        rootNodes.add(node);
+      }
+    });
+
     return rootNodes;
   }
 
+  private static FileNode createOrGetNode(Map<String, FileNode> map, Path repoPath, Path path) {
+    return map.computeIfAbsent(path.toString(), k -> new FileNode(repoPath, path));
+  }
+
+  private static void ensureParentChildLink(Map<String, FileNode> map, Path repoPath, Path childPath, FileNode childNode) {
+    Path parentPath = childPath.getParent();
+    if (parentPath != null && !parentPath.equals(repoPath)) {
+      FileNode parentNode = createOrGetNode(map, repoPath, parentPath);
+      parentNode.addChildNode(childNode);
+    }
+  }
+
   public static void add(Path repoPath, Path filePath) throws IOException, GitAPIException {
     try (Git git = Git.open(repoPath.toFile())) {
       // Stage the file addition/modification

core/new-gui/package.json

@@ -61,6 +61,11 @@
     "ngx-color-picker": "12.0.1",
     "ngx-json-viewer": "3.2.1",
     "ngx-markdown": "16.0.0",
+    "papaparse": "5.4.1",
+    "ng2-pdf-viewer": "9.1.5",
+    "ngx-file-drop": "16.0.0",
+    "ngx-image-viewer": "1.0.13",
+    "read-excel-file": "5.7.1",
     "path-browserify": "^1.0.1",
     "plotly.js-basic-dist-min": "^2.29.0",
     "popper.js": "1.16.1",
@@ -101,6 +106,7 @@
     "@types/node": "~18.15.5",
     "@types/quill": "2.0.9",
     "@types/uuid": "8.3.4",
+    "@types/papaparse": "5.3.5",
     "@typescript-eslint/eslint-plugin": "7.0.2",
     "@typescript-eslint/parser": "7.0.2",
     "babel-plugin-dynamic-import-node": "^2.3.3",

core/new-gui/src/app/app-routing.module.ts

@@ -16,6 +16,8 @@ import { AdminGuardService } from "./dashboard/admin/service/admin-guard.service
 import { SearchComponent } from "./dashboard/user/component/search/search.component";
 import { FlarumComponent } from "./dashboard/user/component/flarum/flarum.component";
 import { GmailComponent } from "./dashboard/admin/component/gmail/gmail.component";
+import { UserDatasetExplorerComponent } from "./dashboard/user/component/user-dataset/user-dataset-explorer/user-dataset-explorer.component";
+import { UserDatasetComponent } from "./dashboard/user/component/user-dataset/user-dataset.component";
 /*
  *  This file defines the url path
  *  The workflow workspace is set as default path
@@ -62,6 +64,19 @@ if (environment.userSystemEnabled) {
         path: "user-file",
         component: UserFileComponent,
       },
+      {
+        path: "dataset",
+        component: UserDatasetComponent,
+      },
+      // the below two URLs route to the same Component. The component will render the page accordingly
+      {
+        path: "dataset/:did",
+        component: UserDatasetExplorerComponent,
+      },
+      {
+        path: "dataset/create",
+        component: UserDatasetExplorerComponent,
+      },
       {
         path: "user-quota",
         component: UserQuotaComponent,