Add Environment GUI

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/DatasetSearchQueryBuilder.scala

@@ -6,7 +6,6 @@ import edu.uci.ics.texera.web.model.jooq.generated.Tables.{DATASET, DATASET_USER
 import edu.uci.ics.texera.web.model.jooq.generated.enums.DatasetUserAccessPrivilege
 import edu.uci.ics.texera.web.model.jooq.generated.tables.pojos.Dataset
 import edu.uci.ics.texera.web.resource.dashboard.DashboardResource.DashboardClickableFileEntry
-import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource
 import edu.uci.ics.texera.web.resource.dashboard.FulltextSearchQueryUtils.{
   getContainsFilter,
   getDateFilter,
@@ -29,6 +28,8 @@ object DatasetSearchQueryBuilder extends SearchQueryBuilder {
     datasetUserAccess = DATASET_USER_ACCESS.PRIVILEGE
   )
 
+  // Notice that this only select those datasets that users have access record.
+  // For those public datasets, need external union to merge them
   override protected def constructFromClause(
       uid: UInteger,
       params: DashboardResource.SearchQueryParams
@@ -37,9 +38,7 @@ object DatasetSearchQueryBuilder extends SearchQueryBuilder {
       .leftJoin(DATASET_USER_ACCESS)
       .on(DATASET_USER_ACCESS.DID.eq(DATASET.DID))
       .where(
-        DATASET.IS_PUBLIC
-          .eq(DatasetResource.DATASET_IS_PUBLIC)
-          .or(DATASET_USER_ACCESS.UID.eq(uid))
+        DATASET_USER_ACCESS.UID.eq(uid)
       )
   }
 

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/DatasetResource.scala

@@ -25,6 +25,8 @@ import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetAccessResou
   userOwnDataset
 }
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{
+  DATASET_IS_PRIVATE,
+  DATASET_IS_PUBLIC,
   DashboardDataset,
   DashboardDatasetVersion,
   DatasetDescriptionModification,
@@ -39,7 +41,8 @@ import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{
   getDashboardDataset,
   getDatasetByID,
   getDatasetLatestVersion,
-  getDatasetVersionHashByID
+  getDatasetVersionHashByID,
+  retrievePublicDatasets
 }
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.FileNode
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage
@@ -73,7 +76,7 @@ import scala.jdk.CollectionConverters._
 
 object DatasetResource {
   val DATASET_IS_PUBLIC: Byte = 1;
-
+  val DATASET_IS_PRIVATE: Byte = 0;
   val FILE_OPERATION_UPLOAD_PREFIX = "file:upload:"
   val FILE_OPERATION_REMOVE_PREFIX = "file:remove"
 
@@ -273,6 +276,11 @@ object DatasetResource {
     FileNode.getAllFileRelativePaths(fileNodes)
   }
 
+  def retrievePublicDatasets(ctx: DSLContext): util.List[Dataset] = {
+    val datasetDao = new DatasetDao(ctx.configuration())
+    datasetDao.fetchByIsPublic(DATASET_IS_PUBLIC)
+  }
+
   case class DashboardDataset(
       dataset: Dataset,
       accessPrivilege: EnumType,
@@ -440,6 +448,32 @@ class DatasetResource {
     }
   }
 
+  @POST
+  @Path("/{did}/update/publicity")
+  def toggleDatasetPublicity(
+      @PathParam("did") did: UInteger,
+      @Auth sessionUser: SessionUser
+  ): Response = {
+    withTransaction(context) { ctx =>
+      val datasetDao = new DatasetDao(ctx.configuration())
+      val uid = sessionUser.getUid
+
+      if (!userHasWriteAccess(ctx, did, uid)) {
+        throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)
+      }
+
+      val existedDataset = getDatasetByID(ctx, did, uid)
+      if (existedDataset.getIsPublic == DATASET_IS_PUBLIC) {
+        existedDataset.setIsPublic(DATASET_IS_PRIVATE)
+      } else {
+        existedDataset.setIsPublic(DATASET_IS_PUBLIC)
+      }
+
+      datasetDao.update(existedDataset)
+      Response.ok().build()
+    }
+  }
+
   @POST
   @Path("/{did}/version/create")
   @Consumes(Array(MediaType.MULTIPART_FORM_DATA))
@@ -492,7 +526,22 @@ class DatasetResource {
       user,
       SearchQueryParams(resourceType = "dataset")
     )
-    result.results.map(_.dataset.get)
+    var accessibleDatasets = result.results.map(_.dataset.get)
+    val publicDatasets = retrievePublicDatasets(context)
+
+    publicDatasets.forEach { publicDataset =>
+      if (!accessibleDatasets.exists(_.dataset.getDid == publicDataset.getDid)) {
+        // Assuming DashboardDataset has a property did for comparison
+        val dashboardDataset = DashboardDataset(
+          isOwner = false,
+          dataset = publicDataset,
+          accessPrivilege = DatasetUserAccessPrivilege.READ
+        )
+        accessibleDatasets = accessibleDatasets :+ dashboardDataset
+      }
+    }
+
+    accessibleDatasets
   }
 
   @GET

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/service/GitVersionControlLocalFileStorage.java

@@ -130,6 +130,39 @@ public static void retrieveFileContentOfVersion(Path baseRepoPath, String commit
     JGitVersionControl.readFileContentOfCommit(baseRepoPath, commitHash, filePath, outputStream);
   }
 
+  /**
+   * Creates a temporary file and writes the content of a specific version of a file, identified by its commit hash, into this temporary file.
+   * This method is useful for retrieving and working with specific versions of a file from a Git repository in a temporary and isolated manner.
+   *
+   * The temporary file is created in the system's default temporary-file directory, with a prefix "versionedFile" and a ".tmp" suffix.
+   * The created temporary file is marked for deletion on JVM exit, ensuring no leftover files during runtime, though it's the caller's responsibility to manage the file if it needs to persist longer.
+   *
+   * <p>This method is THREAD SAFE, ensuring safe use across multiple threads without causing data inconsistency or corruption.
+   *
+   * @param baseRepoPath The path to the repository where the file version is to be retrieved from. This should be a valid path to a local repository managed by Git.
+   * @param commitHash The commit hash that identifies the specific version of the file to retrieve. This commit must exist in the repository's history.
+   * @param filePath The path of the file within the repository, relative to the repository's root directory. This file should exist in the commit specified.
+   * @return The {@link Path} to the created temporary file, which contains the content of the specified file version. This path is absolute, ensuring it can be accessed directly.
+   * @throws IOException If an I/O error occurs during file operations, including issues with creating the temporary file or writing to it.
+   * @throws GitAPIException If the operation to retrieve file content from the Git repository fails. This could be due to issues with accessing the repository, the commit hash, or the file path specified.
+   */
+  public static Path writeVersionedFileToTempFile(Path baseRepoPath, String commitHash, Path filePath) throws IOException, GitAPIException {
+    // Generate a temporary file
+    Path tempFile = Files.createTempFile("versionedFile", ".tmp");
+
+    // Ensure the file gets deleted on JVM exit
+    tempFile.toFile().deleteOnExit();
+
+    // Use the retrieveFileContentOfVersion method to write the file content into the temp file
+    try (OutputStream outputStream = Files.newOutputStream(tempFile)) {
+      retrieveFileContentOfVersion(baseRepoPath, commitHash, filePath, outputStream);
+    }
+
+    // Return the absolute path of the temporary file
+    return tempFile.toAbsolutePath();
+  }
+
+
   /**
    * Check if there is any uncommitted change in the given repo
 

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/dataset/type/DatasetFileDesc.scala

@@ -0,0 +1,18 @@
+package edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`
+
+import edu.uci.ics.texera.web.resource.dashboard.user.dataset.service.GitVersionControlLocalFileStorage
+import java.nio.file.Path
+
+// This file
+class DatasetFileDesc(val fileName: Path, val datasetPath: Path, val versionHash: String) {
+  def tempFilePath(): Path = {
+    GitVersionControlLocalFileStorage.writeVersionedFileToTempFile(
+      datasetPath,
+      versionHash,
+      datasetPath.resolve(fileName)
+    )
+  }
+
+  override def toString: String =
+    s"DatasetFileDesc(fileName=$fileName, datasetPath=$datasetPath, versionHash=$versionHash)"
+}

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/environment/EnvironmentResource.scala

@@ -20,6 +20,7 @@ import edu.uci.ics.texera.web.model.jooq.generated.tables.daos.{
   EnvironmentOfWorkflowDao
 }
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.retrieveDatasetVersionFilePaths
+import edu.uci.ics.texera.web.resource.dashboard.user.dataset.`type`.DatasetFileDesc
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.{
   DatasetAccessResource,
   DatasetResource
@@ -48,11 +49,13 @@ import org.jooq.DSLContext
 import org.jooq.types.UInteger
 
 import java.net.URLDecoder
+import java.nio.file.Paths
 import javax.annotation.security.RolesAllowed
 import javax.ws.rs.core.{MediaType, Response}
 import javax.ws.rs.{GET, POST, Path, PathParam, Produces}
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 import scala.jdk.CollectionConverters.CollectionHasAsScala
+import scala.util.matching.Regex
 
 object EnvironmentResource {
   private val context = SqlServer.createDSLContext()
@@ -94,6 +97,57 @@ object EnvironmentResource {
       .into(classOf[Environment])
   }
 
+  // return the descriptor of the target file.
+  // The filename is passed from the frontend, the did is contained in the filename in the format of /{dataset-name}/{filepath}
+  def getEnvironmentDatasetFilePathAndVersion(
+      uid: UInteger,
+      eid: UInteger,
+      fileName: String
+  ): DatasetFileDesc = {
+    withTransaction(context) { ctx =>
+      {
+        // Adjust the pattern to match the new fileName format
+        val datasetNamePattern: Regex = """/([^/]+)/.*""".r
+
+        // Extract 'datasetName' using the pattern
+        val datasetName = datasetNamePattern.findFirstMatchIn(fileName) match {
+          case Some(matched) => matched.group(1) // Extract the first group which is 'datasetName'
+          case None =>
+            throw new RuntimeException(
+              "The fileName format is not correct"
+            ) // Default value or handle error
+        }
+
+        // Extract the file path
+        val filePath = Paths.get(
+          fileName.substring(fileName.indexOf(s"/$datasetName/") + s"/$datasetName/".length)
+        )
+        val datasetsOfEnvironment = retrieveDatasetsAndVersions(ctx, uid, eid)
+
+        // Initialize datasetFileDesc as None
+        var datasetFileDesc: Option[DatasetFileDesc] = None
+
+        // Iterate over datasetsOfEnvironment to find a match based on datasetName
+        datasetsOfEnvironment.foreach { datasetAndVersion =>
+          if (datasetAndVersion.dataset.getName == datasetName) {
+            datasetFileDesc = Some(
+              new DatasetFileDesc(
+                filePath,
+                Paths.get(datasetAndVersion.dataset.getStoragePath),
+                datasetAndVersion.version.getVersionHash
+              )
+            )
+          }
+        }
+
+        // Check if datasetFileDesc is set, if not, throw an exception
+        datasetFileDesc.getOrElse(
+          throw new RuntimeException("Given file is not found in the environment")
+        )
+      }
+    }
+  }
+
   private def getEnvironmentByEid(ctx: DSLContext, eid: UInteger): Environment = {
     val environmentDao: EnvironmentDao = new EnvironmentDao(ctx.configuration())
     val env = environmentDao.fetchOneByEid(eid)
@@ -226,7 +280,7 @@ object EnvironmentResource {
       val datasetName = entry.dataset.getName
       val fileList = retrieveDatasetVersionFilePaths(ctx, uid, did, dvid)
       val resList: ListBuffer[String] = new ListBuffer[String]
-      fileList.forEach(file => resList.append(s"/$datasetName-$did/$file"))
+      fileList.forEach(file => resList.append(s"/$datasetName/$file"))
       resList.toList
     })
   }

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala

@@ -82,6 +82,11 @@ object WorkflowResource {
     )
   }
 
+  def getEnvironmentEidOfWorkflow(wid: UInteger): UInteger = {
+    val environmentOfWorkflow = environmentOfWorkflowDao.fetchByWid(wid)
+    environmentOfWorkflow.get(0).getEid
+  }
+
   case class DashboardWorkflow(
       isOwner: Boolean,
       accessLevel: String,

core/amber/src/main/scala/edu/uci/ics/texera/web/service/ExecutionsMetadataPersistService.scala

@@ -34,7 +34,8 @@ object ExecutionsMetadataPersistService extends LazyLogging {
       workflowId: WorkflowIdentity,
       uid: Option[UInteger],
       executionName: String,
-      environmentVersion: String
+      environmentVersion: String,
+      environmentEid: UInteger
   ): ExecutionIdentity = {
     if (!AmberConfig.isUserSystemEnabled) return DEFAULT_EXECUTION_ID
     // first retrieve the latest version of this workflow
@@ -46,7 +47,9 @@ object ExecutionsMetadataPersistService extends LazyLogging {
     newExecution.setVid(vid)
     newExecution.setUid(uid.orNull)
     newExecution.setStartingTime(new Timestamp(System.currentTimeMillis()))
+    // TODO: consider put environment version as a part of the environment
     newExecution.setEnvironmentVersion(environmentVersion)
+    newExecution.setEnvironmentEid(environmentEid)
     workflowExecutionsDao.insert(newExecution)
     ExecutionIdentity(newExecution.getEid.longValue())
   }

core/amber/src/main/scala/edu/uci/ics/texera/web/service/WorkflowService.scala

@@ -15,6 +15,7 @@ import edu.uci.ics.amber.engine.common.virtualidentity.{
 }
 import edu.uci.ics.texera.web.model.websocket.event.TexeraWebSocketEvent
 import edu.uci.ics.texera.web.model.websocket.request.WorkflowExecuteRequest
+import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowResource
 import edu.uci.ics.texera.web.service.WorkflowService.mkWorkflowStateId
 import edu.uci.ics.texera.web.storage.ExecutionStateStore.updateWorkflowState
 import edu.uci.ics.texera.web.storage.{ExecutionStateStore, WorkflowStateStore}
@@ -152,11 +153,15 @@ class WorkflowService(
     val workflowContext: WorkflowContext = createWorkflowContext(uidOpt)
     var controllerConf = ControllerConfig.default
 
+    // fetch the workflow's environment eid
+    val environmentEid =
+      WorkflowResource.getEnvironmentEidOfWorkflow(UInteger.valueOf(workflowContext.workflowId.id))
     workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution(
       workflowContext.workflowId,
       workflowContext.userId,
       req.executionName,
-      convertToJson(req.engineVersion)
+      convertToJson(req.engineVersion),
+      environmentEid
     )
 
     if (AmberConfig.isUserSystemEnabled) {

core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/source/scan/ScanSourceOpDesc.scala

@@ -4,7 +4,8 @@ import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty, JsonPropertyD
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle
 import edu.uci.ics.amber.engine.common.workflow.OutputPort
-import edu.uci.ics.texera.web.resource.dashboard.user.file.UserFileAccessResource
+import edu.uci.ics.texera.web.resource.dashboard.user.environment.EnvironmentResource.getEnvironmentDatasetFilePathAndVersion
+import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowResource
 import edu.uci.ics.texera.workflow.common.WorkflowContext
 import edu.uci.ics.texera.workflow.common.metadata.{OperatorGroupConstants, OperatorInfo}
 import edu.uci.ics.texera.workflow.common.operators.source.SourceOperatorDescriptor
@@ -63,17 +64,15 @@ abstract class ScanSourceOpDesc extends SourceOperatorDescriptor {
 
     if (getContext.userId.isDefined) {
       // if context has a valid user ID, the fileName will be in the following format:
-      //    ownerName/fileName
+      //    /datasetName/fileName
       // resolve fileName to be the actual file path.
-      val splitNames = fileName.get.split("/")
-      filePath = UserFileAccessResource
-        .getFilePath(
-          email = splitNames.apply(0),
-          fileName = splitNames.apply(1),
-          getContext.userId.get,
-          UInteger.valueOf(getContext.workflowId.id)
-        )
-
+      // fetch the environment id that workflow is in
+      val environmentEid = WorkflowResource.getEnvironmentEidOfWorkflow(
+        UInteger.valueOf(workflowContext.workflowId.id)
+      )
+      val datasetFileDescriptor =
+        getEnvironmentDatasetFilePathAndVersion(getContext.userId.get, environmentEid, fileName.get)
+      filePath = Some(datasetFileDescriptor.tempFilePath().toString)
     } else {
       // otherwise, the fileName will be inputted by user, which is the filePath.
       filePath = fileName

core/new-gui/src/app/app.module.ts

@@ -134,6 +134,7 @@ import { NgxFileDropModule } from "ngx-file-drop";
 import { NzTreeModule } from "ng-zorro-antd/tree";
 import { NzTreeViewModule } from "ng-zorro-antd/tree-view";
 import { NzNoAnimationModule } from "ng-zorro-antd/core/no-animation";
+import { EnvironmentComponent } from "./workspace/component/left-panel/environment/environment.component";
 
 registerLocaleData(en);
 
@@ -148,6 +149,7 @@ registerLocaleData(en);
     PropertyEditorComponent,
     VersionsListComponent,
     TimeTravelComponent,
+    EnvironmentComponent,
     WorkflowEditorComponent,
     ResultPanelComponent,
     OperatorLabelComponent,