Add workflow-operator to micro-services

core/micro-services/build.sbt

@@ -1,8 +1,10 @@
-lazy val WorkflowCore = project in file("workflow-core")
+lazy val Dao = project in file("dao")
+lazy val WorkflowCore = (project in file("workflow-core")).dependsOn(Dao)
+lazy val WorkflowOperator = (project in file("workflow-operator")).dependsOn(WorkflowCore)
 
 // root project definition
 lazy val MicroServices = (project in file("."))
-  .aggregate(WorkflowCore)
+  .aggregate(Dao, WorkflowCore, WorkflowOperator)
   .settings(
     name := "micro-services",
     version := "0.1.0",

core/micro-services/dao/build.sbt

@@ -0,0 +1,91 @@
+/////////////////////////////////////////////////////////////////////////////
+// Project Settings
+/////////////////////////////////////////////////////////////////////////////
+
+name := "dao"
+organization := "edu.uci.ics"
+version := "0.1.0"
+scalaVersion := "2.13.12"
+
+enablePlugins(JavaAppPackaging)
+
+// Enable semanticdb for Scalafix
+ThisBuild / semanticdbEnabled := true
+ThisBuild / semanticdbVersion := scalafixSemanticdb.revision
+
+// Manage dependency conflicts by always using the latest revision
+ThisBuild / conflictManager := ConflictManager.latestRevision
+
+// Restrict parallel execution of tests to avoid conflicts
+Global / concurrentRestrictions += Tags.limit(Tags.Test, 1)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Compiler Options
+/////////////////////////////////////////////////////////////////////////////
+
+// Scala compiler options
+Compile / scalacOptions ++= Seq(
+  "-Xelide-below", "WARNING",       // Turn on optimizations with "WARNING" as the threshold
+  "-feature",                       // Check feature warnings
+  "-deprecation",                   // Check deprecation warnings
+  "-Ywarn-unused:imports"           // Check for unused imports
+)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// ScalaPB Configuration
+/////////////////////////////////////////////////////////////////////////////
+
+// Exclude some proto files
+PB.generate / excludeFilter := "scalapb.proto"
+
+// Set the protoc version for ScalaPB
+ThisBuild / PB.protocVersion := "3.19.4"
+
+// ScalaPB code generation for .proto files
+Compile / PB.targets := Seq(
+  scalapb.gen(singleLineToProtoString = true) -> (Compile / sourceManaged).value
+)
+
+// Mark the ScalaPB-generated directory as a generated source root
+Compile / managedSourceDirectories += (Compile / sourceManaged).value
+
+// ScalaPB library dependencies
+libraryDependencies ++= Seq(
+  "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf",
+  "com.thesamet.scalapb" %% "scalapb-json4s" % "0.12.0"  // For ScalaPB 0.11.x
+)
+
+// Enable protobuf compilation in Test
+Test / PB.protoSources += PB.externalSourcePath.value
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Test-related Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "org.scalamock" %% "scalamock" % "5.2.0" % Test,                  // ScalaMock
+  "org.scalatest" %% "scalatest" % "3.2.15" % Test,                 // ScalaTest
+  "junit" % "junit" % "4.13.2" % Test,                              // JUnit
+  "com.novocode" % "junit-interface" % "0.11" % Test                // SBT interface for JUnit
+)
+
+/////////////////////////////////////////////////////////////////////////////
+// Jooq-related Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "org.jooq" % "jooq" % "3.14.16",
+  "org.jooq" % "jooq-codegen" % "3.12.4"
+)
+
+/////////////////////////////////////////////////////////////////////////////
+// Additional Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "mysql" % "mysql-connector-java" % "8.0.23",          // MySQL connector
+  "org.yaml" % "snakeyaml" % "1.29"                     // YAML reader
+)

core/micro-services/dao/src/main/resources/dao-config.yaml

@@ -0,0 +1,4 @@
+jdbc:
+  url: "jdbc:mysql://localhost:3306/texera_db?serverTimezone=UTC"
+  username: "root"
+  password: "123456"

core/micro-services/dao/src/main/resources/jooq-conf.xml

@@ -0,0 +1,45 @@
+<configuration>
+    <generator>
+        <generate>
+            <generatedAnnotation>false</generatedAnnotation>
+
+            <daos>true</daos>
+            <interfaces>true</interfaces>
+        </generate>
+        <!-- The default code generator. You can override this one, to generate your own code style.
+             Supported generators:
+             - org.jooq.codegen.JavaGenerator
+             - org.jooq.codegen.ScalaGenerator
+             Defaults to org.jooq.codegen.JavaGenerator -->
+        <name>org.jooq.codegen.JavaGenerator</name>
+
+        <database>
+            <!-- The database type. The format here is:
+                 org.jooq.meta.[database].[database]Database -->
+            <name>org.jooq.meta.mysql.MySQLDatabase</name>
+
+            <!-- The database schema (or in the absence of schema support, in your RDBMS this
+                 can be the owner, user, database name) to be generated -->
+            <inputSchema>texera_db</inputSchema>
+
+            <!-- All elements that are generated from your schema
+                 (A Java regular expression. Use the pipe to separate several expressions)
+                 Watch out for case-sensitivity. Depending on your database, this might be important! -->
+            <includes>.*</includes>
+
+            <!-- All elements that are excluded from your schema
+                 (A Java regular expression. Use the pipe to separate several expressions).
+                 Excludes match before includes, i.e. excludes have a higher priority -->
+            <excludes>(test_.*)|(ignore_.*)</excludes>
+
+        </database>
+
+        <target>
+            <!-- The destination package of your generated classes (within the destination directory) -->
+            <packageName>edu.uci.ics.texera.model.jooq.generated</packageName>
+
+            <!-- The destination directory of your generated classes. Using Maven directory layout here -->
+            <directory>dao/src/main/scala</directory>
+        </target>
+    </generator>
+</configuration>

core/micro-services/dao/src/main/resources/sql/texera_ddl.sql

@@ -0,0 +1,212 @@
+CREATE SCHEMA IF NOT EXISTS `texera_db`;
+USE `texera_db`;
+
+DROP TABLE IF EXISTS `workflow_runtime_statistics`;
+DROP TABLE IF EXISTS `workflow_user_access`;
+DROP TABLE IF EXISTS `workflow_of_user`;
+DROP TABLE IF EXISTS `user_config`;
+DROP TABLE IF EXISTS `user`;
+DROP TABLE IF EXISTS `workflow`;
+DROP TABLE IF EXISTS `workflow_version`;
+DROP TABLE IF EXISTS `project`;
+DROP TABLE IF EXISTS `workflow_of_project`;
+DROP TABLE IF EXISTS `workflow_executions`;
+DROP TABLE IF EXISTS `dataset`;
+DROP TABLE IF EXISTS `dataset_user_access`;
+DROP TABLE IF EXISTS `dataset_version`;
+
+SET PERSIST time_zone = '+00:00'; -- this line is mandatory
+SET PERSIST sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));
+
+CREATE TABLE IF NOT EXISTS user
+(
+    `uid`           INT UNSIGNED AUTO_INCREMENT NOT NULL,
+    `name`          VARCHAR(256)                NOT NULL,
+    `email`         VARCHAR(256) UNIQUE,
+    `password`      VARCHAR(256),
+    `google_id`     VARCHAR(256) UNIQUE,
+    `role`          ENUM('INACTIVE', 'RESTRICTED', 'REGULAR', 'ADMIN') NOT NULL DEFAULT 'INACTIVE',
+    `google_avatar` VARCHAR(100) null,
+    PRIMARY KEY (`uid`),
+    CONSTRAINT CK_nulltest
+        CHECK (`password` IS NOT NULL OR `google_id` IS NOT NULL)
+) ENGINE = INNODB,
+-- start auto increment userID from 1 because userID 0 means user not exists
+  AUTO_INCREMENT = 1;
+
+CREATE TABLE IF NOT EXISTS user_config
+(
+    `uid`   INT UNSIGNED NOT NULL,
+    `key`   varchar(256) NOT NULL,
+    `value` text         NOT NULL,
+    PRIMARY KEY (`uid`, `key`),
+    FOREIGN KEY (`uid`) REFERENCES user (`uid`) ON DELETE CASCADE
+) ENGINE = InnoDB;
+
+CREATE TABLE IF NOT EXISTS workflow
+(
+    `name`               VARCHAR(128)                NOT NULL,
+	`description`        VARCHAR(500),
+    `wid`                INT UNSIGNED AUTO_INCREMENT NOT NULL,
+    `content`            LONGTEXT                    NOT NULL,
+    `creation_time`      TIMESTAMP                   NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `last_modified_time` TIMESTAMP                   NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    PRIMARY KEY (`wid`)
+) ENGINE = INNODB,
+  AUTO_INCREMENT = 1;
+
+CREATE TABLE IF NOT EXISTS workflow_of_user
+(
+    `uid` INT UNSIGNED NOT NULL,
+    `wid` INT UNSIGNED NOT NULL,
+    PRIMARY KEY (`uid`, `wid`),
+    FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON DELETE CASCADE,
+    FOREIGN KEY (`wid`) REFERENCES `workflow` (`wid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS workflow_user_access
+(
+    `uid`             INT UNSIGNED NOT NULL,
+    `wid`             INT UNSIGNED NOT NULL,
+    `privilege`          ENUM('NONE', 'READ', 'WRITE') NOT NULL DEFAULT 'NONE',
+    PRIMARY KEY (`uid`, `wid`),
+    FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON DELETE CASCADE,
+    FOREIGN KEY (`wid`) REFERENCES `workflow` (`wid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS workflow_version
+(
+    `vid`             INT UNSIGNED AUTO_INCREMENT NOT NULL,
+    `wid`             INT UNSIGNED NOT NULL,
+    `content`            TEXT                        NOT NULL,
+    `creation_time`      TIMESTAMP                   NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (`vid`),
+    FOREIGN KEY (`wid`) REFERENCES `workflow` (`wid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS project
+(
+    `pid`             INT UNSIGNED AUTO_INCREMENT      NOT NULL,
+    `name`            VARCHAR(128)                     NOT NULL,
+    `description`     VARCHAR(10000),
+    `owner_id`        INT UNSIGNED                     NOT NULL,
+    `creation_time`   TIMESTAMP                        NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `color`           VARCHAR(6),
+    UNIQUE(`owner_id`, `name`),
+    PRIMARY KEY (`pid`),
+    FOREIGN KEY (`owner_id`) REFERENCES user (`uid`)   ON DELETE CASCADE
+) ENGINE = INNODB,
+  AUTO_INCREMENT = 1;
+
+CREATE TABLE IF NOT EXISTS workflow_of_project
+(
+     `wid`            INT UNSIGNED                     NOT NULL,
+     `pid`            INT UNSIGNED                     NOT NULL,
+     PRIMARY KEY (`wid`, `pid`),
+     FOREIGN KEY (`wid`) REFERENCES `workflow` (`wid`) ON DELETE CASCADE,
+     FOREIGN KEY (`pid`) REFERENCES `project` (`pid`)  ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS project_user_access
+(
+    `uid`             INT UNSIGNED NOT NULL,
+    `pid`             INT UNSIGNED NOT NULL,
+    `privilege`          ENUM('NONE', 'READ', 'WRITE') NOT NULL DEFAULT 'NONE',
+    PRIMARY KEY (`uid`, `pid`),
+    FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON DELETE CASCADE,
+    FOREIGN KEY (`pid`) REFERENCES `project` (`pid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS workflow_executions
+(
+    `eid`                    INT UNSIGNED AUTO_INCREMENT NOT NULL,
+    `vid`                    INT UNSIGNED NOT NULL,
+    `uid`                    INT UNSIGNED NOT NULL,
+    `status`                 TINYINT NOT NULL DEFAULT 1,
+    `result`                 TEXT, /* pointer to volume */
+    `starting_time`          TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `last_update_time`       TIMESTAMP,
+    `bookmarked`             BOOLEAN DEFAULT FALSE,
+    `name`				     VARCHAR(128) NOT NULL DEFAULT 'Untitled Execution',
+    `environment_version`    VARCHAR(128) NOT NULL,
+    `log_location`           TEXT, /* uri to log storage */
+    PRIMARY KEY (`eid`),
+    FOREIGN KEY (`vid`) REFERENCES `workflow_version` (`vid`) ON DELETE CASCADE,
+    FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS public_project
+(
+    `pid`             INT UNSIGNED NOT NULL,
+    `uid`             INT UNSIGNED,
+    PRIMARY KEY (`pid`),
+    FOREIGN KEY (`pid`) REFERENCES `project` (`pid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS workflow_runtime_statistics
+(
+    `workflow_id`      INT UNSIGNED             NOT NULL,
+    `execution_id`     INT UNSIGNED             NOT NULL,
+    `operator_id`      VARCHAR(100)             NOT NULL,
+    `time`             TIMESTAMP(6)             NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+    `input_tuple_cnt`  INT UNSIGNED             NOT NULL DEFAULT 0,
+    `output_tuple_cnt` INT UNSIGNED             NOT NULL DEFAULT 0,
+    `status`           TINYINT                  NOT NULL DEFAULT 1,
+    `data_processing_time` BIGINT UNSIGNED      NOT NULL DEFAULT 0,
+    `control_processing_time` BIGINT UNSIGNED   NOT NULL DEFAULT 0,
+    `idle_time`        BIGINT UNSIGNED          NOT NULL DEFAULT 0,
+    `num_workers`      INT UNSIGNED             NOT NULL DEFAULT 0,
+    PRIMARY KEY (`workflow_id`, `execution_id`, `operator_id`, `time`),
+    FOREIGN KEY (`workflow_id`) REFERENCES `workflow` (`wid`) ON DELETE CASCADE,
+    FOREIGN KEY (`execution_id`) REFERENCES `workflow_executions` (`eid`) ON DELETE CASCADE
+) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS dataset
+(
+    `did`             INT UNSIGNED AUTO_INCREMENT NOT NULL,
+    `owner_uid`       INT UNSIGNED NOT NULL,
+    `name`            VARCHAR(128) NOT NULL,
+    `is_public`       TINYINT NOT NULL DEFAULT 1,
+    `description`     VARCHAR(512) NOT NULL,
+    `creation_time`   TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(`did`),
+    FOREIGN KEY (`owner_uid`) REFERENCES `user` (`uid`) ON DELETE CASCADE
+    ) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS dataset_user_access
+(
+    `did`             INT UNSIGNED NOT NULL,
+    `uid`             INT UNSIGNED NOT NULL,
+    `privilege`    ENUM('NONE', 'READ', 'WRITE') NOT NULL DEFAULT 'NONE',
+    PRIMARY KEY(`did`, `uid`),
+    FOREIGN KEY (`did`) REFERENCES `dataset` (`did`) ON DELETE CASCADE,
+    FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON DELETE CASCADE
+    ) ENGINE = INNODB;
+
+CREATE TABLE IF NOT EXISTS dataset_version
+(
+    `dvid`            INT UNSIGNED AUTO_INCREMENT NOT NULL,
+    `did`             INT UNSIGNED NOT NULL,
+    `creator_uid`     INT UNSIGNED NOT NULL,
+    `name`            VARCHAR(128) NOT NULL,
+    `version_hash`    VARCHAR(64) NOT NULL,
+    `creation_time`   TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(`dvid`),
+    FOREIGN KEY (`did`) REFERENCES `dataset` (`did`) ON DELETE CASCADE
+    )  ENGINE = INNODB;
+
+
+-- create fulltext search indexes
+
+CREATE FULLTEXT INDEX `idx_workflow_name_description_content` ON `texera_db`.`workflow` (name, description, content);
+
+CREATE FULLTEXT INDEX `idx_user_name` ON `texera_db`.`user` (name);
+
+CREATE FULLTEXT INDEX `idx_user_project_name_description` ON `texera_db`.`project` (name, description);
+
+CREATE FULLTEXT INDEX `idx_dataset_name_description` ON `texera_db`.`dataset` (name, description);
+
+CREATE FULLTEXT INDEX `idx_dataset_version_name` ON `texera_db`.`dataset_version` (name);
+
+ALTER TABLE workflow
+ADD is_published BOOLEAN NOT NULL DEFAULT false;