Encapsulate workflow basic dependencies into a workflow-core package

.github/workflows/github-action-build.yml

@@ -80,6 +80,30 @@ jobs:
       - name: Run backend tests
         run: cd core/amber && sbt -v -J-Xmx2G test
 
+  micro-services:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        java-version: [ 11 ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Texera
+        uses: actions/checkout@v2
+      - name: Setup Java
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java-version }}
+      - uses: coursier/cache-action@v6
+        with:
+          extraSbtFiles: '["core/micro-services/*.sbt", "core/micro-services/project/**.{scala,sbt}", "core/micro-services/project/build.properties" ]'
+      - name: Lint with scalafmt
+        run: cd core/micro-services && sbt scalafmtCheckAll
+      - name: Compile with sbt
+        run: cd core/micro-services && sbt clean package
+      - name: Run backend tests
+        run: cd core/micro-services && sbt test
+
   python_udf:
     runs-on: ${{ matrix.os }}
     strategy:

core/micro-services/.scalafix.conf

@@ -0,0 +1,9 @@
+rules = [
+  ProcedureSyntax,
+  RemoveUnused,
+]
+RemoveUnused.imports = true
+RemoveUnused.privates = true
+RemoveUnused.locals = false
+RemoveUnused.patternvars = false
+RemoveUnused.params = false

core/micro-services/.scalafmt.conf

@@ -0,0 +1,2 @@
+version=2.6.4
+maxColumn = 100

core/micro-services/build.sbt

@@ -1,21 +1,12 @@
+lazy val WorkflowCore = project in file("workflow-core")
+
 // root project definition
 lazy val MicroServices = (project in file("."))
+  .aggregate(WorkflowCore)
   .settings(
     name := "micro-services",
-    version := "0.1.0"
+    version := "0.1.0",
+    organization := "edu.uci.ics",
+    scalaVersion := "2.13.12",
+    publishMavenStyle := true
   )
-
-// The template of the subproject: WorkflowCompilingService(as an example)
-// lazy val WorkflowCompilingService = (project in file("workflow-compiling-service"))
-//  .settings(
-//    name := "WorkflowCompilingService",
-//    version := "0.1.0"
-//    libraryDependencies ++= Seq(
-//      "io.dropwizard" % "dropwizard-core" % "4.0.7",
-//      "com.typesafe" % "config" % "1.4.1",
-//      "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.2",       // Jackson Databind for JSON processing
-//      "com.fasterxml.jackson.core" % "jackson-annotations" % "2.17.2",    // Jackson Annotations for JSON properties
-//      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.17.2" // Jackson Scala module
-//    )
-//  )
-// once this subproject is defined, aggregate it to the MicroServices definition

core/micro-services/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.9.9

core/micro-services/project/plugins.sbt

@@ -0,0 +1,7 @@
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2")
+addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.11.0")
+
+// for scalapb code gen
+addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.6")
+libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.1"
+addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16")

core/micro-services/workflow-core/build.sbt

@@ -0,0 +1,113 @@
+/////////////////////////////////////////////////////////////////////////////
+// Project Settings
+/////////////////////////////////////////////////////////////////////////////
+
+name := "workflow-core"
+organization := "edu.uci.ics"
+version := "0.1.0"
+scalaVersion := "2.13.12"
+
+enablePlugins(JavaAppPackaging)
+
+// Enable semanticdb for Scalafix
+ThisBuild / semanticdbEnabled := true
+ThisBuild / semanticdbVersion := scalafixSemanticdb.revision
+
+// Manage dependency conflicts by always using the latest revision
+ThisBuild / conflictManager := ConflictManager.latestRevision
+
+// Restrict parallel execution of tests to avoid conflicts
+Global / concurrentRestrictions += Tags.limit(Tags.Test, 1)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Compiler Options
+/////////////////////////////////////////////////////////////////////////////
+
+// Scala compiler options
+Compile / scalacOptions ++= Seq(
+  "-Xelide-below", "WARNING",       // Turn on optimizations with "WARNING" as the threshold
+  "-feature",                       // Check feature warnings
+  "-deprecation",                   // Check deprecation warnings
+  "-Ywarn-unused:imports"           // Check for unused imports
+)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// ScalaPB Configuration
+/////////////////////////////////////////////////////////////////////////////
+
+// Exclude some proto files
+PB.generate / excludeFilter := "scalapb.proto"
+
+// Set the protoc version for ScalaPB
+ThisBuild / PB.protocVersion := "3.19.4"
+
+// ScalaPB code generation for .proto files
+Compile / PB.targets := Seq(
+  scalapb.gen(singleLineToProtoString = true) -> (Compile / sourceManaged).value
+)
+
+// Mark the ScalaPB-generated directory as a generated source root
+Compile / managedSourceDirectories += (Compile / sourceManaged).value
+
+// ScalaPB library dependencies
+libraryDependencies ++= Seq(
+  "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf",
+  "com.thesamet.scalapb" %% "scalapb-json4s" % "0.12.0"  // For ScalaPB 0.11.x
+)
+
+// Enable protobuf compilation in Test
+Test / PB.protoSources += PB.externalSourcePath.value
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Test-related Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "org.scalamock" %% "scalamock" % "5.2.0" % Test,                  // ScalaMock
+  "org.scalatest" %% "scalatest" % "3.2.15" % Test,                 // ScalaTest
+  "junit" % "junit" % "4.13.2" % Test,                              // JUnit
+  "com.novocode" % "junit-interface" % "0.11" % Test                // SBT interface for JUnit
+)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Jackson-related Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+val jacksonVersion = "2.15.1"
+libraryDependencies ++= Seq(
+  "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,        // Jackson Databind
+  "com.fasterxml.jackson.module" % "jackson-module-kotlin" % jacksonVersion % Test,   // Jackson Kotlin Module
+  "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % jacksonVersion % Test, // Jackson JDK8 Datatypes
+  "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion % Test, // Jackson JSR310
+  "com.fasterxml.jackson.datatype" % "jackson-datatype-joda" % jacksonVersion % Test,   // Jackson Joda
+  "com.fasterxml.jackson.module" % "jackson-module-jsonSchema" % jacksonVersion,      // JSON Schema Module
+  "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion,           // Scala Module
+  "com.fasterxml.jackson.module" % "jackson-module-no-ctor-deser" % jacksonVersion     // No Constructor Deserializer
+)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// MongoDB-related Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "org.mongodb" % "mongodb-driver-sync" % "5.0.0",               // MongoDB driver
+  "org.apache.commons" % "commons-jcs3-core" % "3.2"             // Apache Commons JCS
+)
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Additional Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "com.github.sisyphsu" % "dateparser" % "1.0.11",               // DateParser
+  "com.google.guava" % "guava" % "31.1-jre",                     // Guava
+  "org.ehcache" % "sizeof" % "0.4.3",                            // Ehcache SizeOf
+  "org.jgrapht" % "jgrapht-core" % "1.4.0",                      // JGraphT Core
+  "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5"      // Scala Logging
+)

core/micro-services/workflow-core/src/main/protobuf/edu/uci/ics/amber/virtualidentity.proto

@@ -0,0 +1,45 @@
+syntax = "proto3";
+
+package edu.uci.ics.amber;
+
+import "scalapb/scalapb.proto";
+
+option (scalapb.options) = {
+  scope: FILE,
+  preserve_unknown_fields: false
+  no_default_values_in_constructor: true
+};
+
+message WorkflowIdentity {
+  int64 id = 1;
+}
+
+message ExecutionIdentity {
+  int64 id = 1;
+}
+
+message ActorVirtualIdentity {
+  string name = 1;
+}
+
+message ChannelIdentity {
+  ActorVirtualIdentity fromWorkerId = 1 [(scalapb.field).no_box = true];
+  ActorVirtualIdentity toWorkerId = 2 [(scalapb.field).no_box = true];
+  bool isControl = 3;
+}
+
+message OperatorIdentity {
+  string id = 1;
+}
+
+message PhysicalOpIdentity{
+  OperatorIdentity logicalOpId = 1 [(scalapb.field).no_box = true];
+  string layerName = 2;
+}
+
+message ChannelMarkerIdentity{
+  string id = 1;
+}
+
+
+

core/micro-services/workflow-core/src/main/protobuf/edu/uci/ics/amber/workflow.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+package edu.uci.ics.amber;
+
+import "edu/uci/ics/amber/virtualidentity.proto";
+import "scalapb/scalapb.proto";
+
+option (scalapb.options) = {
+  scope: FILE,
+  preserve_unknown_fields: false,
+  no_default_values_in_constructor: false
+};
+
+message PortIdentity {
+  int32 id = 1;
+  bool internal = 2;
+}
+
+message InputPort {
+  PortIdentity id = 1 [(scalapb.field).no_box = true];
+  string displayName = 2;
+  bool allowMultiLinks = 3;
+  repeated PortIdentity dependencies = 4;
+}
+
+message OutputPort {
+  PortIdentity id = 1 [(scalapb.field).no_box = true];
+  string displayName = 2;
+  bool blocking = 3;
+}
+
+
+message PhysicalLink {
+  PhysicalOpIdentity fromOpId = 1 [(scalapb.field).no_box = true];
+  PortIdentity fromPortId = 2 [(scalapb.field).no_box = true];
+  PhysicalOpIdentity toOpId = 3 [(scalapb.field).no_box = true];
+  PortIdentity toPortId = 4 [(scalapb.field).no_box = true];
+}