From 4ffb358e8fed5f87b837be99da5a3b77c806efbe Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Tue, 21 Jan 2025 16:19:19 -0800 Subject: [PATCH 01/18] backend service endpoint for computing unit metrics --- .../templates/metrics-server.yaml | 202 ++++++++++++++++++ .../build.sbt | 3 +- ...WorkflowComputingUnitManagingService.scala | 3 +- .../WorkflowComputingUnitMetricResource.scala | 31 +++ .../util/KubernetesMetricService.scala | 62 ++++++ 5 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 core/scripts/texera-helmchart/templates/metrics-server.yaml create mode 100644 core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala create mode 100644 core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala diff --git a/core/scripts/texera-helmchart/templates/metrics-server.yaml b/core/scripts/texera-helmchart/templates/metrics-server.yaml new file mode 100644 index 00000000000..31ab7e011b3 --- /dev/null +++ b/core/scripts/texera-helmchart/templates/metrics-server.yaml @@ -0,0 +1,202 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=10250 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.7.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 10250 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 diff --git a/core/workflow-computing-unit-managing-service/build.sbt b/core/workflow-computing-unit-managing-service/build.sbt index a4924ac3314..e0bdda78c89 100644 --- a/core/workflow-computing-unit-managing-service/build.sbt +++ b/core/workflow-computing-unit-managing-service/build.sbt @@ -21,7 +21,8 @@ libraryDependencies ++= Seq( "mysql" % "mysql-connector-java" % "8.0.33", "com.softwaremill.sttp.client4" %% "core" % "4.0.0-M6", "com.lihaoyi" %% "upickle" % "3.1.0", - "com.typesafe" % "config" % "1.4.2" + "com.typesafe" % "config" % "1.4.2", + "io.fabric8" % "kubernetes-client" % "6.12.1" ) // Compiler Options diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala index 298718902d3..8a022fb05c3 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala @@ -2,7 +2,7 @@ package edu.uci.ics.texera.service import com.fasterxml.jackson.module.scala.DefaultScalaModule import edu.uci.ics.amber.util.PathUtils.workflowComputingUnitManagingServicePath -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource +import edu.uci.ics.texera.service.resource.{WorkflowComputingUnitManagingResource, WorkflowComputingUnitMetricResource} import io.dropwizard.core.setup.{Bootstrap, Environment} import io.dropwizard.core.Application @@ -22,6 +22,7 @@ class WorkflowComputingUnitManagingService // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey().register(new WorkflowComputingUnitManagingResource) + environment.jersey().register(new WorkflowComputingUnitMetricResource) } } diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala new file mode 100644 index 00000000000..3ca4fbdb25e --- /dev/null +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala @@ -0,0 +1,31 @@ +package edu.uci.ics.texera.service.resource + +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitMetricResource.WorkflowComputingUnitMetricParam +import edu.uci.ics.texera.service.util.KubernetesMetricService._ +import jakarta.ws.rs._ +import jakarta.ws.rs.core.MediaType + +object WorkflowComputingUnitMetricResource { + + case class WorkflowComputingUnitMetricParam(computingUnitName: String, namespace: String) +} + +@Produces(Array(MediaType.APPLICATION_JSON)) +@Path("/resource-metrics") +class WorkflowComputingUnitMetricResource { + + /** + * Retrieves the computing unit metrics for a given name in the specified namespace. + * + * @param params The parameters containing the computingUnitName and namespace + * @return The computing unit metrics for a given name in a specified namespace + */ + @GET + @Consumes(Array(MediaType.APPLICATION_JSON)) + @Produces(Array(MediaType.APPLICATION_JSON)) + @Path("/pod-metrics") + def getComputingUnitMetric(params: WorkflowComputingUnitMetricParam): Map[String, String] = { + getPodMetrics(params.computingUnitName, params.namespace) + } + +} diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala new file mode 100644 index 00000000000..6d3afe1505b --- /dev/null +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala @@ -0,0 +1,62 @@ +package edu.uci.ics.texera.service.util + +import config.WorkflowComputingUnitManagingServiceConf +import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetricsList +import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder} +import scala.jdk.CollectionConverters._ + +object KubernetesMetricService { + + // Initialize the Kubernetes client + val client: KubernetesClient = new KubernetesClientBuilder().build() + + /** + * Retrieves the pod metric for a given name in the specified namespace. + * + * @param namespace The namespace of the pod to be returned + * @param targetPodName The name of the pod to be returned + * @return The Pod metrics for a given name in a specified namespace. + */ + def getPodMetrics(targetPodName: String, namespace: String): Map[String, String] = { + val podMetricsList: PodMetricsList = client.top().pods().metrics(namespace) + + podMetricsList.getItems.asScala.collectFirst { + case podMetrics if podMetrics.getMetadata.getName == targetPodName => + podMetrics.getContainers.asScala.collectFirst { + case container => + val usageMap = container.getUsage.asScala.map { + case ("cpu", value) => + val cpuInCores = convertCPUToCores(value.getAmount) + println(s"CPU Usage: $cpuInCores cores") + "cpu" -> f"$cpuInCores cores" + case ("memory", value) => + val memoryInMB = convertMemoryToMB(value.getAmount) + println(s"Memory Usage: $memoryInMB MB") + "memory" -> f"$memoryInMB%.2f MB" + case (key, value) => + println(s"Other Metric - $key: ${value.getAmount}") + key -> value.getAmount + }.toMap + usageMap + }.getOrElse(Map.empty) + }.getOrElse { + println(s"No metrics found for pod: $targetPodName in namespace: $namespace") + Map.empty + } + } + + private def convertMemoryToMB(memoryUsage: String): Double = { + val memoryUsageInKi = memoryUsage.toDoubleOption.getOrElse(0.0) + memoryUsageInKi / 1024 + } + + private def convertCPUToCores(cpuUsage: String): Double = { + val cpuUsageInNano = cpuUsage.toDoubleOption.getOrElse(0.0) + cpuUsageInNano / 1_000_000_000 + } + +// def main(args: Array[String]): Unit = { +// val podName: String = "computing-unit-1" +// println(getPodMetrics(podName, WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace).toString) +// } +} From 318360d8ac4a2a0c81e9c5f6978a1543030e0ec4 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sun, 26 Jan 2025 23:50:53 -0800 Subject: [PATCH 02/18] updated coded for resource metrics with request handling --- core/gui/proxy.config.json | 6 ++++- .../computing-unit-selection.component.ts | 24 ++++++++++++++++++- ...orkflow-computing-unit-managing.service.ts | 18 ++++++++++++-- .../types/workflow-computing-unit.ts | 6 +++++ .../WorkflowComputingUnitMetricResource.scala | 22 ++++++++++++----- .../util/KubernetesMetricService.scala | 20 +++++++++------- 6 files changed, 77 insertions(+), 19 deletions(-) diff --git a/core/gui/proxy.config.json b/core/gui/proxy.config.json index c3fadc17c04..6e4023d7aab 100755 --- a/core/gui/proxy.config.json +++ b/core/gui/proxy.config.json @@ -4,7 +4,11 @@ "secure": false, "changeOrigin": true }, - + "/api/resource-metrics": { + "target": "http://localhost:8888", + "secure": false, + "changeOrigin": true + }, "/api": { "target": "http://texera.example.com:30080", "secure": false, diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 68e9c9f81ab..e3921ea6536 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -1,6 +1,6 @@ import { Component, Input, OnInit } from "@angular/core"; import { interval } from "rxjs"; -import { switchMap } from "rxjs/operators"; +import {map, switchMap} from "rxjs/operators"; import { WorkflowComputingUnitManagingService } from "../../service/workflow-computing-unit/workflow-computing-unit-managing.service"; import { DashboardWorkflowComputingUnit } from "../../types/workflow-computing-unit"; import { NotificationService } from "../../../common/service/notification/notification.service"; @@ -68,6 +68,28 @@ export class ComputingUnitSelectionComponent implements OnInit { ) { this.selectedComputingUnit = null; } + this.updateComputingUnitMetrics(); + } + + /** + * Update the currently selected computing units metrics + */ + updateComputingUnitMetrics(): void { + const cuid = this.selectedComputingUnit?.computingUnit.cuid; + if (!cuid) { + return // for now + } + this.computingUnitService.getComputingUnitMetrics(cuid) + .pipe( + map(metrics => ({ + cpu: metrics["cpu"], + memory: metrics["memory"] + })) + ) + .subscribe(({ cpu, memory }) => { + console.log(`CPU: ${cpu}`); + console.log(`Memory: ${memory}`); + }); } /** diff --git a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts index 0188813bbf6..37dc03476ae 100644 --- a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts +++ b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts @@ -1,13 +1,18 @@ import { Injectable } from "@angular/core"; -import { HttpClient } from "@angular/common/http"; +import {HttpClient, HttpParams} from "@angular/common/http"; import { Observable } from "rxjs"; import { AppSettings } from "../../../common/app-setting"; -import { DashboardWorkflowComputingUnit } from "../../types/workflow-computing-unit"; +import { + DashboardWorkflowComputingUnit, + WorkflowComputingUnitMetrics +} from "../../types/workflow-computing-unit"; export const COMPUTING_UNIT_BASE_URL = "computing-unit"; +export const COMPUTING_UNIT_METRICS_BASE_URL = "resource-metrics"; export const COMPUTING_UNIT_CREATE_URL = `${COMPUTING_UNIT_BASE_URL}/create`; export const COMPUTING_UNIT_TERMINATE_URL = `${COMPUTING_UNIT_BASE_URL}/terminate`; export const COMPUTING_UNIT_LIST_URL = `${COMPUTING_UNIT_BASE_URL}`; +export const COMPUTING_UNIT_METRIC_URL = `${COMPUTING_UNIT_METRICS_BASE_URL}/pod-metrics` @Injectable({ providedIn: "root", @@ -50,4 +55,13 @@ export class WorkflowComputingUnitManagingService { `${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_LIST_URL}` ); } + + /** + * Get a computing units resource metrics + * + */ + public getComputingUnitMetrics(cuid: number): Observable { + let params = new HttpParams().set("computingUnitCUID",cuid) + return this.http.get(`${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_METRIC_URL}`, { params }); + } } diff --git a/core/gui/src/app/workspace/types/workflow-computing-unit.ts b/core/gui/src/app/workspace/types/workflow-computing-unit.ts index 238e8013606..b5861ae9dea 100644 --- a/core/gui/src/app/workspace/types/workflow-computing-unit.ts +++ b/core/gui/src/app/workspace/types/workflow-computing-unit.ts @@ -11,3 +11,9 @@ export interface DashboardWorkflowComputingUnit { uri: string; status: string; } + +export interface WorkflowComputingUnitMetrics { + cuid: number; + cpu: number; + memory: number; +} diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala index 3ca4fbdb25e..74a27125b45 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala @@ -1,13 +1,17 @@ package edu.uci.ics.texera.service.resource -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitMetricResource.WorkflowComputingUnitMetricParam +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitMetricResource.{WorkflowComputingUnitMetricParam, WorkflowComputingUnitMetrics} import edu.uci.ics.texera.service.util.KubernetesMetricService._ import jakarta.ws.rs._ import jakarta.ws.rs.core.MediaType object WorkflowComputingUnitMetricResource { - case class WorkflowComputingUnitMetricParam(computingUnitName: String, namespace: String) + case class WorkflowComputingUnitMetrics ( + cuid: Int, + cpu: Double, + memory: Double + ) } @Produces(Array(MediaType.APPLICATION_JSON)) @@ -17,15 +21,21 @@ class WorkflowComputingUnitMetricResource { /** * Retrieves the computing unit metrics for a given name in the specified namespace. * - * @param params The parameters containing the computingUnitName and namespace + * @param computingUnitCUID The computing units uid for retrieving metrics * @return The computing unit metrics for a given name in a specified namespace */ @GET - @Consumes(Array(MediaType.APPLICATION_JSON)) @Produces(Array(MediaType.APPLICATION_JSON)) @Path("/pod-metrics") - def getComputingUnitMetric(params: WorkflowComputingUnitMetricParam): Map[String, String] = { - getPodMetrics(params.computingUnitName, params.namespace) + def getComputingUnitMetric(@QueryParam("computingUnitCUID") computingUnitCUID: Int): WorkflowComputingUnitMetrics = { + println(computingUnitCUID) + val metrics: Map[String, Any] = getPodMetrics(computingUnitCUID) + + WorkflowComputingUnitMetrics( + cuid = computingUnitCUID, + cpu = metrics.get("cpu").collect { case value: Double => value }.getOrElse(0.0), + memory = metrics.get("memory").collect { case value: Double => value }.getOrElse(0.0) + ) } } diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala index 6d3afe1505b..d9c26a562f4 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala @@ -10,38 +10,40 @@ object KubernetesMetricService { // Initialize the Kubernetes client val client: KubernetesClient = new KubernetesClientBuilder().build() + private val namespace = WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace + /** * Retrieves the pod metric for a given name in the specified namespace. * - * @param namespace The namespace of the pod to be returned - * @param targetPodName The name of the pod to be returned + * @param cuid The computing unit id of the pod * @return The Pod metrics for a given name in a specified namespace. */ - def getPodMetrics(targetPodName: String, namespace: String): Map[String, String] = { + def getPodMetrics(cuid: Int): Map[String, Any] = { val podMetricsList: PodMetricsList = client.top().pods().metrics(namespace) + val targetPodName = KubernetesClientService.generatePodName(cuid) podMetricsList.getItems.asScala.collectFirst { case podMetrics if podMetrics.getMetadata.getName == targetPodName => podMetrics.getContainers.asScala.collectFirst { case container => - val usageMap = container.getUsage.asScala.map { + container.getUsage.asScala.collect { case ("cpu", value) => val cpuInCores = convertCPUToCores(value.getAmount) println(s"CPU Usage: $cpuInCores cores") - "cpu" -> f"$cpuInCores cores" + "cpu" -> cpuInCores case ("memory", value) => val memoryInMB = convertMemoryToMB(value.getAmount) println(s"Memory Usage: $memoryInMB MB") - "memory" -> f"$memoryInMB%.2f MB" + "memory" -> memoryInMB case (key, value) => println(s"Other Metric - $key: ${value.getAmount}") + // Other metrics may not all be Double key -> value.getAmount }.toMap - usageMap - }.getOrElse(Map.empty) + }.getOrElse(Map.empty[String, Double]) }.getOrElse { println(s"No metrics found for pod: $targetPodName in namespace: $namespace") - Map.empty + Map.empty[String, Double] } } From 742081ff5e499f00b03c9a43ff447cb58fb96d92 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Mon, 27 Jan 2025 15:45:28 -0800 Subject: [PATCH 03/18] updated pod metrics service --- .../service/resource/WorkflowComputingUnitMetricResource.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala index 74a27125b45..5a436692923 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala @@ -1,6 +1,6 @@ package edu.uci.ics.texera.service.resource -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitMetricResource.{WorkflowComputingUnitMetricParam, WorkflowComputingUnitMetrics} +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitMetricResource.WorkflowComputingUnitMetrics import edu.uci.ics.texera.service.util.KubernetesMetricService._ import jakarta.ws.rs._ import jakarta.ws.rs.core.MediaType From e75c0cc3d4f3c2d6e82178105ee3f06c225660c6 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Tue, 4 Feb 2025 10:19:43 -0800 Subject: [PATCH 04/18] update resource metrics implementation and ui --- core/gui/proxy.config.json | 5 - core/gui/src/app/app.module.ts | 2 + .../computing-unit-selection.component.html | 129 +++++++++++------- .../computing-unit-selection.component.scss | 40 +++--- .../computing-unit-selection.component.ts | 26 +--- ...orkflow-computing-unit-managing.service.ts | 7 +- .../types/workflow-computing-unit.ts | 6 +- core/gui/src/styles.scss | 15 ++ ...WorkflowComputingUnitManagingService.scala | 3 +- ...orkflowComputingUnitManagingResource.scala | 32 +++-- .../WorkflowComputingUnitMetricResource.scala | 41 ------ .../util/KubernetesMetricService.scala | 11 +- 12 files changed, 154 insertions(+), 163 deletions(-) delete mode 100644 core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala diff --git a/core/gui/proxy.config.json b/core/gui/proxy.config.json index 6e4023d7aab..979a0f9d5dd 100755 --- a/core/gui/proxy.config.json +++ b/core/gui/proxy.config.json @@ -4,11 +4,6 @@ "secure": false, "changeOrigin": true }, - "/api/resource-metrics": { - "target": "http://localhost:8888", - "secure": false, - "changeOrigin": true - }, "/api": { "target": "http://texera.example.com:30080", "secure": false, diff --git a/core/gui/src/app/app.module.ts b/core/gui/src/app/app.module.ts index f62768cf00b..7b5fb86432d 100644 --- a/core/gui/src/app/app.module.ts +++ b/core/gui/src/app/app.module.ts @@ -144,6 +144,7 @@ import { SocialLoginModule, SocialAuthServiceConfig, GoogleSigninButtonModule } import { GoogleLoginProvider } from "@abacritt/angularx-social-login"; import { lastValueFrom } from "rxjs"; import { HubSearchResultComponent } from "./hub/component/hub-search-result/hub-search-result.component"; +import {NzProgressModule} from "ng-zorro-antd/progress"; registerLocaleData(en); @@ -295,6 +296,7 @@ registerLocaleData(en); NzDividerModule, SocialLoginModule, GoogleSigninButtonModule, + NzProgressModule, ], providers: [ provideNzI18n(en_US), diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index c15d7b95d40..2f3b58410a0 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -1,52 +1,83 @@ - - -
- - - - - - - - {{ unit.computingUnit.name }} -
- +
+
+
+ CPU +
- - - - -
- +
+ Memory + +
- + + + +
+ + + + + + + + {{ unit.computingUnit.name }} +
+ +
+ + + + + +
+ +
+ +
diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss index ca50bc7dce3..f3e740a7973 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss @@ -1,33 +1,37 @@ -nz-select { - width: 300px; +.computing-units-selection { + display: grid; + grid-template-columns: repeat(2, 1fr); + justify-content: center; + align-items: center; + width: 400px; + gap: 10px; } -nz-option { - height: 100%; +#computing-units-list { + width: 250px; } -.computing-unit-option { - display: flex; +.metrics-container { + display: grid; align-items: center; - justify-content: space-between; + grid-template-rows: repeat(2, 1fr); + gap: 3px; + height: 32px; width: 100%; } -.unit-details { - display: flex; +.metric-item { + display: grid; + grid-template-columns: repeat(2, 1fr); + width: 100%; + height: 16px; align-items: center; - gap: 8px; - flex-grow: 1; } -.terminate-box { - width: 16px; /* Size of the red box */ +#cpu-progress-bar, #memory-progress-bar { height: 16px; - background-color: red; - border-radius: 2px; /* Optional, if you want slightly rounded corners */ - cursor: pointer; /* Pointer cursor to indicate it's clickable */ } -.terminate-box:hover { - opacity: 0.8; /* Slight opacity on hover for visual feedback */ +.metric-label { + font-size: 10px; } diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index e3921ea6536..45230fe5331 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -1,8 +1,8 @@ import { Component, Input, OnInit } from "@angular/core"; import { interval } from "rxjs"; -import {map, switchMap} from "rxjs/operators"; +import {switchMap} from "rxjs/operators"; import { WorkflowComputingUnitManagingService } from "../../service/workflow-computing-unit/workflow-computing-unit-managing.service"; -import { DashboardWorkflowComputingUnit } from "../../types/workflow-computing-unit"; +import {DashboardWorkflowComputingUnit} from "../../types/workflow-computing-unit"; import { NotificationService } from "../../../common/service/notification/notification.service"; import { WorkflowWebsocketService } from "../../service/workflow-websocket/workflow-websocket.service"; import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; @@ -68,28 +68,6 @@ export class ComputingUnitSelectionComponent implements OnInit { ) { this.selectedComputingUnit = null; } - this.updateComputingUnitMetrics(); - } - - /** - * Update the currently selected computing units metrics - */ - updateComputingUnitMetrics(): void { - const cuid = this.selectedComputingUnit?.computingUnit.cuid; - if (!cuid) { - return // for now - } - this.computingUnitService.getComputingUnitMetrics(cuid) - .pipe( - map(metrics => ({ - cpu: metrics["cpu"], - memory: metrics["memory"] - })) - ) - .subscribe(({ cpu, memory }) => { - console.log(`CPU: ${cpu}`); - console.log(`Memory: ${memory}`); - }); } /** diff --git a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts index 37dc03476ae..2d2dd9902d4 100644 --- a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts +++ b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts @@ -12,7 +12,6 @@ export const COMPUTING_UNIT_METRICS_BASE_URL = "resource-metrics"; export const COMPUTING_UNIT_CREATE_URL = `${COMPUTING_UNIT_BASE_URL}/create`; export const COMPUTING_UNIT_TERMINATE_URL = `${COMPUTING_UNIT_BASE_URL}/terminate`; export const COMPUTING_UNIT_LIST_URL = `${COMPUTING_UNIT_BASE_URL}`; -export const COMPUTING_UNIT_METRIC_URL = `${COMPUTING_UNIT_METRICS_BASE_URL}/pod-metrics` @Injectable({ providedIn: "root", @@ -58,10 +57,10 @@ export class WorkflowComputingUnitManagingService { /** * Get a computing units resource metrics - * + * @returns an Observable of WorkflowComputingUnitMetrics + * @param cuid */ public getComputingUnitMetrics(cuid: number): Observable { - let params = new HttpParams().set("computingUnitCUID",cuid) - return this.http.get(`${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_METRIC_URL}`, { params }); + return this.http.get(`${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_BASE_URL}/${cuid}/metrics`) } } diff --git a/core/gui/src/app/workspace/types/workflow-computing-unit.ts b/core/gui/src/app/workspace/types/workflow-computing-unit.ts index b5861ae9dea..4d8944d6b7c 100644 --- a/core/gui/src/app/workspace/types/workflow-computing-unit.ts +++ b/core/gui/src/app/workspace/types/workflow-computing-unit.ts @@ -10,10 +10,10 @@ export interface DashboardWorkflowComputingUnit { computingUnit: WorkflowComputingUnit; uri: string; status: string; + metrics: WorkflowComputingUnitMetrics; } export interface WorkflowComputingUnitMetrics { - cuid: number; - cpu: number; - memory: number; + cpuUsage: number; + memoryUsage: number; } diff --git a/core/gui/src/styles.scss b/core/gui/src/styles.scss index 6c2ac14306b..22f52fc4c70 100644 --- a/core/gui/src/styles.scss +++ b/core/gui/src/styles.scss @@ -76,3 +76,18 @@ hr { .annotation-highlight { background-color: #6a5acd; } + +// For compute-unit-selection.html +#computing-units-list > .ant-select-selector { + height: 100% !important; +} + +#cpu-progress-bar .ant-progress-inner, +#memory-progress-bar .ant-progress-inner { + vertical-align: super !important; +} + +#cpu-progress-bar *, +#memory-progress-bar * { + line-height: 16px !important; +} diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala index 8a022fb05c3..298718902d3 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/WorkflowComputingUnitManagingService.scala @@ -2,7 +2,7 @@ package edu.uci.ics.texera.service import com.fasterxml.jackson.module.scala.DefaultScalaModule import edu.uci.ics.amber.util.PathUtils.workflowComputingUnitManagingServicePath -import edu.uci.ics.texera.service.resource.{WorkflowComputingUnitManagingResource, WorkflowComputingUnitMetricResource} +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource import io.dropwizard.core.setup.{Bootstrap, Environment} import io.dropwizard.core.Application @@ -22,7 +22,6 @@ class WorkflowComputingUnitManagingService // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey().register(new WorkflowComputingUnitManagingResource) - environment.jersey().register(new WorkflowComputingUnitMetricResource) } } diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala index 79b7b018215..44225a5e60b 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala @@ -6,14 +6,9 @@ import edu.uci.ics.texera.dao.SqlServer.withTransaction import edu.uci.ics.texera.dao.jooq.generated.tables.daos.WorkflowComputingUnitDao import edu.uci.ics.texera.dao.jooq.generated.tables.WorkflowComputingUnit.WORKFLOW_COMPUTING_UNIT import edu.uci.ics.texera.dao.jooq.generated.tables.pojos.WorkflowComputingUnit -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{ - DashboardWorkflowComputingUnit, - WorkflowComputingUnitCreationParams, - WorkflowComputingUnitTerminationParams, - TerminationResponse, - context -} +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{DashboardWorkflowComputingUnit, TerminationResponse, WorkflowComputingUnitCreationParams, WorkflowComputingUnitMetrics, WorkflowComputingUnitTerminationParams, context} import edu.uci.ics.texera.service.util.KubernetesClientService +import edu.uci.ics.texera.service.util.KubernetesMetricService.getPodMetrics import jakarta.ws.rs._ import jakarta.ws.rs.core.MediaType import org.jooq.DSLContext @@ -31,10 +26,13 @@ object WorkflowComputingUnitManagingResource { case class WorkflowComputingUnitTerminationParams(uri: String, name: String) + case class WorkflowComputingUnitMetrics(cpuUsage: Double, memoryUsage: Double) + case class DashboardWorkflowComputingUnit( computingUnit: WorkflowComputingUnit, uri: String, - status: String + status: String, + metrics: WorkflowComputingUnitMetrics ) case class TerminationResponse(message: String, uri: String) @@ -81,7 +79,8 @@ class WorkflowComputingUnitManagingResource { DashboardWorkflowComputingUnit( insertedUnit, KubernetesClientService.generatePodURI(cuid).toString, - pod.getStatus.getPhase + pod.getStatus.getPhase, + getComputingUnitMetric(cuid.toString) ) } } @@ -112,7 +111,8 @@ class WorkflowComputingUnitManagingResource { DashboardWorkflowComputingUnit( computingUnit = unit, uri = KubernetesClientService.generatePodURI(cuid).toString, - status = if (pod != null && pod.getStatus != null) pod.getStatus.getPhase else "Unknown" + status = if (pod != null && pod.getStatus != null) pod.getStatus.getPhase else "Unknown", + getComputingUnitMetric(cuid.toString) ) }) @@ -147,4 +147,16 @@ class WorkflowComputingUnitManagingResource { TerminationResponse(s"Successfully terminated compute unit with URI $podURI", podURI) } + + @GET + @Produces(Array(MediaType.APPLICATION_JSON)) + @Path("/{cuid}/metrics") + def getComputingUnitMetric(@PathParam("cuid") cuid: String): WorkflowComputingUnitMetrics = { + val metrics: Map[String, Any] = getPodMetrics(cuid.toInt) + + WorkflowComputingUnitMetrics( + cpuUsage = metrics.get("cpu").collect { case value: Double => value }.getOrElse(0.0), + memoryUsage = metrics.get("memory").collect { case value: Double => value }.getOrElse(0.0) + ) + } } diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala deleted file mode 100644 index 5a436692923..00000000000 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitMetricResource.scala +++ /dev/null @@ -1,41 +0,0 @@ -package edu.uci.ics.texera.service.resource - -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitMetricResource.WorkflowComputingUnitMetrics -import edu.uci.ics.texera.service.util.KubernetesMetricService._ -import jakarta.ws.rs._ -import jakarta.ws.rs.core.MediaType - -object WorkflowComputingUnitMetricResource { - - case class WorkflowComputingUnitMetrics ( - cuid: Int, - cpu: Double, - memory: Double - ) -} - -@Produces(Array(MediaType.APPLICATION_JSON)) -@Path("/resource-metrics") -class WorkflowComputingUnitMetricResource { - - /** - * Retrieves the computing unit metrics for a given name in the specified namespace. - * - * @param computingUnitCUID The computing units uid for retrieving metrics - * @return The computing unit metrics for a given name in a specified namespace - */ - @GET - @Produces(Array(MediaType.APPLICATION_JSON)) - @Path("/pod-metrics") - def getComputingUnitMetric(@QueryParam("computingUnitCUID") computingUnitCUID: Int): WorkflowComputingUnitMetrics = { - println(computingUnitCUID) - val metrics: Map[String, Any] = getPodMetrics(computingUnitCUID) - - WorkflowComputingUnitMetrics( - cuid = computingUnitCUID, - cpu = metrics.get("cpu").collect { case value: Double => value }.getOrElse(0.0), - memory = metrics.get("memory").collect { case value: Double => value }.getOrElse(0.0) - ) - } - -} diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala index d9c26a562f4..36c36556ee3 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala @@ -10,6 +10,8 @@ object KubernetesMetricService { // Initialize the Kubernetes client val client: KubernetesClient = new KubernetesClientBuilder().build() + private val MEGABYTES_PER_GIGABYTE = 1024 + private val NANOCORES_PER_CPU_CORE = 1_000_000_000 private val namespace = WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace /** @@ -49,16 +51,11 @@ object KubernetesMetricService { private def convertMemoryToMB(memoryUsage: String): Double = { val memoryUsageInKi = memoryUsage.toDoubleOption.getOrElse(0.0) - memoryUsageInKi / 1024 + memoryUsageInKi / MEGABYTES_PER_GIGABYTE } private def convertCPUToCores(cpuUsage: String): Double = { val cpuUsageInNano = cpuUsage.toDoubleOption.getOrElse(0.0) - cpuUsageInNano / 1_000_000_000 + cpuUsageInNano / NANOCORES_PER_CPU_CORE } - -// def main(args: Array[String]): Unit = { -// val podName: String = "computing-unit-1" -// println(getPodMetrics(podName, WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace).toString) -// } } From e4cffd8f9c03d9bbde80ce0f575fcdf5a2425410 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sat, 8 Feb 2025 16:29:03 -0800 Subject: [PATCH 05/18] computing unit selection and metrics ui fix --- .../computing-unit-selection.component.html | 53 +++++++++++---- .../computing-unit-selection.component.scss | 65 +++++++++++++++++++ 2 files changed, 104 insertions(+), 14 deletions(-) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index 2f3b58410a0..f08258844c7 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -1,5 +1,11 @@
-
+
+ +
+ + + +
+ +
+ - - -
- + +
+
+

CPU

+

+ {{(selectedComputingUnit?.metrics?.cpuUsage || 0).toFixed(2)}} + Cores +

- -
+
+

Memory

+

+ {{(selectedComputingUnit?.metrics?.memoryUsage || 0).toFixed(2)}} + MB +

+
+
+ diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss index f3e740a7973..0a70fd68ce3 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss @@ -7,6 +7,36 @@ gap: 10px; } +nz-option { + height: 100%; +} + +.computing-unit-option { + display: flex; + align-items: center; + justify-content: space-between; + width: 100%; +} + +.unit-details { + display: flex; + align-items: center; + gap: 8px; + flex-grow: 1; +} + +.terminate-box { + width: 16px; /* Size of the red box */ + height: 16px; + background-color: red; + border-radius: 2px; /* Optional, if you want slightly rounded corners */ + cursor: pointer; /* Pointer cursor to indicate it's clickable */ +} + +.terminate-box:hover { + opacity: 0.8; /* Slight opacity on hover for visual feedback */ +} + #computing-units-list { width: 250px; } @@ -18,6 +48,8 @@ gap: 3px; height: 32px; width: 100%; + padding: 0; + border: none; } .metric-item { @@ -35,3 +67,36 @@ .metric-label { font-size: 10px; } + +.resource-metrics { + width: 200px; + display: grid; + grid-template-columns: repeat(2, 1fr); + grid-template-rows: repeat(1, 1fr); + justify-content: start; + align-items: center; + gap: 5px; +} + +.general-metric { + display: flex; + flex-direction: column; + width: 100%; + background-color: #f9fafb; + border-radius: 3px; + padding: 10px; + gap: 3px; +} + +.metric-unit { + font-size: 8px; +} + +.metric-name { + font-size: 10px; + margin: 0; +} + +.metric-value { + margin: 0; +} From f489c3026267d6a9a70a3e1b320afa0c6cceb37b Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Mon, 10 Feb 2025 16:30:32 -0800 Subject: [PATCH 06/18] center aligned computing unit selection --- core/gui/src/styles.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/gui/src/styles.scss b/core/gui/src/styles.scss index 22f52fc4c70..417919e1f00 100644 --- a/core/gui/src/styles.scss +++ b/core/gui/src/styles.scss @@ -82,6 +82,10 @@ hr { height: 100% !important; } +#computing-units-list .ant-select-selection-item { + line-height: 30px !important; +} + #cpu-progress-bar .ant-progress-inner, #memory-progress-bar .ant-progress-inner { vertical-align: super !important; From 435b325277854aad6edcbfc9abe9fb709216834b Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sun, 16 Feb 2025 00:52:55 -0800 Subject: [PATCH 07/18] pod resource metrics and ui changes --- .../computing-unit-selection.component.html | 36 ++++---- .../computing-unit-selection.component.scss | 37 ++++++-- .../computing-unit-selection.component.ts | 84 +++++++++++++++++-- .../types/workflow-computing-unit.ts | 22 +++-- ...orkflowComputingUnitManagingResource.scala | 55 ++++++++++-- .../util/KubernetesClientService.scala | 45 ++++++++-- .../util/KubernetesMetricService.scala | 49 ++++++----- 7 files changed, 249 insertions(+), 79 deletions(-) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index f08258844c7..05c4c52f290 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -1,7 +1,7 @@

Memory

- {{(selectedComputingUnit?.metrics?.memoryUsage || 0).toFixed(2)}} - MB + {{getMemoryValue() | number:'1.2-2' || '0.00'}} + / {{getMemoryLimit()}} {{getMemoryLimitUnit()}}

diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss index 0a70fd68ce3..af91b8642ac 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss @@ -3,7 +3,7 @@ grid-template-columns: repeat(2, 1fr); justify-content: center; align-items: center; - width: 400px; + width: 350px; gap: 10px; } @@ -16,29 +16,48 @@ nz-option { align-items: center; justify-content: space-between; width: 100%; + gap: 10px; } .unit-details { display: flex; align-items: center; + justify-content: space-between; gap: 8px; flex-grow: 1; } -.terminate-box { - width: 16px; /* Size of the red box */ +.terminate-icon { + color: red; + cursor: pointer; + width: 16px; height: 16px; - background-color: red; - border-radius: 2px; /* Optional, if you want slightly rounded corners */ - cursor: pointer; /* Pointer cursor to indicate it's clickable */ + display: flex; + align-items: center; + justify-content: center; + border: 2px solid red; + border-radius: 50%; + padding: 1px; +} + +.create-unit { + width: 65%; + margin-top: 10px; + margin-bottom: 10px; } -.terminate-box:hover { +.dropdown-footer { + display: flex; + align-items: center; + justify-content: center; +} + +.terminate-icon:hover { opacity: 0.8; /* Slight opacity on hover for visual feedback */ } #computing-units-list { - width: 250px; + width: 225px; } .metrics-container { @@ -69,7 +88,7 @@ nz-option { } .resource-metrics { - width: 200px; + width: 250px; display: grid; grid-template-columns: repeat(2, 1fr); grid-template-rows: repeat(1, 1fr); diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 45230fe5331..197c3991d09 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -1,13 +1,15 @@ -import { Component, Input, OnInit } from "@angular/core"; -import { interval } from "rxjs"; +import {Component, Input, OnInit} from "@angular/core"; +import {interval} from "rxjs"; import {switchMap} from "rxjs/operators"; -import { WorkflowComputingUnitManagingService } from "../../service/workflow-computing-unit/workflow-computing-unit-managing.service"; +import { + WorkflowComputingUnitManagingService +} from "../../service/workflow-computing-unit/workflow-computing-unit-managing.service"; import {DashboardWorkflowComputingUnit} from "../../types/workflow-computing-unit"; -import { NotificationService } from "../../../common/service/notification/notification.service"; -import { WorkflowWebsocketService } from "../../service/workflow-websocket/workflow-websocket.service"; -import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; -import { isDefined } from "../../../common/util/predicate"; -import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; +import {NotificationService} from "../../../common/service/notification/notification.service"; +import {WorkflowWebsocketService} from "../../service/workflow-websocket/workflow-websocket.service"; +import {WorkflowActionService} from "../../service/workflow-graph/model/workflow-action.service"; +import {isDefined} from "../../../common/util/predicate"; +import {UntilDestroy, untilDestroyed} from "@ngneat/until-destroy"; @UntilDestroy() @Component({ @@ -104,7 +106,7 @@ export class ComputingUnitSelectionComponent implements OnInit { .pipe(untilDestroyed(this)) .subscribe({ next: (res: Response) => { - this.notificationService.success(`Terminated computing unit with URI: ${uri}`); + this.notificationService.success(`Terminated ${this.getComputingUnitName(uri)}`); this.refreshComputingUnits(); }, error: (err: unknown) => this.notificationService.error("Failed to terminate computing unit"), @@ -126,10 +128,74 @@ export class ComputingUnitSelectionComponent implements OnInit { } } + getComputingUnitName(unitURI: String) : String { + // computing-unit-85.workflow-computing-unit-svc.workflow-computing-unit-pool.svc.cluster.local + const computingUnit = unitURI.split('.')[0] // "computing-unit-85" + return computingUnit + .split('-') + .map((word, index) => + index < 2 ? word.charAt(0).toUpperCase() + word.slice(1) : word + ) + .join(' ') + } + /** * Get badge color based on the unit's status. */ getBadgeColor(status: string): string { return status === "Running" ? "green" : "yellow"; } + + getCpuLimit(): number { + // return 1 by default to avoid division by zero error + return +(this.selectedComputingUnit?.resourceLimits?.cpuLimit?.value || 1); + } + + getCpuLimitUnit(): String { + return this.selectedComputingUnit?.resourceLimits?.cpuLimit?.unit || "Cores"; + } + + getMemoryLimit(): number { + return +(this.selectedComputingUnit?.resourceLimits?.memoryLimit?.value || 1); + } + + getMemoryLimitUnit(): String { + return this.selectedComputingUnit?.resourceLimits?.memoryLimit?.unit || "MiB"; + } + + getCpuValue(): number { + return +(this.selectedComputingUnit?.metrics?.cpuUsage?.value || 0); + } + + getMemoryValue(): number { + return +(this.selectedComputingUnit?.metrics?.memoryUsage?.value || 0); + } + + getCpuPercentage(): number { + return this.getCpuValue() / this.getCpuLimit() * 100; + } + + getCpuStatus(): "success" | "exception" | "active" | "normal" { + const usage = this.getCpuValue(); + const limit = this.getCpuLimit(); + return usage >= limit ? "exception" : "active"; + } + + getMemoryPercentage(): number { + return this.getMemoryValue() / this.getMemoryLimit() * 100; + } + + getMemoryStatus(): "success" | "exception" | "active" | "normal" { + const usage = this.getMemoryValue(); + const limit = this.getMemoryLimit(); + return usage >= limit ? "exception" : "active"; + } + + getCpuUnit(): String { + return this.selectedComputingUnit?.metrics?.cpuUsage?.unit || ""; + } + + getMemoryUnit(): String { + return this.selectedComputingUnit?.metrics?.memoryUsage?.unit || ""; + } } diff --git a/core/gui/src/app/workspace/types/workflow-computing-unit.ts b/core/gui/src/app/workspace/types/workflow-computing-unit.ts index 4d8944d6b7c..7da01599331 100644 --- a/core/gui/src/app/workspace/types/workflow-computing-unit.ts +++ b/core/gui/src/app/workspace/types/workflow-computing-unit.ts @@ -6,14 +6,26 @@ export interface WorkflowComputingUnit { terminateTime: number | undefined; } +export interface ResourceValue { + metric: string; + value: string; + unit: string; +} + +export interface WorkflowComputingUnitResourceLimit { + cpuLimit: ResourceValue; + memoryLimit: ResourceValue; +} + +export interface WorkflowComputingUnitMetrics { + cpuUsage: ResourceValue; + memoryUsage: ResourceValue; +} + export interface DashboardWorkflowComputingUnit { computingUnit: WorkflowComputingUnit; uri: string; status: string; metrics: WorkflowComputingUnitMetrics; -} - -export interface WorkflowComputingUnitMetrics { - cpuUsage: number; - memoryUsage: number; + resourceLimits: WorkflowComputingUnitResourceLimit; } diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala index 44225a5e60b..13b50105f6f 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala @@ -6,7 +6,7 @@ import edu.uci.ics.texera.dao.SqlServer.withTransaction import edu.uci.ics.texera.dao.jooq.generated.tables.daos.WorkflowComputingUnitDao import edu.uci.ics.texera.dao.jooq.generated.tables.WorkflowComputingUnit.WORKFLOW_COMPUTING_UNIT import edu.uci.ics.texera.dao.jooq.generated.tables.pojos.WorkflowComputingUnit -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{DashboardWorkflowComputingUnit, TerminationResponse, WorkflowComputingUnitCreationParams, WorkflowComputingUnitMetrics, WorkflowComputingUnitTerminationParams, context} +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{DashboardWorkflowComputingUnit, ResourceValue, TerminationResponse, WorkflowComputingUnitCreationParams, WorkflowComputingUnitMetrics, WorkflowComputingUnitResourceLimit, WorkflowComputingUnitTerminationParams, context} import edu.uci.ics.texera.service.util.KubernetesClientService import edu.uci.ics.texera.service.util.KubernetesMetricService.getPodMetrics import jakarta.ws.rs._ @@ -26,13 +26,24 @@ object WorkflowComputingUnitManagingResource { case class WorkflowComputingUnitTerminationParams(uri: String, name: String) - case class WorkflowComputingUnitMetrics(cpuUsage: Double, memoryUsage: Double) + case class ResourceValue(metric: String, value: String, unit: String) + + case class WorkflowComputingUnitResourceLimit( + cpuLimit: Option[ResourceValue], + memoryLimit: Option[ResourceValue] + ) + + case class WorkflowComputingUnitMetrics( + cpuUsage: Option[ResourceValue], + memoryUsage: Option[ResourceValue] + ) case class DashboardWorkflowComputingUnit( computingUnit: WorkflowComputingUnit, uri: String, status: String, - metrics: WorkflowComputingUnitMetrics + metrics: WorkflowComputingUnitMetrics, + resourceLimits: WorkflowComputingUnitResourceLimit ) case class TerminationResponse(message: String, uri: String) @@ -80,7 +91,8 @@ class WorkflowComputingUnitManagingResource { insertedUnit, KubernetesClientService.generatePodURI(cuid).toString, pod.getStatus.getPhase, - getComputingUnitMetric(cuid.toString) + getComputingUnitMetric(cuid.toString), + getComputingUnitLimits ) } } @@ -112,7 +124,8 @@ class WorkflowComputingUnitManagingResource { computingUnit = unit, uri = KubernetesClientService.generatePodURI(cuid).toString, status = if (pod != null && pod.getStatus != null) pod.getStatus.getPhase else "Unknown", - getComputingUnitMetric(cuid.toString) + getComputingUnitMetric(cuid.toString), + getComputingUnitLimits ) }) @@ -148,15 +161,39 @@ class WorkflowComputingUnitManagingResource { TerminationResponse(s"Successfully terminated compute unit with URI $podURI", podURI) } + /** + * Retrieves the CPU and memory metrics for a computing unit identified by its `cuid`. + * + * @param cuid The computing unit ID. + * @return A `WorkflowComputingUnitMetrics` object with CPU and memory usage data. + */ @GET @Produces(Array(MediaType.APPLICATION_JSON)) @Path("/{cuid}/metrics") def getComputingUnitMetric(@PathParam("cuid") cuid: String): WorkflowComputingUnitMetrics = { - val metrics: Map[String, Any] = getPodMetrics(cuid.toInt) + val metrics: Map[String, Map[String, String]] = getPodMetrics(cuid.toInt) WorkflowComputingUnitMetrics( - cpuUsage = metrics.get("cpu").collect { case value: Double => value }.getOrElse(0.0), - memoryUsage = metrics.get("memory").collect { case value: Double => value }.getOrElse(0.0) + cpuUsage = Option(ResourceValue( + "cpu", + metrics.getOrElse("cpu", Map("value" -> "0", "unit" -> "Cores"))("value"), + metrics.getOrElse("cpu", Map("value" -> "0", "unit" -> "Cores"))("unit") + )), + memoryUsage = Option(ResourceValue( + "memory", + metrics.getOrElse("memory", Map("value" -> "0", "unit" -> "MiB"))("value"), + metrics.getOrElse("memory", Map("value" -> "0", "unit" -> "MiB"))("unit") + )) ) } -} + + private def getComputingUnitLimits: WorkflowComputingUnitResourceLimit = { + val cpuValue: Map[String, String] = KubernetesClientService.getPodCPULimit + val memoryValue: Map[String, String] = KubernetesClientService.getPodMemoryLimit + + WorkflowComputingUnitResourceLimit( + cpuLimit = Option(ResourceValue("cpu", cpuValue("value"), cpuValue("unit"))), + memoryLimit = Option(ResourceValue("memory", memoryValue("value"), memoryValue("unit"))) + ) + } +} \ No newline at end of file diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala index 487661369f6..a6f91b5413f 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala @@ -1,12 +1,9 @@ package edu.uci.ics.texera.service.util import config.WorkflowComputingUnitManagingServiceConf -import config.WorkflowComputingUnitManagingServiceConf.{ - computeUnitImageName, - computeUnitPortNumber, - computeUnitServiceName -} +import config.WorkflowComputingUnitManagingServiceConf.{computeUnitImageName, computeUnitPortNumber, computeUnitServiceName} import edu.uci.ics.amber.core.storage.StorageConfig +import io.kubernetes.client.custom.Quantity import io.kubernetes.client.openapi.apis.CoreV1Api import io.kubernetes.client.openapi.models._ import io.kubernetes.client.openapi.{ApiClient, Configuration} @@ -28,6 +25,12 @@ object KubernetesClientService { private val poolNamespace: String = WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace + private val cpuLimit: Quantity = new Quantity("1") // in Cores + private val memoryLimit: Quantity = new Quantity("1Gi") // in Gibibytes + + private val cpuRequest: Quantity = new Quantity("0.5") // in Cores + private val memoryRequest: Quantity = new Quantity("0.5Gi") // in Gibibytes + /** * Generates a URI for the pod based on the computing unit ID (cuid). * @@ -160,6 +163,21 @@ object KubernetesClientService { new V1EnvVar().name("JDBC_PASSWORD").value(StorageConfig.jdbcPassword) ) ) + .resources( + new V1ResourceRequirements() + .limits( + util.Map.of( + "cpu", cpuLimit, + "memory", memoryLimit + ) + ) + .requests( + util.Map.of( + "cpu", cpuRequest, + "memory", memoryRequest + ) + ) + ) ) ) .hostname(podName) @@ -179,6 +197,23 @@ object KubernetesClientService { coreApi.deleteNamespacedPod(generatePodName(cuid), poolNamespace).execute() } + /** + * Gets a Pods CPU limit + * + * @return A map representing the pod CPU limit value and unit. + */ + def getPodCPULimit: Map[String, String] = Map("value" -> cpuLimit.getNumber.toString, "unit" -> "Cores") // Default is Cores + + /** + * Gets a Pods memory limit + * + * @return A map representing the pod memory limit value and unit. + */ + def getPodMemoryLimit: Map[String, String] = { + // Quantity returns the byte representation of memory no matter what + Map("value" -> MemoryUnit.convert(memoryLimit.getNumber.doubleValue(), MemoryUnit.Byte, MemoryUnit.Mebibyte).toString, "unit" -> "MiB") + } + /** * Waits for the pod to reach the desired status. * diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala index 36c36556ee3..6bad812ad59 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala @@ -2,7 +2,9 @@ package edu.uci.ics.texera.service.util import config.WorkflowComputingUnitManagingServiceConf import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetricsList +import io.fabric8.kubernetes.api.model.Quantity import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder} + import scala.jdk.CollectionConverters._ object KubernetesMetricService { @@ -10,17 +12,15 @@ object KubernetesMetricService { // Initialize the Kubernetes client val client: KubernetesClient = new KubernetesClientBuilder().build() - private val MEGABYTES_PER_GIGABYTE = 1024 - private val NANOCORES_PER_CPU_CORE = 1_000_000_000 private val namespace = WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace /** - * Retrieves the pod metric for a given name in the specified namespace. + * Retrieves the pod metric for a given ID in the specified namespace. * * @param cuid The computing unit id of the pod * @return The Pod metrics for a given name in a specified namespace. */ - def getPodMetrics(cuid: Int): Map[String, Any] = { + def getPodMetrics(cuid: Int): Map[String, Map[String, String]] = { val podMetricsList: PodMetricsList = client.top().pods().metrics(namespace) val targetPodName = KubernetesClientService.generatePodName(cuid) @@ -29,33 +29,32 @@ object KubernetesMetricService { podMetrics.getContainers.asScala.collectFirst { case container => container.getUsage.asScala.collect { - case ("cpu", value) => - val cpuInCores = convertCPUToCores(value.getAmount) - println(s"CPU Usage: $cpuInCores cores") - "cpu" -> cpuInCores - case ("memory", value) => - val memoryInMB = convertMemoryToMB(value.getAmount) - println(s"Memory Usage: $memoryInMB MB") - "memory" -> memoryInMB - case (key, value) => - println(s"Other Metric - $key: ${value.getAmount}") - // Other metrics may not all be Double - key -> value.getAmount + case (metric, value) => + println(s"Metric - $metric: ${value}") + // CPU is in nanocores and Memory is in Kibibyte + metric -> mapMetricWithUnit(metric, value) }.toMap - }.getOrElse(Map.empty[String, Double]) + }.getOrElse(Map.empty[String, Map[String, String]]) }.getOrElse { println(s"No metrics found for pod: $targetPodName in namespace: $namespace") - Map.empty[String, Double] + Map.empty[String, Map[String, String]] } } - private def convertMemoryToMB(memoryUsage: String): Double = { - val memoryUsageInKi = memoryUsage.toDoubleOption.getOrElse(0.0) - memoryUsageInKi / MEGABYTES_PER_GIGABYTE - } + /** + * Maps metric value with its associated unit, converting the unit to a more readable format. + * + * @param metric The name of the metric (e.g., "cpu", "memory"). + * @param quantity The value of the metric, represented as a Quantity object. + * @return A map containing the metric value in a readable format and the corresponding unit. + */ + private def mapMetricWithUnit(metric: String, quantity: Quantity): Map[String, String] = { + val (value, unit) = metric match { + case "cpu" => (CpuUnit.convert(quantity.getAmount.toDouble, CpuUnit.Nanocores, CpuUnit.Cores), "Cores") + case "memory" => (MemoryUnit.convert(quantity.getAmount.toDouble, MemoryUnit.Kibibyte, MemoryUnit.Mebibyte), "MiB") + case _ => (quantity.getAmount, "unknown") + } - private def convertCPUToCores(cpuUsage: String): Double = { - val cpuUsageInNano = cpuUsage.toDoubleOption.getOrElse(0.0) - cpuUsageInNano / NANOCORES_PER_CPU_CORE + Map("value" -> value.toString, "unit" -> unit) } } From c019fe51b55d5f638baa84e6ff61be2adbc11092 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Tue, 18 Feb 2025 13:30:45 -0800 Subject: [PATCH 08/18] added helper file for workflowComputingUnitManagingResource --- .../service/util/KubernetesHelper.scala | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala new file mode 100644 index 00000000000..f3ca7d48619 --- /dev/null +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala @@ -0,0 +1,27 @@ +package edu.uci.ics.texera.service.util + +import scala.math.pow + +// Can use enums in Scala 3 +sealed abstract class CpuUnit(val nanocores: Double) +object CpuUnit { + case object Nanocores extends CpuUnit(1.0) + case object Millicores extends CpuUnit(1e6) + case object Cores extends CpuUnit(1e9) + + def convert(value: Double, from: CpuUnit, to: CpuUnit): Double = { + value * (from.nanocores / to.nanocores) + } +} + +sealed abstract class MemoryUnit(val bytes: Double) +object MemoryUnit { + case object Byte extends MemoryUnit(1) + case object Kibibyte extends MemoryUnit(1024) + case object Mebibyte extends MemoryUnit(pow(1024, 2).intValue) + case object Gibibyte extends MemoryUnit(pow(1024, 3).intValue) + + def convert(value: Double, from: MemoryUnit, to: MemoryUnit): Double = { + value * (from.bytes / to.bytes) + } +} From 1af80be4be7f0b8d7a104a35586efa7e908c8adb Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sat, 22 Feb 2025 12:18:47 -0800 Subject: [PATCH 09/18] after compute unit selection title normal weight --- .../power-button/computing-unit-selection.component.html | 4 +--- core/gui/src/styles.scss | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index 05c4c52f290..18b12c3c242 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -38,6 +38,7 @@ nzSize="default" nzShowSearch nzAllowClear + nzDropdownClassName="ant-select-item-title-weight" [nzDropdownRender]="renderTemplate" nzPlaceHolder="Select a computing unit" [(ngModel)]="selectedComputingUnit" @@ -52,9 +53,6 @@ - diff --git a/core/gui/src/styles.scss b/core/gui/src/styles.scss index 417919e1f00..7ad6cb45e46 100644 --- a/core/gui/src/styles.scss +++ b/core/gui/src/styles.scss @@ -95,3 +95,7 @@ hr { #memory-progress-bar * { line-height: 16px !important; } + +.ant-select-item-title-weight .ant-select-item{ + font-weight: normal !important; +} From 21fe62da4a6a01ea7be58918d8990bfa00d776cb Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sun, 23 Feb 2025 01:41:38 -0800 Subject: [PATCH 10/18] changes to yaml configuration and image creation --- core/amber/computing-unit.dockerfile | 2 ++ core/amber/webserver.dockerfile | 2 +- core/scripts/server.sh | 1 + .../workflow-computing-unit-manager-service-account.yaml | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/amber/computing-unit.dockerfile b/core/amber/computing-unit.dockerfile index 96529b22d4f..a5b21f04d48 100644 --- a/core/amber/computing-unit.dockerfile +++ b/core/amber/computing-unit.dockerfile @@ -13,6 +13,8 @@ RUN apt-get update && apt-get install -y \ netcat \ unzip \ python3-pip \ + libpq-dev \ + python3-dev \ && apt-get clean # Install Python dependencies diff --git a/core/amber/webserver.dockerfile b/core/amber/webserver.dockerfile index 76b3ce9f9c2..e0c0d035ff3 100644 --- a/core/amber/webserver.dockerfile +++ b/core/amber/webserver.dockerfile @@ -26,7 +26,7 @@ COPY core/ . RUN rm -rf amber/user-resources/* RUN apt-get update -RUN apt-get install -y netcat unzip python3-pip +RUN apt-get install -y netcat unzip python3-pip libpq-dev python3-dev RUN pip3 install python-lsp-server python-lsp-server[websockets] RUN pip3 install -r amber/requirements.txt diff --git a/core/scripts/server.sh b/core/scripts/server.sh index 61201d64e06..282b8cc5689 100755 --- a/core/scripts/server.sh +++ b/core/scripts/server.sh @@ -1,2 +1,3 @@ +#!/bin/bash cd amber target/texera-0.1-SNAPSHOT/bin/texera-web-application \ No newline at end of file diff --git a/core/scripts/texera-helmchart/templates/workflow-computing-unit-manager-service-account.yaml b/core/scripts/texera-helmchart/templates/workflow-computing-unit-manager-service-account.yaml index 4c19642df40..cd14d575c06 100644 --- a/core/scripts/texera-helmchart/templates/workflow-computing-unit-manager-service-account.yaml +++ b/core/scripts/texera-helmchart/templates/workflow-computing-unit-manager-service-account.yaml @@ -14,6 +14,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["metrics.k8s.io"] # Added metrics permissions + resources: ["pods"] + verbs: ["list", "get"] # Added metrics permissions --- apiVersion: rbac.authorization.k8s.io/v1 From 1b24db00eefe7b592e133f630f309d06a8b203cb Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sun, 23 Feb 2025 23:51:29 -0800 Subject: [PATCH 11/18] texera helm chart modifications --- .../texera-helmchart/files/texera_ddl.sql | 45 ++++++++++--------- .../workflow-computing-units-deployment.yaml | 1 + 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/core/scripts/texera-helmchart/files/texera_ddl.sql b/core/scripts/texera-helmchart/files/texera_ddl.sql index 3c188e22c52..71dd433aa6a 100644 --- a/core/scripts/texera-helmchart/files/texera_ddl.sql +++ b/core/scripts/texera-helmchart/files/texera_ddl.sql @@ -1,7 +1,6 @@ CREATE SCHEMA IF NOT EXISTS `texera_db`; USE `texera_db`; -DROP TABLE IF EXISTS `workflow_runtime_statistics`; DROP TABLE IF EXISTS `workflow_user_access`; DROP TABLE IF EXISTS `workflow_of_user`; DROP TABLE IF EXISTS `user_config`; @@ -14,6 +13,8 @@ DROP TABLE IF EXISTS `workflow_executions`; DROP TABLE IF EXISTS `dataset`; DROP TABLE IF EXISTS `dataset_user_access`; DROP TABLE IF EXISTS `dataset_version`; +DROP TABLE IF EXISTS operator_executions; +DROP TABLE IF EXISTS operator_runtime_statistics; SET PERSIST time_zone = '+00:00'; -- this line is mandatory SET PERSIST sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); @@ -143,24 +144,6 @@ CREATE TABLE IF NOT EXISTS public_project FOREIGN KEY (`pid`) REFERENCES `project` (`pid`) ON DELETE CASCADE ) ENGINE = INNODB; -CREATE TABLE IF NOT EXISTS workflow_runtime_statistics -( - `workflow_id` INT UNSIGNED NOT NULL, - `execution_id` INT UNSIGNED NOT NULL, - `operator_id` VARCHAR(100) NOT NULL, - `time` TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), - `input_tuple_cnt` INT UNSIGNED NOT NULL DEFAULT 0, - `output_tuple_cnt` INT UNSIGNED NOT NULL DEFAULT 0, - `status` TINYINT NOT NULL DEFAULT 1, - `data_processing_time` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `control_processing_time` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `idle_time` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `num_workers` INT UNSIGNED NOT NULL DEFAULT 0, - PRIMARY KEY (`workflow_id`, `execution_id`, `operator_id`, `time`), - FOREIGN KEY (`workflow_id`) REFERENCES `workflow` (`wid`) ON DELETE CASCADE, - FOREIGN KEY (`execution_id`) REFERENCES `workflow_executions` (`eid`) ON DELETE CASCADE -) ENGINE = INNODB; - CREATE TABLE IF NOT EXISTS dataset ( `did` INT UNSIGNED AUTO_INCREMENT NOT NULL, @@ -259,4 +242,26 @@ CREATE TABLE IF NOT EXISTS workflow_computing_unit `creation_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, `terminate_time` TIMESTAMP DEFAULT NULL, PRIMARY KEY (`cuid`) -) ENGINE = INNODB; \ No newline at end of file +) ENGINE = INNODB; + +CREATE TABLE IF NOT EXISTS operator_executions ( + operator_execution_id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + workflow_execution_id INT UNSIGNED NOT NULL, + operator_id VARCHAR(100) NOT NULL, + UNIQUE (workflow_execution_id, operator_id), + FOREIGN KEY (workflow_execution_id) REFERENCES workflow_executions (eid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS operator_runtime_statistics ( + operator_execution_id BIGINT UNSIGNED NOT NULL, + time TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + input_tuple_cnt BIGINT UNSIGNED NOT NULL DEFAULT 0, + output_tuple_cnt BIGINT UNSIGNED NOT NULL DEFAULT 0, + status TINYINT NOT NULL DEFAULT 1, + data_processing_time BIGINT UNSIGNED NOT NULL DEFAULT 0, + control_processing_time BIGINT UNSIGNED NOT NULL DEFAULT 0, + idle_time BIGINT UNSIGNED NOT NULL DEFAULT 0, + num_workers INT UNSIGNED NOT NULL DEFAULT 0, + PRIMARY KEY (operator_execution_id, time), + FOREIGN KEY (operator_execution_id) REFERENCES operator_executions (operator_execution_id) ON DELETE CASCADE +); \ No newline at end of file diff --git a/core/scripts/texera-helmchart/templates/workflow-computing-units-deployment.yaml b/core/scripts/texera-helmchart/templates/workflow-computing-units-deployment.yaml index 63d1482cc57..90f89c3cc56 100644 --- a/core/scripts/texera-helmchart/templates/workflow-computing-units-deployment.yaml +++ b/core/scripts/texera-helmchart/templates/workflow-computing-units-deployment.yaml @@ -18,6 +18,7 @@ spec: containers: - name: {{ .Values.workflowComputingUnit.name }} image: {{ .Values.workflowComputingUnit.imageName }} + imagePullPolicy: Never ports: - containerPort: {{ .Values.workflowComputingUnitPool.service.port }} env: From 54ed658a2b3392b53f1035e96163bb8d4801f359 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Tue, 25 Feb 2025 13:23:03 -0800 Subject: [PATCH 12/18] refactored computing unit limits --- ...orkflowComputingUnitManagingResource.scala | 48 ++++++++----------- .../util/KubernetesClientService.scala | 40 +++------------- .../service/util/KubernetesHelper.scala | 27 ----------- .../util/KubernetesMetricService.scala | 41 +++++++++------- 4 files changed, 52 insertions(+), 104 deletions(-) delete mode 100644 core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala index 13b50105f6f..841b76dfdbf 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala @@ -6,9 +6,9 @@ import edu.uci.ics.texera.dao.SqlServer.withTransaction import edu.uci.ics.texera.dao.jooq.generated.tables.daos.WorkflowComputingUnitDao import edu.uci.ics.texera.dao.jooq.generated.tables.WorkflowComputingUnit.WORKFLOW_COMPUTING_UNIT import edu.uci.ics.texera.dao.jooq.generated.tables.pojos.WorkflowComputingUnit -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{DashboardWorkflowComputingUnit, ResourceValue, TerminationResponse, WorkflowComputingUnitCreationParams, WorkflowComputingUnitMetrics, WorkflowComputingUnitResourceLimit, WorkflowComputingUnitTerminationParams, context} +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{DashboardWorkflowComputingUnit, TerminationResponse, WorkflowComputingUnitCreationParams, WorkflowComputingUnitMetrics, WorkflowComputingUnitResourceLimit, WorkflowComputingUnitTerminationParams, context} import edu.uci.ics.texera.service.util.KubernetesClientService -import edu.uci.ics.texera.service.util.KubernetesMetricService.getPodMetrics +import edu.uci.ics.texera.service.util.KubernetesMetricService.{getPodLimits, getPodMetrics} import jakarta.ws.rs._ import jakarta.ws.rs.core.MediaType import org.jooq.DSLContext @@ -22,20 +22,18 @@ object WorkflowComputingUnitManagingResource { .getInstance(StorageConfig.jdbcUrl, StorageConfig.jdbcUsername, StorageConfig.jdbcPassword) .createDSLContext() - case class WorkflowComputingUnitCreationParams(name: String, unitType: String) + case class WorkflowComputingUnitCreationParams(name: String, unitType: String, cpuLimit: String, memoryLimit: String) case class WorkflowComputingUnitTerminationParams(uri: String, name: String) - case class ResourceValue(metric: String, value: String, unit: String) - case class WorkflowComputingUnitResourceLimit( - cpuLimit: Option[ResourceValue], - memoryLimit: Option[ResourceValue] + cpuLimit: String, + memoryLimit: String ) case class WorkflowComputingUnitMetrics( - cpuUsage: Option[ResourceValue], - memoryUsage: Option[ResourceValue] + cpuUsage: String, + memoryUsage: String ) case class DashboardWorkflowComputingUnit( @@ -84,7 +82,7 @@ class WorkflowComputingUnitManagingResource { val insertedUnit = wcDao.fetchOneByCuid(UInteger.valueOf(cuid)) // Create the pod with the generated CUID - val pod = KubernetesClientService.createPod(cuid) + val pod = KubernetesClientService.createPod(cuid, param.cpuLimit, param.memoryLimit) // Return the dashboard response DashboardWorkflowComputingUnit( @@ -92,7 +90,7 @@ class WorkflowComputingUnitManagingResource { KubernetesClientService.generatePodURI(cuid).toString, pod.getStatus.getPhase, getComputingUnitMetric(cuid.toString), - getComputingUnitLimits + WorkflowComputingUnitResourceLimit(param.cpuLimit, param.memoryLimit) ) } } @@ -125,7 +123,7 @@ class WorkflowComputingUnitManagingResource { uri = KubernetesClientService.generatePodURI(cuid).toString, status = if (pod != null && pod.getStatus != null) pod.getStatus.getPhase else "Unknown", getComputingUnitMetric(cuid.toString), - getComputingUnitLimits + getComputingUnitLimits(cuid.toString) ) }) @@ -171,29 +169,23 @@ class WorkflowComputingUnitManagingResource { @Produces(Array(MediaType.APPLICATION_JSON)) @Path("/{cuid}/metrics") def getComputingUnitMetric(@PathParam("cuid") cuid: String): WorkflowComputingUnitMetrics = { - val metrics: Map[String, Map[String, String]] = getPodMetrics(cuid.toInt) + val metrics: Map[String, String] = getPodMetrics(cuid.toInt) WorkflowComputingUnitMetrics( - cpuUsage = Option(ResourceValue( - "cpu", - metrics.getOrElse("cpu", Map("value" -> "0", "unit" -> "Cores"))("value"), - metrics.getOrElse("cpu", Map("value" -> "0", "unit" -> "Cores"))("unit") - )), - memoryUsage = Option(ResourceValue( - "memory", - metrics.getOrElse("memory", Map("value" -> "0", "unit" -> "MiB"))("value"), - metrics.getOrElse("memory", Map("value" -> "0", "unit" -> "MiB"))("unit") - )) + metrics.getOrElse("cpu", ""), + metrics.getOrElse("memory", "") ) } - private def getComputingUnitLimits: WorkflowComputingUnitResourceLimit = { - val cpuValue: Map[String, String] = KubernetesClientService.getPodCPULimit - val memoryValue: Map[String, String] = KubernetesClientService.getPodMemoryLimit + @GET + @Produces(Array(MediaType.APPLICATION_JSON)) + @Path("/{cuid}/limits") + def getComputingUnitLimits(@PathParam("cuid") cuid: String): WorkflowComputingUnitResourceLimit = { + val podLimits: Map[String, String] = getPodLimits(cuid.toInt) WorkflowComputingUnitResourceLimit( - cpuLimit = Option(ResourceValue("cpu", cpuValue("value"), cpuValue("unit"))), - memoryLimit = Option(ResourceValue("memory", memoryValue("value"), memoryValue("unit"))) + podLimits.getOrElse("cpu", ""), + podLimits.getOrElse("memory", "") ) } } \ No newline at end of file diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala index a6f91b5413f..2cd0a5b2511 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala @@ -25,12 +25,6 @@ object KubernetesClientService { private val poolNamespace: String = WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace - private val cpuLimit: Quantity = new Quantity("1") // in Cores - private val memoryLimit: Quantity = new Quantity("1Gi") // in Gibibytes - - private val cpuRequest: Quantity = new Quantity("0.5") // in Cores - private val memoryRequest: Quantity = new Quantity("0.5Gi") // in Gibibytes - /** * Generates a URI for the pod based on the computing unit ID (cuid). * @@ -123,12 +117,15 @@ object KubernetesClientService { * @param cuid The computing unit ID. * @return The newly created V1Pod object. */ - def createPod(cuid: Int): V1Pod = { + def createPod(cuid: Int, cpuLimit: String, memoryLimit: String): V1Pod = { val podName = generatePodName(cuid) if (getPodFromLabel(poolNamespace, s"name=$podName") != null) { throw new Exception(s"Pod with cuid $cuid already exists") } + val cpu: Quantity = new Quantity(cpuLimit) + val memory: Quantity = new Quantity(memoryLimit) + val pod: V1Pod = new V1Pod() .apiVersion("v1") .kind("Pod") @@ -167,16 +164,10 @@ object KubernetesClientService { new V1ResourceRequirements() .limits( util.Map.of( - "cpu", cpuLimit, - "memory", memoryLimit - ) - ) - .requests( - util.Map.of( - "cpu", cpuRequest, - "memory", memoryRequest + "cpu", cpu, + "memory", memory ) - ) + ) // may want to add requests as well to make efficient use of the CPU resources ) ) ) @@ -197,23 +188,6 @@ object KubernetesClientService { coreApi.deleteNamespacedPod(generatePodName(cuid), poolNamespace).execute() } - /** - * Gets a Pods CPU limit - * - * @return A map representing the pod CPU limit value and unit. - */ - def getPodCPULimit: Map[String, String] = Map("value" -> cpuLimit.getNumber.toString, "unit" -> "Cores") // Default is Cores - - /** - * Gets a Pods memory limit - * - * @return A map representing the pod memory limit value and unit. - */ - def getPodMemoryLimit: Map[String, String] = { - // Quantity returns the byte representation of memory no matter what - Map("value" -> MemoryUnit.convert(memoryLimit.getNumber.doubleValue(), MemoryUnit.Byte, MemoryUnit.Mebibyte).toString, "unit" -> "MiB") - } - /** * Waits for the pod to reach the desired status. * diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala deleted file mode 100644 index f3ca7d48619..00000000000 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesHelper.scala +++ /dev/null @@ -1,27 +0,0 @@ -package edu.uci.ics.texera.service.util - -import scala.math.pow - -// Can use enums in Scala 3 -sealed abstract class CpuUnit(val nanocores: Double) -object CpuUnit { - case object Nanocores extends CpuUnit(1.0) - case object Millicores extends CpuUnit(1e6) - case object Cores extends CpuUnit(1e9) - - def convert(value: Double, from: CpuUnit, to: CpuUnit): Double = { - value * (from.nanocores / to.nanocores) - } -} - -sealed abstract class MemoryUnit(val bytes: Double) -object MemoryUnit { - case object Byte extends MemoryUnit(1) - case object Kibibyte extends MemoryUnit(1024) - case object Mebibyte extends MemoryUnit(pow(1024, 2).intValue) - case object Gibibyte extends MemoryUnit(pow(1024, 3).intValue) - - def convert(value: Double, from: MemoryUnit, to: MemoryUnit): Double = { - value * (from.bytes / to.bytes) - } -} diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala index 6bad812ad59..62dd97f690e 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala @@ -2,7 +2,7 @@ package edu.uci.ics.texera.service.util import config.WorkflowComputingUnitManagingServiceConf import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetricsList -import io.fabric8.kubernetes.api.model.Quantity +import io.fabric8.kubernetes.api.model.{PodList, Quantity} import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder} import scala.jdk.CollectionConverters._ @@ -20,7 +20,7 @@ object KubernetesMetricService { * @param cuid The computing unit id of the pod * @return The Pod metrics for a given name in a specified namespace. */ - def getPodMetrics(cuid: Int): Map[String, Map[String, String]] = { + def getPodMetrics(cuid: Int): Map[String, String] = { val podMetricsList: PodMetricsList = client.top().pods().metrics(namespace) val targetPodName = KubernetesClientService.generatePodName(cuid) @@ -32,29 +32,38 @@ object KubernetesMetricService { case (metric, value) => println(s"Metric - $metric: ${value}") // CPU is in nanocores and Memory is in Kibibyte - metric -> mapMetricWithUnit(metric, value) + metric -> value.toString }.toMap - }.getOrElse(Map.empty[String, Map[String, String]]) + }.getOrElse(Map.empty[String, String]) }.getOrElse { println(s"No metrics found for pod: $targetPodName in namespace: $namespace") - Map.empty[String, Map[String, String]] + Map.empty[String, String] } } /** - * Maps metric value with its associated unit, converting the unit to a more readable format. + * Retrieves the pod limits for a given ID in the specified namespace. * - * @param metric The name of the metric (e.g., "cpu", "memory"). - * @param quantity The value of the metric, represented as a Quantity object. - * @return A map containing the metric value in a readable format and the corresponding unit. + * @param cuid The computing unit id of the pod + * @return The Pod limits for a given name in a specified namespace. */ - private def mapMetricWithUnit(metric: String, quantity: Quantity): Map[String, String] = { - val (value, unit) = metric match { - case "cpu" => (CpuUnit.convert(quantity.getAmount.toDouble, CpuUnit.Nanocores, CpuUnit.Cores), "Cores") - case "memory" => (MemoryUnit.convert(quantity.getAmount.toDouble, MemoryUnit.Kibibyte, MemoryUnit.Mebibyte), "MiB") - case _ => (quantity.getAmount, "unknown") - } + def getPodLimits(cuid: Int): Map[String, String] = { + val podList: PodList = client.pods().inNamespace(namespace).list() + val targetPodName = KubernetesClientService.generatePodName(cuid) + + val pod = podList.getItems.asScala.find( + pod => { pod.getMetadata.getName.equals(targetPodName) } + ) - Map("value" -> value.toString, "unit" -> unit) + val limits: Map[String, String] = pod.flatMap { + pod => pod.getSpec.getContainers.asScala.headOption.map { + container => container.getResources.getLimits.asScala.map { + case (key, value) => + key -> value.toString + }.toMap + } + }.getOrElse(Map.empty[String, String]) + println(limits.toString()) + limits } } From 26f7ecdf6518a103da6eddb6b2eb1088987c836a Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Tue, 25 Feb 2025 21:57:39 -0800 Subject: [PATCH 13/18] workflow computing unit refactoring --- .../computing-unit-selection.component.ts | 151 +++++++++++++++--- ...orkflow-computing-unit-managing.service.ts | 6 +- .../types/workflow-computing-unit.ts | 14 +- 3 files changed, 138 insertions(+), 33 deletions(-) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 197c3991d09..d639fb55b36 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -78,7 +78,7 @@ export class ComputingUnitSelectionComponent implements OnInit { startComputingUnit(): void { const computeUnitName = `Compute for Workflow ${this.workflowId}`; this.computingUnitService - .createComputingUnit(computeUnitName) + .createComputingUnit(computeUnitName, "1", "2Gi") .pipe(untilDestroyed(this)) .subscribe({ next: (unit: DashboardWorkflowComputingUnit) => { @@ -128,15 +128,105 @@ export class ComputingUnitSelectionComponent implements OnInit { } } - getComputingUnitName(unitURI: String) : String { - // computing-unit-85.workflow-computing-unit-svc.workflow-computing-unit-pool.svc.cluster.local - const computingUnit = unitURI.split('.')[0] // "computing-unit-85" + /** + * Gets the computing unit name from the units URI + * @param unitURI (i.e. "computing-unit-85.workflow-computing-unit-svc.workflow-computing-unit-pool.svc.cluster.local") + * @return "Computing unit 85" + */ + getComputingUnitName(unitURI: string) : string { + const computingUnit = unitURI.split(".")[0] return computingUnit - .split('-') + .split("-") .map((word, index) => index < 2 ? word.charAt(0).toUpperCase() + word.slice(1) : word ) - .join(' ') + .join(" ") + } + + /** + * Parses computing units resource unit + * @param resource (i.e. "12412512n") + * @return associated unit with resource (i.e. "n", "Mi", "Gi", ...) + */ + parseResourceUnit(resource: string) : string { + const match = resource.match(/[a-z].*/i); + return match ? match[0] : ""; + } + + /** + * Parses computing units numerical value + * @param resource (i.e. "12412512n") + * @return associated number with resource (i.e. 12412512) + */ + parseResourceNumber(resource: string) : number { + const match = resource.match(/[0-9.]*/); + return match ? Number(match[0]) : 0; + } + + /** + * Convert computing cpu unit resource number to a specific unit + * @param from (i.e. "12412512n") + * @param toUnit (i.e. cores) + * @return i.e. 1.2412512 Cores + */ + cpuResourceConversion(from: string, toUnit: string) : string { + // CPU conversion constants (base unit: nanocores) + type CpuUnit = "n" | "u" | "m" | ""; + const cpuUnits: Record = { + "n": 1, // nanocores + "u": 10**3, // microcores + "m": 10**6, // millicores + "": 10**9 // cores + }; + + const fromNumber: number = this.parseResourceNumber(from); + const fromUnit: string = this.parseResourceUnit(from); + + if (!(fromUnit in cpuUnits) || !(toUnit in cpuUnits)) { + return ""; + } + return `${(fromNumber * (cpuUnits[fromUnit as CpuUnit] / cpuUnits[toUnit as CpuUnit]))} ${toUnit}`; + } + + /** + * Convert computing unit memory resource number to a specific unit + * @param from (i.e. "523Mi") + * @param toUnit (i.e. "Gi") + * @return i.e. 0.524 Gi + */ + memoryResourceConversion(from: string, toUnit: string) : string { + // Memory conversion constants (base unit: bytes) + type MemoryUnit = "Ki" | "Mi" | "Gi" | ""; + const memoryUnits = { + "": 1, // bytes + "Ki": 1024, // KiB + "Mi": 1024**2, // MiB + "Gi": 1024**3 // GiB + }; + + const fromNumber: number = this.parseResourceNumber(from); + const fromUnit: string = this.parseResourceUnit(from); + + if (!(fromUnit in memoryUnits) || !(toUnit in memoryUnits)) { + return ""; + } + return `${fromNumber * (memoryUnits[fromUnit as MemoryUnit] / memoryUnits[toUnit as MemoryUnit])} ${toUnit}`; + } + + getCurrentComputingUnitCpuUsage(): string { + return this.selectedComputingUnit?.metrics.cpuUsage || ""; + } + + getCurrentComputingUnitMemoryUsage(): string { + return this.selectedComputingUnit?.metrics.memoryUsage || ""; + } + + getCurrentComputingUnitCpuLimit(): string { + return this.selectedComputingUnit?.resourceLimits.cpuLimit || ""; + } + + getCurrentComputingUnitMemoryLimit(): string { + return this.selectedComputingUnit?.resourceLimits.memoryLimit || ""; } /** @@ -147,32 +237,46 @@ export class ComputingUnitSelectionComponent implements OnInit { } getCpuLimit(): number { - // return 1 by default to avoid division by zero error - return +(this.selectedComputingUnit?.resourceLimits?.cpuLimit?.value || 1); + return this.parseResourceNumber(this.getCurrentComputingUnitCpuLimit()); } - getCpuLimitUnit(): String { - return this.selectedComputingUnit?.resourceLimits?.cpuLimit?.unit || "Cores"; + getCpuLimitUnit(): string { + let unit = this.parseResourceUnit(this.getCurrentComputingUnitCpuLimit()); + if (!unit) { + return this.getCpuLimit() == 1 ? "Core" : "Cores"; + } + return this.parseResourceUnit(this.getCurrentComputingUnitCpuLimit()); } getMemoryLimit(): number { - return +(this.selectedComputingUnit?.resourceLimits?.memoryLimit?.value || 1); + return this.parseResourceNumber(this.getCurrentComputingUnitMemoryLimit()); } - getMemoryLimitUnit(): String { - return this.selectedComputingUnit?.resourceLimits?.memoryLimit?.unit || "MiB"; + getMemoryLimitUnit(): string { + return this.parseResourceUnit(this.getCurrentComputingUnitMemoryLimit()); } getCpuValue(): number { - return +(this.selectedComputingUnit?.metrics?.cpuUsage?.value || 0); + // convert to appropriate unit based on the limit unit + const cpuLimitUnit: string = this.getCpuLimitUnit(); + const convertedValue: string = this.cpuResourceConversion(this.getCurrentComputingUnitCpuUsage(), cpuLimitUnit); + return this.parseResourceNumber(convertedValue); } getMemoryValue(): number { - return +(this.selectedComputingUnit?.metrics?.memoryUsage?.value || 0); + // convert to appropriate unit based on the limit + const memoryLimitUnit: string = this.getMemoryLimitUnit(); + const convertedValue: string = this.memoryResourceConversion(this.getCurrentComputingUnitMemoryUsage(), memoryLimitUnit); + return this.parseResourceNumber(convertedValue); } getCpuPercentage(): number { - return this.getCpuValue() / this.getCpuLimit() * 100; + // handle divison by zero + const cpuLimit = this.getCpuLimit(); + if (cpuLimit <= 0) { + return 0; + } + return this.getCpuValue() / cpuLimit * 100; } getCpuStatus(): "success" | "exception" | "active" | "normal" { @@ -182,7 +286,12 @@ export class ComputingUnitSelectionComponent implements OnInit { } getMemoryPercentage(): number { - return this.getMemoryValue() / this.getMemoryLimit() * 100; + // handle divison by zero + const memoryLimit = this.getMemoryLimit(); + if (memoryLimit <= 0) { + return 0; + } + return this.getMemoryValue() / memoryLimit * 100; } getMemoryStatus(): "success" | "exception" | "active" | "normal" { @@ -191,11 +300,11 @@ export class ComputingUnitSelectionComponent implements OnInit { return usage >= limit ? "exception" : "active"; } - getCpuUnit(): String { - return this.selectedComputingUnit?.metrics?.cpuUsage?.unit || ""; + getCpuUnit(): string { + return this.parseResourceUnit(this.getCurrentComputingUnitCpuUsage()); } - getMemoryUnit(): String { - return this.selectedComputingUnit?.metrics?.memoryUsage?.unit || ""; + getMemoryUnit(): string { + return this.parseResourceUnit(this.getCurrentComputingUnitMemoryUsage()); } } diff --git a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts index 2d2dd9902d4..94ead506837 100644 --- a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts +++ b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts @@ -22,11 +22,13 @@ export class WorkflowComputingUnitManagingService { /** * Create a new workflow computing unit (pod). * @param name The name for the computing unit. + * @param cpuLimit The cpu resource limit for the computing unit. + * @param memoryLimit The memory resource limit for the computing unit. * @param unitType * @returns An Observable of the created WorkflowComputingUnit. */ - public createComputingUnit(name: string, unitType: string = "k8s_pod"): Observable { - const body = { name, unitType }; + public createComputingUnit(name: string, cpuLimit: string, memoryLimit: string, unitType: string = "k8s_pod"): Observable { + const body = { name, cpuLimit, memoryLimit, unitType }; return this.http.post( `${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_CREATE_URL}`, diff --git a/core/gui/src/app/workspace/types/workflow-computing-unit.ts b/core/gui/src/app/workspace/types/workflow-computing-unit.ts index 7da01599331..33565b50a27 100644 --- a/core/gui/src/app/workspace/types/workflow-computing-unit.ts +++ b/core/gui/src/app/workspace/types/workflow-computing-unit.ts @@ -6,20 +6,14 @@ export interface WorkflowComputingUnit { terminateTime: number | undefined; } -export interface ResourceValue { - metric: string; - value: string; - unit: string; -} - export interface WorkflowComputingUnitResourceLimit { - cpuLimit: ResourceValue; - memoryLimit: ResourceValue; + cpuLimit: string; + memoryLimit: string; } export interface WorkflowComputingUnitMetrics { - cpuUsage: ResourceValue; - memoryUsage: ResourceValue; + cpuUsage: string; + memoryUsage: string; } export interface DashboardWorkflowComputingUnit { From 4eb70944c54c5b8c30a766921490a6f61526b8c1 Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sun, 2 Mar 2025 01:16:45 -0800 Subject: [PATCH 14/18] computing unit modifications --- .../computing-unit-selection.component.html | 105 ++++++++---------- .../computing-unit-selection.component.scss | 48 +++++++- .../computing-unit-selection.component.ts | 4 + 3 files changed, 95 insertions(+), 62 deletions(-) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index 18b12c3c242..74e5dc78be6 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -1,11 +1,13 @@ -
- +
- - -
- - - + - - - {{ getComputingUnitName(unit.uri) }} -
+
    +
  • +
    + + + + {{ getComputingUnitName(unit.uri) }} + + +
    - -
- - + (click)="terminateComputingUnit(unit.computingUnit.cuid); $event.stopPropagation()"> +
+ +
  • +
  • + Computing Unit +
    • + +
    - -
    - -
    -     -
    diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss index af91b8642ac..651465f164a 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss @@ -1,9 +1,17 @@ .computing-units-selection { display: grid; - grid-template-columns: repeat(2, 1fr); + grid-template-columns: 1fr; /* Single column by default */ + justify-content: center; + align-items: center; + width: 100%; +} + +.computing-units-selection.metrics-visible { /* Renamed for clarity */ + grid-template-columns: repeat(2, 1fr); /* Two columns when visible */ justify-content: center; align-items: center; - width: 350px; + justify-items: end; + width: 100%; gap: 10px; } @@ -11,6 +19,38 @@ nz-option { height: 100%; } +.computing-units-dropdown { + min-width: 250px; + max-width: 350px; +} + +.computing-unit-option-content { + display: flex; + align-items: center; + width: 100%; + + nz-badge { + margin-right: 8px; + } + + .unit-details { + flex: 1; + } + + .terminate-icon { + visibility: hidden; + cursor: pointer; + + &:hover { + color: #ff4d4f; + } + } + + &:hover .terminate-icon { + visibility: visible; + } +} + .computing-unit-option { display: flex; align-items: center; @@ -56,10 +96,6 @@ nz-option { opacity: 0.8; /* Slight opacity on hover for visual feedback */ } -#computing-units-list { - width: 225px; -} - .metrics-container { display: grid; align-items: center; diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index d639fb55b36..406074c9b49 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -128,6 +128,10 @@ export class ComputingUnitSelectionComponent implements OnInit { } } + isComputingUnitRunning(): boolean { + return this.selectedComputingUnit != null && this.selectedComputingUnit.status === "Running"; + } + /** * Gets the computing unit name from the units URI * @param unitURI (i.e. "computing-unit-85.workflow-computing-unit-svc.workflow-computing-unit-pool.svc.cluster.local") From 88af367ae7b742fb186bac12c405f7762ea8870a Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Sun, 2 Mar 2025 20:55:46 -0800 Subject: [PATCH 15/18] refactored computing unit ui --- .../computing-unit-selection.component.html | 69 +++++++++++---- .../computing-unit-selection.component.scss | 86 ++++++------------- .../computing-unit-selection.component.ts | 39 ++++++++- core/gui/src/styles.scss | 15 ++-- 4 files changed, 122 insertions(+), 87 deletions(-) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index 74e5dc78be6..f4b722968a7 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -42,7 +42,7 @@ [nzDropdownMenu]="menu" [nzPlacement]="'bottomRight'" class="computing-units-dropdown-button"> - + Connected @@ -55,32 +55,69 @@
  • -
    +
    - - - {{ getComputingUnitName(unit.uri) }} - + + {{ getComputingUnitName(unit.uri) }} -
    +
  • -
  • - Computing Unit +
  • +
    + + Computing Unit +
    • + + Create Computing Unit + +
    +
    + Select Memory + + + + + +
    + +
    + Select CPU + + + + + +
    +
    +     + + + + +     +
    diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss index 651465f164a..c3686ad5f81 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss @@ -1,13 +1,13 @@ .computing-units-selection { display: grid; - grid-template-columns: 1fr; /* Single column by default */ + grid-template-columns: 1fr; justify-content: center; align-items: center; width: 100%; } -.computing-units-selection.metrics-visible { /* Renamed for clarity */ - grid-template-columns: repeat(2, 1fr); /* Two columns when visible */ +.computing-units-selection.metrics-visible { + grid-template-columns: repeat(2, 1fr); justify-content: center; align-items: center; justify-items: end; @@ -15,85 +15,51 @@ gap: 10px; } -nz-option { - height: 100%; -} - .computing-units-dropdown { min-width: 250px; max-width: 350px; } -.computing-unit-option-content { - display: flex; - align-items: center; - width: 100%; - - nz-badge { - margin-right: 8px; - } - - .unit-details { - flex: 1; - } - - .terminate-icon { - visibility: hidden; - cursor: pointer; - - &:hover { - color: #ff4d4f; - } - } - - &:hover .terminate-icon { - visibility: visible; - } +#computing-unit-option.unit-selected { + background-color: #f1eeee; } -.computing-unit-option { +.create-computing-unit { display: flex; - align-items: center; - justify-content: space-between; - width: 100%; gap: 10px; -} - -.unit-details { - display: flex; + justify-content: start; align-items: center; - justify-content: space-between; - gap: 8px; - flex-grow: 1; } -.terminate-icon { - color: red; - cursor: pointer; - width: 16px; - height: 16px; +.computing-unit-name { display: flex; align-items: center; justify-content: center; - border: 2px solid red; - border-radius: 50%; - padding: 1px; } -.create-unit { - width: 65%; - margin-top: 10px; - margin-bottom: 10px; +.terminate-icon:hover { + opacity: 0.8; } -.dropdown-footer { - display: flex; +.memory-selection, +.cpu-selection { + width: 200px; +} + +.create-compute-unit-container { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 10px; + justify-content: start; align-items: center; - justify-content: center; } -.terminate-icon:hover { - opacity: 0.8; /* Slight opacity on hover for visual feedback */ +.select-unit { + display: flex; + flex-direction: column; + gap: 10px; + justify-content: center; + align-items: start; } .metrics-container { diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 406074c9b49..c388e403867 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -25,6 +25,10 @@ export class ComputingUnitSelectionComponent implements OnInit { computingUnits: DashboardWorkflowComputingUnit[] = []; private readonly REFRESH_INTERVAL_MS = 2000; + addComputeUnitModalVisible = false; + selectedMemory: string = "2Gi"; + selectedCpu: string = "2"; + constructor( private computingUnitService: WorkflowComputingUnitManagingService, private notificationService: NotificationService, @@ -77,8 +81,10 @@ export class ComputingUnitSelectionComponent implements OnInit { */ startComputingUnit(): void { const computeUnitName = `Compute for Workflow ${this.workflowId}`; + const computeCPU = this.selectedCpu; + const computeMemory = this.selectedMemory; this.computingUnitService - .createComputingUnit(computeUnitName, "1", "2Gi") + .createComputingUnit(computeUnitName, computeCPU, computeMemory) .pipe(untilDestroyed(this)) .subscribe({ next: (unit: DashboardWorkflowComputingUnit) => { @@ -132,6 +138,37 @@ export class ComputingUnitSelectionComponent implements OnInit { return this.selectedComputingUnit != null && this.selectedComputingUnit.status === "Running"; } + computeStatus(): string { + if (!this.selectedComputingUnit) { + return "processing"; + } + switch (this.selectedComputingUnit.status) { + case "Running": + return "success"; + case "Pending" || "Terminating": + return "warning"; + default: + return "error"; + } + } + + isSelectedUnit(unit: DashboardWorkflowComputingUnit): boolean { + return unit.uri === this.selectedComputingUnit?.uri; + } + + showAddComputeUnitModalVisible(): void { + this.addComputeUnitModalVisible = true; + } + + handleAddComputeUnitModalOk(): void { + this.startComputingUnit(); + this.addComputeUnitModalVisible = false; + } + + handleAddComputeUnitModalCancel(): void { + this.addComputeUnitModalVisible = false; + } + /** * Gets the computing unit name from the units URI * @param unitURI (i.e. "computing-unit-85.workflow-computing-unit-svc.workflow-computing-unit-pool.svc.cluster.local") diff --git a/core/gui/src/styles.scss b/core/gui/src/styles.scss index 7ad6cb45e46..fd9a924c32a 100644 --- a/core/gui/src/styles.scss +++ b/core/gui/src/styles.scss @@ -78,14 +78,6 @@ hr { } // For compute-unit-selection.html -#computing-units-list > .ant-select-selector { - height: 100% !important; -} - -#computing-units-list .ant-select-selection-item { - line-height: 30px !important; -} - #cpu-progress-bar .ant-progress-inner, #memory-progress-bar .ant-progress-inner { vertical-align: super !important; @@ -96,6 +88,9 @@ hr { line-height: 16px !important; } -.ant-select-item-title-weight .ant-select-item{ - font-weight: normal !important; +#computing-unit-option .ant-menu-title-content { + display: flex !important; + width: 100% !important; + align-items: center !important; + justify-content: space-between !important; } From e4049ae682e7fa917298230fa66d7563319b075e Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Mon, 3 Mar 2025 15:09:26 -0800 Subject: [PATCH 16/18] disabled unit selection if currently creating --- .../power-button/computing-unit-selection.component.html | 1 + .../power-button/computing-unit-selection.component.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index f4b722968a7..144e61d7a3f 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -56,6 +56,7 @@ nz-menu-item *ngFor="let unit of computingUnits" id="computing-unit-option" + [nzDisabled] = cannotSelectUnit(unit) [ngClass]="{ 'unit-selected': isSelectedUnit(unit) }" (click)="selectedComputingUnit = unit; onComputingUnitChange(unit)">
    diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index c388e403867..66b3720bef1 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -152,6 +152,10 @@ export class ComputingUnitSelectionComponent implements OnInit { } } + cannotSelectUnit(unit: DashboardWorkflowComputingUnit): boolean { + return !unit || unit.status !== "Running"; + } + isSelectedUnit(unit: DashboardWorkflowComputingUnit): boolean { return unit.uri === this.selectedComputingUnit?.uri; } From 7080b929c46edf8c9d6499530f351df40f066a0e Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Mon, 10 Mar 2025 19:40:10 -0700 Subject: [PATCH 17/18] width fix for google signin button --- core/gui/src/app/dashboard/component/dashboard.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/gui/src/app/dashboard/component/dashboard.component.html b/core/gui/src/app/dashboard/component/dashboard.component.html index 5473ef1f408..4dff29a12a9 100644 --- a/core/gui/src/app/dashboard/component/dashboard.component.html +++ b/core/gui/src/app/dashboard/component/dashboard.component.html @@ -152,7 +152,7 @@ *ngIf="!isLogin && googleLogin" type="standard" size="large" - width="200"> + [width]="200">
    From b19d982a668e395720ce6431914cc89eabb6144f Mon Sep 17 00:00:00 2001 From: Noah Wang Date: Wed, 12 Mar 2025 22:34:45 -0700 Subject: [PATCH 18/18] format fix --- .../texera/web/resource/ResultResource.scala | 2 +- .../web/service/ExecutionResultService.scala | 2 +- core/gui/src/app/app.module.ts | 2 +- .../computing-unit-selection.component.html | 116 ++++++++++++------ .../computing-unit-selection.component.scss | 3 +- .../computing-unit-selection.component.ts | 67 +++++----- ...orkflow-computing-unit-managing.service.ts | 18 +-- ...orkflowComputingUnitManagingResource.scala | 46 ++++--- .../util/KubernetesClientService.scala | 12 +- .../util/KubernetesMetricService.scala | 78 ++++++------ .../amber/core/storage/StorageConfig.scala | 2 +- .../result/iceberg/IcebergDocument.scala | 1 - .../iceberg/IcebergTableStatsSpec.scala | 3 +- .../networkGraph/NetworkGraphOpDesc.scala | 2 +- 14 files changed, 213 insertions(+), 141 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/ResultResource.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/ResultResource.scala index 206546e8d56..ffe83824395 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/ResultResource.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/resource/ResultResource.scala @@ -5,7 +5,7 @@ import edu.uci.ics.amber.core.virtualidentity.WorkflowIdentity import edu.uci.ics.texera.web.auth.SessionUser import edu.uci.ics.texera.web.model.websocket.request.ResultExportRequest import edu.uci.ics.texera.web.model.websocket.response.ResultExportResponse -import edu.uci.ics.texera.web.service.{ResultExportService, WorkflowService} +import edu.uci.ics.texera.web.service.ResultExportService import io.dropwizard.auth.Auth import javax.ws.rs._ diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/web/service/ExecutionResultService.scala b/core/amber/src/main/scala/edu/uci/ics/texera/web/service/ExecutionResultService.scala index 7d0f6b86e83..5596f236f41 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/web/service/ExecutionResultService.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/web/service/ExecutionResultService.scala @@ -4,7 +4,7 @@ import akka.actor.Cancellable import com.fasterxml.jackson.annotation.{JsonTypeInfo, JsonTypeName} import com.fasterxml.jackson.databind.node.ObjectNode import com.typesafe.scalalogging.LazyLogging -import edu.uci.ics.amber.core.storage.DocumentFactory.{ICEBERG, MONGODB} +import edu.uci.ics.amber.core.storage.DocumentFactory.ICEBERG import edu.uci.ics.amber.core.storage.VFSResourceType.MATERIALIZED_RESULT import edu.uci.ics.amber.core.storage.model.VirtualDocument import edu.uci.ics.amber.core.storage.{DocumentFactory, StorageConfig, VFSURIFactory} diff --git a/core/gui/src/app/app.module.ts b/core/gui/src/app/app.module.ts index 7b5fb86432d..164d9c926f5 100644 --- a/core/gui/src/app/app.module.ts +++ b/core/gui/src/app/app.module.ts @@ -144,7 +144,7 @@ import { SocialLoginModule, SocialAuthServiceConfig, GoogleSigninButtonModule } import { GoogleLoginProvider } from "@abacritt/angularx-social-login"; import { lastValueFrom } from "rxjs"; import { HubSearchResultComponent } from "./hub/component/hub-search-result/hub-search-result.component"; -import {NzProgressModule} from "ng-zorro-antd/progress"; +import { NzProgressModule } from "ng-zorro-antd/progress"; registerLocaleData(en); diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html index 144e61d7a3f..3be76bcc176 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -1,4 +1,6 @@ -
    +
    + [nzShowInfo]="false">
    @@ -27,11 +28,10 @@ id="memory-progress-bar" [nzPercent]="(getMemoryPercentage() | number:'1.0-0')" [nzStrokeColor]="'#1890ff'" - [nzStatus]=getMemoryStatus() + [nzStatus]="getMemoryStatus()" nzType="line" [nzStrokeWidth]="8" - [nzShowInfo]="false" - > + [nzShowInfo]="false">
    @@ -42,26 +42,39 @@ [nzDropdownMenu]="menu" [nzPlacement]="'bottomRight'" class="computing-units-dropdown-button"> - - + + Connected Connect - + - -
      + +
      • - - + + {{ getComputingUnitName(unit.uri) }}
        @@ -74,9 +87,13 @@ (click)="terminateComputingUnit(unit.computingUnit.cuid); $event.stopPropagation()">
  • -
  • +
  • - + Computing Unit
    • @@ -89,33 +106,58 @@ [nzTitle]="addComputeUnitModalTitle" [nzContent]="addComputeUnitModalContent" [nzFooter]="addComputeUnitModalFooter" - (nzOnCancel)="handleAddComputeUnitModalCancel()" -> + (nzOnCancel)="handleAddComputeUnitModalCancel()"> Create Computing Unit
    -
    - Select Memory - - - - - -
    +
    + Select Memory + + + + + +
    -
    - Select CPU - - - - - -
    +
    + Select CPU + + + + + +
    - - + + diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss index c3686ad5f81..00f186b0349 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.scss @@ -81,7 +81,8 @@ align-items: center; } -#cpu-progress-bar, #memory-progress-bar { +#cpu-progress-bar, +#memory-progress-bar { height: 16px; } diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 66b3720bef1..6229d82b4f7 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -1,15 +1,13 @@ -import {Component, Input, OnInit} from "@angular/core"; -import {interval} from "rxjs"; -import {switchMap} from "rxjs/operators"; -import { - WorkflowComputingUnitManagingService -} from "../../service/workflow-computing-unit/workflow-computing-unit-managing.service"; -import {DashboardWorkflowComputingUnit} from "../../types/workflow-computing-unit"; -import {NotificationService} from "../../../common/service/notification/notification.service"; -import {WorkflowWebsocketService} from "../../service/workflow-websocket/workflow-websocket.service"; -import {WorkflowActionService} from "../../service/workflow-graph/model/workflow-action.service"; -import {isDefined} from "../../../common/util/predicate"; -import {UntilDestroy, untilDestroyed} from "@ngneat/until-destroy"; +import { Component, Input, OnInit } from "@angular/core"; +import { interval } from "rxjs"; +import { switchMap } from "rxjs/operators"; +import { WorkflowComputingUnitManagingService } from "../../service/workflow-computing-unit/workflow-computing-unit-managing.service"; +import { DashboardWorkflowComputingUnit } from "../../types/workflow-computing-unit"; +import { NotificationService } from "../../../common/service/notification/notification.service"; +import { WorkflowWebsocketService } from "../../service/workflow-websocket/workflow-websocket.service"; +import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; +import { isDefined } from "../../../common/util/predicate"; +import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; @UntilDestroy() @Component({ @@ -178,14 +176,12 @@ export class ComputingUnitSelectionComponent implements OnInit { * @param unitURI (i.e. "computing-unit-85.workflow-computing-unit-svc.workflow-computing-unit-pool.svc.cluster.local") * @return "Computing unit 85" */ - getComputingUnitName(unitURI: string) : string { - const computingUnit = unitURI.split(".")[0] + getComputingUnitName(unitURI: string): string { + const computingUnit = unitURI.split(".")[0]; return computingUnit .split("-") - .map((word, index) => - index < 2 ? word.charAt(0).toUpperCase() + word.slice(1) : word - ) - .join(" ") + .map((word, index) => (index < 2 ? word.charAt(0).toUpperCase() + word.slice(1) : word)) + .join(" "); } /** @@ -193,7 +189,7 @@ export class ComputingUnitSelectionComponent implements OnInit { * @param resource (i.e. "12412512n") * @return associated unit with resource (i.e. "n", "Mi", "Gi", ...) */ - parseResourceUnit(resource: string) : string { + parseResourceUnit(resource: string): string { const match = resource.match(/[a-z].*/i); return match ? match[0] : ""; } @@ -203,7 +199,7 @@ export class ComputingUnitSelectionComponent implements OnInit { * @param resource (i.e. "12412512n") * @return associated number with resource (i.e. 12412512) */ - parseResourceNumber(resource: string) : number { + parseResourceNumber(resource: string): number { const match = resource.match(/[0-9.]*/); return match ? Number(match[0]) : 0; } @@ -214,14 +210,14 @@ export class ComputingUnitSelectionComponent implements OnInit { * @param toUnit (i.e. cores) * @return i.e. 1.2412512 Cores */ - cpuResourceConversion(from: string, toUnit: string) : string { + cpuResourceConversion(from: string, toUnit: string): string { // CPU conversion constants (base unit: nanocores) type CpuUnit = "n" | "u" | "m" | ""; const cpuUnits: Record = { - "n": 1, // nanocores - "u": 10**3, // microcores - "m": 10**6, // millicores - "": 10**9 // cores + n: 1, // nanocores + u: 10 ** 3, // microcores + m: 10 ** 6, // millicores + "": 10 ** 9, // cores }; const fromNumber: number = this.parseResourceNumber(from); @@ -230,7 +226,7 @@ export class ComputingUnitSelectionComponent implements OnInit { if (!(fromUnit in cpuUnits) || !(toUnit in cpuUnits)) { return ""; } - return `${(fromNumber * (cpuUnits[fromUnit as CpuUnit] / cpuUnits[toUnit as CpuUnit]))} ${toUnit}`; + return `${fromNumber * (cpuUnits[fromUnit as CpuUnit] / cpuUnits[toUnit as CpuUnit])} ${toUnit}`; } /** @@ -239,14 +235,14 @@ export class ComputingUnitSelectionComponent implements OnInit { * @param toUnit (i.e. "Gi") * @return i.e. 0.524 Gi */ - memoryResourceConversion(from: string, toUnit: string) : string { + memoryResourceConversion(from: string, toUnit: string): string { // Memory conversion constants (base unit: bytes) type MemoryUnit = "Ki" | "Mi" | "Gi" | ""; const memoryUnits = { - "": 1, // bytes - "Ki": 1024, // KiB - "Mi": 1024**2, // MiB - "Gi": 1024**3 // GiB + "": 1, // bytes + Ki: 1024, // KiB + Mi: 1024 ** 2, // MiB + Gi: 1024 ** 3, // GiB }; const fromNumber: number = this.parseResourceNumber(from); @@ -311,7 +307,10 @@ export class ComputingUnitSelectionComponent implements OnInit { getMemoryValue(): number { // convert to appropriate unit based on the limit const memoryLimitUnit: string = this.getMemoryLimitUnit(); - const convertedValue: string = this.memoryResourceConversion(this.getCurrentComputingUnitMemoryUsage(), memoryLimitUnit); + const convertedValue: string = this.memoryResourceConversion( + this.getCurrentComputingUnitMemoryUsage(), + memoryLimitUnit + ); return this.parseResourceNumber(convertedValue); } @@ -321,7 +320,7 @@ export class ComputingUnitSelectionComponent implements OnInit { if (cpuLimit <= 0) { return 0; } - return this.getCpuValue() / cpuLimit * 100; + return (this.getCpuValue() / cpuLimit) * 100; } getCpuStatus(): "success" | "exception" | "active" | "normal" { @@ -336,7 +335,7 @@ export class ComputingUnitSelectionComponent implements OnInit { if (memoryLimit <= 0) { return 0; } - return this.getMemoryValue() / memoryLimit * 100; + return (this.getMemoryValue() / memoryLimit) * 100; } getMemoryStatus(): "success" | "exception" | "active" | "normal" { diff --git a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts index 94ead506837..d8e0aefae18 100644 --- a/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts +++ b/core/gui/src/app/workspace/service/workflow-computing-unit/workflow-computing-unit-managing.service.ts @@ -1,11 +1,8 @@ import { Injectable } from "@angular/core"; -import {HttpClient, HttpParams} from "@angular/common/http"; +import { HttpClient, HttpParams } from "@angular/common/http"; import { Observable } from "rxjs"; import { AppSettings } from "../../../common/app-setting"; -import { - DashboardWorkflowComputingUnit, - WorkflowComputingUnitMetrics -} from "../../types/workflow-computing-unit"; +import { DashboardWorkflowComputingUnit, WorkflowComputingUnitMetrics } from "../../types/workflow-computing-unit"; export const COMPUTING_UNIT_BASE_URL = "computing-unit"; export const COMPUTING_UNIT_METRICS_BASE_URL = "resource-metrics"; @@ -27,7 +24,12 @@ export class WorkflowComputingUnitManagingService { * @param unitType * @returns An Observable of the created WorkflowComputingUnit. */ - public createComputingUnit(name: string, cpuLimit: string, memoryLimit: string, unitType: string = "k8s_pod"): Observable { + public createComputingUnit( + name: string, + cpuLimit: string, + memoryLimit: string, + unitType: string = "k8s_pod" + ): Observable { const body = { name, cpuLimit, memoryLimit, unitType }; return this.http.post( @@ -63,6 +65,8 @@ export class WorkflowComputingUnitManagingService { * @param cuid */ public getComputingUnitMetrics(cuid: number): Observable { - return this.http.get(`${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_BASE_URL}/${cuid}/metrics`) + return this.http.get( + `${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_BASE_URL}/${cuid}/metrics` + ); } } diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala index 841b76dfdbf..3f492987254 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/resource/WorkflowComputingUnitManagingResource.scala @@ -6,7 +6,15 @@ import edu.uci.ics.texera.dao.SqlServer.withTransaction import edu.uci.ics.texera.dao.jooq.generated.tables.daos.WorkflowComputingUnitDao import edu.uci.ics.texera.dao.jooq.generated.tables.WorkflowComputingUnit.WORKFLOW_COMPUTING_UNIT import edu.uci.ics.texera.dao.jooq.generated.tables.pojos.WorkflowComputingUnit -import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{DashboardWorkflowComputingUnit, TerminationResponse, WorkflowComputingUnitCreationParams, WorkflowComputingUnitMetrics, WorkflowComputingUnitResourceLimit, WorkflowComputingUnitTerminationParams, context} +import edu.uci.ics.texera.service.resource.WorkflowComputingUnitManagingResource.{ + DashboardWorkflowComputingUnit, + TerminationResponse, + WorkflowComputingUnitCreationParams, + WorkflowComputingUnitMetrics, + WorkflowComputingUnitResourceLimit, + WorkflowComputingUnitTerminationParams, + context +} import edu.uci.ics.texera.service.util.KubernetesClientService import edu.uci.ics.texera.service.util.KubernetesMetricService.{getPodLimits, getPodMetrics} import jakarta.ws.rs._ @@ -22,18 +30,23 @@ object WorkflowComputingUnitManagingResource { .getInstance(StorageConfig.jdbcUrl, StorageConfig.jdbcUsername, StorageConfig.jdbcPassword) .createDSLContext() - case class WorkflowComputingUnitCreationParams(name: String, unitType: String, cpuLimit: String, memoryLimit: String) + case class WorkflowComputingUnitCreationParams( + name: String, + unitType: String, + cpuLimit: String, + memoryLimit: String + ) case class WorkflowComputingUnitTerminationParams(uri: String, name: String) case class WorkflowComputingUnitResourceLimit( - cpuLimit: String, - memoryLimit: String - ) + cpuLimit: String, + memoryLimit: String + ) case class WorkflowComputingUnitMetrics( - cpuUsage: String, - memoryUsage: String + cpuUsage: String, + memoryUsage: String ) case class DashboardWorkflowComputingUnit( @@ -121,7 +134,8 @@ class WorkflowComputingUnitManagingResource { DashboardWorkflowComputingUnit( computingUnit = unit, uri = KubernetesClientService.generatePodURI(cuid).toString, - status = if (pod != null && pod.getStatus != null) pod.getStatus.getPhase else "Unknown", + status = + if (pod != null && pod.getStatus != null) pod.getStatus.getPhase else "Unknown", getComputingUnitMetric(cuid.toString), getComputingUnitLimits(cuid.toString) ) @@ -160,11 +174,11 @@ class WorkflowComputingUnitManagingResource { } /** - * Retrieves the CPU and memory metrics for a computing unit identified by its `cuid`. - * - * @param cuid The computing unit ID. - * @return A `WorkflowComputingUnitMetrics` object with CPU and memory usage data. - */ + * Retrieves the CPU and memory metrics for a computing unit identified by its `cuid`. + * + * @param cuid The computing unit ID. + * @return A `WorkflowComputingUnitMetrics` object with CPU and memory usage data. + */ @GET @Produces(Array(MediaType.APPLICATION_JSON)) @Path("/{cuid}/metrics") @@ -180,7 +194,9 @@ class WorkflowComputingUnitManagingResource { @GET @Produces(Array(MediaType.APPLICATION_JSON)) @Path("/{cuid}/limits") - def getComputingUnitLimits(@PathParam("cuid") cuid: String): WorkflowComputingUnitResourceLimit = { + def getComputingUnitLimits( + @PathParam("cuid") cuid: String + ): WorkflowComputingUnitResourceLimit = { val podLimits: Map[String, String] = getPodLimits(cuid.toInt) WorkflowComputingUnitResourceLimit( @@ -188,4 +204,4 @@ class WorkflowComputingUnitManagingResource { podLimits.getOrElse("memory", "") ) } -} \ No newline at end of file +} diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala index 2cd0a5b2511..f2b6d07fc9e 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesClientService.scala @@ -1,7 +1,11 @@ package edu.uci.ics.texera.service.util import config.WorkflowComputingUnitManagingServiceConf -import config.WorkflowComputingUnitManagingServiceConf.{computeUnitImageName, computeUnitPortNumber, computeUnitServiceName} +import config.WorkflowComputingUnitManagingServiceConf.{ + computeUnitImageName, + computeUnitPortNumber, + computeUnitServiceName +} import edu.uci.ics.amber.core.storage.StorageConfig import io.kubernetes.client.custom.Quantity import io.kubernetes.client.openapi.apis.CoreV1Api @@ -164,8 +168,10 @@ object KubernetesClientService { new V1ResourceRequirements() .limits( util.Map.of( - "cpu", cpu, - "memory", memory + "cpu", + cpu, + "memory", + memory ) ) // may want to add requests as well to make efficient use of the CPU resources ) diff --git a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala index 62dd97f690e..ec49ed4ad45 100644 --- a/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala +++ b/core/workflow-computing-unit-managing-service/src/main/scala/edu/uci/ics/texera/service/util/KubernetesMetricService.scala @@ -2,7 +2,7 @@ package edu.uci.ics.texera.service.util import config.WorkflowComputingUnitManagingServiceConf import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetricsList -import io.fabric8.kubernetes.api.model.{PodList, Quantity} +import io.fabric8.kubernetes.api.model.PodList import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder} import scala.jdk.CollectionConverters._ @@ -15,54 +15,60 @@ object KubernetesMetricService { private val namespace = WorkflowComputingUnitManagingServiceConf.computeUnitPoolNamespace /** - * Retrieves the pod metric for a given ID in the specified namespace. - * - * @param cuid The computing unit id of the pod - * @return The Pod metrics for a given name in a specified namespace. - */ + * Retrieves the pod metric for a given ID in the specified namespace. + * + * @param cuid The computing unit id of the pod + * @return The Pod metrics for a given name in a specified namespace. + */ def getPodMetrics(cuid: Int): Map[String, String] = { val podMetricsList: PodMetricsList = client.top().pods().metrics(namespace) val targetPodName = KubernetesClientService.generatePodName(cuid) - podMetricsList.getItems.asScala.collectFirst { - case podMetrics if podMetrics.getMetadata.getName == targetPodName => - podMetrics.getContainers.asScala.collectFirst { - case container => - container.getUsage.asScala.collect { - case (metric, value) => - println(s"Metric - $metric: ${value}") - // CPU is in nanocores and Memory is in Kibibyte - metric -> value.toString - }.toMap - }.getOrElse(Map.empty[String, String]) - }.getOrElse { - println(s"No metrics found for pod: $targetPodName in namespace: $namespace") - Map.empty[String, String] - } + podMetricsList.getItems.asScala + .collectFirst { + case podMetrics if podMetrics.getMetadata.getName == targetPodName => + podMetrics.getContainers.asScala + .collectFirst { + case container => + container.getUsage.asScala.collect { + case (metric, value) => + println(s"Metric - $metric: ${value}") + // CPU is in nanocores and Memory is in Kibibyte + metric -> value.toString + }.toMap + } + .getOrElse(Map.empty[String, String]) + } + .getOrElse { + println(s"No metrics found for pod: $targetPodName in namespace: $namespace") + Map.empty[String, String] + } } /** - * Retrieves the pod limits for a given ID in the specified namespace. - * - * @param cuid The computing unit id of the pod - * @return The Pod limits for a given name in a specified namespace. - */ + * Retrieves the pod limits for a given ID in the specified namespace. + * + * @param cuid The computing unit id of the pod + * @return The Pod limits for a given name in a specified namespace. + */ def getPodLimits(cuid: Int): Map[String, String] = { val podList: PodList = client.pods().inNamespace(namespace).list() val targetPodName = KubernetesClientService.generatePodName(cuid) - val pod = podList.getItems.asScala.find( - pod => { pod.getMetadata.getName.equals(targetPodName) } - ) + val pod = podList.getItems.asScala.find(pod => { + pod.getMetadata.getName.equals(targetPodName) + }) - val limits: Map[String, String] = pod.flatMap { - pod => pod.getSpec.getContainers.asScala.headOption.map { - container => container.getResources.getLimits.asScala.map { - case (key, value) => - key -> value.toString - }.toMap + val limits: Map[String, String] = pod + .flatMap { pod => + pod.getSpec.getContainers.asScala.headOption.map { container => + container.getResources.getLimits.asScala.map { + case (key, value) => + key -> value.toString + }.toMap + } } - }.getOrElse(Map.empty[String, String]) + .getOrElse(Map.empty[String, String]) println(limits.toString()) limits } diff --git a/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/StorageConfig.scala b/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/StorageConfig.scala index b3d6335b246..f95264611a2 100644 --- a/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/StorageConfig.scala +++ b/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/StorageConfig.scala @@ -49,4 +49,4 @@ object StorageConfig { // File storage configurations val fileStorageDirectoryPath: Path = corePath.resolve("amber").resolve("user-resources").resolve("workflow-results") -} \ No newline at end of file +} diff --git a/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/result/iceberg/IcebergDocument.scala b/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/result/iceberg/IcebergDocument.scala index 862c953b2eb..88530aa4c8c 100644 --- a/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/result/iceberg/IcebergDocument.scala +++ b/core/workflow-core/src/main/scala/edu/uci/ics/amber/core/storage/result/iceberg/IcebergDocument.scala @@ -12,7 +12,6 @@ import org.apache.iceberg.types.{Conversions, Types} import java.net.URI import java.util.concurrent.locks.{ReentrantLock, ReentrantReadWriteLock} -import scala.collection.mutable import scala.jdk.CollectionConverters._ import java.nio.ByteBuffer import java.time.{Instant, LocalDate, ZoneOffset} diff --git a/core/workflow-core/src/test/scala/edu/uci/ics/amber/storage/result/iceberg/IcebergTableStatsSpec.scala b/core/workflow-core/src/test/scala/edu/uci/ics/amber/storage/result/iceberg/IcebergTableStatsSpec.scala index 2aca3408c0e..08d618309fe 100644 --- a/core/workflow-core/src/test/scala/edu/uci/ics/amber/storage/result/iceberg/IcebergTableStatsSpec.scala +++ b/core/workflow-core/src/test/scala/edu/uci/ics/amber/storage/result/iceberg/IcebergTableStatsSpec.scala @@ -18,10 +18,9 @@ import org.scalatest.{BeforeAndAfterAll, Suite} import java.net.URI import java.sql.Timestamp -import java.time.{Instant, LocalDate, ZoneId, ZoneOffset} +import java.time.{LocalDate, ZoneId} import java.time.format.DateTimeFormatter import java.util.UUID -import scala.jdk.CollectionConverters._ class IcebergTableStatsSpec extends AnyFlatSpec with BeforeAndAfterAll with Suite { diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala index 4f57d167d53..5e9682bec19 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala @@ -1,7 +1,7 @@ package edu.uci.ics.amber.operator.visualization.networkGraph import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} -import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import edu.uci.ics.amber.core.tuple.{AttributeType, Schema} import edu.uci.ics.amber.core.workflow.OutputPort.OutputMode import edu.uci.ics.amber.core.workflow.{InputPort, OutputPort, PortIdentity}