Add computing unit manager service

.gitignore

@@ -108,4 +108,35 @@ Composer-Setup.exe
 .metals/
 .bloop/
 .ammonite/
-metals.sbt
+metals.sbt
+
+
+# Ignore Helm-specific artifacts in texera-helmchart
+core/scripts/texera-helmchart/*.tgz
+core/scripts/texera-helmchart/charts/
+core/scripts/texera-helmchart/tmpcharts/
+core/scripts/texera-helmchart/*.lock
+core/scripts/texera-helmchart/test-output/
+core/scripts/texera-helmchart/tests/test-*.yaml
+core/scripts/texera-helmchart/*.helmignore
+
+# Ignore general OS-specific files and editor files in texera-helmchart
+core/scripts/texera-helmchart/.DS_Store
+core/scripts/texera-helmchart/Thumbs.db
+core/scripts/texera-helmchart/*.swp
+core/scripts/texera-helmchart/*.swo
+core/scripts/texera-helmchart/*.bak
+core/scripts/texera-helmchart/*~
+core/scripts/texera-helmchart/.idea/
+core/scripts/texera-helmchart/.vscode/
+core/scripts/texera-helmchart/*.log
+
+# Temporary files in texera-helmchart
+core/scripts/texera-helmchart/*.tmp
+core/scripts/texera-helmchart/*.out
+core/scripts/texera-helmchart/*.cache
+core/scripts/texera-helmchart/*.pyc
+core/scripts/texera-helmchart/__pycache__/
+
+# Build artifacts in texera-helmchart
+core/scripts/texera-helmchart/dist/

core/amber/computing-unit.dockerfile

@@ -0,0 +1,40 @@
+FROM sbtscala/scala-sbt:eclipse-temurin-jammy-11.0.17_8_1.9.3_2.13.11
+
+# Set working directory
+WORKDIR /core
+
+# Copy all projects under core to /core
+COPY core/ .
+
+RUN rm -rf amber/user-resources/*
+
+# Update system and install dependencies
+RUN apt-get update && apt-get install -y \
+    netcat \
+    unzip \
+    python3-pip \
+    libpq-dev \
+    && apt-get clean
+
+# Install Python dependencies
+RUN pip3 install --upgrade pip setuptools wheel
+RUN pip3 install python-lsp-server python-lsp-server[websockets]
+
+# Install requirements with a fallback for wordcloud
+RUN pip3 install -r amber/requirements.txt
+RUN pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r amber/operator-requirements.txt || \
+    pip3 install --no-cache-dir wordcloud==1.9.2
+
+# Additional setup
+WORKDIR /core
+# Add .git for runtime calls to jgit from OPversion
+COPY .git ../.git
+
+# Build services
+RUN scripts/build-services.sh
+
+# Set the default command
+CMD ["scripts/workflow-computing-unit.sh"]
+
+# Expose the required port
+EXPOSE 8085

core/amber/src/main/resources/application.conf

@@ -18,7 +18,7 @@ network-buffering {
 }
 
 reconfiguration {
-    enable-transactional-reconfiguration = false
+    enable-transactional-reconfiguration = true
 }
 
 cache {
@@ -27,10 +27,11 @@ cache {
 }
 
 user-sys {
-    enabled = false
+    enabled = true
     google {
         clientId = ""
-        smtp {
+        clientSecret = ""
+	smtp {
             gmail = ""
             password = ""
         }
@@ -40,17 +41,17 @@ user-sys {
         exp-in-days = 30
         # generate the secret again for each deployment using the following:
         # 'openssl rand -hex 16'  or  'xxd -l16 -ps /dev/urandom'
-        256-bit-secret = "8a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d"
+        256-bit-secret = "5259e3eb6a58cb89b53790406c24c8f5"
     }
 }
 
 result-cleanup {
-    ttl-in-seconds = 86400 # time to live for a collection is 2 days
-    collection-check-interval-in-seconds = 86400 # 2 days
+    ttl-in-seconds = 3600 # time to live for a collection is 2 days
+    collection-check-interval-in-seconds = 3600 # 2 days
 }
 
 web-server {
-    workflow-state-cleanup-in-seconds = 30
+    workflow-state-cleanup-in-seconds = 3600
     python-console-buffer-size = 100
     workflow-result-pulling-in-seconds = 3
     clean-all-execution-results-on-server-start = false
@@ -74,7 +75,7 @@ fault-tolerance {
 }
 
 schedule-generator {
-    enable-cost-based-schedule-generator = false
+    enable-cost-based-schedule-generator = true
     use-global-search = false
     use-top-down-search = false
     search-timeout-milliseconds = 1000
@@ -86,4 +87,4 @@ ai-assistant-server{
     ai-service-key = ""
     # Put your Ai service url here (If you are using OpenAI, then the url should be "https://api.openai.com/v1")
     ai-service-url = ""
-}
+}

core/amber/src/main/resources/udf.conf

@@ -1,6 +1,6 @@
 python {
     # python3 executable path
-    path = ""
+    path = "/usr/bin/python3"
 
     log {
         streamHandler {

core/amber/src/main/scala/edu/uci/ics/amber/engine/common/AmberConfig.scala

@@ -2,9 +2,14 @@ package edu.uci.ics.amber.engine.common
 
 import akka.actor.Address
 import com.typesafe.config.{Config, ConfigFactory}
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.revwalk.RevWalk
+import org.eclipse.jgit.revwalk.filter.RevFilter
 
 import java.io.File
 import java.net.URI
+import scala.io.Source
+import scala.jdk.CollectionConverters.IteratorHasAsScala
 
 object AmberConfig {
 
@@ -15,6 +20,35 @@ object AmberConfig {
   private var conf: Config = _
 
   var masterNodeAddr: Address = Address("akka", "Amber", "localhost", 2552)
+  val lastDeployTimestamp:String = try {
+    // Read the timestamp string from the file
+    val source = Source.fromFile(Utils.amberHomePath.resolve("timestamp.txt").toString)
+    val timestamp = source.getLines().mkString
+    source.close()
+    // Print the timestamp
+    println(s"Timestamp read from file: $timestamp")
+    timestamp
+  } catch {
+    case e: Exception =>
+      println(s"An error occurred: ${e.getMessage}")
+      "Timestamp unavailable"
+  }
+
+
+
+  val latestCommitFromMaster = try {
+    // Read the timestamp string from the file
+    val source = Source.fromFile(Utils.amberHomePath.resolve("commit_title_before_merge.txt").toString)
+    val commitMsg = source.getLines().mkString
+    source.close()
+    // Print the timestamp
+    println(s"Commit message from file: $commitMsg")
+    commitMsg
+  } catch {
+    case e: Exception =>
+      println(s"An error occurred: ${e.getMessage}")
+      "Commit message unavailable"
+  }
 
   // perform lazy reload
   private def getConfSource: Config = {

core/amber/src/main/scala/edu/uci/ics/amber/engine/common/AmberRuntime.scala

@@ -54,6 +54,7 @@ object AmberRuntime {
         akka.cluster.seed-nodes = [ "akka://Amber@$localIpAddress:2552" ]
         """)
       .withFallback(akkaConfig)
+      .resolve()
     AmberConfig.masterNodeAddr = createMasterAddress(localIpAddress)
     createAmberSystem(masterConfig)
   }
@@ -75,6 +76,7 @@ object AmberRuntime {
         akka.cluster.seed-nodes = [ "akka://Amber@$addr:2552" ]
         """)
       .withFallback(akkaConfig)
+      .resolve()
     AmberConfig.masterNodeAddr = createMasterAddress(addr)
     createAmberSystem(workerConfig)
   }

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/hub/HubResource.scala

@@ -3,15 +3,7 @@ package edu.uci.ics.texera.web.resource.dashboard.hub
 import edu.uci.ics.texera.dao.SqlServer
 import edu.uci.ics.texera.dao.jooq.generated.Tables._
 import HubResource.{
-  fetchDashboardDatasetsByDids,
-  fetchDashboardWorkflowsByWids,
-  getUserLCCount,
-  isLikedHelper,
-  recordLikeActivity,
-  recordUserActivity,
-  userRequest,
-  validateEntityType
-}
+  fetchDashboardDatasetsByDids,fetchDashboardWorkflowsByWids, getUserLCCount, isLikedHelper, recordLikeActivity, recordUserActivity, userRequest, validateEntityType}
 import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowResource.{
   DashboardWorkflow,
   baseWorkflowSelect,
@@ -31,6 +23,12 @@ import edu.uci.ics.texera.dao.jooq.generated.tables.DatasetUserAccess.DATASET_US
 import edu.uci.ics.texera.dao.jooq.generated.tables.User.USER
 import edu.uci.ics.texera.dao.jooq.generated.tables.pojos.{Dataset, DatasetUserAccess}
 import edu.uci.ics.texera.web.resource.dashboard.DashboardResource.DashboardClickableFileEntry
+import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.{
+  DashboardDataset,
+  baseDatasetSelect,
+  mapDashboardDataset
+}
+import edu.uci.ics.amber.engine.common.AmberConfig
 import edu.uci.ics.texera.web.resource.dashboard.user.dataset.DatasetResource.DashboardDataset
 
 object HubResource {
@@ -305,6 +303,20 @@ class HubResource {
     .getInstance()
     .createDSLContext()
 
+  @GET
+  @Path("/git-describe")
+  @Produces(Array(MediaType.TEXT_PLAIN))
+  def getGitHead: String = {
+    AmberConfig.latestCommitFromMaster
+  }
+
+  @GET
+  @Path("/last-deploy")
+  @Produces(Array(MediaType.TEXT_PLAIN))
+  def getLastDeploy: String = {
+    AmberConfig.lastDeployTimestamp
+  }
+
   @GET
   @Path("/count")
   def getPublishedWorkflowCount(@QueryParam("entityType") entityType: String): Integer = {

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala

@@ -3,12 +3,18 @@ package edu.uci.ics.texera.web.resource.dashboard.user.workflow
 import edu.uci.ics.amber.core.storage.VFSURIFactory.decodeURI
 import edu.uci.ics.amber.core.storage.result.ExecutionResourcesMapping
 import edu.uci.ics.amber.core.storage.{DocumentFactory, VFSResourceType, VFSURIFactory}
+import edu.uci.ics.amber.core.storage.{DocumentFactory, VFSURIFactory}
 import edu.uci.ics.amber.core.tuple.Tuple
 import edu.uci.ics.amber.core.virtualidentity._
 import edu.uci.ics.amber.core.workflow.PortIdentity
 import edu.uci.ics.amber.engine.architecture.logreplay.{ReplayDestination, ReplayLogRecord}
 import edu.uci.ics.amber.engine.common.AmberConfig
 import edu.uci.ics.amber.engine.common.storage.SequentialRecordStorage
+import edu.uci.ics.amber.core.virtualidentity.{ChannelMarkerIdentity, ExecutionIdentity, OperatorIdentity, WorkflowIdentity}
+import edu.uci.ics.amber.core.workflow.PortIdentity
+import edu.uci.ics.amber.engine.architecture.logreplay.{ReplayDestination, ReplayLogRecord}
+import edu.uci.ics.amber.engine.common.AmberConfig
+import edu.uci.ics.amber.engine.common.storage.SequentialRecordStorage
 import edu.uci.ics.texera.dao.SqlServer
 import edu.uci.ics.texera.dao.jooq.generated.Tables._
 import edu.uci.ics.texera.dao.jooq.generated.tables.daos.WorkflowExecutionsDao
@@ -17,6 +23,7 @@ import edu.uci.ics.texera.web.auth.SessionUser
 import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource._
 import edu.uci.ics.texera.web.service.ExecutionsMetadataPersistService
 import io.dropwizard.auth.Auth
+import org.jooq.types.ULong
 
 import java.net.URI
 import java.sql.Timestamp

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala

@@ -3,15 +3,11 @@ package edu.uci.ics.texera.web.resource.dashboard.user.workflow
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import com.typesafe.scalalogging.LazyLogging
+import edu.uci.ics.amber.engine.common.AmberConfig
 import edu.uci.ics.texera.dao.SqlServer
 import edu.uci.ics.texera.dao.jooq.generated.Tables._
 import edu.uci.ics.texera.dao.jooq.generated.enums.PrivilegeEnum
-import edu.uci.ics.texera.dao.jooq.generated.tables.daos.{
-  WorkflowDao,
-  WorkflowOfProjectDao,
-  WorkflowOfUserDao,
-  WorkflowUserAccessDao
-}
+import edu.uci.ics.texera.dao.jooq.generated.tables.daos.{WorkflowDao, WorkflowOfProjectDao, WorkflowOfUserDao, WorkflowUserAccessDao}
 import edu.uci.ics.texera.dao.jooq.generated.tables.pojos._
 import edu.uci.ics.texera.web.auth.SessionUser
 import edu.uci.ics.texera.web.resource.dashboard.hub.HubResource.recordCloneActivity
@@ -51,6 +47,7 @@ object WorkflowResource {
   )
   final private lazy val workflowOfProjectDao = new WorkflowOfProjectDao(context.configuration)
 
+
   def getWorkflowName(wid: Integer): String = {
     val workflow = workflowDao.fetchOneByWid(wid)
     if (workflow == null) {

core/amber/timestamp.txt

@@ -0,0 +1 @@
+2025-02-11 17:03:49

core/amber/webserver.dockerfile

@@ -0,0 +1,42 @@
+FROM node:18-alpine AS nodegui
+
+WORKDIR /gui
+COPY core/gui/package.json core/gui/yarn.lock ./
+RUN apk add --no-cache python3 make g++ git
+RUN corepack enable && corepack prepare yarn@4.5.1 --activate && yarn set version --yarn-path  4.5.1
+# Fake git-version.js during yarn install to prevent git from causing cache
+# invalidation of dependencies
+# Add tools required for building dependencies
+RUN touch git-version.js && YARN_NODE_LINKER=node-modules yarn install
+
+COPY core/gui .
+# Position of .git doesn't matter since it's only there for the revision hash
+COPY .git ./.git
+RUN apk add --no-cache git && \
+	node git-version.js && \
+	apk del git && \
+	yarn run build
+
+FROM sbtscala/scala-sbt:eclipse-temurin-jammy-11.0.17_8_1.9.3_2.13.11
+
+# copy all projects under core to /core
+WORKDIR /core
+COPY core/ .
+
+RUN rm -rf amber/user-resources/*
+
+RUN apt-get update
+RUN apt-get install -y netcat unzip python3-pip
+#RUN pip3 install python-lsp-server python-lsp-server[websockets]
+#RUN pip3 install -r amber/requirements.txt
+
+WORKDIR /core
+# Add .git for runtime calls to jgit from OPversion
+COPY .git ../.git
+COPY --from=nodegui /gui/dist ./gui/dist
+
+RUN scripts/build-services.sh
+
+CMD ["scripts/server.sh"]
+
+EXPOSE 8080

core/build.sbt

@@ -3,6 +3,14 @@ lazy val WorkflowCore = (project in file("workflow-core"))
   .dependsOn(DAO)
   .configs(Test)
   .dependsOn(DAO % "test->test") // test scope dependency
+lazy val WorkflowComputingUnitManagingService = (project in file("workflow-computing-unit-managing-service"))
+  .dependsOn(WorkflowCore)
+  .settings(
+    dependencyOverrides ++= Seq(
+      // override it as io.dropwizard 4 require 2.16.1 or higher
+      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.17.0",
+)
+  )
 lazy val FileService = (project in file("file-service"))
   .dependsOn(WorkflowCore)
   .settings(
@@ -47,7 +55,7 @@ lazy val WorkflowExecutionService = (project in file("amber"))
 
 // root project definition
 lazy val CoreProject = (project in file("."))
-  .aggregate(DAO, WorkflowCore, FileService, WorkflowOperator, WorkflowCompilingService, WorkflowExecutionService)
+  .aggregate(DAO, WorkflowComputingUnitManagingService, WorkflowCore, FileService, WorkflowOperator, WorkflowCompilingService, WorkflowExecutionService)
   .settings(
     name := "core",
     version := "0.1.0",