[CTM-142] New terra base image [DO NOT MERGE]

automation/src/test/resources/reference.conf

@@ -1,4 +1,5 @@
 leonardo {
+    baseImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-base:0.0.1"
     rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7"
     pythonImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6"
     hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.14"

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoConfig.scala

@@ -12,14 +12,14 @@ object LeonardoConfig extends CommonConfig {
   object Leonardo {
     val apiUrl: String = leonardo.getString("apiUrl")
     val notebooksServiceAccountEmail: String = leonardo.getString("notebooksServiceAccountEmail")
+    val baseImageUrl: String = leonardo.getString("baseImageUrl")
     val rImageUrl: String = leonardo.getString("rImageUrl")
     val pythonImageUrl: String = leonardo.getString("pythonImageUrl")
     val hailImageUrl: String = leonardo.getString("hailImageUrl")
     val gatkImageUrl: String = leonardo.getString("gatkImageUrl")
     val aouImageUrl: String = leonardo.getString("aouImageUrl")
-    val baseImageUrl: String = leonardo.getString("baseImageUrl")
     val rstudioBioconductorImage =
-      ContainerImage(leonardo.getString("rstudioBioconductorImageUrl"), ContainerRegistry.GCR)
+      ContainerImage(leonardo.getString("rstudioBioconductorImageUrl"), ContainerRegistry.GAR)
 
     private val topic = ProjectTopicName.of(gcs.getString("serviceProject"), leonardo.getString("topicName"))
     val location: Location = Location(leonardo.getString("location"))

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/RuntimeFixtureSpec.scala

@@ -115,7 +115,7 @@ trait RuntimeFixtureSpec
           billingProject,
           runtimeName,
           getRuntimeRequest(cloudService.getOrElse(CloudService.GCE),
-                            toolDockerImage.map(i => ContainerImage(i, ContainerRegistry.GCR)),
+                            toolDockerImage.map(i => ContainerImage(i, ContainerRegistry.GAR)),
                             welderRegistry
           )
         )

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeCreationDiskSpec.scala

@@ -213,7 +213,7 @@ class RuntimeCreationDiskSpec extends BillingProjectFixtureSpec with ParallelTes
       )
 
       val createRuntime2Request = createRuntimeRequest.copy(toolDockerImage =
-        Some(ContainerImage(LeonardoConfig.Leonardo.pythonImageUrl, ContainerRegistry.GCR))
+        Some(ContainerImage(LeonardoConfig.Leonardo.pythonImageUrl, ContainerRegistry.GAR))
       ) // this just needs to be a different image from default image Leonardo uses, which is gatk
 
       val createRuntimeCloneRequest = createRuntime2Request.copy(

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeDataprocSpec.scala

@@ -106,7 +106,7 @@ class RuntimeDataprocSpec extends BillingProjectFixtureSpec with ParallelTestExe
               false
             )
           ),
-          toolDockerImage = Some(ContainerImage(LeonardoConfig.Leonardo.hailImageUrl, ContainerRegistry.GCR))
+          toolDockerImage = Some(ContainerImage(LeonardoConfig.Leonardo.hailImageUrl, ContainerRegistry.GAR))
         )
         getRuntimeResponse <- LeonardoApiClient.createRuntimeWithWait(project, runtimeName, createRuntimeRequest)
         runtime = ClusterCopy.fromGetRuntimeResponseCopy(getRuntimeResponse)

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/containerModels.scala

@@ -12,9 +12,11 @@ sealed trait ContainerRegistry extends EnumEntry with Product with Serializable
 object ContainerRegistry extends Enum[ContainerRegistry] {
   val values = findValues
 
-  final case object GCR extends ContainerRegistry {
+  final case object GAR extends ContainerRegistry {
     val regex: Regex =
-      """^((?:us\.|eu\.|asia\.)?gcr.io)/([\w.-]+/[\w.-]+)(?::(\w[\w.-]+))?(?:@([\w+.-]+:[A-Fa-f0-9]{32,}))?$""".r
+      // with GCR switching to GAR but keeping the old addresses valid, this accepts both gcr.io and gar.io addresses
+      """^((?:us\.|eu\.|asia\.)?(?:gcr|gar)\.io)/([\w.-]+/[\w.-]+)(?::(\w[\w.-]+))?(?:@([\w+.-]+:[A-Fa-f0-9]{32,}))?$""".r
+
     override def toString: String = "GCR"
   }
 

http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml

@@ -33,8 +33,9 @@ services:
       R_LIBS: "${NOTEBOOKS_DIR}/packages"
       # The next two lines aren't great. But they're for updating PYTHONPATH, PATH in older than (inclusive) us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.2
       # We should remove the two lines once we no longer support older images. In the meantime, we need to be careful updating Jupyter base images.
+      # NOTE: its very important to update these paths to access any tools added or moved in the base image
       PYTHONPATH: "/etc/jupyter/custom:/usr/lib/spark/python:${NOTEBOOKS_DIR}/packages"
-      PATH: "/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${NOTEBOOKS_DIR}/.local/bin:${NOTEBOOKS_DIR}/packages/bin"
+      PATH: "/root/.local/bin:/etc/jupyter/bin:/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${NOTEBOOKS_DIR}/.local/bin:${NOTEBOOKS_DIR}/packages/bin"
     env_file:
       - /var/custom_env_vars.env
     # See https://docs.docker.com/engine/reference/run/#user-memory-constraints

http/src/main/resources/init-resources/jupyter-docker-compose.yaml

@@ -51,8 +51,9 @@ services:
       # The next two lines aren't great. But they're for updating PYTHONPATH, PATH in older than (inclusive) us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.2.
       # We should remove the two lines once we no longer support older images.
       # When we update base image in terra-docker next time, we should verify the paths are still valid
+      # NOTE: its very important to update these paths to access any tools added or moved in the base image
       PYTHONPATH: "/etc/jupyter/custom:/usr/lib/spark/python:${NOTEBOOKS_DIR}/packages"
-      PATH: "/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${NOTEBOOKS_DIR}/packages/bin"
+      PATH: "/root/.local/bin:/etc/jupyter/bin:/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${NOTEBOOKS_DIR}/packages/bin"
     env_file:
       - /var/custom_env_vars.env
     # See https://docs.docker.com/engine/reference/run/#user-memory-constraints

http/src/main/resources/init-resources/startup.sh

@@ -365,8 +365,18 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then
     # This is to make it so that older images will still work after we change notebooks location to home dir
     docker exec ${JUPYTER_SERVER_NAME} sed -i '/^# to mount there as it effectively deletes existing files on the image/,+5d' ${JUPYTER_HOME}/jupyter_notebook_config.py
 
+  # Determine jupyter path based on image type
+  # images based on the new terra-base have jupyter installed in /etc/jupyter/bin/jupyter
+  # this marker is added as one of the last steps in the new base image here:
+  # https://github.com/DataBiosphere/terra-docker/blob/9aa343ae91ad1c94ca5a9f9e5bee02cc5d943b58/terra-base/Dockerfile#L235
+  if docker exec $JUPYTER_SERVER_NAME test -f /etc/jupyter/.terra-base-marker; then
+    JUPYTER_PATH="/etc/jupyter/bin/jupyter"
+  else
+    JUPYTER_PATH="/opt/conda/bin/jupyter"
+  fi
+
     # Start Jupyter server
-    docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "export WELDER_ENABLED=$WELDER_ENABLED && export NOTEBOOKS_DIR=$NOTEBOOKS_DIR && (/etc/jupyter/scripts/run-jupyter.sh $NOTEBOOKS_DIR || /opt/conda/bin/jupyter notebook)"
+    docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "export WELDER_ENABLED=$WELDER_ENABLED && export NOTEBOOKS_DIR=$NOTEBOOKS_DIR && (/etc/jupyter/scripts/run-jupyter.sh $NOTEBOOKS_DIR || $JUPYTER_PATH notebook)"
 fi
 
 # Configuring RStudio, if enabled

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala

@@ -107,7 +107,7 @@ class HttpDockerDAO[F[_]] private (httpClient: Client[F])(implicit logger: Logge
   ): F[Option[Token]] =
     parsedImage.registry match {
       // If it's a GCR repo, use the pet token
-      case ContainerRegistry.GCR => F.pure(petTokenOpt.map(Token))
+      case ContainerRegistry.GAR => F.pure(petTokenOpt.map(Token))
       // If it's a Dockerhub repo, need to request a token from Dockerhub
       case ContainerRegistry.DockerHub =>
         httpClient.expectOptionOr[Token](
@@ -150,14 +150,14 @@ class HttpDockerDAO[F[_]] private (httpClient: Client[F])(implicit logger: Logge
 
   private[dao] def parseImage(image: ContainerImage)(implicit ev: Ask[F, TraceId]): F[ParsedImage] =
     image.imageUrl match {
-      case GCR.regex(registry, imageName, tagOpt, shaOpt) =>
+      case GAR.regex(registry, imageName, tagOpt, shaOpt) =>
         val version = Option(tagOpt)
           .map(Tag)
           .orElse(Option(shaOpt).map(Sha))
         for {
           traceId <- ev.ask
           res <- version.fold(F.raiseError[ParsedImage](ImageParseException(traceId, image)))(i =>
-            F.pure(ParsedImage(GCR, Uri.unsafeFromString(s"https://$registry/v2"), imageName, i))
+            F.pure(ParsedImage(GAR, Uri.unsafeFromString(s"https://$registry/v2"), imageName, i))
           )
         } yield res
       case DockerHub.regex(imageName, tagOpt, shaOpt) =>

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala

@@ -159,7 +159,7 @@ abstract private[util] class BaseRuntimeInterpreter[F[_]](
           .toRight(new Exception(s"Unable to update welder because current welder image is not available"))
           .flatMap(x =>
             x.registry match {
-              case Some(ContainerRegistry.GCR) | Some(ContainerRegistry.GHCR) =>
+              case Some(ContainerRegistry.GAR) | Some(ContainerRegistry.GHCR) =>
                 Right(config.imageConfig.welderGcrImage.imageUrl)
               case Some(ContainerRegistry.DockerHub) => Right(config.imageConfig.welderDockerHubImage.imageUrl)
               case None => Left(new Exception(s"Unable to update Welder: registry for ${x.imageUrl} not parsable"))

http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAOSpec.scala

@@ -6,7 +6,7 @@ import cats.effect.unsafe.implicits.global
 import io.circe.CursorOp.DownField
 import io.circe.DecodingFailure
 import io.circe.parser.decode
-import org.broadinstitute.dsde.workbench.leonardo.ContainerRegistry.{DockerHub, GCR, GHCR}
+import org.broadinstitute.dsde.workbench.leonardo.ContainerRegistry.{DockerHub, GAR, GHCR}
 import org.broadinstitute.dsde.workbench.leonardo.RuntimeImageType.{Jupyter, RStudio}
 import org.broadinstitute.dsde.workbench.leonardo.dao.HttpDockerDAO._
 import org.broadinstitute.dsde.workbench.leonardo.model.InvalidImage
@@ -30,10 +30,10 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll
 //      "broadinstitute/leonardo-notebooks@sha256:bb959cf74f31d2a10f7bb8ee0f0754138d7c90f7ed8a92c3697ac994ff8b40b7"
 //    ),
     // gcr with tag
-    ContainerImage("us.gcr.io/broad-dsp-gcr-public/leonardo-jupyter:dev", GCR),
-    ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6", GCR),
-    ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7", GCR),
-    ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.9", GCR),
+    ContainerImage("us.gcr.io/broad-dsp-gcr-public/leonardo-jupyter:dev", GAR),
+    ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6", GAR),
+    ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7", GAR),
+    ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.9", GAR),
     // gcr with sha
     // TODO shas are currently not working
 //    GCR(
@@ -55,7 +55,7 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll
 //      "rocker/rstudio@sha256:5aea617714eb38a97a21de652ab667c6d7bb486d7468a4ab6b4d515154fec383"
 //    ),
     // gcr with tag
-    ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base:0.0.1", GCR)
+    ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base:0.0.1", GAR)
     // gcr with sha
     // TODO shas are currently not working
 //    GCR(
@@ -83,7 +83,7 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll
   }
 
   it should s"detect ImageParseException" in withDockerDAO { dockerDAO =>
-    val image = ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base", GCR) // non existent tag
+    val image = ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base", GAR) // non existent tag
     val res = for {
       ctx <- appContext.ask[AppContext]
       response <- dockerDAO.detectTool(image, None, ctx.now).attempt
@@ -93,7 +93,7 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll
 
   it should s"detect invalid GCR image if image doesn't have proper environment variables set" in withDockerDAO {
     dockerDAO =>
-      val image = ContainerImage("us.gcr.io/broad-dsp-gcr-public/welder-server:latest", GCR) // not a supported tool
+      val image = ContainerImage("us.gcr.io/broad-dsp-gcr-public/welder-server:latest", GAR) // not a supported tool
       val res = for {
         ctx <- appContext.ask[AppContext]
         response <- dockerDAO.detectTool(image, None, ctx.now).attempt

http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterpSpec.scala

@@ -378,7 +378,7 @@ class RuntimeServiceInterpTest
   it should "successfully create a cluster with an rstudio image" in isolatedDbTest {
     val cloudContext = CloudContext.Gcp(GoogleProject("googleProject"))
     val runtimeName = RuntimeName("clusterName2")
-    val rstudioImage = ContainerImage("some-rstudio-image", ContainerRegistry.GCR)
+    val rstudioImage = ContainerImage("some-rstudio-image", ContainerRegistry.GAR)
     val request = emptyCreateRuntimeReq.copy(
       toolDockerImage = Some(rstudioImage)
     )
@@ -564,7 +564,7 @@ class RuntimeServiceInterpTest
           userInfo,
           cloudContext,
           runtimeName2,
-          emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GCR))
+          emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GAR))
         )
         .attempt
       r3 <- runtimeService
@@ -634,7 +634,7 @@ class RuntimeServiceInterpTest
           userInfo,
           cloudContext,
           runtimeName2,
-          emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GCR))
+          emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GAR))
         )
         .attempt
 

http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/model/LeonardoModelSpec.scala

@@ -169,14 +169,16 @@ class LeonardoModelSpec extends LeonardoTestSuite with AnyFlatSpecLike {
   }
 
   "DockerRegistry regex" should "match expected image url format" in {
-    ContainerRegistry.GCR.regex.pattern.asPredicate().test("us.gcr.io/google/ubuntu1804:latest") shouldBe true
-    ContainerRegistry.GCR.regex.pattern.asPredicate().test("us.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
-    ContainerRegistry.GCR.regex.pattern.asPredicate().test("us/broad-dsp-gcr-public/ubuntu1804") shouldBe false
-    ContainerRegistry.GCR.regex.pattern.asPredicate().test("eu.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
-    ContainerRegistry.GCR.regex.pattern
+    ContainerRegistry.GAR.regex.pattern.asPredicate().test("us.gcr.io/google/ubuntu1804:latest") shouldBe true
+    ContainerRegistry.GAR.regex.pattern.asPredicate().test("us.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
+    ContainerRegistry.GAR.regex.pattern.asPredicate().test("us.gar.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
+    ContainerRegistry.GAR.regex.pattern.asPredicate().test("us/broad-dsp-gcr-public/ubuntu1804") shouldBe false
+    ContainerRegistry.GAR.regex.pattern.asPredicate().test("eu.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
+    ContainerRegistry.GAR.regex.pattern.asPredicate().test("eu.gar.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
+    ContainerRegistry.GAR.regex.pattern
       .asPredicate()
       .test("asia.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true
-    ContainerRegistry.GCR.regex.pattern
+    ContainerRegistry.GAR.regex.pattern
       .asPredicate()
       .test("unknown.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe false
 
@@ -212,7 +214,7 @@ class LeonardoModelSpec extends LeonardoTestSuite with AnyFlatSpecLike {
 
   "ContainerImage.stringToJupyterDockerImage" should "match GCR first, and then dockerhub" in {
     ContainerImage.fromImageUrl("us.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe (Some(
-      ContainerImage("us.gcr.io/broad-dsp-gcr-public/ubuntu1804", ContainerRegistry.GCR)
+      ContainerImage("us.gcr.io/broad-dsp-gcr-public/ubuntu1804", ContainerRegistry.GAR)
     ))
     ContainerImage.fromImageUrl("asd/asdf") shouldBe (Some(
       ContainerImage("asd/asdf", ContainerRegistry.DockerHub)

jenkins/dataproc-custom-images/prepare-custom-leonardo-jupyter-dataproc-image.sh

@@ -17,6 +17,7 @@ set -e -x
 #
 # Note: You can check which version of the AOU image is used in prod here: https://github.com/all-of-us/workbench/blob/main/api/config/config_prod.json#L15C1-L16C1
 
+terra_base="us.gcr.io/broad-dsp-gcr-public/terra-base:0.0.1"
 terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6"
 terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7"
 terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.7"
@@ -33,7 +34,7 @@ cryptomining_detector="us.gcr.io/broad-dsp-gcr-public/cryptomining-detector:0.0.
 
 # This array determines which of the above images are baked into the custom dataproc 2.2.x image
 # the entry must match the var name above, which must correspond to a valid docker URI
-docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou openidc_proxy anvil_rstudio_bioconductor cryptomining_detector"
+docker_image_var_names="welder_server terra-base terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou openidc_proxy anvil_rstudio_bioconductor cryptomining_detector"
 
 # Comment the above and uncomment this to create the dataproc 2.1.x image
 # You would also need to revert the dataproc versions in the create_dataproc_image.sh like this:

jenkins/gce-custom-images/prepare_gce_image.sh

@@ -15,7 +15,7 @@ set -e -x
 #
 # Constants and Global Vars
 #
-
+terra_base="us.gcr.io/broad-dsp-gcr-public/terra-base:0.0.1"
 terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6"
 terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7"
 terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.7"
@@ -39,7 +39,7 @@ cryptomining_detector="us.gcr.io/broad-dsp-gcr-public/cryptomining-detector:0.0.
 
 # This array determines which of the above images are baked into the custom image
 # the entry must match the var name above, which must correspond to a valid docker URI
-docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_gatk terra_jupyter_aou terra_jupyter_aou_old openidc_proxy anvil_rstudio_bioconductor cryptomining_detector cos_gpu_installer google_cloud_toolbox docker_composer docker_composer_with_auth"
+docker_image_var_names="welder_server terra_base terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_gatk terra_jupyter_aou terra_jupyter_aou_old openidc_proxy anvil_rstudio_bioconductor cryptomining_detector cos_gpu_installer google_cloud_toolbox docker_composer docker_composer_with_auth"
 
 #
 # Functions