From fb10ca526decd6228ca7e785fbfdc91747eaa98b Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 13:36:02 +0530 Subject: [PATCH 01/66] Initial setup for standalone OpenWhisk --- core/standalone/build.gradle | 44 ++++++++++++++++ .../src/main/resources/standalone.conf | 24 +++++++++ .../apache/openwhisk/standalone/Main.scala | 50 +++++++++++++++++++ settings.gradle | 1 + 4 files changed, 119 insertions(+) create mode 100644 core/standalone/build.gradle create mode 100644 core/standalone/src/main/resources/standalone.conf create mode 100644 core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle new file mode 100644 index 00000000000..ffc763c5087 --- /dev/null +++ b/core/standalone/build.gradle @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'org.springframework.boot' version '2.0.2.RELEASE' + id 'scala' +} + +apply plugin: 'org.scoverage' +apply plugin: 'maven' + +project.archivesBaseName = "openwhisk-standalone" + +repositories { + mavenCentral() +} + +jar { + enabled = true +} + +bootJar { + mainClassName = 'org.apache.openwhisk.standalone.Main' +} + +dependencies { + compile project(':core:controller') + compile 'org.rogach:scallop_2.12:3.3.1' + scoverage gradle.scoverage.deps +} diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf new file mode 100644 index 00000000000..b6a97923ff9 --- /dev/null +++ b/core/standalone/src/main/resources/standalone.conf @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include classpath("application.conf") + +whisk { + spi { + ArtifactStoreProvider = "org.apache.openwhisk.core.database.memory.MemoryArtifactStoreProvider" + } +} \ No newline at end of file diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala new file mode 100644 index 00000000000..4ce68810f6b --- /dev/null +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.standalone + +import java.io.File + +import org.apache.openwhisk.core.controller.Controller +import org.rogach.scallop.ScallopConf + +class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { + banner("OpenWhisk standalone launcher") + + val configFile = opt[File](descr = "application.conf which overwrites the default whisk.conf") + + verify() +} + +object Main { + + def main(args: Array[String]): Unit = { + val conf = new Conf(args) + initConfig(conf) + Controller.main(Array("standalone")) + } + + private def initConfig(conf: Conf): Unit = { + conf.configFile.toOption match { + case Some(f) => + require(f.exists(), s"Config file $f does not exist") + System.setProperty("config.file", f.getAbsolutePath) + case None => + System.setProperty("config.resource", "standalone.conf") + } + } +} diff --git a/settings.gradle b/settings.gradle index d4ebf0bf63c..40c59bbca9b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,6 +19,7 @@ include 'common:scala' include 'core:controller' include 'core:invoker' +include 'core:standalone' include 'tests' include 'tests:performance:gatling_tests' From 74538d9f915d876826547a46b5ff296d36c3d54d Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 14:51:52 +0530 Subject: [PATCH 02/66] Enable lean providers --- core/standalone/src/main/resources/standalone.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index b6a97923ff9..d9878622ebc 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -20,5 +20,7 @@ include classpath("application.conf") whisk { spi { ArtifactStoreProvider = "org.apache.openwhisk.core.database.memory.MemoryArtifactStoreProvider" + MessagingProvider = "org.apache.openwhisk.connector.lean.LeanMessagingProvider" + LoadBalancerProvider = "org.apache.openwhisk.core.loadBalancer.LeanBalancer" } } \ No newline at end of file From 9f3acbb91815a11a4a7b6ca4373d0029004561a3 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 14:56:04 +0530 Subject: [PATCH 03/66] Add support for setting whisk properties from typesafe config --- .../org/apache/openwhisk/common/Config.scala | 20 +++++++++++++++---- .../apache/openwhisk/core/WhiskConfig.scala | 2 ++ .../src/main/resources/standalone.conf | 14 +++++++++++++ .../apache/openwhisk/standalone/Main.scala | 15 ++++++++++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala b/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala index b9fd333982a..91d5e657733 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala @@ -116,14 +116,14 @@ class Config(requiredProperties: Map[String, String], optionalProperties: Set[St */ protected def getProperties(): scala.collection.mutable.Map[String, String] = { val required = scala.collection.mutable.Map[String, String]() ++= requiredProperties - Config.readPropertiesFromEnvironment(required, env) + Config.readPropertiesFromSystemAndEnv(required, env) // for optional value, assign them a default from the required properties list // to prevent loss of a default value on a required property that may not otherwise be defined val optional = scala.collection.mutable.Map[String, String]() ++= optionalProperties.map { k => k -> required.getOrElse(k, null) } - Config.readPropertiesFromEnvironment(optional, env) + Config.readPropertiesFromSystemAndEnv(optional, env) required ++ optional } @@ -133,13 +133,15 @@ class Config(requiredProperties: Map[String, String], optionalProperties: Set[St * Singleton object which provides global methods to manage configuration. */ object Config { + val prefix = "whisk-config." /** * Reads a Map of key-value pairs from the environment -- store them in the * mutable properties object. */ - def readPropertiesFromEnvironment(properties: scala.collection.mutable.Map[String, String], env: Map[String, String])( - implicit logging: Logging) = { + def readPropertiesFromSystemAndEnv(properties: scala.collection.mutable.Map[String, String], + env: Map[String, String])(implicit logging: Logging) = { + readPropertiesFromSystem(properties) for (p <- properties.keys) { val envp = p.replace('.', '_').toUpperCase val envv = env.get(envp) @@ -150,6 +152,16 @@ object Config { } } + def readPropertiesFromSystem(properties: scala.collection.mutable.Map[String, String])(implicit logging: Logging) = { + for (p <- properties.keys) { + val sysv = Option(System.getProperty(prefix + p)) + if (sysv.isDefined) { + logging.info(this, s"system set value for $p") + properties += p -> sysv.get.trim + } + } + } + /** * Checks that the properties object defines all the required properties. * diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index 4a0fb4f20a5..3cc41dfeca0 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -255,4 +255,6 @@ object ConfigKeys { val metrics = "whisk.metrics" val featureFlags = "whisk.feature-flags" + + val whiskConfig = "whisk.config" } diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index d9878622ebc..f46bf1f07ab 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -23,4 +23,18 @@ whisk { MessagingProvider = "org.apache.openwhisk.connector.lean.LeanMessagingProvider" LoadBalancerProvider = "org.apache.openwhisk.core.loadBalancer.LeanBalancer" } + + info { + build-no = "standalone" + date = "???" + } + + config { + whisk-api-host-name = "localhost" + controller-instances = 1 + limits-actions-sequence-maxLength = 50 + limits-triggers-fires-perMinute = 60 + limits-actions-invokes-perMinute = 60 + limits-actions-invokes-concurrent = 30 + } } \ No newline at end of file diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index 4ce68810f6b..055500a37bc 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -19,8 +19,11 @@ package org.apache.openwhisk.standalone import java.io.File +import org.apache.openwhisk.common.Config +import org.apache.openwhisk.core.ConfigKeys import org.apache.openwhisk.core.controller.Controller import org.rogach.scallop.ScallopConf +import pureconfig.loadConfigOrThrow class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner("OpenWhisk standalone launcher") @@ -34,11 +37,12 @@ object Main { def main(args: Array[String]): Unit = { val conf = new Conf(args) - initConfig(conf) + initConfigLocation(conf) + loadWhiskConfig() Controller.main(Array("standalone")) } - private def initConfig(conf: Conf): Unit = { + private def initConfigLocation(conf: Conf): Unit = { conf.configFile.toOption match { case Some(f) => require(f.exists(), s"Config file $f does not exist") @@ -47,4 +51,11 @@ object Main { System.setProperty("config.resource", "standalone.conf") } } + + def configKey(k: String): String = Config.prefix + k.replace('-', '.') + + def loadWhiskConfig(): Unit = { + val config = loadConfigOrThrow[Map[String, String]](ConfigKeys.whiskConfig) + config.foreach { case (k, v) => System.setProperty(configKey(k), v) } + } } From d58c5f1b69e5e9239a5b7791317448a2db949229 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 16:33:24 +0530 Subject: [PATCH 04/66] Add support for configuring manifest --- .../apache/openwhisk/standalone/Main.scala | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index 055500a37bc..c0cbd85900a 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -18,9 +18,11 @@ package org.apache.openwhisk.standalone import java.io.File +import java.nio.charset.StandardCharsets.UTF_8 +import org.apache.commons.io.FileUtils import org.apache.openwhisk.common.Config -import org.apache.openwhisk.core.ConfigKeys +import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} import org.apache.openwhisk.core.controller.Controller import org.rogach.scallop.ScallopConf import pureconfig.loadConfigOrThrow @@ -29,15 +31,44 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner("OpenWhisk standalone launcher") val configFile = opt[File](descr = "application.conf which overwrites the default whisk.conf") + val manifest = opt[File](descr = "Manifest json defining the supported runtimes") verify() } object Main { + val defaultRuntime = """{ + | "runtimes": { + | "nodejs": [ + | { + | "kind": "nodejs:10", + | "default": true, + | "image": { + | "prefix": "openwhisk", + | "name": "action-nodejs-v10", + | "tag": "latest" + | }, + | "deprecated": false, + | "attached": { + | "attachmentName": "codefile", + | "attachmentType": "text/plain" + | }, + | "stemCells": [ + | { + | "count": 1, + | "memory": "256 MB" + | } + | ] + | } + | ] + | } + |} + |""".stripMargin def main(args: Array[String]): Unit = { val conf = new Conf(args) initConfigLocation(conf) + configureRuntimeManifest(conf) loadWhiskConfig() Controller.main(Array("standalone")) } @@ -58,4 +89,14 @@ object Main { val config = loadConfigOrThrow[Map[String, String]](ConfigKeys.whiskConfig) config.foreach { case (k, v) => System.setProperty(configKey(k), v) } } + + def configureRuntimeManifest(conf: Conf): Unit = { + val manifest = conf.manifest.toOption match { + case Some(file) => + FileUtils.readFileToString(file, UTF_8) + case None => + defaultRuntime + } + System.setProperty(configKey(WhiskConfig.runtimesManifest), manifest) + } } From bcd48ba28fb2c70667e97111cbdf4b831ef6f59b Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 17:53:39 +0530 Subject: [PATCH 05/66] Make MemoryArtifactStore work in singleton mode --- .../scala/src/main/resources/reference.conf | 12 +++++++++++ .../apache/openwhisk/core/WhiskConfig.scala | 1 + .../database/memory/MemoryArtifactStore.scala | 21 +++++++++++++++---- tests/src/test/resources/application.conf.j2 | 7 +++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/common/scala/src/main/resources/reference.conf b/common/scala/src/main/resources/reference.conf index 51779cd31f6..473113e1535 100644 --- a/common/scala/src/main/resources/reference.conf +++ b/common/scala/src/main/resources/reference.conf @@ -82,3 +82,15 @@ dispatchers { throughput = 5 } } + +whisk { + memory-store { + # If set to false then a new store instance is created for each `makeStore` call + # even for same entity type + singleton-mode = true + + # If false then in memory state would be cleared. Default is false so as to ensure + # any changes made reamin visible in other live references + ignore-shutdown = true + } +} diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index 3cc41dfeca0..3d7ba5d46eb 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -257,4 +257,5 @@ object ConfigKeys { val featureFlags = "whisk.feature-flags" val whiskConfig = "whisk.config" + val memoryStore = "whisk.memory-store" } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala index 049bcbe0de0..0c20a095978 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala @@ -39,6 +39,8 @@ import scala.reflect.ClassTag import scala.util.{Failure, Success, Try} object MemoryArtifactStoreProvider extends ArtifactStoreProvider { + case class MemoryStoreConfig(singletonMode: Boolean, ignoreShutdown: Boolean) + private val stores = new TrieMap[String, MemoryArtifactStore[_]]() override def makeStore[D <: DocumentSerializer: ClassTag](useBatching: Boolean)( implicit jsonFormat: RootJsonFormat[D], docReader: DocumentReader, @@ -55,10 +57,18 @@ object MemoryArtifactStoreProvider extends ArtifactStoreProvider { logging: Logging, materializer: ActorMaterializer): ArtifactStore[D] = { + val memoryStoreConfig = loadConfigOrThrow[MemoryStoreConfig](actorSystem.settings.config, ConfigKeys.memoryStore) val classTag = implicitly[ClassTag[D]] val (dbName, handler, viewMapper) = handlerAndMapper(classTag) val inliningConfig = loadConfigOrThrow[InliningConfig](ConfigKeys.db) - new MemoryArtifactStore(dbName, handler, viewMapper, inliningConfig, attachmentStore) + val store = new MemoryArtifactStore( + dbName, + handler, + viewMapper, + inliningConfig, + attachmentStore, + memoryStoreConfig.ignoreShutdown) + if (memoryStoreConfig.singletonMode) stores.getOrElseUpdate(dbName, store).asInstanceOf[ArtifactStore[D]] else store } private def handlerAndMapper[D](entityType: ClassTag[D])( @@ -85,7 +95,8 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str documentHandler: DocumentHandler, viewMapper: MemoryViewMapper, val inliningConfig: InliningConfig, - val attachmentStore: AttachmentStore)( + val attachmentStore: AttachmentStore, + ignoreShutdown: Boolean)( implicit system: ActorSystem, val logging: Logging, jsonFormat: RootJsonFormat[DocumentAbstraction], @@ -285,8 +296,10 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str } override def shutdown(): Unit = { - artifacts.clear() - attachmentStore.shutdown() + if (!ignoreShutdown) { + artifacts.clear() + attachmentStore.shutdown() + } } override protected[database] def get(id: DocId)(implicit transid: TransactionId): Future[Option[JsObject]] = { diff --git a/tests/src/test/resources/application.conf.j2 b/tests/src/test/resources/application.conf.j2 index 0a4feb26afb..a11a3ef45f8 100644 --- a/tests/src/test/resources/application.conf.j2 +++ b/tests/src/test/resources/application.conf.j2 @@ -62,6 +62,13 @@ whisk { throughput = 400 } + whisk { + memory-store { + singleton-mode = false + ignore-shutdown = false + } + } + controller { protocol = {{ controller.protocol }} https { From e543afab0ccb742de711d236821e90e79d4a7aef Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 17:55:03 +0530 Subject: [PATCH 06/66] Allow bootstrapping default set of users Refactor Controller such that actor system can be passed from outside --- .../core/controller/Controller.scala | 8 +++- core/standalone/build.gradle | 1 + .../src/main/resources/standalone.conf | 7 ++++ .../apache/openwhisk/standalone/Main.scala | 41 +++++++++++++++++-- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala index 44a70998890..0caf0feffb4 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala @@ -207,10 +207,14 @@ object Controller { "runtimes" -> runtimes.toJson) def main(args: Array[String]): Unit = { - ConfigMXBean.register() - Kamon.loadReportersFromConfig() implicit val actorSystem = ActorSystem("controller-actor-system") implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) + start(args) + } + + def start(args: Array[String])(implicit actorSystem: ActorSystem, logger: Logging): Unit = { + ConfigMXBean.register() + Kamon.loadReportersFromConfig() // Prepare Kamon shutdown CoordinatedShutdown(actorSystem).addTask(CoordinatedShutdown.PhaseActorSystemTerminate, "shutdownKamon") { () => diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index ffc763c5087..e5e98c0af92 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -39,6 +39,7 @@ bootJar { dependencies { compile project(':core:controller') + compile project(':tools:admin') compile 'org.rogach:scallop_2.12:3.3.1' scoverage gradle.scoverage.deps } diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index f46bf1f07ab..0174f5ac3ea 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -22,6 +22,7 @@ whisk { ArtifactStoreProvider = "org.apache.openwhisk.core.database.memory.MemoryArtifactStoreProvider" MessagingProvider = "org.apache.openwhisk.connector.lean.LeanMessagingProvider" LoadBalancerProvider = "org.apache.openwhisk.core.loadBalancer.LeanBalancer" + ContainerFactoryProvider = "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider" } info { @@ -37,4 +38,10 @@ whisk { limits-actions-invokes-perMinute = 60 limits-actions-invokes-concurrent = 30 } + + # Default set of users which are bootstrapped upon start + users { + whisk-system = "789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP" + guest = "23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP" + } } \ No newline at end of file diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index c0cbd85900a..c90331966a4 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -20,15 +20,22 @@ package org.apache.openwhisk.standalone import java.io.File import java.nio.charset.StandardCharsets.UTF_8 +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import org.apache.commons.io.FileUtils -import org.apache.openwhisk.common.Config -import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} +import org.apache.openwhisk.common.{AkkaLogging, Config, Logging} +import org.apache.openwhisk.core.cli.WhiskAdmin import org.apache.openwhisk.core.controller.Controller +import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} import org.rogach.scallop.ScallopConf import pureconfig.loadConfigOrThrow +import scala.concurrent.Await +import scala.concurrent.duration._ + class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner("OpenWhisk standalone launcher") + //TODO Make server port configurable val configFile = opt[File](descr = "application.conf which overwrites the default whisk.conf") val manifest = opt[File](descr = "Manifest json defining the supported runtimes") @@ -67,10 +74,25 @@ object Main { def main(args: Array[String]): Unit = { val conf = new Conf(args) + initialize(conf) + + //Create actor system only after initializing the config + implicit val actorSystem = ActorSystem("standalone-actor-system") + implicit val materializer = ActorMaterializer.create(actorSystem) + implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) + + bootstrapUsers() + startController() + } + + def initialize(conf: Conf): Unit = { initConfigLocation(conf) configureRuntimeManifest(conf) loadWhiskConfig() - Controller.main(Array("standalone")) + } + + def startController()(implicit actorSystem: ActorSystem, logger: Logging): Unit = { + Controller.start(Array("standalone")) } private def initConfigLocation(conf: Conf): Unit = { @@ -99,4 +121,17 @@ object Main { } System.setProperty(configKey(WhiskConfig.runtimesManifest), manifest) } + + def bootstrapUsers()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer, logging: Logging): Unit = { + val users = loadConfigOrThrow[Map[String, String]]("whisk.users") + + users.foreach { + case (name, key) => + val subject = name.replace('-', '.') + val conf = new org.apache.openwhisk.core.cli.Conf(Seq("user", "create", "--auth", key, subject)) + val admin = WhiskAdmin(conf) + Await.ready(admin.executeCommand(), 60.seconds) + logging.info(this, s"Created user [$subject]") + } + } } From 19e2c5ed7444de050739cdcdf2c69cae2923763b Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 18:02:25 +0530 Subject: [PATCH 07/66] Allow configuring server port --- .../apache/openwhisk/standalone/Main.scala | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index c90331966a4..857f13caf88 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -21,6 +21,7 @@ import java.io.File import java.nio.charset.StandardCharsets.UTF_8 import akka.actor.ActorSystem +import akka.event.slf4j.SLF4JLogging import akka.stream.ActorMaterializer import org.apache.commons.io.FileUtils import org.apache.openwhisk.common.{AkkaLogging, Config, Logging} @@ -35,15 +36,15 @@ import scala.concurrent.duration._ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner("OpenWhisk standalone launcher") - //TODO Make server port configurable val configFile = opt[File](descr = "application.conf which overwrites the default whisk.conf") val manifest = opt[File](descr = "Manifest json defining the supported runtimes") + val port = opt[Int](descr = "Server port", default = Some(8080)) verify() } -object Main { +object Main extends SLF4JLogging { val defaultRuntime = """{ | "runtimes": { | "nodejs": [ @@ -85,7 +86,14 @@ object Main { startController() } + def configureServerPort(conf: Conf) = { + val port = conf.port() + log.info(s"Starting OpenWhisk standalone on port $port") + setConfigProp(WhiskConfig.servicePort, port.toString) + } + def initialize(conf: Conf): Unit = { + configureServerPort(conf) initConfigLocation(conf) configureRuntimeManifest(conf) loadWhiskConfig() @@ -109,7 +117,7 @@ object Main { def loadWhiskConfig(): Unit = { val config = loadConfigOrThrow[Map[String, String]](ConfigKeys.whiskConfig) - config.foreach { case (k, v) => System.setProperty(configKey(k), v) } + config.foreach { case (k, v) => setConfigProp(k, v) } } def configureRuntimeManifest(conf: Conf): Unit = { @@ -119,7 +127,11 @@ object Main { case None => defaultRuntime } - System.setProperty(configKey(WhiskConfig.runtimesManifest), manifest) + setConfigProp(WhiskConfig.runtimesManifest, manifest) + } + + def setConfigProp(key: String, value: String): Unit = { + System.setProperty(configKey(key), value) } def bootstrapUsers()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer, logging: Logging): Unit = { From 59e11913e0f9a67799324ce67de37a6542278bf0 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 18:20:15 +0530 Subject: [PATCH 08/66] Disable jar target This ensures that standalone jar gets created as main artifact --- core/standalone/build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index e5e98c0af92..b5d787c0970 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -29,10 +29,6 @@ repositories { mavenCentral() } -jar { - enabled = true -} - bootJar { mainClassName = 'org.apache.openwhisk.standalone.Main' } From e4574eb8ec82268cf9e03ee8f6e23af4f808ceb9 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 18:21:19 +0530 Subject: [PATCH 09/66] Make OS specific setting programatic --- core/standalone/src/main/resources/standalone.conf | 1 - .../scala/org/apache/openwhisk/standalone/Main.scala | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index 0174f5ac3ea..f21fdef1b80 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -22,7 +22,6 @@ whisk { ArtifactStoreProvider = "org.apache.openwhisk.core.database.memory.MemoryArtifactStoreProvider" MessagingProvider = "org.apache.openwhisk.connector.lean.LeanMessagingProvider" LoadBalancerProvider = "org.apache.openwhisk.core.loadBalancer.LeanBalancer" - ContainerFactoryProvider = "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider" } info { diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index 857f13caf88..844d7ca0f4a 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -24,6 +24,7 @@ import akka.actor.ActorSystem import akka.event.slf4j.SLF4JLogging import akka.stream.ActorMaterializer import org.apache.commons.io.FileUtils +import org.apache.commons.lang3.SystemUtils import org.apache.openwhisk.common.{AkkaLogging, Config, Logging} import org.apache.openwhisk.core.cli.WhiskAdmin import org.apache.openwhisk.core.controller.Controller @@ -94,6 +95,7 @@ object Main extends SLF4JLogging { def initialize(conf: Conf): Unit = { configureServerPort(conf) + configureOSSpecificOpts() initConfigLocation(conf) configureRuntimeManifest(conf) loadWhiskConfig() @@ -146,4 +148,13 @@ object Main extends SLF4JLogging { logging.info(this, s"Created user [$subject]") } } + + def configureOSSpecificOpts(): Unit = { + if (SystemUtils.IS_OS_MAC) { + System.setProperty("whisk.docker.container-factory.use-runc", "False") + System.setProperty( + "whisk.spi.ContainerFactoryProvider", + "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider") + } + } } From 0237e5474d076f8ed22833dc2403720d52b3a235 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 18 Jun 2019 18:33:23 +0530 Subject: [PATCH 10/66] Enable prometheus metrics by default --- core/standalone/src/main/resources/standalone.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index f21fdef1b80..10879e547d5 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -17,7 +17,17 @@ include classpath("application.conf") +kamon { + reporters = [] +} + whisk { + metrics { + kamon-enabled = true + kamon-tags-enabled = true + prometheus-enabled = true + } + spi { ArtifactStoreProvider = "org.apache.openwhisk.core.database.memory.MemoryArtifactStoreProvider" MessagingProvider = "org.apache.openwhisk.connector.lean.LeanMessagingProvider" From 4716ac9e606f09fd82dd12739321d78f2d606177 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 19 Jun 2019 09:47:12 +0530 Subject: [PATCH 11/66] Fix the description --- .../src/main/scala/org/apache/openwhisk/standalone/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index 844d7ca0f4a..1e570f9e127 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -38,7 +38,7 @@ import scala.concurrent.duration._ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner("OpenWhisk standalone launcher") - val configFile = opt[File](descr = "application.conf which overwrites the default whisk.conf") + val configFile = opt[File](descr = "application.conf which overrides the default standalone.conf") val manifest = opt[File](descr = "Manifest json defining the supported runtimes") val port = opt[Int](descr = "Server port", default = Some(8080)) From 5d11c5d40b5c0484fed462c4483b94b980c45ead Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Thu, 20 Jun 2019 12:10:22 +0530 Subject: [PATCH 12/66] Use WhiskerControl port 3233 as default! --- .../src/main/scala/org/apache/openwhisk/standalone/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala index 1e570f9e127..aab197957b8 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala @@ -40,7 +40,7 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { val configFile = opt[File](descr = "application.conf which overrides the default standalone.conf") val manifest = opt[File](descr = "Manifest json defining the supported runtimes") - val port = opt[Int](descr = "Server port", default = Some(8080)) + val port = opt[Int](descr = "Server port", default = Some(3233)) verify() } From 13930ebe91461d6e77e594aa398c1312776e4d7c Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Thu, 20 Jun 2019 15:46:38 +0530 Subject: [PATCH 13/66] Allow disabling read of whisk.properties This is required for test run to avoid launched server influenced by whisk.properties generated for the tests --- .../scala/org/apache/openwhisk/core/WhiskConfig.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index 3d7ba5d46eb..a064d2b5032 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -49,10 +49,14 @@ class WhiskConfig(requiredProperties: Map[String, String], */ override protected def getProperties() = { val properties = super.getProperties() - WhiskConfig.readPropertiesFromFile(properties, Option(propertiesFile) getOrElse (WhiskConfig.whiskPropertiesFile)) + if (!disableReadFromFile()) { + WhiskConfig.readPropertiesFromFile(properties, Option(propertiesFile) getOrElse (WhiskConfig.whiskPropertiesFile)) + } properties } + private def disableReadFromFile() = java.lang.Boolean.getBoolean(WhiskConfig.disableWhiskPropsFileRead) + val servicePort = this(WhiskConfig.servicePort) val dockerEndpoint = this(WhiskConfig.dockerEndpoint) val dockerPort = this(WhiskConfig.dockerPort) @@ -85,6 +89,7 @@ class WhiskConfig(requiredProperties: Map[String, String], } object WhiskConfig { + val disableWhiskPropsFileRead = Config.prefix + "disable.whisks.props.file.read" /** * Reads a key from system environment as if it was part of WhiskConfig. @@ -170,7 +175,7 @@ object WhiskConfig { val kafkaHostList = "kafka.hosts" val zookeeperHostList = "zookeeper.hosts" - private val edgeHostApiPort = "edge.host.apiport" + val edgeHostApiPort = "edge.host.apiport" val invokerHostsList = "invoker.hosts" val dbHostsList = "db.hostsList" From 03170d0ef5740a3e8ba0bb7977451323af202ce0 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Thu, 20 Jun 2019 18:19:19 +0530 Subject: [PATCH 14/66] Ensure that MemoryStore is only created when needed This avoid superfluous log message about false inits --- .../database/memory/MemoryArtifactStore.scala | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala index 0c20a095978..fb2f1d1a280 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala @@ -61,14 +61,17 @@ object MemoryArtifactStoreProvider extends ArtifactStoreProvider { val classTag = implicitly[ClassTag[D]] val (dbName, handler, viewMapper) = handlerAndMapper(classTag) val inliningConfig = loadConfigOrThrow[InliningConfig](ConfigKeys.db) - val store = new MemoryArtifactStore( - dbName, - handler, - viewMapper, - inliningConfig, - attachmentStore, - memoryStoreConfig.ignoreShutdown) - if (memoryStoreConfig.singletonMode) stores.getOrElseUpdate(dbName, store).asInstanceOf[ArtifactStore[D]] else store + val storeFactory = () => + new MemoryArtifactStore( + dbName, + handler, + viewMapper, + inliningConfig, + attachmentStore, + memoryStoreConfig.ignoreShutdown) + if (memoryStoreConfig.singletonMode) + stores.getOrElseUpdate(dbName, storeFactory.apply()).asInstanceOf[ArtifactStore[D]] + else storeFactory.apply() } private def handlerAndMapper[D](entityType: ClassTag[D])( @@ -107,6 +110,8 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str with DocumentProvider with AttachmentSupport[DocumentAbstraction] { + logging.info(this, s"Created MemoryStore for [$dbName]. ignoreShutdown=$ignoreShutdown") + override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher private val artifacts = new TrieMap[String, Artifact] From bf6b25f2d60fdf59627b518030a6fb1ffe502d60 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 10:28:27 +0530 Subject: [PATCH 15/66] Fetch logs via `docker logs` command on Mac --- .../core/containerpool/Container.scala | 2 + .../docker/DockerForMacContainerFactory.scala | 45 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala index 1da3e2f2c35..45662c29038 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala @@ -82,6 +82,8 @@ trait Container { protected var containerHttpMaxConcurrent: Int = 1 protected var containerHttpTimeout: FiniteDuration = 60.seconds + def containerId: ContainerId = id + /** Stops the container from consuming CPU cycles. NOT thread-safe - caller must synchronize. */ def suspend()(implicit transid: TransactionId): Future[Unit] = { //close connection first, then close connection pool diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala index fb7bc2e9b81..b6dca4dc764 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala @@ -17,11 +17,20 @@ package org.apache.openwhisk.core.containerpool.docker +import java.time.Instant + import akka.actor.ActorSystem -import org.apache.openwhisk.common.{Logging, TransactionId} +import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId} import org.apache.openwhisk.core.WhiskConfig import org.apache.openwhisk.core.containerpool._ -import org.apache.openwhisk.core.entity.InvokerInstanceId +import org.apache.openwhisk.core.containerpool.logging.{DockerToActivationLogStore, LogStore, LogStoreProvider} +import org.apache.openwhisk.core.entity.{ + ActivationLogs, + ExecutableWhiskAction, + Identity, + InvokerInstanceId, + WhiskActivation +} import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration._ @@ -52,6 +61,8 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex with DockerApiWithFileAccess { implicit private val ec: ExecutionContext = executionContext + private val waitForLogs: FiniteDuration = 2.seconds + private val logTimeSpanMargin = 1.second override def run(image: String, args: Seq[String] = Seq.empty[String])( implicit transid: TransactionId): Future[ContainerId] = { @@ -72,4 +83,34 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex case stdout => Future.successful(ContainerAddress("localhost", stdout.toInt)) } } + + def collectLogs(id: ContainerId, since: Instant, untill: Instant)(implicit transid: TransactionId): Future[String] = { + //Add a slight buffer to account for delay writes of logs + val end = untill.plusSeconds(logTimeSpanMargin.toSeconds) + runCmd( + Seq("logs", id.asString, "--since", since.getEpochSecond.toString, "--until", end.getEpochSecond.toString), + waitForLogs) + } +} + +object DockerForMacLogStoreProvider extends LogStoreProvider { + override def instance(actorSystem: ActorSystem): LogStore = { + //Logger is currently not passed implicitly to LogStoreProvider. So create one explicitly + implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) + new DockerForMacLogStore(actorSystem) + } +} + +class DockerForMacLogStore(system: ActorSystem)(implicit log: Logging) extends DockerToActivationLogStore(system) { + private val client = new DockerForMacClient()(system.dispatcher)(log, system) + override def collectLogs(transid: TransactionId, + user: Identity, + activation: WhiskActivation, + container: Container, + action: ExecutableWhiskAction): Future[ActivationLogs] = { + //TODO Lookup for Log markers to be more precise in log collection + client + .collectLogs(container.containerId, activation.start, activation.end)(transid) + .map(logs => ActivationLogs(logs.linesIterator.toVector)) + } } From f988f4521a84579223ec98f71043d106ad6c321d Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 11:00:01 +0530 Subject: [PATCH 16/66] Revert config support for MemoryArtifactStore for now MemoryArtifactStore would always return same instance for same db name --- .../scala/src/main/resources/reference.conf | 12 --------- .../apache/openwhisk/core/WhiskConfig.scala | 1 - .../database/memory/MemoryArtifactStore.scala | 27 +++++-------------- tests/src/test/resources/application.conf.j2 | 7 ----- 4 files changed, 7 insertions(+), 40 deletions(-) diff --git a/common/scala/src/main/resources/reference.conf b/common/scala/src/main/resources/reference.conf index 473113e1535..51779cd31f6 100644 --- a/common/scala/src/main/resources/reference.conf +++ b/common/scala/src/main/resources/reference.conf @@ -82,15 +82,3 @@ dispatchers { throughput = 5 } } - -whisk { - memory-store { - # If set to false then a new store instance is created for each `makeStore` call - # even for same entity type - singleton-mode = true - - # If false then in memory state would be cleared. Default is false so as to ensure - # any changes made reamin visible in other live references - ignore-shutdown = true - } -} diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index a064d2b5032..44e646864ca 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -262,5 +262,4 @@ object ConfigKeys { val featureFlags = "whisk.feature-flags" val whiskConfig = "whisk.config" - val memoryStore = "whisk.memory-store" } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala index fb2f1d1a280..6b8e2ad76d6 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala @@ -39,7 +39,6 @@ import scala.reflect.ClassTag import scala.util.{Failure, Success, Try} object MemoryArtifactStoreProvider extends ArtifactStoreProvider { - case class MemoryStoreConfig(singletonMode: Boolean, ignoreShutdown: Boolean) private val stores = new TrieMap[String, MemoryArtifactStore[_]]() override def makeStore[D <: DocumentSerializer: ClassTag](useBatching: Boolean)( implicit jsonFormat: RootJsonFormat[D], @@ -57,23 +56,15 @@ object MemoryArtifactStoreProvider extends ArtifactStoreProvider { logging: Logging, materializer: ActorMaterializer): ArtifactStore[D] = { - val memoryStoreConfig = loadConfigOrThrow[MemoryStoreConfig](actorSystem.settings.config, ConfigKeys.memoryStore) val classTag = implicitly[ClassTag[D]] val (dbName, handler, viewMapper) = handlerAndMapper(classTag) val inliningConfig = loadConfigOrThrow[InliningConfig](ConfigKeys.db) - val storeFactory = () => - new MemoryArtifactStore( - dbName, - handler, - viewMapper, - inliningConfig, - attachmentStore, - memoryStoreConfig.ignoreShutdown) - if (memoryStoreConfig.singletonMode) - stores.getOrElseUpdate(dbName, storeFactory.apply()).asInstanceOf[ArtifactStore[D]] - else storeFactory.apply() + val storeFactory = () => new MemoryArtifactStore(dbName, handler, viewMapper, inliningConfig, attachmentStore) + stores.getOrElseUpdate(dbName, storeFactory.apply()).asInstanceOf[ArtifactStore[D]] } + def purgeAll(): Unit = stores.clear() + private def handlerAndMapper[D](entityType: ClassTag[D])( implicit actorSystem: ActorSystem, logging: Logging, @@ -98,8 +89,7 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str documentHandler: DocumentHandler, viewMapper: MemoryViewMapper, val inliningConfig: InliningConfig, - val attachmentStore: AttachmentStore, - ignoreShutdown: Boolean)( + val attachmentStore: AttachmentStore)( implicit system: ActorSystem, val logging: Logging, jsonFormat: RootJsonFormat[DocumentAbstraction], @@ -110,7 +100,7 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str with DocumentProvider with AttachmentSupport[DocumentAbstraction] { - logging.info(this, s"Created MemoryStore for [$dbName]. ignoreShutdown=$ignoreShutdown") + logging.info(this, s"Created MemoryStore for [$dbName]") override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher @@ -301,10 +291,7 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str } override def shutdown(): Unit = { - if (!ignoreShutdown) { - artifacts.clear() - attachmentStore.shutdown() - } + attachmentStore.shutdown() } override protected[database] def get(id: DocId)(implicit transid: TransactionId): Future[Option[JsObject]] = { diff --git a/tests/src/test/resources/application.conf.j2 b/tests/src/test/resources/application.conf.j2 index a11a3ef45f8..0a4feb26afb 100644 --- a/tests/src/test/resources/application.conf.j2 +++ b/tests/src/test/resources/application.conf.j2 @@ -62,13 +62,6 @@ whisk { throughput = 400 } - whisk { - memory-store { - singleton-mode = false - ignore-shutdown = false - } - } - controller { protocol = {{ controller.protocol }} https { From eb4eced1c80cffe24d60d99f1e1b4c5edc92b7a8 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 11:12:23 +0530 Subject: [PATCH 17/66] Rename the main class Also ensure that a system property is only set if no existing value found. This ensures that in special case user can still supercede the value being set via explicitly providing a system property override --- core/standalone/build.gradle | 2 +- .../{Main.scala => StandaloneOpenWhisk.scala} | 62 +++++++++++++------ 2 files changed, 43 insertions(+), 21 deletions(-) rename core/standalone/src/main/scala/org/apache/openwhisk/standalone/{Main.scala => StandaloneOpenWhisk.scala} (73%) diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index b5d787c0970..90f5e493e35 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -30,7 +30,7 @@ repositories { } bootJar { - mainClassName = 'org.apache.openwhisk.standalone.Main' + mainClassName = 'org.apache.openwhisk.standalone.StandaloneOpenWhisk' } dependencies { diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala similarity index 73% rename from core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala rename to core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index aab197957b8..a1cc45f5fad 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/Main.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -45,7 +45,7 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { verify() } -object Main extends SLF4JLogging { +object StandaloneOpenWhisk extends SLF4JLogging { val defaultRuntime = """{ | "runtimes": { | "nodejs": [ @@ -83,14 +83,7 @@ object Main extends SLF4JLogging { implicit val materializer = ActorMaterializer.create(actorSystem) implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) - bootstrapUsers() - startController() - } - - def configureServerPort(conf: Conf) = { - val port = conf.port() - log.info(s"Starting OpenWhisk standalone on port $port") - setConfigProp(WhiskConfig.servicePort, port.toString) + startServer() } def initialize(conf: Conf): Unit = { @@ -101,8 +94,19 @@ object Main extends SLF4JLogging { loadWhiskConfig() } - def startController()(implicit actorSystem: ActorSystem, logger: Logging): Unit = { - Controller.start(Array("standalone")) + def startServer()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer, logging: Logging): Unit = { + bootstrapUsers() + startController() + } + + private def configureServerPort(conf: Conf) = { + val port = conf.port() + log.info(s"Starting OpenWhisk standalone on port $port") + System.setProperty(WhiskConfig.disableWhiskPropsFileRead, "true") + setConfigProp(WhiskConfig.servicePort, port.toString) + setConfigProp(WhiskConfig.wskApiPort, port.toString) + setConfigProp(WhiskConfig.wskApiProtocol, "http") + setConfigProp(WhiskConfig.wskApiHostname, "localhost") } private def initConfigLocation(conf: Conf): Unit = { @@ -115,14 +119,14 @@ object Main extends SLF4JLogging { } } - def configKey(k: String): String = Config.prefix + k.replace('-', '.') + private def configKey(k: String): String = Config.prefix + k.replace('-', '.') - def loadWhiskConfig(): Unit = { + private def loadWhiskConfig(): Unit = { val config = loadConfigOrThrow[Map[String, String]](ConfigKeys.whiskConfig) config.foreach { case (k, v) => setConfigProp(k, v) } } - def configureRuntimeManifest(conf: Conf): Unit = { + private def configureRuntimeManifest(conf: Conf): Unit = { val manifest = conf.manifest.toOption match { case Some(file) => FileUtils.readFileToString(file, UTF_8) @@ -132,11 +136,17 @@ object Main extends SLF4JLogging { setConfigProp(WhiskConfig.runtimesManifest, manifest) } - def setConfigProp(key: String, value: String): Unit = { - System.setProperty(configKey(key), value) + private def setConfigProp(key: String, value: String): Unit = { + setSysPro(configKey(key), value) } - def bootstrapUsers()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer, logging: Logging): Unit = { + private def startController()(implicit actorSystem: ActorSystem, logger: Logging): Unit = { + Controller.start(Array("standalone"), "http") + } + + private def bootstrapUsers()(implicit actorSystem: ActorSystem, + materializer: ActorMaterializer, + logging: Logging): Unit = { val users = loadConfigOrThrow[Map[String, String]]("whisk.users") users.foreach { @@ -149,12 +159,24 @@ object Main extends SLF4JLogging { } } - def configureOSSpecificOpts(): Unit = { + private def configureOSSpecificOpts(): Unit = { if (SystemUtils.IS_OS_MAC) { - System.setProperty("whisk.docker.container-factory.use-runc", "False") - System.setProperty( + setSysPro("whisk.docker.container-factory.use-runc", "False") + setSysPro( "whisk.spi.ContainerFactoryProvider", "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider") + setSysPro( + "whisk.spi.LogStoreProvider", + "org.apache.openwhisk.core.containerpool.docker.DockerForMacLogStoreProvider") + } + } + + private def setSysPro(key: String, value: String): Unit = { + Option(System.getProperty(key)) match { + case Some(x) if x != value => + log.info(s"Founding existing value for system property '$key'- Going to set '$value' , found '$x'") + case _ => + System.setProperty(key, value) } } } From dae96cbf8d78a12b78a21cdf88227733992e317f Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 11:43:39 +0530 Subject: [PATCH 18/66] Add a OpenWhisk cool banner --- .../standalone/StandaloneOpenWhisk.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index a1cc45f5fad..6507a2bec7e 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -36,7 +36,8 @@ import scala.concurrent.Await import scala.concurrent.duration._ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { - banner("OpenWhisk standalone launcher") + banner(StandaloneOpenWhisk.banner) + footer("\nOpenWhisk standalone server") val configFile = opt[File](descr = "application.conf which overrides the default standalone.conf") val manifest = opt[File](descr = "Manifest json defining the supported runtimes") @@ -46,6 +47,16 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { } object StandaloneOpenWhisk extends SLF4JLogging { + val banner = + """ + | ____ ___ _ _ _ _ _ + | /\ \ / _ \ _ __ ___ _ __ | | | | |__ (_)___| | __ + | /\ /__\ \ | | | | '_ \ / _ \ '_ \| | | | '_ \| / __| |/ / + | / \____ \ / | |_| | |_) | __/ | | | |/\| | | | | \__ \ < + | \ \ / \/ \___/| .__/ \___|_| |_|__/\__|_| |_|_|___/_|\_\ + | \___\/ tm |_| + """.stripMargin + val defaultRuntime = """{ | "runtimes": { | "nodejs": [ @@ -76,8 +87,8 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) + println(banner) initialize(conf) - //Create actor system only after initializing the config implicit val actorSystem = ActorSystem("standalone-actor-system") implicit val materializer = ActorMaterializer.create(actorSystem) From c7b72b229312b8cfe4f3550f191a9f511761417e Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 11:51:00 +0530 Subject: [PATCH 19/66] Remove dependency on other uncomiited changes --- .../org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 6507a2bec7e..1089106e042 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -152,7 +152,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { } private def startController()(implicit actorSystem: ActorSystem, logger: Logging): Unit = { - Controller.start(Array("standalone"), "http") + Controller.start(Array("standalone")) } private def bootstrapUsers()(implicit actorSystem: ActorSystem, From 32fdfde63d7d98aec0bb666fb7061a3f329b4fc0 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 12:44:40 +0530 Subject: [PATCH 20/66] Extract the cli based log store as a generic impl for all OS --- .../docker/DockerCliLogStore.scala | 73 +++++++++++++++++++ .../docker/DockerForMacContainerFactory.scala | 45 +----------- .../standalone/StandaloneOpenWhisk.scala | 7 +- 3 files changed, 79 insertions(+), 46 deletions(-) create mode 100644 core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala new file mode 100644 index 00000000000..6e1655792d4 --- /dev/null +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.core.containerpool.docker + +import java.time.Instant + +import akka.actor.ActorSystem +import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId} +import org.apache.openwhisk.core.containerpool.logging.{DockerToActivationLogStore, LogStore, LogStoreProvider} +import org.apache.openwhisk.core.containerpool.{Container, ContainerId} +import org.apache.openwhisk.core.entity.{ActivationLogs, ExecutableWhiskAction, Identity, WhiskActivation} + +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} + +/** + * Docker based log store implementation which fetches logs via cli command. + * This mode is inefficient and is only provided for running in developer modes + */ +object DockerCliLogStoreProvider extends LogStoreProvider { + override def instance(actorSystem: ActorSystem): LogStore = { + //Logger is currently not passed implicitly to LogStoreProvider. So create one explicitly + implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) + new DockerCliLogStore(actorSystem) + } +} + +class DockerCliLogStore(system: ActorSystem)(implicit log: Logging) extends DockerToActivationLogStore(system) { + private val client = new ExtendedDockerClient()(system.dispatcher)(log, system) + override def collectLogs(transid: TransactionId, + user: Identity, + activation: WhiskActivation, + container: Container, + action: ExecutableWhiskAction): Future[ActivationLogs] = { + //TODO Lookup for Log markers to be more precise in log collection + client + .collectLogs(container.containerId, activation.start, activation.end)(transid) + .map(logs => ActivationLogs(logs.linesIterator.toVector)) + } +} + +class ExtendedDockerClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(implicit log: Logging, + as: ActorSystem) + extends DockerClientWithFileAccess(dockerHost)(executionContext) + with DockerApiWithFileAccess { + + implicit private val ec: ExecutionContext = executionContext + private val waitForLogs: FiniteDuration = 2.seconds + private val logTimeSpanMargin = 1.second + + def collectLogs(id: ContainerId, since: Instant, untill: Instant)(implicit transid: TransactionId): Future[String] = { + //Add a slight buffer to account for delay writes of logs + val end = untill.plusSeconds(logTimeSpanMargin.toSeconds) + runCmd( + Seq("logs", id.asString, "--since", since.getEpochSecond.toString, "--until", end.getEpochSecond.toString), + waitForLogs) + } +} diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala index b6dca4dc764..fb7bc2e9b81 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala @@ -17,20 +17,11 @@ package org.apache.openwhisk.core.containerpool.docker -import java.time.Instant - import akka.actor.ActorSystem -import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId} +import org.apache.openwhisk.common.{Logging, TransactionId} import org.apache.openwhisk.core.WhiskConfig import org.apache.openwhisk.core.containerpool._ -import org.apache.openwhisk.core.containerpool.logging.{DockerToActivationLogStore, LogStore, LogStoreProvider} -import org.apache.openwhisk.core.entity.{ - ActivationLogs, - ExecutableWhiskAction, - Identity, - InvokerInstanceId, - WhiskActivation -} +import org.apache.openwhisk.core.entity.InvokerInstanceId import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration._ @@ -61,8 +52,6 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex with DockerApiWithFileAccess { implicit private val ec: ExecutionContext = executionContext - private val waitForLogs: FiniteDuration = 2.seconds - private val logTimeSpanMargin = 1.second override def run(image: String, args: Seq[String] = Seq.empty[String])( implicit transid: TransactionId): Future[ContainerId] = { @@ -83,34 +72,4 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex case stdout => Future.successful(ContainerAddress("localhost", stdout.toInt)) } } - - def collectLogs(id: ContainerId, since: Instant, untill: Instant)(implicit transid: TransactionId): Future[String] = { - //Add a slight buffer to account for delay writes of logs - val end = untill.plusSeconds(logTimeSpanMargin.toSeconds) - runCmd( - Seq("logs", id.asString, "--since", since.getEpochSecond.toString, "--until", end.getEpochSecond.toString), - waitForLogs) - } -} - -object DockerForMacLogStoreProvider extends LogStoreProvider { - override def instance(actorSystem: ActorSystem): LogStore = { - //Logger is currently not passed implicitly to LogStoreProvider. So create one explicitly - implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) - new DockerForMacLogStore(actorSystem) - } -} - -class DockerForMacLogStore(system: ActorSystem)(implicit log: Logging) extends DockerToActivationLogStore(system) { - private val client = new DockerForMacClient()(system.dispatcher)(log, system) - override def collectLogs(transid: TransactionId, - user: Identity, - activation: WhiskActivation, - container: Container, - action: ExecutableWhiskAction): Future[ActivationLogs] = { - //TODO Lookup for Log markers to be more precise in log collection - client - .collectLogs(container.containerId, activation.start, activation.end)(transid) - .map(logs => ActivationLogs(logs.linesIterator.toVector)) - } } diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 1089106e042..6181975dfe2 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -176,10 +176,11 @@ object StandaloneOpenWhisk extends SLF4JLogging { setSysPro( "whisk.spi.ContainerFactoryProvider", "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider") - setSysPro( - "whisk.spi.LogStoreProvider", - "org.apache.openwhisk.core.containerpool.docker.DockerForMacLogStoreProvider") } + + //Use cli based log store for all setups as its more stable to use + // and does not require root user access + setSysPro("whisk.spi.LogStoreProvider", "org.apache.openwhisk.core.containerpool.docker.DockerCliLogStoreProvider") } private def setSysPro(key: String, value: String): Unit = { From ae22d17c0a5c533c87f7416befa106f049a208b8 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 12:57:14 +0530 Subject: [PATCH 21/66] Bound only to localhost interface and not 0.0.0.0 for better security --- .../scala/org/apache/openwhisk/http/BasicHttpService.scala | 4 ++-- core/controller/src/main/resources/reference.conf | 1 + .../org/apache/openwhisk/core/controller/Controller.scala | 5 ++++- core/standalone/src/main/resources/standalone.conf | 7 +++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala b/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala index e734d7ab943..9e9d96557ef 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala @@ -167,11 +167,11 @@ object BasicHttpService { /** * Starts an HTTP(S) route handler on given port and registers a shutdown hook. */ - def startHttpService(route: Route, port: Int, config: Option[HttpsConfig] = None)( + def startHttpService(route: Route, port: Int, config: Option[HttpsConfig] = None, interface: String = "0.0.0.0")( implicit actorSystem: ActorSystem, materializer: ActorMaterializer): Unit = { val connectionContext = config.map(Https.connectionContext(_)).getOrElse(HttpConnectionContext) - val httpBinding = Http().bindAndHandle(route, "0.0.0.0", port, connectionContext = connectionContext) + val httpBinding = Http().bindAndHandle(route, interface, port, connectionContext = connectionContext) addShutdownHook(httpBinding) } diff --git a/core/controller/src/main/resources/reference.conf b/core/controller/src/main/resources/reference.conf index a8040207b1a..55322dec201 100644 --- a/core/controller/src/main/resources/reference.conf +++ b/core/controller/src/main/resources/reference.conf @@ -29,5 +29,6 @@ whisk { } controller { protocol: http + interface: "0.0.0.0" } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala index 0caf0feffb4..88e97130911 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala @@ -170,6 +170,7 @@ class Controller(val instance: ControllerInstanceId, object Controller { protected val protocol = loadConfigOrThrow[String]("whisk.controller.protocol") + protected val interface = loadConfigOrThrow[String]("whisk.controller.interface") // requiredProperties is a Map whose keys define properties that must be bound to // a value, and whose values are default values. A null value in the Map means there is @@ -267,7 +268,9 @@ object Controller { val httpsConfig = if (Controller.protocol == "https") Some(loadConfigOrThrow[HttpsConfig]("whisk.controller.https")) else None - BasicHttpService.startHttpService(controller.route, port, httpsConfig)(actorSystem, controller.materializer) + BasicHttpService.startHttpService(controller.route, port, httpsConfig, interface)( + actorSystem, + controller.materializer) case Failure(t) => abort(s"Invalid runtimes manifest: $t") diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index 10879e547d5..b911157eb92 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -48,6 +48,13 @@ whisk { limits-actions-invokes-concurrent = 30 } + controller { + protocol: http + + # Bound only to localhost by default for better security + interface: localhost + } + # Default set of users which are bootstrapped upon start users { whisk-system = "789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP" From f795b88f0443ea7fac9b32ce66005202f9df61ee Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 17:10:18 +0530 Subject: [PATCH 22/66] Make it work on Windows too!!! --- .../docker/DockerCliLogStore.scala | 3 +- .../containerpool/docker/DockerClient.scala | 4 +- .../docker/DockerForMacContainerFactory.scala | 6 +- .../DockerForWindowsContainerFactory.scala | 66 +++++++++++++++++++ .../src/main/resources/standalone.conf | 5 ++ .../standalone/StandaloneOpenWhisk.scala | 9 ++- 6 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala index 6e1655792d4..ff835df834a 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala @@ -57,7 +57,8 @@ class DockerCliLogStore(system: ActorSystem)(implicit log: Logging) extends Dock class ExtendedDockerClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(implicit log: Logging, as: ActorSystem) extends DockerClientWithFileAccess(dockerHost)(executionContext) - with DockerApiWithFileAccess { + with DockerApiWithFileAccess + with WindowsDockerClient { implicit private val ec: ExecutionContext = executionContext private val waitForLogs: FiniteDuration = 2.seconds diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala index d204258c247..fbf9d6e85ff 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala @@ -87,7 +87,7 @@ class DockerClient(dockerHost: Option[String] = None, // Determines how to run docker. Failure to find a Docker binary implies // a failure to initialize this instance of DockerClient. protected val dockerCmd: Seq[String] = { - val alternatives = List("/usr/bin/docker", "/usr/local/bin/docker") + val alternatives = List("/usr/bin/docker", "/usr/local/bin/docker") ++ executableAlternatives val dockerBin = Try { alternatives.find(a => Files.isExecutable(Paths.get(a))).get @@ -99,6 +99,8 @@ class DockerClient(dockerHost: Option[String] = None, Seq(dockerBin) ++ host } + protected def executableAlternatives: List[String] = List.empty + // Invoke docker CLI to determine client version. // If the docker client version cannot be determined, an exception will be thrown and instance initialization will fail. // Rationale: if we cannot invoke `docker version` successfully, it is unlikely subsequent `docker` invocations will succeed. diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala index fb7bc2e9b81..08b72b002cb 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala @@ -64,12 +64,12 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex override def inspectIPAddress(id: ContainerId, network: String)( implicit transid: TransactionId): Future[ContainerAddress] = { super - .runCmd( - Seq("inspect", "--format", """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""", id.asString), - 10.seconds) + .runCmd(Seq("inspect", "--format", inspectCommand, id.asString), 10.seconds) .flatMap { case "" => Future.failed(new NoSuchElementException) case stdout => Future.successful(ContainerAddress("localhost", stdout.toInt)) } } + + def inspectCommand: String = """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""" } diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala new file mode 100644 index 00000000000..a79224dc8be --- /dev/null +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.core.containerpool.docker + +import akka.actor.ActorSystem +import org.apache.openwhisk.common.Logging +import org.apache.openwhisk.core.WhiskConfig +import org.apache.openwhisk.core.containerpool._ +import org.apache.openwhisk.core.entity.InvokerInstanceId +import pureconfig.loadConfig + +import scala.concurrent.ExecutionContext + +/** + * This factory provides a Docker for Mac client which exposes action container's ports on the host. + */ +object DockerForWindowsContainerFactory extends ContainerFactoryProvider { + override def instance(actorSystem: ActorSystem, + logging: Logging, + config: WhiskConfig, + instanceId: InvokerInstanceId, + parameters: Map[String, Set[String]]): ContainerFactory = { + + new DockerContainerFactory(instanceId, parameters)( + actorSystem, + actorSystem.dispatcher, + logging, + new DockerForWindowsClient()(actorSystem.dispatcher)(logging, actorSystem), + new RuncClient()(actorSystem.dispatcher)(logging, actorSystem)) + } + +} + +trait WindowsDockerClient { + self: DockerClient => + + override protected def executableAlternatives: List[String] = { + val executable = loadConfig[String]("whisk.docker.executable").map(Some(_)).getOrElse(None) + List(Some("C:\\Program Files\\Docker\\Docker\\resources\\bin\\docker.exe"), executable).flatten + } +} + +class DockerForWindowsClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)( + implicit log: Logging, + as: ActorSystem) + extends DockerForMacClient(dockerHost)(executionContext) + with WindowsDockerClient { + //Due to some Docker + Windows + Go parsing quirks need to add double quotes around whole command + //See https://github.com/moby/moby/issues/27592#issuecomment-255227097 + override def inspectCommand: String = "\"{{(index (index .NetworkSettings.Ports \\\"8080/tcp\\\") 0).HostPort}}\"" +} diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index b911157eb92..782c09ea232 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -60,4 +60,9 @@ whisk { whisk-system = "789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP" guest = "23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP" } + + docker { + # Path to docker executuable. Generally its /var/lib/docker + # executable = + } } \ No newline at end of file diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 6181975dfe2..650b6586e5c 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -172,11 +172,18 @@ object StandaloneOpenWhisk extends SLF4JLogging { private def configureOSSpecificOpts(): Unit = { if (SystemUtils.IS_OS_MAC) { - setSysPro("whisk.docker.container-factory.use-runc", "False") setSysPro( "whisk.spi.ContainerFactoryProvider", "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider") } + if (SystemUtils.IS_OS_WINDOWS) { + setSysPro( + "whisk.spi.ContainerFactoryProvider", + "org.apache.openwhisk.core.containerpool.docker.DockerForWindowsContainerFactory") + } + + //Disable runc by default to keep things stable + setSysPro("whisk.docker.container-factory.use-runc", "False") //Use cli based log store for all setups as its more stable to use // and does not require root user access From 380c97cee1e3504f9a1814e343ca5ca3489302eb Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 18:57:26 +0530 Subject: [PATCH 23/66] Add blank line to make build going --- core/standalone/src/main/resources/standalone.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index 782c09ea232..535b05e414f 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -65,4 +65,4 @@ whisk { # Path to docker executuable. Generally its /var/lib/docker # executable = } -} \ No newline at end of file +} From 6fe78885511a55bda98580c0277054277203c80f Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 19:11:28 +0530 Subject: [PATCH 24/66] Package the default runtimes.json --- core/standalone/build.gradle | 6 ++++++ .../openwhisk/standalone/StandaloneOpenWhisk.scala | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index 90f5e493e35..6e1cda5c473 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -29,6 +29,12 @@ repositories { mavenCentral() } +processResources { + from(new File(project.rootProject.projectDir, "ansible/files/runtimes.json")) { + into(".") + } +} + bootJar { mainClassName = 'org.apache.openwhisk.standalone.StandaloneOpenWhisk' } diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 650b6586e5c..988974d6a5a 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -23,7 +23,7 @@ import java.nio.charset.StandardCharsets.UTF_8 import akka.actor.ActorSystem import akka.event.slf4j.SLF4JLogging import akka.stream.ActorMaterializer -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.commons.lang3.SystemUtils import org.apache.openwhisk.common.{AkkaLogging, Config, Logging} import org.apache.openwhisk.core.cli.WhiskAdmin @@ -34,6 +34,7 @@ import pureconfig.loadConfigOrThrow import scala.concurrent.Await import scala.concurrent.duration._ +import scala.util.Try class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner(StandaloneOpenWhisk.banner) @@ -141,8 +142,10 @@ object StandaloneOpenWhisk extends SLF4JLogging { val manifest = conf.manifest.toOption match { case Some(file) => FileUtils.readFileToString(file, UTF_8) - case None => - defaultRuntime + case None => { + //Fallback to a default runtime in case resource not found. Say while running from IDE + Try(IOUtils.resourceToString("/runtimes.json", UTF_8)).getOrElse(defaultRuntime) + } } setConfigProp(WhiskConfig.runtimesManifest, manifest) } From ff53e39973bb91916d86d78693bc36fd09532389 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 19:33:19 +0530 Subject: [PATCH 25/66] Change generated jar name --- core/standalone/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index 6e1cda5c473..bf795195c6c 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -36,6 +36,7 @@ processResources { } bootJar { + archiveName = "${project.archivesBaseName}.jar" mainClassName = 'org.apache.openwhisk.standalone.StandaloneOpenWhisk' } From 01e00411c9975ae6370489fe7ab90fdd043d6a9f Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 19:57:09 +0530 Subject: [PATCH 26/66] Use custom transactionId for user bootstrap --- .../org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 988974d6a5a..8b782fac64b 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -25,7 +25,7 @@ import akka.event.slf4j.SLF4JLogging import akka.stream.ActorMaterializer import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.commons.lang3.SystemUtils -import org.apache.openwhisk.common.{AkkaLogging, Config, Logging} +import org.apache.openwhisk.common.{AkkaLogging, Config, Logging, TransactionId} import org.apache.openwhisk.core.cli.WhiskAdmin import org.apache.openwhisk.core.controller.Controller import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} @@ -162,7 +162,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { materializer: ActorMaterializer, logging: Logging): Unit = { val users = loadConfigOrThrow[Map[String, String]]("whisk.users") - + implicit val userTid: TransactionId = TransactionId("userBootstrap") users.foreach { case (name, key) => val subject = name.replace('-', '.') From c9d0d3569ccf5007d40d568860c2e74a6ed4ff74 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Fri, 21 Jun 2019 19:57:21 +0530 Subject: [PATCH 27/66] Add initial readme --- core/standalone/README.md | 147 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 core/standalone/README.md diff --git a/core/standalone/README.md b/core/standalone/README.md new file mode 100644 index 00000000000..f4f4a7ee35a --- /dev/null +++ b/core/standalone/README.md @@ -0,0 +1,147 @@ + + +# OpenWhisk Standalone Server + +OpenWhisk standalone server is meant to run a simple OpenWhisk server for local development and test purposes. It can be +executed as a normal java application from command line. + +```bash +java -jar openwhisk-standalone.jar +``` + +This should start the OpenWhisk server on port 3233 by default. This server by default uses a memory based store and does +not depend on any other external service like Kafka and CouchDB. It only needs Docker and Java to for running. + +Few key points related to it + +* Uses in memory store. Once the server is stopped all changes would be lost +* Bootstraps the `guest` and `whisk.system` with default keys +* Supports running on LacOS, Linux and Windows setup +* Can be customized to use any other storage like CouchDB + + +### Build + +To build this standalone server run + +```bash +$ ./gradlew :core:standalone:build +``` + +This would create the runnable jar in `core/standalone/build/libs/` directory + +### Usage + +OpenWhisk standalone server support various launch options + +``` +$ java -jar openwhisk-standalone.jar -h + + + /\ \ / _ \ _ __ ___ _ __ | | | | |__ (_)___| | __ + /\ /__\ \ | | | | '_ \ / _ \ '_ \| | | | '_ \| / __| |/ / + / \____ \ / | |_| | |_) | __/ | | | |/\| | | | | \__ \ < + \ \ / \/ \___/| .__/ \___|_| |_|__/\__|_| |_|_|___/_|\_\ + \___\/ tm |_| + + -c, --config-file application.conf which overrides the default + standalone.conf + -m, --manifest Manifest json defining the supported runtimes + -p, --port Server port + -h, --help Show help message + +OpenWhisk standalone server +``` + +Sections below would illustrate some of the supported options + +To change the default config you can provide a custom `application.conf` file via `-c` option. The application conf file +must always include the default `standalone.conf` + +```hocon +include classpath("standalone.conf") + +whisk { + //Custom config +} +``` + +Then pass this config file + +```bash +java -jar openwhisk-standalone.jar -c custom.conf +``` + +#### Adding custom namespaces + +If you need to register custom namespaces (aka users) then you can pass them via config file like below + +```hocon +include classpath("standalone.conf") + +whisk { + users { + whisk-test = "cafebabe-cafe-babe-cafe-babecafebabe:007zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP" + } +} +``` + +Then pass this config file via `-c` option. You can check the users created from log + +``` +[2019-06-21T19:52:02.923Z] [INFO] [#tid_userBootstrap] [StandaloneOpenWhisk] Created user [guest] +[2019-06-21T19:52:03.008Z] [INFO] [#tid_userBootstrap] [StandaloneOpenWhisk] Created user [whisk.system] +[2019-06-21T19:52:03.094Z] [INFO] [#tid_userBootstrap] [StandaloneOpenWhisk] Created user [whisk.test] +``` + +#### Using custom runtimes + +To use custom runtime pass the runtime manifest via `-m` option + +```json +{ + "runtimes": { + "ruby": [ + { + "kind": "ruby:2.5", + "default": true, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + }, + "image": { + "prefix": "openwhisk", + "name": "action-ruby-v2.5", + "tag": "latest" + } + } + ] + } +} +``` + +The pass this file at launch time + +```bash +java -jar openwhisk-standalone.jar -m custom-runtime.json +``` + +You can then see the runtime config reflect in `http://localhost:3233` \ No newline at end of file From d776f96ac6bb88fe1b116a5e0fbaa79e9a38ef2d Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 06:58:45 +0530 Subject: [PATCH 28/66] Base revision on content hash --- .../database/memory/MemoryArtifactStore.scala | 83 +++++++++++-------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala index 6b8e2ad76d6..2613d31de05 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala @@ -17,13 +17,15 @@ package org.apache.openwhisk.core.database.memory +import java.nio.charset.StandardCharsets.UTF_8 + import akka.actor.ActorSystem import akka.http.scaladsl.model.{ContentType, Uri} import akka.stream.ActorMaterializer import akka.stream.scaladsl.{Sink, Source} import akka.util.ByteString import pureconfig.loadConfigOrThrow -import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat} +import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, JsValue, RootJsonFormat} import org.apache.openwhisk.common.{Logging, LoggingMarkers, TransactionId} import org.apache.openwhisk.core.ConfigKeys import org.apache.openwhisk.core.database.StoreUtils._ @@ -116,22 +118,25 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str val id = asJson.fields(_id).convertTo[String].trim require(!id.isEmpty, "document id must be defined") - val rev: Int = getRevision(asJson) - val docinfoStr = s"id: $id, rev: $rev" + val (oldRev, newRev) = computeRevision(asJson) + val docinfoStr = s"id: $id, rev: ${oldRev.getOrElse("null")}" val start = transid.started(this, LoggingMarkers.DATABASE_SAVE, s"[PUT] '$dbName' saving document: '$docinfoStr'") - val existing = Artifact(id, rev, asJson) - val updated = existing.incrementRev() + val updated = Artifact(id, newRev, asJson) val t = Try[DocInfo] { - if (rev == 0) { - artifacts.putIfAbsent(id, updated) match { - case Some(_) => throw DocumentConflictException("conflict on 'put'") - case None => updated.docInfo - } - } else if (artifacts.replace(id, existing, updated)) { - updated.docInfo - } else { - throw DocumentConflictException("conflict on 'put'") + oldRev match { + case Some(rev) => + val existing = Artifact(id, rev, asJson) + if (artifacts.replace(id, existing, updated)) { + updated.docInfo + } else { + throw DocumentConflictException("conflict on 'put'") + } + case None => + artifacts.putIfAbsent(id, updated) match { + case Some(_) => throw DocumentConflictException("conflict on 'put'") + case None => updated.docInfo + } } } @@ -313,38 +318,50 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str reportFailure(f, start, failure => s"[GET] '$dbName' internal error, doc: '$id', failure: '${failure.getMessage}'") } - private def getRevision(asJson: JsObject) = { - asJson.fields.get(_rev) match { - case Some(JsString(r)) => r.toInt - case _ => 0 + private def computeRevision(js: JsObject): (Option[String], String) = { + js.fields.get(_rev) match { + case Some(JsString(r)) => (Some(r), digest(js)) + case _ => (None, digest(js)) } } + private def digest(js: JsObject) = { + val jsWithoutRev = transform(js, Seq.empty, Seq(_rev)) + val md = emptyDigest() + encodeDigest(md.digest(jsWithoutRev.compactPrint.getBytes(UTF_8))) + } + //Use curried case class to allow equals support only for id and rev //This allows us to implement atomic replace and remove which check //for id,rev equality only - private case class Artifact(id: String, rev: Int)(val doc: JsObject, val computed: JsObject) { - def incrementRev(): Artifact = { - val (newRev, updatedDoc) = incrementAndGet() - copy(rev = newRev)(updatedDoc, computed) //With Couch attachments are lost post update - } - + private case class Artifact(id: String, rev: String)(val doc: JsObject, val computed: JsObject) { def docInfo = DocInfo(DocId(id), DocRevision(rev.toString)) - - private def incrementAndGet() = { - val newRev = rev + 1 - val updatedDoc = JsObject(doc.fields + (_rev -> JsString(newRev.toString))) - (newRev, updatedDoc) - } } private object Artifact { - def apply(id: String, rev: Int, doc: JsObject): Artifact = { - Artifact(id, rev)(doc, documentHandler.computedFields(doc)) + def apply(id: String, rev: String, doc: JsObject): Artifact = { + val docWithRev = transform(doc, Seq((_rev, Some(JsString(rev))))) + Artifact(id, rev)(docWithRev, documentHandler.computedFields(doc)) } def apply(info: DocInfo): Artifact = { - Artifact(info.id.id, info.rev.rev.toInt)(JsObject.empty, JsObject.empty) + Artifact(info.id.id, info.rev.rev)(JsObject.empty, JsObject.empty) } } + + /** + * Transforms a json object by adding and removing fields + * + * @param json base json object to transform + * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored + * @param fieldsToRemove list of field names to remove + * @return transformed json + */ + private def transform(json: JsObject, + fieldsToAdd: Seq[(String, Option[JsValue])], + fieldsToRemove: Seq[String] = Seq.empty): JsObject = { + //TODO Refactor this to util as its used in CosmosDBSTore also + val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove + JsObject(fields) + } } From 39471e0a675d02f061654a292b6c8a3e453ff4e1 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 07:05:26 +0530 Subject: [PATCH 29/66] Fix whitespace --- core/standalone/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/standalone/README.md b/core/standalone/README.md index f4f4a7ee35a..d43bd883c0e 100644 --- a/core/standalone/README.md +++ b/core/standalone/README.md @@ -43,7 +43,7 @@ To build this standalone server run ```bash $ ./gradlew :core:standalone:build -``` +``` This would create the runnable jar in `core/standalone/build/libs/` directory @@ -83,7 +83,7 @@ whisk { } ``` -Then pass this config file +Then pass this config file ```bash java -jar openwhisk-standalone.jar -c custom.conf @@ -144,4 +144,4 @@ The pass this file at launch time java -jar openwhisk-standalone.jar -m custom-runtime.json ``` -You can then see the runtime config reflect in `http://localhost:3233` \ No newline at end of file +You can then see the runtime config reflect in `http://localhost:3233` From f11209dd851c030df2a10fa8f3462ca4ff2d886f Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 12:31:11 +0530 Subject: [PATCH 30/66] Add build info details based on git commit --- core/standalone/build.gradle | 3 +- .../standalone/StandaloneOpenWhisk.scala | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index bf795195c6c..205bedfe38b 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -16,7 +16,8 @@ */ plugins { - id 'org.springframework.boot' version '2.0.2.RELEASE' + id 'org.springframework.boot' version '2.1.6.RELEASE' + id "com.gorylenko.gradle-git-properties" version "2.0.0" id 'scala' } diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 8b782fac64b..ae278eb8e3f 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -17,8 +17,9 @@ package org.apache.openwhisk.standalone -import java.io.File +import java.io.{ByteArrayInputStream, File} import java.nio.charset.StandardCharsets.UTF_8 +import java.util.Properties import akka.actor.ActorSystem import akka.event.slf4j.SLF4JLogging @@ -32,6 +33,7 @@ import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} import org.rogach.scallop.ScallopConf import pureconfig.loadConfigOrThrow +import scala.collection.JavaConverters._ import scala.concurrent.Await import scala.concurrent.duration._ import scala.util.Try @@ -39,6 +41,7 @@ import scala.util.Try class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { banner(StandaloneOpenWhisk.banner) footer("\nOpenWhisk standalone server") + StandaloneOpenWhisk.gitInfo.foreach(g => version(s"Git Commit - ${g.commitId}")) val configFile = opt[File](descr = "application.conf which overrides the default standalone.conf") val manifest = opt[File](descr = "Manifest json defining the supported runtimes") @@ -47,6 +50,8 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { verify() } +case class GitInfo(commitId: String, commitTime: String) + object StandaloneOpenWhisk extends SLF4JLogging { val banner = """ @@ -86,9 +91,11 @@ object StandaloneOpenWhisk extends SLF4JLogging { |} |""".stripMargin + val gitInfo: Option[GitInfo] = loadGitInfo() + def main(args: Array[String]): Unit = { val conf = new Conf(args) - println(banner) + printBanner initialize(conf) //Create actor system only after initializing the config implicit val actorSystem = ActorSystem("standalone-actor-system") @@ -99,6 +106,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { } def initialize(conf: Conf): Unit = { + configureBuildInfo() configureServerPort(conf) configureOSSpecificOpts() initConfigLocation(conf) @@ -193,6 +201,26 @@ object StandaloneOpenWhisk extends SLF4JLogging { setSysPro("whisk.spi.LogStoreProvider", "org.apache.openwhisk.core.containerpool.docker.DockerCliLogStoreProvider") } + private def loadGitInfo() = { + val info = loadPropResource("git.properties") + for { + commit <- info.get("git.commit.id.abbrev") + time <- info.get("git.commit.time") + } yield GitInfo(commit, time) + } + + private def printBanner = { + println(banner) + gitInfo.foreach(g => println(s"Git Commit: ${g.commitId}, Build Date: ${g.commitTime}")) + } + + private def configureBuildInfo(): Unit = { + gitInfo.foreach { g => + setSysPro("whisk.info.build-no", g.commitId) + setSysPro("whisk.info.date", g.commitTime) + } + } + private def setSysPro(key: String, value: String): Unit = { Option(System.getProperty(key)) match { case Some(x) if x != value => @@ -201,4 +229,13 @@ object StandaloneOpenWhisk extends SLF4JLogging { System.setProperty(key, value) } } + + private def loadPropResource(name: String): Map[String, String] = { + Try { + val propString = IOUtils.resourceToString("/" + name, UTF_8) + val props = new Properties() + props.load(new ByteArrayInputStream(propString.getBytes(UTF_8))) + props.asScala.toMap + }.getOrElse(Map.empty) + } } From 9e59f73eddebf5a3eab2c0a7f78053f555f1b15f Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 14:14:41 +0530 Subject: [PATCH 31/66] Purge Memory stores before start --- .../database/memory/MemoryArtifactStoreBehaviorBase.scala | 5 +++++ .../core/database/memory/MemoryAttachmentStoreTests.scala | 5 +++++ .../core/database/s3/S3AttachmentStoreBehaviorBase.scala | 7 ++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala index 2d8a1be8a79..8ca2d7c0a51 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala @@ -39,6 +39,11 @@ trait MemoryArtifactStoreBehaviorBase extends FlatSpec with ArtifactStoreBehavio MemoryArtifactStoreProvider.makeArtifactStore[WhiskAuth](getAttachmentStore[WhiskAuth]()) } + override protected def beforeAll(): Unit = { + MemoryArtifactStoreProvider.purgeAll() + super.beforeAll() + } + override lazy val entityStore = MemoryArtifactStoreProvider.makeArtifactStore[WhiskEntity](getAttachmentStore[WhiskEntity]())( classTag[WhiskEntity], diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala index 32c157d8f89..c10395dfb85 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala @@ -32,6 +32,11 @@ class MemoryAttachmentStoreTests extends FlatSpec with AttachmentStoreBehaviors override def storeType: String = "Memory" + override protected def beforeAll(): Unit = { + MemoryArtifactStoreProvider.purgeAll() + super.beforeAll() + } + override def afterAll(): Unit = { super.afterAll() val count = store.asInstanceOf[MemoryAttachmentStore].attachmentCount diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala index ee9f1fb6af6..9226a4feaf1 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala @@ -22,7 +22,7 @@ import akka.stream.ActorMaterializer import org.scalatest.FlatSpec import org.apache.openwhisk.common.Logging import org.apache.openwhisk.core.database.{AttachmentStore, DocumentSerializer} -import org.apache.openwhisk.core.database.memory.MemoryArtifactStoreBehaviorBase +import org.apache.openwhisk.core.database.memory.{MemoryArtifactStoreBehaviorBase, MemoryArtifactStoreProvider} import org.apache.openwhisk.core.database.test.AttachmentStoreBehaviors import org.apache.openwhisk.core.database.test.behavior.ArtifactStoreAttachmentBehaviors import org.apache.openwhisk.core.entity.WhiskEntity @@ -41,6 +41,11 @@ trait S3AttachmentStoreBehaviorBase override val prefix = s"attachmentTCK_${Random.alphanumeric.take(4).mkString}" + override protected def beforeAll(): Unit = { + MemoryArtifactStoreProvider.purgeAll() + super.beforeAll() + } + override def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore = makeS3Store[D]() From 7255cf982214ddf7a02e547c66bc7be093a36055 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 14:35:55 +0530 Subject: [PATCH 32/66] Update readme with cli setup --- core/standalone/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/standalone/README.md b/core/standalone/README.md index d43bd883c0e..5ed87fc4b23 100644 --- a/core/standalone/README.md +++ b/core/standalone/README.md @@ -26,8 +26,11 @@ executed as a normal java application from command line. java -jar openwhisk-standalone.jar ``` -This should start the OpenWhisk server on port 3233 by default. This server by default uses a memory based store and does -not depend on any other external service like Kafka and CouchDB. It only needs Docker and Java to for running. +This should start the OpenWhisk server on port 3233 by default. Once the server is started then [configure the cli][1] +and then try out the [samples][2]. + +This server by default uses a memory based store and does not depend on any other external service like Kafka and CouchDB. +It only needs Docker and Java to for running. Few key points related to it @@ -145,3 +148,6 @@ java -jar openwhisk-standalone.jar -m custom-runtime.json ``` You can then see the runtime config reflect in `http://localhost:3233` + +[1]: https://github.com/apache/incubator-openwhisk/blob/master/docs/cli.md +[2]: https://github.com/apache/incubator-openwhisk/blob/master/docs/samples.md From 18317d0d517afbfb2250ffd947db67812a55b093 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 14:41:17 +0530 Subject: [PATCH 33/66] Add test to check if recreated doc with diff content has diff rev --- .../test/behavior/ArtifactStoreCRUDBehaviors.scala | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala index 83f5d6a29c0..bbd232bbc97 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala @@ -48,6 +48,20 @@ trait ArtifactStoreCRUDBehaviors extends ArtifactStoreBehaviorBase { doc2.rev.empty shouldBe false } + it should "put delete and then recreate document with same id with different rev" in { + implicit val tid: TransactionId = transid() + val auth = newAuth() + val doc = put(authStore, auth) + + delete(authStore, doc) shouldBe true + + val auth2 = auth.copy(namespaces = Set(wskNS("foo1"))) + val doc2 = put(authStore, auth2) + + doc2.rev should not be doc.rev + doc2.rev.empty shouldBe false + } + it should "throw DocumentConflictException when updated with old revision" in { implicit val tid: TransactionId = transid() val auth = newAuth() From 59ac3f36f4957dd1f209315ba199377325039cc5 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 15:02:14 +0530 Subject: [PATCH 34/66] Move transform method to a utility class --- .../openwhisk/core/database/StoreUtils.scala | 17 ++++++++++++++- .../cosmosdb/CosmosDBArtifactStore.scala | 2 +- .../core/database/cosmosdb/CosmosDBUtil.scala | 18 ++-------------- .../database/memory/MemoryArtifactStore.scala | 21 ++----------------- 4 files changed, 21 insertions(+), 37 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala index 2c168d4a971..1300a4c92bf 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala @@ -24,7 +24,7 @@ import akka.stream.SinkShape import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Keep, Sink} import akka.util.ByteString import spray.json.DefaultJsonProtocol._ -import spray.json.{JsObject, RootJsonFormat} +import spray.json.{JsObject, JsValue, RootJsonFormat} import org.apache.openwhisk.common.{Logging, StartMarker, TransactionId} import org.apache.openwhisk.core.entity.{DocInfo, DocRevision, DocumentReader, WhiskDocument} @@ -97,6 +97,21 @@ private[database] object StoreUtils { s"$encodedAlgoName-$digest" } + /** + * Transforms a json object by adding and removing fields + * + * @param json base json object to transform + * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored + * @param fieldsToRemove list of field names to remove + * @return transformed json + */ + def transform(json: JsObject, + fieldsToAdd: Seq[(String, Option[JsValue])], + fieldsToRemove: Seq[String] = Seq.empty): JsObject = { + val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove + JsObject(fields) + } + private def combineResult[T](digest: Future[String], length: Future[Long], upload: Future[T])( implicit ec: ExecutionContext) = { for { diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala index c635092e63a..1ddc8b8ff3a 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala @@ -28,7 +28,7 @@ import com.microsoft.azure.cosmosdb.internal.Constants.Properties import com.microsoft.azure.cosmosdb.rx.AsyncDocumentClient import kamon.metric.MeasurementUnit import org.apache.openwhisk.common.{LogMarkerToken, Logging, LoggingMarkers, MetricEmitter, Scheduler, TransactionId} -import org.apache.openwhisk.core.database.StoreUtils.{checkDocHasRevision, deserialize, reportFailure} +import org.apache.openwhisk.core.database.StoreUtils._ import org.apache.openwhisk.core.database._ import org.apache.openwhisk.core.database.cosmosdb.CosmosDBArtifactStoreProvider.DocumentClientRef import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants._ diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala index 2953cd8eef9..fbe1b49b673 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala @@ -19,7 +19,8 @@ package org.apache.openwhisk.core.database.cosmosdb import com.microsoft.azure.cosmosdb.internal.Constants.Properties.{AGGREGATE, E_TAG, ID, SELF_LINK} import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants._ -import spray.json.{JsObject, JsString, JsValue} +import org.apache.openwhisk.core.database.StoreUtils.transform +import spray.json.{JsObject, JsString} import scala.collection.immutable.Iterable @@ -124,21 +125,6 @@ private[cosmosdb] trait CosmosDBUtil { transform(stripInternalFields(js), fieldsToAdd, Seq.empty) } - /** - * Transforms a json object by adding and removing fields - * - * @param json base json object to transform - * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored - * @param fieldsToRemove list of field names to remove - * @return transformed json - */ - def transform(json: JsObject, - fieldsToAdd: Seq[(String, Option[JsValue])], - fieldsToRemove: Seq[String] = Seq.empty): JsObject = { - val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove - JsObject(fields) - } - private def stripInternalFields(js: JsObject) = { //Strip out all field name starting with '_' which are considered as db specific internal fields JsObject(js.fields.filter { case (k, _) => !k.startsWith("_") && k != cid }) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala index 2613d31de05..a88cc35372f 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala @@ -24,16 +24,15 @@ import akka.http.scaladsl.model.{ContentType, Uri} import akka.stream.ActorMaterializer import akka.stream.scaladsl.{Sink, Source} import akka.util.ByteString -import pureconfig.loadConfigOrThrow -import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, JsValue, RootJsonFormat} import org.apache.openwhisk.common.{Logging, LoggingMarkers, TransactionId} import org.apache.openwhisk.core.ConfigKeys import org.apache.openwhisk.core.database.StoreUtils._ import org.apache.openwhisk.core.database._ import org.apache.openwhisk.core.entity.Attachments.Attached import org.apache.openwhisk.core.entity._ -import org.apache.openwhisk.core.entity.size._ import org.apache.openwhisk.http.Messages +import pureconfig.loadConfigOrThrow +import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat} import scala.collection.concurrent.TrieMap import scala.concurrent.{ExecutionContext, Future} @@ -348,20 +347,4 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str Artifact(info.id.id, info.rev.rev)(JsObject.empty, JsObject.empty) } } - - /** - * Transforms a json object by adding and removing fields - * - * @param json base json object to transform - * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored - * @param fieldsToRemove list of field names to remove - * @return transformed json - */ - private def transform(json: JsObject, - fieldsToAdd: Seq[(String, Option[JsValue])], - fieldsToRemove: Seq[String] = Seq.empty): JsObject = { - //TODO Refactor this to util as its used in CosmosDBSTore also - val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove - JsObject(fields) - } } From 1f99dc49c5625907ac817ecd8429e75734f14ac8 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sat, 22 Jun 2019 15:44:43 +0530 Subject: [PATCH 35/66] Readd bytesize import --- .../openwhisk/core/database/memory/MemoryArtifactStore.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala index a88cc35372f..4e9c096c455 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala @@ -30,6 +30,7 @@ import org.apache.openwhisk.core.database.StoreUtils._ import org.apache.openwhisk.core.database._ import org.apache.openwhisk.core.entity.Attachments.Attached import org.apache.openwhisk.core.entity._ +import org.apache.openwhisk.core.entity.size._ import org.apache.openwhisk.http.Messages import pureconfig.loadConfigOrThrow import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat} From ee3c3d2e74aed5a9f889b2b20dece5f06c6f958d Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sun, 23 Jun 2019 12:02:23 +0530 Subject: [PATCH 36/66] Include log timestamps and filter till sentinel --- .../containerpool/docker/DockerCliLogStore.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala index ff835df834a..89f6a8f2bca 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala @@ -21,6 +21,7 @@ import java.time.Instant import akka.actor.ActorSystem import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId} +import org.apache.openwhisk.core.containerpool.Container.ACTIVATION_LOG_SENTINEL import org.apache.openwhisk.core.containerpool.logging.{DockerToActivationLogStore, LogStore, LogStoreProvider} import org.apache.openwhisk.core.containerpool.{Container, ContainerId} import org.apache.openwhisk.core.entity.{ActivationLogs, ExecutableWhiskAction, Identity, WhiskActivation} @@ -47,10 +48,9 @@ class DockerCliLogStore(system: ActorSystem)(implicit log: Logging) extends Dock activation: WhiskActivation, container: Container, action: ExecutableWhiskAction): Future[ActivationLogs] = { - //TODO Lookup for Log markers to be more precise in log collection client .collectLogs(container.containerId, activation.start, activation.end)(transid) - .map(logs => ActivationLogs(logs.linesIterator.toVector)) + .map(logs => ActivationLogs(logs.linesIterator.takeWhile(!_.contains(ACTIVATION_LOG_SENTINEL)).toVector)) } } @@ -68,7 +68,14 @@ class ExtendedDockerClient(dockerHost: Option[String] = None)(executionContext: //Add a slight buffer to account for delay writes of logs val end = untill.plusSeconds(logTimeSpanMargin.toSeconds) runCmd( - Seq("logs", id.asString, "--since", since.getEpochSecond.toString, "--until", end.getEpochSecond.toString), + Seq( + "logs", + id.asString, + "--since", + since.getEpochSecond.toString, + "--until", + end.getEpochSecond.toString, + "--timestamps"), waitForLogs) } } From dc5938b2b49b4a78750251bc80af9d5419fc75ee Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sun, 23 Jun 2019 12:28:42 +0530 Subject: [PATCH 37/66] Use host.docker.internal as api host for Mac and Windows --- .../standalone/StandaloneOpenWhisk.scala | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index ae278eb8e3f..64b8445bc4f 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -95,7 +95,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) - printBanner + printBanner() initialize(conf) //Create actor system only after initializing the config implicit val actorSystem = ActorSystem("standalone-actor-system") @@ -126,7 +126,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { setConfigProp(WhiskConfig.servicePort, port.toString) setConfigProp(WhiskConfig.wskApiPort, port.toString) setConfigProp(WhiskConfig.wskApiProtocol, "http") - setConfigProp(WhiskConfig.wskApiHostname, "localhost") + setConfigProp(WhiskConfig.wskApiHostname, localHostName) } private def initConfigLocation(conf: Conf): Unit = { @@ -201,6 +201,15 @@ object StandaloneOpenWhisk extends SLF4JLogging { setSysPro("whisk.spi.LogStoreProvider", "org.apache.openwhisk.core.containerpool.docker.DockerCliLogStoreProvider") } + private def localHostName = { + //For connecting back to controller on container host following name needs to be used + // on Windows and Mac + // https://docs.docker.com/docker-for-windows/networking/#use-cases-and-workarounds + if (SystemUtils.IS_OS_MAC || SystemUtils.IS_OS_WINDOWS) + "host.docker.internal" + else "localhost" + } + private def loadGitInfo() = { val info = loadPropResource("git.properties") for { @@ -209,7 +218,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { } yield GitInfo(commit, time) } - private def printBanner = { + private def printBanner() = { println(banner) gitInfo.foreach(g => println(s"Git Commit: ${g.commitId}, Build Date: ${g.commitTime}")) } From c833779504d95b702a71e1031e9541eaf5f26f83 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sun, 23 Jun 2019 13:22:43 +0530 Subject: [PATCH 38/66] Enable colored logging --- .../src/main/resources/logback-standalone.xml | 36 +++++++++++ .../standalone/LogbackConfigurator.scala | 59 +++++++++++++++++++ .../standalone/StandaloneOpenWhisk.scala | 23 +++++--- 3 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 core/standalone/src/main/resources/logback-standalone.xml create mode 100644 core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala diff --git a/core/standalone/src/main/resources/logback-standalone.xml b/core/standalone/src/main/resources/logback-standalone.xml new file mode 100644 index 00000000000..4121b7e9571 --- /dev/null +++ b/core/standalone/src/main/resources/logback-standalone.xml @@ -0,0 +1,36 @@ + + + + + + + + [%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}] %highlight([%p]) %msg%n + + + + + + + + + + + + + \ No newline at end of file diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala new file mode 100644 index 00000000000..03b18fcc4bf --- /dev/null +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.standalone + +import java.io.ByteArrayInputStream +import java.nio.charset.StandardCharsets.UTF_8 + +import ch.qos.logback.classic.LoggerContext +import ch.qos.logback.classic.joran.JoranConfigurator +import ch.qos.logback.core.joran.spi.JoranException +import ch.qos.logback.core.util.StatusPrinter +import org.apache.commons.io.IOUtils +import org.slf4j.LoggerFactory + +import scala.util.Try + +/** + * Resets the Logback config if logging is configure via non standard file + */ +object LogbackConfigurator { + + def configureLogbackFromResource(resourceName: String): Unit = { + Try(configureLogback(IOUtils.resourceToString("/" + resourceName, UTF_8))).failed.foreach(t => + println(s"Could not load resource $resourceName- ${t.getMessage}")) + } + + private def configureLogback(fileContent: String): Unit = { + val context = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + + try { + val configurator = new JoranConfigurator + configurator.setContext(context) + // Call context.reset() to clear any previous configuration, e.g. default + // configuration. For multi-step configuration, omit calling context.reset(). + context.reset() + val is = new ByteArrayInputStream(fileContent.getBytes(UTF_8)) + configurator.doConfigure(is) + } catch { + case _: JoranException => + // StatusPrinter will handle this + } + StatusPrinter.printInCaseOfErrorsOrWarnings(context) + } +} diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 64b8445bc4f..47785ff6277 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -95,6 +95,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) + configureLogging() printBanner() initialize(conf) //Create actor system only after initializing the config @@ -159,7 +160,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { } private def setConfigProp(key: String, value: String): Unit = { - setSysPro(configKey(key), value) + setSysProp(configKey(key), value) } private def startController()(implicit actorSystem: ActorSystem, logger: Logging): Unit = { @@ -183,22 +184,22 @@ object StandaloneOpenWhisk extends SLF4JLogging { private def configureOSSpecificOpts(): Unit = { if (SystemUtils.IS_OS_MAC) { - setSysPro( + setSysProp( "whisk.spi.ContainerFactoryProvider", "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider") } if (SystemUtils.IS_OS_WINDOWS) { - setSysPro( + setSysProp( "whisk.spi.ContainerFactoryProvider", "org.apache.openwhisk.core.containerpool.docker.DockerForWindowsContainerFactory") } //Disable runc by default to keep things stable - setSysPro("whisk.docker.container-factory.use-runc", "False") + setSysProp("whisk.docker.container-factory.use-runc", "False") //Use cli based log store for all setups as its more stable to use // and does not require root user access - setSysPro("whisk.spi.LogStoreProvider", "org.apache.openwhisk.core.containerpool.docker.DockerCliLogStoreProvider") + setSysProp("whisk.spi.LogStoreProvider", "org.apache.openwhisk.core.containerpool.docker.DockerCliLogStoreProvider") } private def localHostName = { @@ -225,12 +226,12 @@ object StandaloneOpenWhisk extends SLF4JLogging { private def configureBuildInfo(): Unit = { gitInfo.foreach { g => - setSysPro("whisk.info.build-no", g.commitId) - setSysPro("whisk.info.date", g.commitTime) + setSysProp("whisk.info.build-no", g.commitId) + setSysProp("whisk.info.date", g.commitTime) } } - private def setSysPro(key: String, value: String): Unit = { + private def setSysProp(key: String, value: String): Unit = { Option(System.getProperty(key)) match { case Some(x) if x != value => log.info(s"Founding existing value for system property '$key'- Going to set '$value' , found '$x'") @@ -247,4 +248,10 @@ object StandaloneOpenWhisk extends SLF4JLogging { props.asScala.toMap }.getOrElse(Map.empty) } + + private def configureLogging(): Unit = { + if (System.getProperty("logback.configurationFile") == null) { + LogbackConfigurator.configureLogbackFromResource("logback-standalone.xml") + } + } } From 1fb514e4989afa6b30036421c30f3a47c3e8da98 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sun, 23 Jun 2019 13:27:12 +0530 Subject: [PATCH 39/66] Add support for --verbose flag to enable debug logging easily --- .../standalone/LogbackConfigurator.scala | 16 +++++++++++++++- .../standalone/StandaloneOpenWhisk.scala | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala index 03b18fcc4bf..b819c00f521 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala @@ -20,7 +20,7 @@ package org.apache.openwhisk.standalone import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets.UTF_8 -import ch.qos.logback.classic.LoggerContext +import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.classic.joran.JoranConfigurator import ch.qos.logback.core.joran.spi.JoranException import ch.qos.logback.core.util.StatusPrinter @@ -34,6 +34,20 @@ import scala.util.Try */ object LogbackConfigurator { + def initLogging(conf: Conf): Unit = { + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).setLevel(toLevel(conf.verbose())) + } + + private def toLevel(v: Int) = { + v match { + case 0 => Level.WARN + case 1 => Level.INFO + case 2 => Level.DEBUG + case _ => Level.ALL + } + } + def configureLogbackFromResource(resourceName: String): Unit = { Try(configureLogback(IOUtils.resourceToString("/" + resourceName, UTF_8))).failed.foreach(t => println(s"Could not load resource $resourceName- ${t.getMessage}")) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 47785ff6277..ebb89b05976 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -47,6 +47,7 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { val manifest = opt[File](descr = "Manifest json defining the supported runtimes") val port = opt[Int](descr = "Server port", default = Some(3233)) + val verbose = tally() verify() } @@ -95,7 +96,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) - configureLogging() + configureLogging(conf) printBanner() initialize(conf) //Create actor system only after initializing the config @@ -249,9 +250,10 @@ object StandaloneOpenWhisk extends SLF4JLogging { }.getOrElse(Map.empty) } - private def configureLogging(): Unit = { + private def configureLogging(conf: Conf): Unit = { if (System.getProperty("logback.configurationFile") == null) { LogbackConfigurator.configureLogbackFromResource("logback-standalone.xml") } + LogbackConfigurator.initLogging(conf) } } From 00e4576097220367e4f52240388f72c881938fad Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Sun, 23 Jun 2019 13:54:36 +0530 Subject: [PATCH 40/66] Color transactionId and source also. Also support disabling color logging if needed --- .../org/apache/openwhisk/common/Logging.scala | 2 +- .../standalone/LogbackConfigurator.scala | 27 ++++++++++++++++--- .../standalone/StandaloneOpenWhisk.scala | 13 +++++++-- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala index e27d043da24..825df9011b0 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala @@ -156,7 +156,7 @@ object LogMarker { } } -private object Logging { +object Logging { /** * Given a class object, return its simple name less the trailing dollar sign. diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala index b819c00f521..14dae79e9e6 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala @@ -20,11 +20,15 @@ package org.apache.openwhisk.standalone import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets.UTF_8 +import akka.event.Logging.LogLevel +import akka.event.LoggingAdapter import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.classic.joran.JoranConfigurator import ch.qos.logback.core.joran.spi.JoranException +import ch.qos.logback.core.pattern.color.ANSIConstants._ import ch.qos.logback.core.util.StatusPrinter import org.apache.commons.io.IOUtils +import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId} import org.slf4j.LoggerFactory import scala.util.Try @@ -41,9 +45,8 @@ object LogbackConfigurator { private def toLevel(v: Int) = { v match { - case 0 => Level.WARN - case 1 => Level.INFO - case 2 => Level.DEBUG + case 0 => Level.INFO + case 1 => Level.DEBUG case _ => Level.ALL } } @@ -71,3 +74,21 @@ object LogbackConfigurator { StatusPrinter.printInCaseOfErrorsOrWarnings(context) } } + +/** + * Similar to AkkaLogging but with color support + */ +class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends Logging { + private val setDefaultColor = ESC_START + "0;" + DEFAULT_FG + ESC_END + def emit(loglevel: LogLevel, id: TransactionId, from: AnyRef, message: => String) = { + if (loggingAdapter.isEnabled(loglevel)) { + val logmsg: String = message // generates the message + if (logmsg.nonEmpty) { // log it only if its not empty + val name = if (from.isInstanceOf[String]) from else Logging.getCleanSimpleClassName(from.getClass) + loggingAdapter.log(loglevel, s"[${clr(id.toString, BOLD)}] [${clr(name.toString, CYAN_FG)}] $logmsg") + } + } + } + + def clr(s: String, code: String) = s"$ESC_START$code$ESC_END$s$setDefaultColor" +} diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index ebb89b05976..d44571d7cff 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -48,6 +48,7 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { val port = opt[Int](descr = "Server port", default = Some(3233)) val verbose = tally() + val disableColorLogging = opt[Boolean](descr = "Disables colored logging", noshort = true) verify() } @@ -102,7 +103,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { //Create actor system only after initializing the config implicit val actorSystem = ActorSystem("standalone-actor-system") implicit val materializer = ActorMaterializer.create(actorSystem) - implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) + implicit val logger: Logging = createLogging(actorSystem, conf) startServer() } @@ -251,9 +252,17 @@ object StandaloneOpenWhisk extends SLF4JLogging { } private def configureLogging(conf: Conf): Unit = { - if (System.getProperty("logback.configurationFile") == null) { + if (System.getProperty("logback.configurationFile") == null && !conf.disableColorLogging()) { LogbackConfigurator.configureLogbackFromResource("logback-standalone.xml") } LogbackConfigurator.initLogging(conf) } + + private def createLogging(actorSystem: ActorSystem, conf: Conf): Logging = { + val adapter = akka.event.Logging.getLogger(actorSystem, this) + if (conf.disableColorLogging()) + new AkkaLogging(adapter) + else + new ColoredAkkaLogging(adapter) + } } From 89a8658ddac0bbfce0e954d505f46c917033b3a2 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 10:11:38 +0530 Subject: [PATCH 41/66] Make the banner colored --- .../standalone/LogbackConfigurator.scala | 17 ++++++++++------- .../standalone/StandaloneOpenWhisk.scala | 10 +++++++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala index 14dae79e9e6..a9da175ee21 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala @@ -22,15 +22,15 @@ import java.nio.charset.StandardCharsets.UTF_8 import akka.event.Logging.LogLevel import akka.event.LoggingAdapter -import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.classic.joran.JoranConfigurator +import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.core.joran.spi.JoranException -import ch.qos.logback.core.pattern.color.ANSIConstants._ import ch.qos.logback.core.util.StatusPrinter import org.apache.commons.io.IOUtils -import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId} +import org.apache.openwhisk.common.{Logging, TransactionId} import org.slf4j.LoggerFactory +import scala.io.AnsiColor import scala.util.Try /** @@ -78,17 +78,20 @@ object LogbackConfigurator { /** * Similar to AkkaLogging but with color support */ -class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends Logging { - private val setDefaultColor = ESC_START + "0;" + DEFAULT_FG + ESC_END +class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends Logging with AnsiColor { + import ColorOutput.clr + def emit(loglevel: LogLevel, id: TransactionId, from: AnyRef, message: => String) = { if (loggingAdapter.isEnabled(loglevel)) { val logmsg: String = message // generates the message if (logmsg.nonEmpty) { // log it only if its not empty val name = if (from.isInstanceOf[String]) from else Logging.getCleanSimpleClassName(from.getClass) - loggingAdapter.log(loglevel, s"[${clr(id.toString, BOLD)}] [${clr(name.toString, CYAN_FG)}] $logmsg") + loggingAdapter.log(loglevel, s"[${clr(id.toString, BOLD)}] [${clr(name.toString, CYAN)}] $logmsg") } } } +} - def clr(s: String, code: String) = s"$ESC_START$code$ESC_END$s$setDefaultColor" +object ColorOutput extends AnsiColor { + def clr(s: String, code: String) = s"$code$s$RESET" } diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index d44571d7cff..27b9a445c51 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -30,12 +30,14 @@ import org.apache.openwhisk.common.{AkkaLogging, Config, Logging, TransactionId} import org.apache.openwhisk.core.cli.WhiskAdmin import org.apache.openwhisk.core.controller.Controller import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} +import org.apache.openwhisk.standalone.ColorOutput.clr import org.rogach.scallop.ScallopConf import pureconfig.loadConfigOrThrow import scala.collection.JavaConverters._ import scala.concurrent.Await import scala.concurrent.duration._ +import scala.io.AnsiColor import scala.util.Try class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { @@ -98,7 +100,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) configureLogging(conf) - printBanner() + printBanner(conf) initialize(conf) //Create actor system only after initializing the config implicit val actorSystem = ActorSystem("standalone-actor-system") @@ -221,8 +223,10 @@ object StandaloneOpenWhisk extends SLF4JLogging { } yield GitInfo(commit, time) } - private def printBanner() = { - println(banner) + private def printBanner(conf: Conf) = { + val bannerTxt = + if (conf.disableColorLogging()) banner else clr(banner, AnsiColor.CYAN) + println(bannerTxt) gitInfo.foreach(g => println(s"Git Commit: ${g.commitId}, Build Date: ${g.commitTime}")) } From d3b2cb222967817be2d68f68e506003f025697cc Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 10:16:43 +0530 Subject: [PATCH 42/66] Expose extension point for format log message --- .../org/apache/openwhisk/common/Logging.scala | 6 ++++-- .../standalone/LogbackConfigurator.scala | 16 ++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala index 825df9011b0..01c97542451 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala @@ -95,10 +95,12 @@ class AkkaLogging(loggingAdapter: LoggingAdapter) extends Logging { val logmsg: String = message // generates the message if (logmsg.nonEmpty) { // log it only if its not empty val name = if (from.isInstanceOf[String]) from else Logging.getCleanSimpleClassName(from.getClass) - loggingAdapter.log(loglevel, s"[$id] [$name] $logmsg") + loggingAdapter.log(loglevel, format(id, name.toString, logmsg)) } } } + + protected def format(id: TransactionId, name: String, logmsg: String) = s"[$id] [$name] $logmsg" } /** @@ -156,7 +158,7 @@ object LogMarker { } } -object Logging { +private object Logging { /** * Given a class object, return its simple name less the trailing dollar sign. diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala index a9da175ee21..2f03d391322 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala @@ -20,14 +20,13 @@ package org.apache.openwhisk.standalone import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets.UTF_8 -import akka.event.Logging.LogLevel import akka.event.LoggingAdapter import ch.qos.logback.classic.joran.JoranConfigurator import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.core.joran.spi.JoranException import ch.qos.logback.core.util.StatusPrinter import org.apache.commons.io.IOUtils -import org.apache.openwhisk.common.{Logging, TransactionId} +import org.apache.openwhisk.common.{AkkaLogging, TransactionId} import org.slf4j.LoggerFactory import scala.io.AnsiColor @@ -78,18 +77,11 @@ object LogbackConfigurator { /** * Similar to AkkaLogging but with color support */ -class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends Logging with AnsiColor { +class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends AkkaLogging(loggingAdapter) with AnsiColor { import ColorOutput.clr - def emit(loglevel: LogLevel, id: TransactionId, from: AnyRef, message: => String) = { - if (loggingAdapter.isEnabled(loglevel)) { - val logmsg: String = message // generates the message - if (logmsg.nonEmpty) { // log it only if its not empty - val name = if (from.isInstanceOf[String]) from else Logging.getCleanSimpleClassName(from.getClass) - loggingAdapter.log(loglevel, s"[${clr(id.toString, BOLD)}] [${clr(name.toString, CYAN)}] $logmsg") - } - } - } + override protected def format(id: TransactionId, name: String, logmsg: String) = + s"[${clr(id.toString, BOLD)}] [${clr(name.toString, CYAN)}] $logmsg" } object ColorOutput extends AnsiColor { From f1f22ae5bfa8e0476eb729e4197424ec0f75f1d6 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 10:33:06 +0530 Subject: [PATCH 43/66] Api host name is now defined via system property --- core/standalone/src/main/resources/standalone.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index 535b05e414f..5dfcfe14f80 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -40,7 +40,6 @@ whisk { } config { - whisk-api-host-name = "localhost" controller-instances = 1 limits-actions-sequence-maxLength = 50 limits-triggers-fires-perMinute = 60 From 144fd626275b809ac7305f7ed3b31124400fb0f5 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 14:12:58 +0530 Subject: [PATCH 44/66] Add a set of pre flight checks to confirm if OpenWhisk can run properly on this host --- .../standalone/PreFlightChecks.scala | 116 ++++++++++++++++++ .../standalone/StandaloneOpenWhisk.scala | 7 +- 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala new file mode 100644 index 00000000000..8e82ddb3327 --- /dev/null +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.standalone + +import com.typesafe.config.{Config, ConfigFactory} +import org.apache.commons.lang3.StringUtils +import org.apache.openwhisk.standalone.StandaloneOpenWhisk.usersConfigKey +import pureconfig.loadConfigOrThrow + +import scala.io.AnsiColor +import scala.sys.process._ +import scala.util.Try + +case class PreFlightChecks(conf: Conf) extends AnsiColor { + import ColorOutput.clr + private val noopLogger = ProcessLogger(_ => ()) + private val separator = "=" * 80 + private val pass = st(true) + private val failed = st(false) + private val cliDownloadUrl = "https://s.apache.org/openwhisk-cli-download" + private val dockerUrl = "https://docs.docker.com/install/" + + def run(): Unit = { + println(separator) + println("Running pre flight checks ...") + println() + checkForDocker() + checkForWsk() + println(separator) + } + + def checkForDocker() = { + val dockerExistsResult = Try("docker --version".!(noopLogger)).getOrElse(-1) + if (dockerExistsResult != 0) { + println(s"$failed 'docker' cli not found.") + println(s"\t Install docker from $dockerUrl") + } else { + println(s"$pass 'docker' cli found.") + //TODO add check for minimal supported docker version + //TODO should we also run docker run hello-world to see if we can execute docker run command + } + } + + def checkForWsk(): Unit = { + val wskExistsResult = Try("wsk property get --cliversion".!(noopLogger)).getOrElse(-1) + if (wskExistsResult != 0) { + println(s"$failed 'wsk' cli not found.") + println(s"\tDownload the cli from $cliDownloadUrl") + } else { + println(s"$pass 'wsk' cli found.") + checkWskProps() + } + } + + def checkWskProps(): Unit = { + val config = loadConfig() + val users = loadConfigOrThrow[Map[String, String]](config, usersConfigKey) + val configuredAuth = "wsk property get --auth".!!.trim + val apihost = "wsk property get --apihost".!!.trim + + val requiredHostValue = s"http://localhost:${conf.port()}" + + val hostMatched = apihost.endsWith(requiredHostValue) + + //We can use -o option to get raw value. However as its a recent addition + //using a lazy approach where we check if output ends with one of the configured auth keys or + val matchedAuth = users.find { case (_, auth) => configuredAuth.endsWith(auth) } + + if (matchedAuth.isDefined && hostMatched) { + println(s"$pass 'wsk' configured for namespace [${matchedAuth.get._1}].") + println(s"$pass 'wsk' configured to connect to $requiredHostValue.") + } else { + val guestUser = users.find { case (ns, _) => ns == "guest" } + //Only if guest user is found suggest wsk command to use that. Otherwise user is using a non default setup + //which may not be used for wsk based access like for tests + guestUser match { + case Some((ns, guestAuth)) => + println(s"$failed Configure wsk via below command to connect to this server as [$ns]") + println() + println(s"wsk property set --apihost '$requiredHostValue' --auth '$guestAuth'") + case None => + } + } + } + + private def loadConfig(): Config = { + conf.configFile.toOption match { + case Some(f) => + require(f.exists(), s"Config file $f does not exist") + ConfigFactory.parseFile(f) + case None => + ConfigFactory.parseResources("standalone.conf") + } + } + + private def st(pass: Boolean) = { + val (msg, code) = if (pass) (StringUtils.center("OK", "FAILURE".length), GREEN) else ("FAILURE", RED) + val transformedMsg = if (conf.disableColorLogging()) msg else clr(msg, code) + s"[$transformedMsg]" + } +} diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 27b9a445c51..2f0f89b6c53 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -57,6 +57,8 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { case class GitInfo(commitId: String, commitTime: String) object StandaloneOpenWhisk extends SLF4JLogging { + val usersConfigKey = "whisk.users" + val banner = """ | ____ ___ _ _ _ _ _ @@ -99,6 +101,9 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) + + PreFlightChecks(conf).run() + configureLogging(conf) printBanner(conf) initialize(conf) @@ -174,7 +179,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { private def bootstrapUsers()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer, logging: Logging): Unit = { - val users = loadConfigOrThrow[Map[String, String]]("whisk.users") + val users = loadConfigOrThrow[Map[String, String]](usersConfigKey) implicit val userTid: TransactionId = TransactionId("userBootstrap") users.foreach { case (name, key) => From a75645cb019cb24c5c86dd42004cb6d3acc44f42 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 14:28:43 +0530 Subject: [PATCH 45/66] Also check if docker is running --- .../standalone/PreFlightChecks.scala | 20 ++++++++++++++----- .../standalone/StandaloneOpenWhisk.scala | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala index 8e82ddb3327..de6933934c3 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala @@ -51,8 +51,19 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { println(s"\t Install docker from $dockerUrl") } else { println(s"$pass 'docker' cli found.") + checkDockerIsRunning() //TODO add check for minimal supported docker version - //TODO should we also run docker run hello-world to see if we can execute docker run command + //TODO should we also run `docker run hello-world` to see if we can execute docker run command + //This command takes 2-4 secs. So running it by default for every run should be avoided + } + } + + private def checkDockerIsRunning(): Unit = { + val dockerInfoResult = Try("docker info".!(noopLogger)).getOrElse(-1) + if (dockerInfoResult != 0) { + println(s"$failed 'docker' not found to be running.") + } else { + println(s"$pass 'docker' is running.") } } @@ -68,18 +79,17 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { } def checkWskProps(): Unit = { - val config = loadConfig() - val users = loadConfigOrThrow[Map[String, String]](config, usersConfigKey) + val users = loadConfigOrThrow[Map[String, String]](loadConfig(), usersConfigKey) + val configuredAuth = "wsk property get --auth".!!.trim val apihost = "wsk property get --apihost".!!.trim val requiredHostValue = s"http://localhost:${conf.port()}" - val hostMatched = apihost.endsWith(requiredHostValue) - //We can use -o option to get raw value. However as its a recent addition //using a lazy approach where we check if output ends with one of the configured auth keys or val matchedAuth = users.find { case (_, auth) => configuredAuth.endsWith(auth) } + val hostMatched = apihost.endsWith(requiredHostValue) if (matchedAuth.isDefined && hostMatched) { println(s"$pass 'wsk' configured for namespace [${matchedAuth.get._1}].") diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 2f0f89b6c53..5a680cefa71 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -102,10 +102,10 @@ object StandaloneOpenWhisk extends SLF4JLogging { def main(args: Array[String]): Unit = { val conf = new Conf(args) + printBanner(conf) PreFlightChecks(conf).run() configureLogging(conf) - printBanner(conf) initialize(conf) //Create actor system only after initializing the config implicit val actorSystem = ActorSystem("standalone-actor-system") From 0931083c8c8db507bd0b38c9e2ee2c1bfae27782 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 14:35:12 +0530 Subject: [PATCH 46/66] Update error message --- .../org/apache/openwhisk/standalone/PreFlightChecks.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala index de6933934c3..676fb8753fe 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala @@ -52,8 +52,9 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { } else { println(s"$pass 'docker' cli found.") checkDockerIsRunning() - //TODO add check for minimal supported docker version - //TODO should we also run `docker run hello-world` to see if we can execute docker run command + //Other things we can possibly check for + //1. add check for minimal supported docker version + //2. should we also run `docker run hello-world` to see if we can execute docker run command //This command takes 2-4 secs. So running it by default for every run should be avoided } } @@ -61,7 +62,7 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { private def checkDockerIsRunning(): Unit = { val dockerInfoResult = Try("docker info".!(noopLogger)).getOrElse(-1) if (dockerInfoResult != 0) { - println(s"$failed 'docker' not found to be running.") + println(s"$failed 'docker' not found to be running. Failed to run 'docker info'") } else { println(s"$pass 'docker' is running.") } From 6159470843b39d07941c16d7790e10ec7b9c2bb2 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 24 Jun 2019 16:55:19 +0530 Subject: [PATCH 47/66] standalone openwhisk dev --- tests/build.gradle | 1 + .../test/scala/common/FreePortFinder.scala | 33 +++++++++ .../scala/common/SystemPropertyGuard.scala | 32 +++++++++ tests/src/test/scala/common/TestFolder.scala | 71 +++++++++++++++++++ .../system/basic/WskRestBasicTests.scala | 6 ++ 5 files changed, 143 insertions(+) create mode 100644 tests/src/test/scala/common/FreePortFinder.scala create mode 100644 tests/src/test/scala/common/SystemPropertyGuard.scala create mode 100644 tests/src/test/scala/common/TestFolder.scala diff --git a/tests/build.gradle b/tests/build.gradle index efeb83da138..a8c950159be 100644 --- a/tests/build.gradle +++ b/tests/build.gradle @@ -202,6 +202,7 @@ dependencies { compile project(':common:scala') compile project(':core:controller') compile project(':core:invoker') + compile project(':core:standalone') compile project(':tools:admin') diff --git a/tests/src/test/scala/common/FreePortFinder.scala b/tests/src/test/scala/common/FreePortFinder.scala new file mode 100644 index 00000000000..0ebd54a97ff --- /dev/null +++ b/tests/src/test/scala/common/FreePortFinder.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import java.net.ServerSocket + +/** + * Utility to find a free port such that any launched service can be configured to use that. + * This helps in ensuring that test do not fail due to conflict with any existing service using a standard port + */ +object FreePortFinder { + + def freePort(): Int = { + val socket = new ServerSocket(0) + try socket.getLocalPort + finally if (socket != null) socket.close() + } +} diff --git a/tests/src/test/scala/common/SystemPropertyGuard.scala b/tests/src/test/scala/common/SystemPropertyGuard.scala new file mode 100644 index 00000000000..bb7778d5d43 --- /dev/null +++ b/tests/src/test/scala/common/SystemPropertyGuard.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import org.scalatest.{BeforeAndAfterAll, Suite} + +/** + * Utility trait to ensures that any system property changed during test gets reverted + */ +trait SystemPropertyGuard extends BeforeAndAfterAll { + self: Suite => + private val sysProps = System.getProperties + + override protected def afterAll(): Unit = { + System.setProperties(sysProps) + } +} diff --git a/tests/src/test/scala/common/TestFolder.scala b/tests/src/test/scala/common/TestFolder.scala new file mode 100644 index 00000000000..8e51b2638f9 --- /dev/null +++ b/tests/src/test/scala/common/TestFolder.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import java.io.File + +import org.scalatest._ + +/** + * Creates a temporary folder for the lifetime of a single test. + * The folder's name will exist in a `File` field named `testFolder`. + */ +trait TestFolder extends TestSuite { self: Suite => + var testFolder: File = _ + + //Default value ensures that temp files are created under build dir + protected def parentFolder: File = new File("build/tmp/scalaTestFolder") + + private def deleteFile(file: File) { + if (!file.exists) return + if (file.isFile) { + file.delete() + } else { + file.listFiles().foreach(deleteFile) + file.delete() + } + } + + abstract override def withFixture(test: NoArgTest): Outcome = { + testFolder = createTemporaryFolderIn(parentFolder) + try { + super.withFixture(test) + } finally { + deleteFile(testFolder) + } + } + + def newFile(): File = File.createTempFile("scalatest", null, rootFolder) + + def newFolder(): File = createTemporaryFolderIn(rootFolder) + + private def rootFolder = { + require(testFolder != null, "the temporary folder has not yet been created") + testFolder + } + + private def createTemporaryFolderIn(parentFolder: File) = { + if (!parentFolder.exists()) { + parentFolder.mkdirs() + } + val createdFolder = File.createTempFile("scalatest", "", parentFolder) + createdFolder.delete + createdFolder.mkdir + createdFolder + } +} diff --git a/tests/src/test/scala/system/basic/WskRestBasicTests.scala b/tests/src/test/scala/system/basic/WskRestBasicTests.scala index 2155e7ad5d1..2001aefe0b3 100644 --- a/tests/src/test/scala/system/basic/WskRestBasicTests.scala +++ b/tests/src/test/scala/system/basic/WskRestBasicTests.scala @@ -51,6 +51,12 @@ class WskRestBasicTests extends TestHelpers with WskTestHelpers with WskActorSys */ def cacheRetry[T](fn: => T) = org.apache.openwhisk.utils.retry(fn, 5, Some(1.second)) + override def withFixture(test: NoArgTest) = { + val outcome = super.withFixture(test) + Thread.sleep(200) + outcome + } + behavior of "Wsk REST" it should "reject creating duplicate entity" in withAssetCleaner(wskprops) { (wp, assetHelper) => From 098e9cdbad7ba0c2b03e99788cb37ca4fa3ee6f8 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 15:50:09 +0530 Subject: [PATCH 48/66] New StandaloneDockerContainerFactory which adapts as per OS --- .../docker/DockerContainerFactory.scala | 2 +- .../DockerForWindowsContainerFactory.scala | 66 ------------ .../StandaloneDockerContainerFactory.scala | 100 ++++++++++++++++++ .../standalone/StandaloneOpenWhisk.scala | 13 +-- 4 files changed, 104 insertions(+), 77 deletions(-) delete mode 100644 core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala create mode 100644 core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala index 9035d964539..94e5119bf0c 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala @@ -42,7 +42,7 @@ class DockerContainerFactory(instance: InvokerInstanceId, parameters: Map[String, Set[String]], containerArgsConfig: ContainerArgsConfig = loadConfigOrThrow[ContainerArgsConfig](ConfigKeys.containerArgs), - runtimesRegistryConfig: RuntimesRegistryConfig = + protected val runtimesRegistryConfig: RuntimesRegistryConfig = loadConfigOrThrow[RuntimesRegistryConfig](ConfigKeys.runtimesRegistry), dockerContainerFactoryConfig: DockerContainerFactoryConfig = loadConfigOrThrow[DockerContainerFactoryConfig](ConfigKeys.dockerContainerFactory))( diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala deleted file mode 100644 index a79224dc8be..00000000000 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForWindowsContainerFactory.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.openwhisk.core.containerpool.docker - -import akka.actor.ActorSystem -import org.apache.openwhisk.common.Logging -import org.apache.openwhisk.core.WhiskConfig -import org.apache.openwhisk.core.containerpool._ -import org.apache.openwhisk.core.entity.InvokerInstanceId -import pureconfig.loadConfig - -import scala.concurrent.ExecutionContext - -/** - * This factory provides a Docker for Mac client which exposes action container's ports on the host. - */ -object DockerForWindowsContainerFactory extends ContainerFactoryProvider { - override def instance(actorSystem: ActorSystem, - logging: Logging, - config: WhiskConfig, - instanceId: InvokerInstanceId, - parameters: Map[String, Set[String]]): ContainerFactory = { - - new DockerContainerFactory(instanceId, parameters)( - actorSystem, - actorSystem.dispatcher, - logging, - new DockerForWindowsClient()(actorSystem.dispatcher)(logging, actorSystem), - new RuncClient()(actorSystem.dispatcher)(logging, actorSystem)) - } - -} - -trait WindowsDockerClient { - self: DockerClient => - - override protected def executableAlternatives: List[String] = { - val executable = loadConfig[String]("whisk.docker.executable").map(Some(_)).getOrElse(None) - List(Some("C:\\Program Files\\Docker\\Docker\\resources\\bin\\docker.exe"), executable).flatten - } -} - -class DockerForWindowsClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)( - implicit log: Logging, - as: ActorSystem) - extends DockerForMacClient(dockerHost)(executionContext) - with WindowsDockerClient { - //Due to some Docker + Windows + Go parsing quirks need to add double quotes around whole command - //See https://github.com/moby/moby/issues/27592#issuecomment-255227097 - override def inspectCommand: String = "\"{{(index (index .NetworkSettings.Ports \\\"8080/tcp\\\") 0).HostPort}}\"" -} diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala new file mode 100644 index 00000000000..86b3c00b139 --- /dev/null +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.core.containerpool.docker + +import akka.actor.ActorSystem +import org.apache.commons.lang3.SystemUtils +import org.apache.openwhisk.common.{Logging, TransactionId} +import org.apache.openwhisk.core.WhiskConfig +import org.apache.openwhisk.core.containerpool.{Container, ContainerFactory, ContainerFactoryProvider} +import org.apache.openwhisk.core.entity.{ByteSize, ExecManifest, InvokerInstanceId} +import pureconfig.loadConfig + +import scala.collection.concurrent.TrieMap +import scala.concurrent.{ExecutionContext, Future} + +object StandaloneDockerContainerFactoryProvider extends ContainerFactoryProvider { + override def instance(actorSystem: ActorSystem, + logging: Logging, + config: WhiskConfig, + instanceId: InvokerInstanceId, + parameters: Map[String, Set[String]]): ContainerFactory = { + val client = + if (SystemUtils.IS_OS_MAC) new DockerForMacClient()(actorSystem.dispatcher)(logging, actorSystem) + else if (SystemUtils.IS_OS_WINDOWS) new DockerForWindowsClient()(actorSystem.dispatcher)(logging, actorSystem) + else new DockerClientWithFileAccess()(actorSystem.dispatcher)(logging, actorSystem) + + new StandaloneDockerContainerFactory(instanceId, parameters)( + actorSystem, + actorSystem.dispatcher, + logging, + client, + new RuncClient()(actorSystem.dispatcher)(logging, actorSystem)) + } +} + +class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: Map[String, Set[String]])( + implicit actorSystem: ActorSystem, + ec: ExecutionContext, + logging: Logging, + docker: DockerApiWithFileAccess, + runc: RuncApi) + extends DockerContainerFactory(instance, parameters) { + private val pulledImages = new TrieMap[String, Boolean]() + + override def createContainer(tid: TransactionId, + name: String, + actionImage: ExecManifest.ImageName, + userProvidedImage: Boolean, + memory: ByteSize, + cpuShares: Int)(implicit config: WhiskConfig, logging: Logging): Future[Container] = { + + //For standalone server usage we would also want to pull the OpenWhisk provided image so as to ensure if + //local setup does not have the image then it pulls it down + //For standard usage its expected that standard images have already been pulled in. + val imageName = actionImage.localImageName(runtimesRegistryConfig.url) + val pulled = if (!userProvidedImage && !pulledImages.contains(imageName)) { + docker.pull(imageName)(tid).map { _ => + logging.info(this, s"Pulled OpenWhisk provided image $imageName") + pulledImages.put(imageName, true) + true + } + } else Future.successful(true) + + pulled.flatMap(_ => super.createContainer(tid, name, actionImage, userProvidedImage, memory, cpuShares)) + } +} + +trait WindowsDockerClient { + self: DockerClient => + + override protected def executableAlternatives: List[String] = { + val executable = loadConfig[String]("whisk.docker.executable").map(Some(_)).getOrElse(None) + List(Some("C:\\Program Files\\Docker\\Docker\\resources\\bin\\docker.exe"), executable).flatten + } +} + +class DockerForWindowsClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)( + implicit log: Logging, + as: ActorSystem) + extends DockerForMacClient(dockerHost)(executionContext) + with WindowsDockerClient { + //Due to some Docker + Windows + Go parsing quirks need to add double quotes around whole command + //See https://github.com/moby/moby/issues/27592#issuecomment-255227097 + override def inspectCommand: String = "\"{{(index (index .NetworkSettings.Ports \\\"8080/tcp\\\") 0).HostPort}}\"" +} diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 5a680cefa71..21692bbf45f 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -192,16 +192,9 @@ object StandaloneOpenWhisk extends SLF4JLogging { } private def configureOSSpecificOpts(): Unit = { - if (SystemUtils.IS_OS_MAC) { - setSysProp( - "whisk.spi.ContainerFactoryProvider", - "org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider") - } - if (SystemUtils.IS_OS_WINDOWS) { - setSysProp( - "whisk.spi.ContainerFactoryProvider", - "org.apache.openwhisk.core.containerpool.docker.DockerForWindowsContainerFactory") - } + setSysProp( + "whisk.spi.ContainerFactoryProvider", + "org.apache.openwhisk.core.containerpool.docker.StandaloneDockerContainerFactoryProvider") //Disable runc by default to keep things stable setSysProp("whisk.docker.container-factory.use-runc", "False") From 984cfa1e5c9322a5832b894cf0f7bdfdf336733a Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 15:51:56 +0530 Subject: [PATCH 49/66] Revert "standalone openwhisk dev" This reverts commit 61594708 --- tests/build.gradle | 1 - .../test/scala/common/FreePortFinder.scala | 33 --------- .../scala/common/SystemPropertyGuard.scala | 32 --------- tests/src/test/scala/common/TestFolder.scala | 71 ------------------- .../system/basic/WskRestBasicTests.scala | 6 -- 5 files changed, 143 deletions(-) delete mode 100644 tests/src/test/scala/common/FreePortFinder.scala delete mode 100644 tests/src/test/scala/common/SystemPropertyGuard.scala delete mode 100644 tests/src/test/scala/common/TestFolder.scala diff --git a/tests/build.gradle b/tests/build.gradle index a8c950159be..efeb83da138 100644 --- a/tests/build.gradle +++ b/tests/build.gradle @@ -202,7 +202,6 @@ dependencies { compile project(':common:scala') compile project(':core:controller') compile project(':core:invoker') - compile project(':core:standalone') compile project(':tools:admin') diff --git a/tests/src/test/scala/common/FreePortFinder.scala b/tests/src/test/scala/common/FreePortFinder.scala deleted file mode 100644 index 0ebd54a97ff..00000000000 --- a/tests/src/test/scala/common/FreePortFinder.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package common - -import java.net.ServerSocket - -/** - * Utility to find a free port such that any launched service can be configured to use that. - * This helps in ensuring that test do not fail due to conflict with any existing service using a standard port - */ -object FreePortFinder { - - def freePort(): Int = { - val socket = new ServerSocket(0) - try socket.getLocalPort - finally if (socket != null) socket.close() - } -} diff --git a/tests/src/test/scala/common/SystemPropertyGuard.scala b/tests/src/test/scala/common/SystemPropertyGuard.scala deleted file mode 100644 index bb7778d5d43..00000000000 --- a/tests/src/test/scala/common/SystemPropertyGuard.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package common - -import org.scalatest.{BeforeAndAfterAll, Suite} - -/** - * Utility trait to ensures that any system property changed during test gets reverted - */ -trait SystemPropertyGuard extends BeforeAndAfterAll { - self: Suite => - private val sysProps = System.getProperties - - override protected def afterAll(): Unit = { - System.setProperties(sysProps) - } -} diff --git a/tests/src/test/scala/common/TestFolder.scala b/tests/src/test/scala/common/TestFolder.scala deleted file mode 100644 index 8e51b2638f9..00000000000 --- a/tests/src/test/scala/common/TestFolder.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package common - -import java.io.File - -import org.scalatest._ - -/** - * Creates a temporary folder for the lifetime of a single test. - * The folder's name will exist in a `File` field named `testFolder`. - */ -trait TestFolder extends TestSuite { self: Suite => - var testFolder: File = _ - - //Default value ensures that temp files are created under build dir - protected def parentFolder: File = new File("build/tmp/scalaTestFolder") - - private def deleteFile(file: File) { - if (!file.exists) return - if (file.isFile) { - file.delete() - } else { - file.listFiles().foreach(deleteFile) - file.delete() - } - } - - abstract override def withFixture(test: NoArgTest): Outcome = { - testFolder = createTemporaryFolderIn(parentFolder) - try { - super.withFixture(test) - } finally { - deleteFile(testFolder) - } - } - - def newFile(): File = File.createTempFile("scalatest", null, rootFolder) - - def newFolder(): File = createTemporaryFolderIn(rootFolder) - - private def rootFolder = { - require(testFolder != null, "the temporary folder has not yet been created") - testFolder - } - - private def createTemporaryFolderIn(parentFolder: File) = { - if (!parentFolder.exists()) { - parentFolder.mkdirs() - } - val createdFolder = File.createTempFile("scalatest", "", parentFolder) - createdFolder.delete - createdFolder.mkdir - createdFolder - } -} diff --git a/tests/src/test/scala/system/basic/WskRestBasicTests.scala b/tests/src/test/scala/system/basic/WskRestBasicTests.scala index 2001aefe0b3..2155e7ad5d1 100644 --- a/tests/src/test/scala/system/basic/WskRestBasicTests.scala +++ b/tests/src/test/scala/system/basic/WskRestBasicTests.scala @@ -51,12 +51,6 @@ class WskRestBasicTests extends TestHelpers with WskTestHelpers with WskActorSys */ def cacheRetry[T](fn: => T) = org.apache.openwhisk.utils.retry(fn, 5, Some(1.second)) - override def withFixture(test: NoArgTest) = { - val outcome = super.withFixture(test) - Thread.sleep(200) - outcome - } - behavior of "Wsk REST" it should "reject creating duplicate entity" in withAssetCleaner(wskprops) { (wp, assetHelper) => From f4c07bcd52f0c01e1286e997f9edb055842650d9 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 20:37:02 +0530 Subject: [PATCH 50/66] Use warn for misconfigured wsk --- .../standalone/LogbackConfigurator.scala | 4 ++-- .../standalone/PreFlightChecks.scala | 23 ++++++++++++------- .../standalone/StandaloneOpenWhisk.scala | 6 +++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala index 2f03d391322..e84a4a32c46 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala @@ -81,9 +81,9 @@ class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends AkkaLogging(log import ColorOutput.clr override protected def format(id: TransactionId, name: String, logmsg: String) = - s"[${clr(id.toString, BOLD)}] [${clr(name.toString, CYAN)}] $logmsg" + s"[${clr(id.toString, BOLD, true)}] [${clr(name.toString, CYAN, true)}] $logmsg" } object ColorOutput extends AnsiColor { - def clr(s: String, code: String) = s"$code$s$RESET" + def clr(s: String, code: String, clrEnabled: Boolean) = if (clrEnabled) s"$code$s$RESET" else s } diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala index 676fb8753fe..b3893ed817f 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala @@ -29,9 +29,11 @@ import scala.util.Try case class PreFlightChecks(conf: Conf) extends AnsiColor { import ColorOutput.clr private val noopLogger = ProcessLogger(_ => ()) + private val clrEnabled = conf.colorEnabled private val separator = "=" * 80 - private val pass = st(true) - private val failed = st(false) + private val pass = st("OK") + private val failed = st("FAILURE") + private val warn = st("WARN") private val cliDownloadUrl = "https://s.apache.org/openwhisk-cli-download" private val dockerUrl = "https://docs.docker.com/install/" @@ -41,6 +43,7 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { println() checkForDocker() checkForWsk() + println() println(separator) } @@ -101,9 +104,9 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { //which may not be used for wsk based access like for tests guestUser match { case Some((ns, guestAuth)) => - println(s"$failed Configure wsk via below command to connect to this server as [$ns]") + println(s"$warn Configure wsk via below command to connect to this server as [$ns]") println() - println(s"wsk property set --apihost '$requiredHostValue' --auth '$guestAuth'") + println(clr(s"wsk property set --apihost '$requiredHostValue' --auth '$guestAuth'", MAGENTA, clrEnabled)) case None => } } @@ -119,9 +122,13 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { } } - private def st(pass: Boolean) = { - val (msg, code) = if (pass) (StringUtils.center("OK", "FAILURE".length), GREEN) else ("FAILURE", RED) - val transformedMsg = if (conf.disableColorLogging()) msg else clr(msg, code) - s"[$transformedMsg]" + private def st(level: String) = { + val maxLength = "FAILURE".length + val (msg, code) = level match { + case "OK" => (StringUtils.center("OK", maxLength), GREEN) + case "WARN" => (StringUtils.center("WARN", maxLength), MAGENTA) + case _ => ("FAILURE", RED) + } + s"[${clr(msg, code, clrEnabled)}]" } } diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 21692bbf45f..21c4674df9e 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -51,7 +51,10 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { val verbose = tally() val disableColorLogging = opt[Boolean](descr = "Disables colored logging", noshort = true) + verify() + + val colorEnabled = !disableColorLogging() } case class GitInfo(commitId: String, commitTime: String) @@ -222,8 +225,7 @@ object StandaloneOpenWhisk extends SLF4JLogging { } private def printBanner(conf: Conf) = { - val bannerTxt = - if (conf.disableColorLogging()) banner else clr(banner, AnsiColor.CYAN) + val bannerTxt = clr(banner, AnsiColor.CYAN, conf.colorEnabled) println(bannerTxt) gitInfo.foreach(g => println(s"Git Commit: ${g.commitId}, Build Date: ${g.commitTime}")) } From 5527f0e4d96f28afdf41b38115c5d6c93300ac3d Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 20:48:47 +0530 Subject: [PATCH 51/66] Validate the files passed as argument to check if it exists --- .../apache/openwhisk/standalone/StandaloneOpenWhisk.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala index 21c4674df9e..ff0de5195e1 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala @@ -45,8 +45,10 @@ class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { footer("\nOpenWhisk standalone server") StandaloneOpenWhisk.gitInfo.foreach(g => version(s"Git Commit - ${g.commitId}")) - val configFile = opt[File](descr = "application.conf which overrides the default standalone.conf") - val manifest = opt[File](descr = "Manifest json defining the supported runtimes") + this.printedName = "openwhisk" + val configFile = + opt[File](descr = "application.conf which overrides the default standalone.conf", validate = _.canRead) + val manifest = opt[File](descr = "Manifest json defining the supported runtimes", validate = _.canRead) val port = opt[Int](descr = "Server port", default = Some(3233)) val verbose = tally() From 26532c4383ac40b038b5fe0d2d5b20efbc6d71c2 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 20:59:22 +0530 Subject: [PATCH 52/66] Only do pull for images having `openwhisk` prefix. For local image generally named `whisk/` no pull would be done --- .../docker/StandaloneDockerContainerFactory.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala index 86b3c00b139..3ebcd8e8f67 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala @@ -68,13 +68,14 @@ class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: //local setup does not have the image then it pulls it down //For standard usage its expected that standard images have already been pulled in. val imageName = actionImage.localImageName(runtimesRegistryConfig.url) - val pulled = if (!userProvidedImage && !pulledImages.contains(imageName)) { - docker.pull(imageName)(tid).map { _ => - logging.info(this, s"Pulled OpenWhisk provided image $imageName") - pulledImages.put(imageName, true) - true - } - } else Future.successful(true) + val pulled = + if (!userProvidedImage && !pulledImages.contains(imageName) && actionImage.prefix.contains("openwhisk")) { + docker.pull(imageName)(tid).map { _ => + logging.info(this, s"Pulled OpenWhisk provided image $imageName") + pulledImages.put(imageName, true) + true + } + } else Future.successful(true) pulled.flatMap(_ => super.createContainer(tid, name, actionImage, userProvidedImage, memory, cpuShares)) } From 0d2ca2a3e0c39a8e765d6f0e2f94afddd2e7443c Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 21:09:50 +0530 Subject: [PATCH 53/66] Add ./gradlew :core:standalone:run command to run jar directly --- core/standalone/build.gradle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index 205bedfe38b..c0efb9fa474 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -25,6 +25,7 @@ apply plugin: 'org.scoverage' apply plugin: 'maven' project.archivesBaseName = "openwhisk-standalone" +def jarName = "${project.archivesBaseName}.jar" repositories { mavenCentral() @@ -37,7 +38,7 @@ processResources { } bootJar { - archiveName = "${project.archivesBaseName}.jar" + archiveName = jarName mainClassName = 'org.apache.openwhisk.standalone.StandaloneOpenWhisk' } @@ -47,3 +48,10 @@ dependencies { compile 'org.rogach:scallop_2.12:3.3.1' scoverage gradle.scoverage.deps } + +task run(dependsOn: jar) << { + javaexec { + main = "-jar" + args new File(jar.archivePath.getParentFile(), jarName) + } +} From fd292cd81b7ec0d58501c5297b44ae39969ecbfd Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 21:12:17 +0530 Subject: [PATCH 54/66] Update the readme --- core/standalone/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/standalone/README.md b/core/standalone/README.md index 5ed87fc4b23..9ba130db879 100644 --- a/core/standalone/README.md +++ b/core/standalone/README.md @@ -36,7 +36,7 @@ Few key points related to it * Uses in memory store. Once the server is stopped all changes would be lost * Bootstraps the `guest` and `whisk.system` with default keys -* Supports running on LacOS, Linux and Windows setup +* Supports running on MacOS, Linux and Windows (experimental) setup * Can be customized to use any other storage like CouchDB @@ -48,7 +48,12 @@ To build this standalone server run $ ./gradlew :core:standalone:build ``` -This would create the runnable jar in `core/standalone/build/libs/` directory +This would create the runnable jar in `core/standalone/build/libs/` directory. If you directly want to build and run the +server then you can use following command + +```bash +$ ./gradlew :core:standalone:run +``` ### Usage From 31c413791dba5a3f9002a7f985f945cbff9e03ab Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 25 Jun 2019 21:17:41 +0530 Subject: [PATCH 55/66] Add standalone target --- tools/build/redo | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/build/redo b/tools/build/redo index cb81b269ef8..54cdccdb880 100755 --- a/tools/build/redo +++ b/tools/build/redo @@ -311,7 +311,13 @@ Components = [ 'run units tests', yaml = False, tasks = 'testUnit', - gradle = 'tests') + gradle = 'tests'), + + makeComponent('standalone', + 'run standalone server', + yaml = False, + tasks = 'run', + gradle = 'core:standalone') ] def getComponent(component): From 31f0881aed890c7942aacd30ac6fee7cc3ddc9ab Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 10:25:25 +0530 Subject: [PATCH 56/66] Use Spring boot default bootRun target to run the jar. Also copy the jar to bin directory --- core/standalone/README.md | 10 ++++++++-- core/standalone/build.gradle | 15 +++++++-------- tools/build/redo | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/core/standalone/README.md b/core/standalone/README.md index 9ba130db879..dcd154bff5b 100644 --- a/core/standalone/README.md +++ b/core/standalone/README.md @@ -48,11 +48,17 @@ To build this standalone server run $ ./gradlew :core:standalone:build ``` -This would create the runnable jar in `core/standalone/build/libs/` directory. If you directly want to build and run the +This would create the runnable jar in `bin/` directory. If you directly want to run the server then you can use following command ```bash -$ ./gradlew :core:standalone:run +$ ./gradlew :core:standalone:bootRun +``` + +To pass argument to the run command use + +```bash +$ ./gradlew :core:standalone:bootRun --args='-m runtimes.json' ``` ### Usage diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle index c0efb9fa474..06b55260c42 100644 --- a/core/standalone/build.gradle +++ b/core/standalone/build.gradle @@ -25,7 +25,6 @@ apply plugin: 'org.scoverage' apply plugin: 'maven' project.archivesBaseName = "openwhisk-standalone" -def jarName = "${project.archivesBaseName}.jar" repositories { mavenCentral() @@ -37,9 +36,15 @@ processResources { } } +task copyBootJarToBin(type:Copy){ + from ("${buildDir}/libs") + into file("${project.rootProject.projectDir}/bin") + rename("${project.archivesBaseName}-${version}.jar", "${project.archivesBaseName}.jar") +} + bootJar { - archiveName = jarName mainClassName = 'org.apache.openwhisk.standalone.StandaloneOpenWhisk' + finalizedBy copyBootJarToBin } dependencies { @@ -49,9 +54,3 @@ dependencies { scoverage gradle.scoverage.deps } -task run(dependsOn: jar) << { - javaexec { - main = "-jar" - args new File(jar.archivePath.getParentFile(), jarName) - } -} diff --git a/tools/build/redo b/tools/build/redo index 54cdccdb880..05344098d92 100755 --- a/tools/build/redo +++ b/tools/build/redo @@ -316,7 +316,7 @@ Components = [ makeComponent('standalone', 'run standalone server', yaml = False, - tasks = 'run', + tasks = 'bootRun', gradle = 'core:standalone') ] From b3784c6632de003b7372f49314f3d7325ad065ef Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 11:29:55 +0530 Subject: [PATCH 57/66] Make test run against standalone server --- .../test/scala/common/FreePortFinder.scala | 33 +++++ .../test/scala/common/WhiskProperties.java | 2 +- .../standalone/StandaloneServerFixture.scala | 122 ++++++++++++++++++ .../standalone/StandaloneServerTests.scala | 28 ++++ .../system/basic/WskRestBasicTests.scala | 2 +- 5 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 tests/src/test/scala/common/FreePortFinder.scala create mode 100644 tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala create mode 100644 tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala diff --git a/tests/src/test/scala/common/FreePortFinder.scala b/tests/src/test/scala/common/FreePortFinder.scala new file mode 100644 index 00000000000..0ebd54a97ff --- /dev/null +++ b/tests/src/test/scala/common/FreePortFinder.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import java.net.ServerSocket + +/** + * Utility to find a free port such that any launched service can be configured to use that. + * This helps in ensuring that test do not fail due to conflict with any existing service using a standard port + */ +object FreePortFinder { + + def freePort(): Int = { + val socket = new ServerSocket(0) + try socket.getLocalPort + finally if (socket != null) socket.close() + } +} diff --git a/tests/src/test/scala/common/WhiskProperties.java b/tests/src/test/scala/common/WhiskProperties.java index b53be03d3f5..1d442eb7c19 100644 --- a/tests/src/test/scala/common/WhiskProperties.java +++ b/tests/src/test/scala/common/WhiskProperties.java @@ -34,7 +34,7 @@ public class WhiskProperties { /** * System property key which refers to OpenWhisk Edge Host url */ - private static final String WHISK_SERVER = "whisk.server"; + public static final String WHISK_SERVER = "whisk.server"; /** * System property key which refers to authentication key to be used for testing diff --git a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala new file mode 100644 index 00000000000..32a18e64ad1 --- /dev/null +++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.standalone + +import java.io.File +import java.net.URI +import java.nio.charset.StandardCharsets.UTF_8 + +import com.google.common.base.Stopwatch +import common.WhiskProperties.WHISK_SERVER +import common.{FreePortFinder, StreamLogging, WhiskProperties} +import io.restassured.RestAssured +import org.apache.commons.io.FileUtils +import org.apache.commons.lang3.SystemUtils +import org.apache.openwhisk.core.WhiskConfig +import org.apache.openwhisk.utils.retry +import org.scalatest.{BeforeAndAfterAll, Pending, Suite, TestSuite} + +import scala.concurrent.duration._ +import scala.sys.process._ + +trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with StreamLogging { + self: Suite => + + private val jarPathProp = "whisk.server.jar" + private var manifestFile: Option[File] = None + private var serverProcess: Process = _ + protected val serverPort: Int = FreePortFinder.freePort() + protected val serverUrl: String = s"http://localhost:$serverPort/" + + //Following tests always fail on Mac but pass when standalone server is running on Linux + //It looks related to how networking works on Mac for Docker container + //For now ignoring there failure + private val ignoredTestsOnMac = Set( + "Wsk Action REST should create, and invoke an action that utilizes a docker container", + "Wsk Action REST should create, and invoke an action that utilizes dockerskeleton with native zip", + "Wsk Action REST should create and invoke a blocking action resulting in an application error response", + "Wsk Action REST should create an action, and invoke an action that returns an empty JSON object") + + override def beforeAll(): Unit = { + System.setProperty(WHISK_SERVER, serverUrl) + //TODO avoid starting the server if url whisk.server property is predefined + super.beforeAll() + manifestFile = getRuntimeManifest() + val args = Seq( + Seq("java", "-jar", standaloneServerJar.getAbsolutePath, "--disable-color-logging"), + Seq("-p", serverPort.toString), + manifestFile.map(f => Seq("-m", f.getAbsolutePath)).getOrElse(Seq.empty)).flatten + + serverProcess = args.run(ProcessLogger(s => printstream.println(s))) + val w = waitForServerToStart() + println(s"Started test server at $serverUrl in [$w]") + } + + override def afterAll(): Unit = { + System.clearProperty(WHISK_SERVER) + super.afterAll() + manifestFile.foreach(FileUtils.deleteQuietly) + serverProcess.destroy() + } + + override def withFixture(test: NoArgTest) = { + val outcome = super.withFixture(test) + if (outcome.isFailed) { + println(logLines.mkString("\n")) + } + stream.reset() + val result = if (outcome.isFailed && SystemUtils.IS_OS_MAC && ignoredTestsOnMac.contains(test.name)) { + println(s"Ignoring known failed test for Mac [${test.name}]") + Pending + } else outcome + result + } + + def waitForServerToStart(): Stopwatch = { + val w = Stopwatch.createStarted() + retry({ + println(s"Waiting for OpenWhisk server to start since $w") + val response = RestAssured.get(new URI(serverUrl)) + require(response.statusCode() == 200) + }, 10, Some(1.second)) + w + } + + private def getRuntimeManifest(): Option[File] = { + Option(WhiskProperties.getProperty(WhiskConfig.runtimesManifest)).map { json => + val f = newFile() + FileUtils.write(f, json, UTF_8) + f + } + } + + private def newFile(): File = File.createTempFile("whisktest", null, null) + + private def standaloneServerJar: File = { + Option(System.getProperty(jarPathProp)) match { + case Some(p) => + val jarFile = new File(p) + assert( + jarFile.canRead, + s"OpenWhisk standalone server jar file [$p] specified via system property [$jarPathProp] not found") + jarFile + case None => + fail(s"No jar file specified via system property [$jarPathProp]") + } + } +} diff --git a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala new file mode 100644 index 00000000000..725033c2a7a --- /dev/null +++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.openwhisk.standalone + +import common.WskProps +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +import system.basic.WskRestBasicTests + +@RunWith(classOf[JUnitRunner]) +class StandaloneServerTests extends WskRestBasicTests with StandaloneServerFixture { + override implicit val wskprops = WskProps().copy(apihost = serverUrl) +} diff --git a/tests/src/test/scala/system/basic/WskRestBasicTests.scala b/tests/src/test/scala/system/basic/WskRestBasicTests.scala index 2155e7ad5d1..134808928ca 100644 --- a/tests/src/test/scala/system/basic/WskRestBasicTests.scala +++ b/tests/src/test/scala/system/basic/WskRestBasicTests.scala @@ -39,7 +39,7 @@ import org.apache.openwhisk.http.Messages @RunWith(classOf[JUnitRunner]) class WskRestBasicTests extends TestHelpers with WskTestHelpers with WskActorSystem { - implicit val wskprops = WskProps() + implicit def wskprops: WskProps = WskProps() val wsk = new WskRestOperations val defaultAction: Some[String] = Some(TestUtils.getTestActionFilename("hello.js")) From 6f09fc0eae461f39a8be0356e576137860b08d06 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 12:37:24 +0530 Subject: [PATCH 58/66] Support disabling pull of standard images all together. Required for test runs --- .../apache/openwhisk/core/WhiskConfig.scala | 1 + .../src/main/resources/application.conf | 5 +++++ .../StandaloneDockerContainerFactory.scala | 19 ++++++++++++++++--- .../src/main/resources/standalone.conf | 4 ++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index 44e646864ca..62830337301 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -222,6 +222,7 @@ object ConfigKeys { val docker = "whisk.docker" val dockerClient = s"$docker.client" val dockerContainerFactory = s"$docker.container-factory" + val standaloneDockerContainerFactory = s"$docker.standalone.container-factory" val runc = "whisk.runc" val runcTimeouts = s"$runc.timeouts" diff --git a/core/invoker/src/main/resources/application.conf b/core/invoker/src/main/resources/application.conf index 938af2a604c..4f362094cf9 100644 --- a/core/invoker/src/main/resources/application.conf +++ b/core/invoker/src/main/resources/application.conf @@ -51,6 +51,11 @@ whisk { use-runc: true } + docker.standalone.container-factory { + #If enabled then pull would also be attempted for standard OpenWhisk images under`openwhisk` prefix + pull-standard-images: false + } + container-pool { user-memory: 1024 m concurrent-peek-factor: 0.5 #factor used to limit message peeking: 0 < factor <= 1.0 - larger number improves concurrent processing, but increases risk of message loss during invoker crash diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala index 3ebcd8e8f67..60df1b8ee03 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala @@ -20,10 +20,10 @@ package org.apache.openwhisk.core.containerpool.docker import akka.actor.ActorSystem import org.apache.commons.lang3.SystemUtils import org.apache.openwhisk.common.{Logging, TransactionId} -import org.apache.openwhisk.core.WhiskConfig +import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} import org.apache.openwhisk.core.containerpool.{Container, ContainerFactory, ContainerFactoryProvider} import org.apache.openwhisk.core.entity.{ByteSize, ExecManifest, InvokerInstanceId} -import pureconfig.loadConfig +import pureconfig.{loadConfig, loadConfigOrThrow} import scala.collection.concurrent.TrieMap import scala.concurrent.{ExecutionContext, Future} @@ -48,6 +48,8 @@ object StandaloneDockerContainerFactoryProvider extends ContainerFactoryProvider } } +case class StandaloneDockerConfig(pullStandardImages: Boolean) + class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: Map[String, Set[String]])( implicit actorSystem: ActorSystem, ec: ExecutionContext, @@ -56,6 +58,7 @@ class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: runc: RuncApi) extends DockerContainerFactory(instance, parameters) { private val pulledImages = new TrieMap[String, Boolean]() + private val factoryConfig = loadConfigOrThrow[StandaloneDockerConfig](ConfigKeys.standaloneDockerContainerFactory) override def createContainer(tid: TransactionId, name: String, @@ -69,7 +72,10 @@ class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: //For standard usage its expected that standard images have already been pulled in. val imageName = actionImage.localImageName(runtimesRegistryConfig.url) val pulled = - if (!userProvidedImage && !pulledImages.contains(imageName) && actionImage.prefix.contains("openwhisk")) { + if (!userProvidedImage + && factoryConfig.pullStandardImages + && !pulledImages.contains(imageName) + && actionImage.prefix.contains("openwhisk")) { docker.pull(imageName)(tid).map { _ => logging.info(this, s"Pulled OpenWhisk provided image $imageName") pulledImages.put(imageName, true) @@ -79,6 +85,13 @@ class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: pulled.flatMap(_ => super.createContainer(tid, name, actionImage, userProvidedImage, memory, cpuShares)) } + + override def init(): Unit = { + logging.info( + this, + s"Standalone docker container factory config pullStandardImages: ${factoryConfig.pullStandardImages}") + super.init() + } } trait WindowsDockerClient { diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf index 5dfcfe14f80..e3a89ad4251 100644 --- a/core/standalone/src/main/resources/standalone.conf +++ b/core/standalone/src/main/resources/standalone.conf @@ -63,5 +63,9 @@ whisk { docker { # Path to docker executuable. Generally its /var/lib/docker # executable = + standalone.container-factory { + #If enabled then pull would also be attempted for standard OpenWhisk images under`openwhisk` prefix + pull-standard-images: true + } } } From f3d57a3a4b4e7cd559f2ad3efcf9ebe9c2663aca Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 12:37:43 +0530 Subject: [PATCH 59/66] Disable pause/resume support for non linux setups --- .../containerpool/docker/DockerForMacContainerFactory.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala index 08b72b002cb..97d265c43a0 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala @@ -72,4 +72,9 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex } def inspectCommand: String = """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""" + + //Pause unpause is causing issue on non Linux setups. So disable by default + override def pause(id: ContainerId)(implicit transid: TransactionId): Future[Unit] = Future.successful(()) + + override def unpause(id: ContainerId)(implicit transid: TransactionId): Future[Unit] = Future.successful(()) } From 828c90c953df4d3fa886b7dabc533ed7f23b38fe Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 12:38:11 +0530 Subject: [PATCH 60/66] Pass on standalone jar path to test --- tests/build.gradle | 26 +++++++++++-------- .../standalone/StandaloneServerFixture.scala | 9 ++++++- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/build.gradle b/tests/build.gradle index efeb83da138..48fc925362e 100644 --- a/tests/build.gradle +++ b/tests/build.gradle @@ -36,17 +36,6 @@ install.dependsOn ':tools:admin:install' project.archivesBaseName = "openwhisk-tests" -tasks.withType(Test) { - systemProperties(System.getProperties()) - - testLogging { - events "passed", "skipped", "failed" - showStandardStreams = true - exceptionFormat = 'full' - } - outputs.upToDateWhen { false } // force tests to run every time -} - def leanExcludes = [ '**/MaxActionDurationTests*', 'invokerShoot/**' @@ -250,6 +239,17 @@ gradle.projectsEvaluated { task testCoverage(type: Test) { classpath = getScoverageClasspath(project) } + tasks.withType(Test) { + systemProperties(System.getProperties()) + systemProperty("whisk.server.jar", getStandaloneJarFilePath()) + + testLogging { + events "passed", "skipped", "failed" + showStandardStreams = true + exceptionFormat = 'full' + } + outputs.upToDateWhen { false } // force tests to run every time + } } task copyMeasurementFiles() { @@ -366,3 +366,7 @@ task testSwaggerCodegen(type: GradleBuild) { buildFile = "${buildDir}/swagger-code-java/build.gradle" tasks = ['build'] } + +def getStandaloneJarFilePath(){ + project(":core:standalone").jar.archivePath +} diff --git a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala index 32a18e64ad1..02c0840e080 100644 --- a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala @@ -42,6 +42,7 @@ trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with Stre private var serverProcess: Process = _ protected val serverPort: Int = FreePortFinder.freePort() protected val serverUrl: String = s"http://localhost:$serverPort/" + private val disablePullConfig = "whisk.docker.standalone.container-factory.pull-standard-images" //Following tests always fail on Mac but pass when standalone server is running on Linux //It looks related to how networking works on Mac for Docker container @@ -56,9 +57,15 @@ trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with Stre System.setProperty(WHISK_SERVER, serverUrl) //TODO avoid starting the server if url whisk.server property is predefined super.beforeAll() + println(s"Running standalone server from ${standaloneServerJar.getAbsolutePath}") manifestFile = getRuntimeManifest() val args = Seq( - Seq("java", "-jar", standaloneServerJar.getAbsolutePath, "--disable-color-logging"), + Seq( + "java", + s"-D$disablePullConfig=false", + "-jar", + standaloneServerJar.getAbsolutePath, + "--disable-color-logging"), Seq("-p", serverPort.toString), manifestFile.map(f => Seq("-m", f.getAbsolutePath)).getOrElse(Seq.empty)).flatten From 6ae5f58d8ac33c8bc4b2aa5a16856385122b613f Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 12:47:45 +0530 Subject: [PATCH 61/66] Disable launching the server if preexisting server provided --- .../standalone/StandaloneServerFixture.scala | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala index 02c0840e080..a709c9ff41d 100644 --- a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala @@ -41,8 +41,9 @@ trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with Stre private var manifestFile: Option[File] = None private var serverProcess: Process = _ protected val serverPort: Int = FreePortFinder.freePort() - protected val serverUrl: String = s"http://localhost:$serverPort/" + protected var serverUrl: String = System.getProperty(WHISK_SERVER, s"http://localhost:$serverPort/") private val disablePullConfig = "whisk.docker.standalone.container-factory.pull-standard-images" + private var serverStartedForTest = false //Following tests always fail on Mac but pass when standalone server is running on Linux //It looks related to how networking works on Mac for Docker container @@ -54,31 +55,41 @@ trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with Stre "Wsk Action REST should create an action, and invoke an action that returns an empty JSON object") override def beforeAll(): Unit = { - System.setProperty(WHISK_SERVER, serverUrl) - //TODO avoid starting the server if url whisk.server property is predefined - super.beforeAll() - println(s"Running standalone server from ${standaloneServerJar.getAbsolutePath}") - manifestFile = getRuntimeManifest() - val args = Seq( - Seq( - "java", - s"-D$disablePullConfig=false", - "-jar", - standaloneServerJar.getAbsolutePath, - "--disable-color-logging"), - Seq("-p", serverPort.toString), - manifestFile.map(f => Seq("-m", f.getAbsolutePath)).getOrElse(Seq.empty)).flatten + val serverUrlViaSysProp = Option(System.getProperty(WHISK_SERVER)) + serverUrlViaSysProp match { + case Some(u) => + serverUrl = u + println(s"Connecting to existing server at $serverUrl") + case None => + System.setProperty(WHISK_SERVER, serverUrl) + //TODO avoid starting the server if url whisk.server property is predefined + super.beforeAll() + println(s"Running standalone server from ${standaloneServerJar.getAbsolutePath}") + manifestFile = getRuntimeManifest() + val args = Seq( + Seq( + "java", + s"-D$disablePullConfig=false", + "-jar", + standaloneServerJar.getAbsolutePath, + "--disable-color-logging"), + Seq("-p", serverPort.toString), + manifestFile.map(f => Seq("-m", f.getAbsolutePath)).getOrElse(Seq.empty)).flatten - serverProcess = args.run(ProcessLogger(s => printstream.println(s))) - val w = waitForServerToStart() - println(s"Started test server at $serverUrl in [$w]") + serverProcess = args.run(ProcessLogger(s => printstream.println(s))) + val w = waitForServerToStart() + serverStartedForTest = true + println(s"Started test server at $serverUrl in [$w]") + } } override def afterAll(): Unit = { - System.clearProperty(WHISK_SERVER) super.afterAll() - manifestFile.foreach(FileUtils.deleteQuietly) - serverProcess.destroy() + if (serverStartedForTest) { + System.clearProperty(WHISK_SERVER) + manifestFile.foreach(FileUtils.deleteQuietly) + serverProcess.destroy() + } } override def withFixture(test: NoArgTest) = { From f368bfd9b27df87f5240154baa64bf2c125b6119 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 13:57:02 +0530 Subject: [PATCH 62/66] Print the wsk and docker cli version --- .../apache/openwhisk/standalone/PreFlightChecks.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala index b3893ed817f..f3a82d3336a 100644 --- a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala +++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala @@ -53,7 +53,7 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { println(s"$failed 'docker' cli not found.") println(s"\t Install docker from $dockerUrl") } else { - println(s"$pass 'docker' cli found.") + println(s"$pass 'docker' cli found. $dockerVersion") checkDockerIsRunning() //Other things we can possibly check for //1. add check for minimal supported docker version @@ -62,6 +62,10 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { } } + private def dockerVersion = version("docker --version '{{.Client.Version}}'") + + private def version(cmd: String) = Try(cmd !! (noopLogger)).map(v => s"(${v.trim})").getOrElse("") + private def checkDockerIsRunning(): Unit = { val dockerInfoResult = Try("docker info".!(noopLogger)).getOrElse(-1) if (dockerInfoResult != 0) { @@ -77,7 +81,7 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { println(s"$failed 'wsk' cli not found.") println(s"\tDownload the cli from $cliDownloadUrl") } else { - println(s"$pass 'wsk' cli found.") + println(s"$pass 'wsk' cli found. $wskCliVersion") checkWskProps() } } @@ -112,6 +116,8 @@ case class PreFlightChecks(conf: Conf) extends AnsiColor { } } + private def wskCliVersion = version("wsk property get --cliversion -o raw") + private def loadConfig(): Config = { conf.configFile.toOption match { case Some(f) => From add1fab1305a104625df643c2a5d16de6bc410c4 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 14:02:35 +0530 Subject: [PATCH 63/66] Update README with details on how to connect to db --- core/standalone/README.md | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/core/standalone/README.md b/core/standalone/README.md index dcd154bff5b..831f276e4d9 100644 --- a/core/standalone/README.md +++ b/core/standalone/README.md @@ -75,11 +75,14 @@ $ java -jar openwhisk-standalone.jar -h \ \ / \/ \___/| .__/ \___|_| |_|__/\__|_| |_|_|___/_|\_\ \___\/ tm |_| - -c, --config-file application.conf which overrides the default - standalone.conf - -m, --manifest Manifest json defining the supported runtimes - -p, --port Server port - -h, --help Show help message + -c, --config-file application.conf which overrides the default + standalone.conf + --disable-color-logging Disables colored logging + -m, --manifest Manifest json defining the supported runtimes + -p, --port Server port + -v, --verbose + -h, --help Show help message + --version Show version of this program OpenWhisk standalone server ``` @@ -160,5 +163,31 @@ java -jar openwhisk-standalone.jar -m custom-runtime.json You can then see the runtime config reflect in `http://localhost:3233` +#### Using CouchDB + +If you need to connect to CouchDB or any other supported artifact store then you can pass the required config + +```hocon +include classpath("standalone.conf") + +whisk { + couchdb { + protocol = "http" + host = "172.17.0.1" + port = "5984" + username = "whisk_admin" + password = "some_passw0rd" + provider = "CouchDB" + databases { + WhiskAuth = "whisk_local_subjects" + WhiskEntity = "whisk_local_whisks" + WhiskActivation = "whisk_local_activations" + } + } +} +``` + +Then pass this config file via `-c` option. + [1]: https://github.com/apache/incubator-openwhisk/blob/master/docs/cli.md [2]: https://github.com/apache/incubator-openwhisk/blob/master/docs/samples.md From 0fda29a21ad7c6ed6161296936de171eda851425 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 14:12:27 +0530 Subject: [PATCH 64/66] Build standalone as part of docker dist so as to be used in test --- tools/travis/distDocker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/travis/distDocker.sh b/tools/travis/distDocker.sh index 7f40d0b3026..a6022fd45d2 100755 --- a/tools/travis/distDocker.sh +++ b/tools/travis/distDocker.sh @@ -29,5 +29,6 @@ TERM=dumb ./gradlew distDocker -PdockerImagePrefix=testing $GRADLE_PROJS_SKIP TERM=dumb ./gradlew :core:controller:distDockerCoverage -PdockerImagePrefix=testing TERM=dumb ./gradlew :core:invoker:distDockerCoverage -PdockerImagePrefix=testing +TERM=dumb ./gradlew :core:standalone:build echo "Time taken for ${0##*/} is $SECONDS secs" From a65b5283a39b11fc579425f8cc254c791ab9ddb5 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 16:39:07 +0530 Subject: [PATCH 65/66] Increase time allowed for server to start to 30 secs and log logs upon timeout failure --- .../standalone/StandaloneServerFixture.scala | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala index a709c9ff41d..df07b271e73 100644 --- a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala @@ -33,6 +33,7 @@ import org.scalatest.{BeforeAndAfterAll, Pending, Suite, TestSuite} import scala.concurrent.duration._ import scala.sys.process._ +import scala.util.control.NonFatal trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with StreamLogging { self: Suite => @@ -107,11 +108,17 @@ trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with Stre def waitForServerToStart(): Stopwatch = { val w = Stopwatch.createStarted() - retry({ - println(s"Waiting for OpenWhisk server to start since $w") - val response = RestAssured.get(new URI(serverUrl)) - require(response.statusCode() == 200) - }, 10, Some(1.second)) + try { + retry({ + println(s"Waiting for OpenWhisk server to start since $w") + val response = RestAssured.get(new URI(serverUrl)) + require(response.statusCode() == 200) + }, 30, Some(1.second)) + } catch { + case NonFatal(e) => + println(logLines.mkString("\n")) + throw e + } w } From 5b99c6e7bf9a0c936aa25da7febf441226865a50 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Wed, 26 Jun 2019 16:44:42 +0530 Subject: [PATCH 66/66] Enable verifySystemShutdown in Mesos tests --- .../containerpool/mesos/test/MesosContainerFactoryTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala index 04577e33b2d..ef0121eaece 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala @@ -98,7 +98,7 @@ class MesosContainerFactoryTest } override def afterAll(): Unit = { - TestKit.shutdownActorSystem(system) + TestKit.shutdownActorSystem(system, verifySystemShutdown = true) super.afterAll() }