SAMZA-2418: Integration tests are failing and do not properly pass params to zopkio

bin/integration-tests.sh

@@ -72,8 +72,7 @@ source $SAMZA_INTEGRATION_TESTS_DIR/bin/activate
 pip install -r $SCRIPTS_DIR/requirements.txt
 
 # treat all trailing parameters (after dirname, test_type) as zopkio switches
-shift
-SWITCHES="${*:3}"
+SWITCHES="${@:3}"
 
 # default to info-level debugging if not specified
 if [[ $SWITCHES != *"console-log-level"* ]]; then

build.gradle

@@ -845,9 +845,6 @@ project(":samza-test_$scalaSuffix") {
     compile.exclude group: 'javax.jms', module: 'jms'
     compile.exclude group: 'com.sun.jdmk', module: 'jmxtools'
     compile.exclude group: 'com.sun.jmx', module: 'jmxri'
-    // Exclude Slf4j bindings for log4j1 and log4j2 since samza-test just logs to console using slf4j-simple
-    compile.exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-    compile.exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl'
   }
 
   dependencies {
@@ -858,6 +855,7 @@ project(":samza-test_$scalaSuffix") {
     compile project(":samza-core_$scalaSuffix")
     compile project(":samza-kafka_$scalaSuffix")
     compile project(":samza-sql_$scalaSuffix")
+    runtime project(":samza-log4j2_$scalaSuffix")
     runtime project(":samza-yarn_$scalaSuffix")
     runtime project(":samza-hdfs_$scalaSuffix")
     compile "org.scala-lang:scala-library:$scalaVersion"

docs/contribute/tests.md

@@ -60,7 +60,7 @@ On Mac, check-all.sh will default to the appropriate path for each environment v
 
 Samza uses [Zopkio](https://github.com/linkedin/Zopkio) to deploy and execute its integration tests. Integration tests can be executed by running:
 
-    ./bin/integration-tests.sh /tmp/samza-tests
+    ./bin/integration-tests.sh /tmp/samza-tests [yarn-integration-tests|standalone-integration-tests]
 
 The parameter defines where the integration tests should install packages both locally and on remote systems. Executing this command will:
 
@@ -90,15 +90,15 @@ Zopkio supports public key authentication if you prefer to use it, or if your en
 
 Once this is done, you can run Zopkio with the \-\-nopassword parameter:
 
-    ./bin/integration-tests.sh /tmp/samza-tests --nopassword
+    ./bin/integration-tests.sh /tmp/samza-tests [yarn-integration-tests|standalone-integration-tests] --nopassword
 
 This will skip the password prompt, and force Zopkio to try public key authentication.
 
 #### Console Logging
 
 The integration-tests.sh script will set the console log level to INFO by default. The level can be changed with:
 
-    ./bin/integration-tests.sh /tmp/samza-tests --console-log-level DEBUG
+    ./bin/integration-tests.sh /tmp/samza-tests [yarn-integration-tests|standalone-integration-tests] --console-log-level DEBUG
 
 Changing this setting will define how verbose Zopkio is during test execution. It does not affect any of the log4j.xml settings in Samza, YARN, Kafka, or ZooKeeper.
 

samza-test/src/main/python/configs/tests.json

@@ -1,5 +1,5 @@
 {
-  "samza_executable": "samza-test_2.11-1.1.1-SNAPSHOT.tgz",
+  "samza_executable": "samza-test_2.11-1.4.0-SNAPSHOT.tgz",
   "samza_install_path": "deploy/smoke_tests",
   "samza_config_factory": "org.apache.samza.config.factories.PropertiesConfigFactory"
 }

samza-test/src/main/python/stream_processor.py

@@ -43,7 +43,7 @@ def __init__(self, host_name, processor_id):
         logger.info('Running processor start command: {0}'.format(self.processor_start_command))
         self.deployment_config = {
             'install_path': os.path.join(runtime.get_active_config('remote_install_path'), 'deploy/{0}'.format(self.processor_id)),
-            'executable': 'samza-test_2.11-1.1.1-SNAPSHOT.tgz',
+            'executable': 'samza-test_2.11-1.4.0-SNAPSHOT.tgz',
             'post_install_cmds': [],
             'start_command': self.processor_start_command,
             'stop_command': '',

samza-test/src/main/resources/log4j2.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  you under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<Configuration>
+  <Appenders>
+    <Console name="STDOUT" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] [%X{REQUEST_UUID}] %c{1} [%p] %m%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="info">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+    <Logger name="kafka" level="off"></Logger>
+    <Logger name="org.apache.kafka" level="off"></Logger>
+    <Logger name="org.apache.samza.system.kafka" level="off"></Logger>
+    <Logger name="org.apache.zookeeper" level="off"></Logger>
+  </Loggers>
+</Configuration>