From 6cc8ccdba8cde7250b714af94350d08ff67ecf50 Mon Sep 17 00:00:00 2001 From: Liquan Pei Date: Sat, 23 Apr 2016 22:45:58 -0700 Subject: [PATCH 1/6] Exclude test jars in kafka-run-class.sh --- bin/kafka-run-class.sh | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 88d43be3188d9..6fd6aa890e699 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -20,6 +20,15 @@ then exit 1 fi +check_file() { + file=$1 + if [ -z "$(echo "$file" | grep "test")" ] ; then + return 0 + else + return 1 + fi +} + base_dir=$(dirname $0)/.. if [ -z "$SCALA_VERSION" ]; then @@ -41,24 +50,32 @@ do fi done -for file in $base_dir/examples/build/libs//kafka-examples*.jar; +for file in $base_dir/examples/build/libs/kafka-examples*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/clients/build/libs/kafka-clients*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/streams/build/libs/kafka-streams*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/streams/examples/build/libs/kafka-streams-examples*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar; @@ -68,7 +85,9 @@ done for file in $base_dir/tools/build/libs/kafka-tools*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for dir in $base_dir/tools/build/dependant-libs-${SCALA_VERSION}*; @@ -80,7 +99,9 @@ for cc_pkg in "api" "runtime" "file" "json" "tools" do for file in $base_dir/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done if [ -d "$base_dir/connect/${cc_pkg}/build/dependant-libs" ] ; then CLASSPATH=$CLASSPATH:$base_dir/connect/${cc_pkg}/build/dependant-libs/* @@ -92,7 +113,9 @@ CLASSPATH=$CLASSPATH:$base_dir/libs/* for file in $base_dir/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar; do - CLASSPATH=$CLASSPATH:$file + if check_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done shopt -u nullglob From 45cf01c83b912820975f511273201d3d38f58265 Mon Sep 17 00:00:00 2001 From: Liquan Pei Date: Mon, 25 Apr 2016 13:32:25 -0700 Subject: [PATCH 2/6] Filter out unnecessary files with binary packages --- bin/kafka-run-class.sh | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 6fd6aa890e699..43a9f9228d9af 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -20,9 +20,11 @@ then exit 1 fi -check_file() { +# Exclude jars not necessary for running commands. +regex="((test|src|scaladoc|javadoc)\.jar|jar.asc)$" +filter_file() { file=$1 - if [ -z "$(echo "$file" | grep "test")" ] ; then + if [ -z "$(echo "$file" | egrep "$regex")" ] ; then return 0 else return 1 @@ -52,28 +54,28 @@ done for file in $base_dir/examples/build/libs/kafka-examples*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/clients/build/libs/kafka-clients*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/streams/build/libs/kafka-streams*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/streams/examples/build/libs/kafka-streams-examples*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -85,7 +87,7 @@ done for file in $base_dir/tools/build/libs/kafka-tools*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -99,7 +101,7 @@ for cc_pkg in "api" "runtime" "file" "json" "tools" do for file in $base_dir/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -109,11 +111,16 @@ do done # classpath addition for release -CLASSPATH=$CLASSPATH:$base_dir/libs/* +for file in $base_dir/libs; +do + if filter_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi +done for file in $base_dir/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar; do - if check_file "$file"; then + if filter_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -168,8 +175,6 @@ if [ "x$KAFKA_DEBUG" != "x" ]; then JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS" fi - - echo "Enabling Java debug options: $JAVA_DEBUG_OPTS" KAFKA_OPTS="$JAVA_DEBUG_OPTS $KAFKA_OPTS" fi From 2791f3b6ab43d00406e97fd521513e74eb9f8925 Mon Sep 17 00:00:00 2001 From: Liquan Pei Date: Tue, 26 Apr 2016 00:20:59 -0700 Subject: [PATCH 3/6] Add - before to the regex --- bin/kafka-run-class.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 43a9f9228d9af..30e91c6c7bfd6 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -21,7 +21,7 @@ then fi # Exclude jars not necessary for running commands. -regex="((test|src|scaladoc|javadoc)\.jar|jar.asc)$" +regex="(-(test|src|scaladoc|javadoc)\.jar|jar.asc)$" filter_file() { file=$1 if [ -z "$(echo "$file" | egrep "$regex")" ] ; then From 27e389230414a4c9ac6a3087808c2a449f93f47b Mon Sep 17 00:00:00 2001 From: Liquan Pei Date: Thu, 28 Apr 2016 12:51:32 -0700 Subject: [PATCH 4/6] Add INCLUDE_TEST_JARS variable --- bin/kafka-run-class.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 30e91c6c7bfd6..c3f2cd19ee034 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -20,9 +20,16 @@ then exit 1 fi +if [ -z "$INCLUDE_TEST_JARS" ]; then + INCLUDE_TEST_JARS=false +fi + # Exclude jars not necessary for running commands. regex="(-(test|src|scaladoc|javadoc)\.jar|jar.asc)$" -filter_file() { +should_include_file() { + if [ "$INCLUDE_TEST_JARS" = true ]; then + return 0 + fi file=$1 if [ -z "$(echo "$file" | egrep "$regex")" ] ; then return 0 @@ -54,28 +61,28 @@ done for file in $base_dir/examples/build/libs/kafka-examples*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/clients/build/libs/kafka-clients*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/streams/build/libs/kafka-streams*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/streams/examples/build/libs/kafka-streams-examples*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -87,7 +94,7 @@ done for file in $base_dir/tools/build/libs/kafka-tools*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -101,7 +108,7 @@ for cc_pkg in "api" "runtime" "file" "json" "tools" do for file in $base_dir/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done @@ -113,19 +120,21 @@ done # classpath addition for release for file in $base_dir/libs; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done for file in $base_dir/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar; do - if filter_file "$file"; then + if should_include_file "$file"; then CLASSPATH=$CLASSPATH:$file fi done shopt -u nullglob +echo $CLASSPATH + # JMX settings if [ -z "$KAFKA_JMX_OPTS" ]; then KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " From f597e493e4e8a2633202039f0ab25319edc8e70e Mon Sep 17 00:00:00 2001 From: Liquan Pei Date: Thu, 28 Apr 2016 21:38:10 -0700 Subject: [PATCH 5/6] Add INCLUDE_TEST_JARS to relavant test services --- tests/kafkatest/services/security/minikdc.py | 5 ++--- tests/kafkatest/services/streams.py | 10 ++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/kafkatest/services/security/minikdc.py b/tests/kafkatest/services/security/minikdc.py index b376e268f7f8b..d83aede11a89b 100644 --- a/tests/kafkatest/services/security/minikdc.py +++ b/tests/kafkatest/services/security/minikdc.py @@ -23,6 +23,7 @@ from io import open import uuid + class MiniKdc(Service): logs = { @@ -54,9 +55,7 @@ def replace_in_file(self, file_path, pattern, subst): remove(file_path) move(abs_path, file_path) - def start_node(self, node): - node.account.ssh("mkdir -p %s" % MiniKdc.WORK_DIR, allow_fail=False) props_file = self.render('minikdc.properties', node=node) node.account.create_file(MiniKdc.PROPS_FILE, props_file) @@ -69,7 +68,7 @@ def start_node(self, node): jar_paths = self.core_jar_paths(node, "dependant-testlibs") + self.core_jar_paths(node, "libs") classpath = ":".join(jar_paths) - cmd = "CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh kafka.security.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE) + cmd = "INCLUDE_TEST_JARS=true CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh kafka.security.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE) self.logger.debug("Attempting to start MiniKdc on %s with command: %s" % (str(node.account), cmd)) with node.account.monitor_log(MiniKdc.LOG_FILE) as monitor: node.account.ssh(cmd) diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py index dcbcc696b8b3f..53d967e1bc8bd 100644 --- a/tests/kafkatest/services/streams.py +++ b/tests/kafkatest/services/streams.py @@ -15,10 +15,11 @@ from ducktape.services.service import Service from ducktape.utils.util import wait_until -from ducktape.errors import DucktapeError from kafkatest.services.kafka.directory import kafka_dir -import signal, random, requests, os.path, json +import signal +import os.path + class StreamsSmokeTestBaseService(Service): """Base class for Streams Smoke Test services providing some common settings and functionality""" @@ -46,7 +47,7 @@ class StreamsSmokeTestBaseService(Service): def __init__(self, context, kafka, command): super(StreamsSmokeTestBaseService, self).__init__(context, 1) self.kafka = kafka - self.args = { 'command': command } + self.args = {'command': command} @property def node(self): @@ -107,7 +108,7 @@ def start_cmd(self, node): args['kafka_dir'] = kafka_dir(node) cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ - "/opt/%(kafka_dir)s/bin/kafka-run-class.sh org.apache.kafka.streams.smoketest.StreamsSmokeTest " \ + "INCLUDE_TEST_JARS=true /opt/%(kafka_dir)s/bin/kafka-run-class.sh org.apache.kafka.streams.smoketest.StreamsSmokeTest " \ " %(command)s %(kafka)s %(zk)s %(state_dir)s " \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args @@ -131,6 +132,7 @@ class StreamsSmokeTestDriverService(StreamsSmokeTestBaseService): def __init__(self, context, kafka): super(StreamsSmokeTestDriverService, self).__init__(context, kafka, "run") + class StreamsSmokeTestJobRunnerService(StreamsSmokeTestBaseService): def __init__(self, context, kafka): super(StreamsSmokeTestJobRunnerService, self).__init__(context, kafka, "process") From a380c156d0613110fbc2bb9aeae3cab530b194c0 Mon Sep 17 00:00:00 2001 From: Liquan Pei Date: Fri, 29 Apr 2016 10:30:37 -0700 Subject: [PATCH 6/6] Remove leftover debug message --- bin/kafka-run-class.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index c3f2cd19ee034..e7f8d2ef9fec3 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -133,8 +133,6 @@ do done shopt -u nullglob -echo $CLASSPATH - # JMX settings if [ -z "$KAFKA_JMX_OPTS" ]; then KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "