Skip to content

KAFKA-3615: Exclude test jars in kafka-run-class.sh #1263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Ishiihara wants to merge 6 commits into from
Closed

KAFKA-3615: Exclude test jars in kafka-run-class.sh #1263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6cc8ccd
Exclude test jars in kafka-run-class.sh
Ishiihara Apr 24, 2016
45cf01c
Filter out unnecessary files with binary packages
Ishiihara Apr 25, 2016
2791f3b
Add - before to the regex
Ishiihara Apr 26, 2016
27e3892
Add INCLUDE_TEST_JARS variable
Ishiihara Apr 28, 2016
f597e49
Add INCLUDE_TEST_JARS to relavant test services
Ishiihara Apr 29, 2016
a380c15
Remove leftover debug message
Ishiihara Apr 29, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 46 additions & 11 deletions bin/kafka-run-class.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,24 @@ then
exit 1
fi

if [ -z "$INCLUDE_TEST_JARS" ]; then
INCLUDE_TEST_JARS=false
fi

# Exclude jars not necessary for running commands.
regex="(-(test|src|scaladoc|javadoc)\.jar|jar.asc)$"
should_include_file() {
if [ "$INCLUDE_TEST_JARS" = true ]; then
return 0
fi
file=$1
if [ -z "$(echo "$file" | egrep "$regex")" ] ; then
return 0
else
return 1
fi
}

base_dir=$(dirname $0)/..

if [ -z "$SCALA_VERSION" ]; then
Expand All @@ -41,24 +59,32 @@ do
fi
done

for file in $base_dir/examples/build/libs//kafka-examples*.jar;
for file in $base_dir/examples/build/libs/kafka-examples*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done

for file in $base_dir/clients/build/libs/kafka-clients*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done

for file in $base_dir/streams/build/libs/kafka-streams*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done

for file in $base_dir/streams/examples/build/libs/kafka-streams-examples*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done

for file in $base_dir/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar;
Expand All @@ -68,7 +94,9 @@ done

for file in $base_dir/tools/build/libs/kafka-tools*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done

for dir in $base_dir/tools/build/dependant-libs-${SCALA_VERSION}*;
Expand All @@ -80,19 +108,28 @@ for cc_pkg in "api" "runtime" "file" "json" "tools"
do
for file in $base_dir/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done
if [ -d "$base_dir/connect/${cc_pkg}/build/dependant-libs" ] ; then
CLASSPATH=$CLASSPATH:$base_dir/connect/${cc_pkg}/build/dependant-libs/*
fi
done

# classpath addition for release
CLASSPATH=$CLASSPATH:$base_dir/libs/*
for file in $base_dir/libs;
Copy link
Copy Markdown
Contributor

@dpkp dpkp May 1, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is broken. It should be for file in $base_dir/libs/*; ... without the glob, you're looping over 1 entry: "libs".

I believe this bug breaks the classpath for all scripts in bin/ in the release artifact

do
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done

for file in $base_dir/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar;
do
CLASSPATH=$CLASSPATH:$file
if should_include_file "$file"; then
CLASSPATH=$CLASSPATH:$file
fi
done
shopt -u nullglob

Expand Down Expand Up @@ -145,8 +182,6 @@ if [ "x$KAFKA_DEBUG" != "x" ]; then
JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS"
fi



echo "Enabling Java debug options: $JAVA_DEBUG_OPTS"
KAFKA_OPTS="$JAVA_DEBUG_OPTS $KAFKA_OPTS"
fi
Expand Down
5 changes: 2 additions & 3 deletions tests/kafkatest/services/security/minikdc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from io import open
import uuid


class MiniKdc(Service):

logs = {
Expand Down Expand Up @@ -54,9 +55,7 @@ def replace_in_file(self, file_path, pattern, subst):
remove(file_path)
move(abs_path, file_path)


def start_node(self, node):

node.account.ssh("mkdir -p %s" % MiniKdc.WORK_DIR, allow_fail=False)
props_file = self.render('minikdc.properties', node=node)
node.account.create_file(MiniKdc.PROPS_FILE, props_file)
Expand All @@ -69,7 +68,7 @@ def start_node(self, node):

jar_paths = self.core_jar_paths(node, "dependant-testlibs") + self.core_jar_paths(node, "libs")
classpath = ":".join(jar_paths)
cmd = "CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh kafka.security.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE)
cmd = "INCLUDE_TEST_JARS=true CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh kafka.security.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE)
self.logger.debug("Attempting to start MiniKdc on %s with command: %s" % (str(node.account), cmd))
with node.account.monitor_log(MiniKdc.LOG_FILE) as monitor:
node.account.ssh(cmd)
Expand Down
10 changes: 6 additions & 4 deletions tests/kafkatest/services/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@

from ducktape.services.service import Service
from ducktape.utils.util import wait_until
from ducktape.errors import DucktapeError

from kafkatest.services.kafka.directory import kafka_dir
import signal, random, requests, os.path, json
import signal
import os.path


class StreamsSmokeTestBaseService(Service):
"""Base class for Streams Smoke Test services providing some common settings and functionality"""
Expand Down Expand Up @@ -46,7 +47,7 @@ class StreamsSmokeTestBaseService(Service):
def __init__(self, context, kafka, command):
super(StreamsSmokeTestBaseService, self).__init__(context, 1)
self.kafka = kafka
self.args = { 'command': command }
self.args = {'command': command}

@property
def node(self):
Expand Down Expand Up @@ -107,7 +108,7 @@ def start_cmd(self, node):
args['kafka_dir'] = kafka_dir(node)

cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \
"/opt/%(kafka_dir)s/bin/kafka-run-class.sh org.apache.kafka.streams.smoketest.StreamsSmokeTest " \
"INCLUDE_TEST_JARS=true /opt/%(kafka_dir)s/bin/kafka-run-class.sh org.apache.kafka.streams.smoketest.StreamsSmokeTest " \
" %(command)s %(kafka)s %(zk)s %(state_dir)s " \
" & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args

Expand All @@ -131,6 +132,7 @@ class StreamsSmokeTestDriverService(StreamsSmokeTestBaseService):
def __init__(self, context, kafka):
super(StreamsSmokeTestDriverService, self).__init__(context, kafka, "run")


class StreamsSmokeTestJobRunnerService(StreamsSmokeTestBaseService):
def __init__(self, context, kafka):
super(StreamsSmokeTestJobRunnerService, self).__init__(context, kafka, "process")