Merging Apache trunk to Microsoft trunk

.gitignore

@@ -38,7 +38,4 @@ tests/results
 .ducktape
 tests/.ducktape
 
-docs/producer_config.html
-docs/consumer_config.html
-docs/kafka_config.html
-docs/connect_config.html
+docs/generated/

README.md

@@ -47,6 +47,9 @@ Change the log4j setting in either `clients/src/test/resources/log4j.properties`
 
     ./gradlew -i -Dtest.single=RequestResponseSerializationTest core:test
 
+### Generating test coverage reports ###
+    ./gradlew reportCoverage
+
 ### Building a binary release gzipped tar ball ###
     ./gradlew clean
     ./gradlew releaseTarGz
@@ -60,10 +63,10 @@ The release file can be found inside `./core/build/distributions/`.
 ### Cleaning the build ###
     ./gradlew clean
 
-### Running a task on a particular version of Scala (either 2.10.6 or 2.11.7) ###
+### Running a task on a particular version of Scala (either 2.10.6 or 2.11.8) ###
 *Note that if building the jars with a version other than 2.10, you need to set the `SCALA_BINARY_VERSION` variable or change it in `bin/kafka-run-class.sh` to run the quick start.*
 
-You can pass either the major version (eg 2.11) or the full version (eg 2.11.7):
+You can pass either the major version (eg 2.11) or the full version (eg 2.11.8):
 
     ./gradlew -PscalaVersion=2.11 jar
     ./gradlew -PscalaVersion=2.11 test

Vagrantfile

@@ -22,6 +22,8 @@ VAGRANTFILE_API_VERSION = "2"
 
 # General config
 enable_dns = false
+# Override to false when bringing up a cluster on AWS
+enable_hostmanager = true
 enable_jmx = false
 num_zookeepers = 1
 num_brokers = 3
@@ -55,7 +57,7 @@ end
 
 # TODO(ksweeney): RAM requirements are not empirical and can probably be significantly lowered.
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.hostmanager.enabled = true
+  config.hostmanager.enabled = enable_hostmanager
   config.hostmanager.manage_host = enable_dns
   config.hostmanager.include_offline = false
 

bin/kafka-run-class.sh

@@ -20,6 +20,24 @@ then
   exit 1
 fi
 
+if [ -z "$INCLUDE_TEST_JARS" ]; then
+  INCLUDE_TEST_JARS=false
+fi
+
+# Exclude jars not necessary for running commands.
+regex="(-(test|src|scaladoc|javadoc)\.jar|jar.asc)$"
+should_include_file() {
+  if [ "$INCLUDE_TEST_JARS" = true ]; then
+    return 0
+  fi
+  file=$1
+  if [ -z "$(echo "$file" | egrep "$regex")" ] ; then
+    return 0
+  else
+    return 1
+  fi
+}
+
 base_dir=$(dirname $0)/..
 
 if [ -z "$SCALA_VERSION" ]; then
@@ -34,27 +52,39 @@ fi
 shopt -s nullglob
 for dir in $base_dir/core/build/dependant-libs-${SCALA_VERSION}*;
 do
-  CLASSPATH=$CLASSPATH:$dir/*
+  if [ -z $CLASSPATH ] ; then
+    CLASSPATH=$dir/*
+  else
+    CLASSPATH=$CLASSPATH:$dir/*
+  fi
 done
 
-for file in $base_dir/examples/build/libs//kafka-examples*.jar;
+for file in $base_dir/examples/build/libs/kafka-examples*.jar;
 do
-  CLASSPATH=$CLASSPATH:$file
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
 done
 
 for file in $base_dir/clients/build/libs/kafka-clients*.jar;
 do
-  CLASSPATH=$CLASSPATH:$file
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
 done
 
 for file in $base_dir/streams/build/libs/kafka-streams*.jar;
 do
-  CLASSPATH=$CLASSPATH:$file
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
 done
 
 for file in $base_dir/streams/examples/build/libs/kafka-streams-examples*.jar;
 do
-  CLASSPATH=$CLASSPATH:$file
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
 done
 
 for file in $base_dir/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar;
@@ -64,7 +94,9 @@ done
 
 for file in $base_dir/tools/build/libs/kafka-tools*.jar;
 do
-  CLASSPATH=$CLASSPATH:$file
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
 done
 
 for dir in $base_dir/tools/build/dependant-libs-${SCALA_VERSION}*;
@@ -76,19 +108,28 @@ for cc_pkg in "api" "runtime" "file" "json" "tools"
 do
   for file in $base_dir/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar;
   do
-    CLASSPATH=$CLASSPATH:$file
+    if should_include_file "$file"; then
+      CLASSPATH=$CLASSPATH:$file
+    fi
   done
   if [ -d "$base_dir/connect/${cc_pkg}/build/dependant-libs" ] ; then
     CLASSPATH=$CLASSPATH:$base_dir/connect/${cc_pkg}/build/dependant-libs/*
   fi
 done
 
 # classpath addition for release
-CLASSPATH=$CLASSPATH:$base_dir/libs/*
+for file in $base_dir/libs/*;
+do
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
+done
 
 for file in $base_dir/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar;
 do
-  CLASSPATH=$CLASSPATH:$file
+  if should_include_file "$file"; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
 done
 shopt -u nullglob
 
@@ -141,8 +182,6 @@ if [ "x$KAFKA_DEBUG" != "x" ]; then
         JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS"
     fi
 
-
-
     echo "Enabling Java debug options: $JAVA_DEBUG_OPTS"
     KAFKA_OPTS="$JAVA_DEBUG_OPTS $KAFKA_OPTS"
 fi

bin/kafka-server-stop.sh

@@ -19,6 +19,6 @@ if [ -z "$PIDS" ]; then
   echo "No kafka server to stop"
   exit 1
 else 
-  kill -SIGTERM $PIDS
+  kill -s TERM $PIDS
 fi
 

bin/zookeeper-server-stop.sh

@@ -19,6 +19,6 @@ if [ -z "$PIDS" ]; then
   echo "No zookeeper server to stop"
   exit 1
 else
-  kill -SIGTERM $PIDS
+  kill -s TERM $PIDS
 fi