From 7632c27cb5583d285aa66a9d2bf0510139666b41 Mon Sep 17 00:00:00 2001 From: Ewen Cheslack-Postava Date: Thu, 12 Nov 2015 10:25:33 -0800 Subject: [PATCH] KAFKA-2807: Fix Kafka Connect packaging and move VerifiableSource/Sink into runtime jar. Gradle does not handle subprojects with the same name (top-level tools vs connect/tools) properly, making the dependency impossible to express correctly since we need to move the ThroughputThrottler class into the top level tools project. Moving the current set of tools into the runtime jar works fine since they are only used for system tests at the moment. --- build.gradle | 101 ++++-------------- checkstyle/import-control.xml | 1 + .../tools/VerifiableSinkConnector.java | 0 .../connect/tools/VerifiableSinkTask.java | 0 .../tools/VerifiableSourceConnector.java | 0 .../connect/tools/VerifiableSourceTask.java | 2 +- settings.gradle | 2 +- .../kafka/tools/ThroughputThrottler.java | 2 +- 8 files changed, 23 insertions(+), 85 deletions(-) rename connect/{tools => runtime}/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java (100%) rename connect/{tools => runtime}/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkTask.java (100%) rename connect/{tools => runtime}/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceConnector.java (100%) rename connect/{tools => runtime}/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java (100%) diff --git a/build.gradle b/build.gradle index 0ee6c41a29209..29e351f24eb84 100644 --- a/build.gradle +++ b/build.gradle @@ -230,7 +230,7 @@ for ( sv in ['2_10_5', '2_11_7'] ) { } } -def connectPkgs = ['connect-api', 'connect-runtime', 'connect-json', 'connect-file', 'connect-tools'] +def connectPkgs = ['connect:api', 'connect:runtime', 'connect:json', 'connect:file'] def pkgs = ['clients', 'examples', 'log4j-appender', 'tools', 'streams'] + connectPkgs tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {} @@ -321,7 +321,7 @@ project(':core') { standardOutput = new File('docs/kafka_config.html').newOutputStream() } - task siteDocsTar(dependsOn: ['genProducerConfigDocs', 'genConsumerConfigDocs', 'genKafkaConfigDocs', ':connect-runtime:genConnectConfigDocs'], type: Tar) { + task siteDocsTar(dependsOn: ['genProducerConfigDocs', 'genConsumerConfigDocs', 'genKafkaConfigDocs', ':connect:runtime:genConnectConfigDocs'], type: Tar) { classifier = 'site-docs' compression = Compression.GZIP from project.file("../docs") @@ -342,16 +342,14 @@ project(':core') { from(project.siteDocsTar) { into("site-docs/") } from(project(':tools').jar) { into("libs/") } from(project(':tools').configurations.runtime) { into("libs/") } - from(project(':connect-api').jar) { into("libs/") } - from(project(':connect-api').configurations.runtime) { into("libs/") } - from(project(':connect-runtime').jar) { into("libs/") } - from(project(':connect-runtime').configurations.runtime) { into("libs/") } - from(project(':connect-json').jar) { into("libs/") } - from(project(':connect-json').configurations.runtime) { into("libs/") } - from(project(':connect-file').jar) { into("libs/") } - from(project(':connect-file').configurations.runtime) { into("libs/") } - from(project(':connect-tools').jar) { into("libs/") } - from(project(':connect-tools').configurations.runtime) { into("libs/") } + from(project(':connect:api').jar) { into("libs/") } + from(project(':connect:api').configurations.runtime) { into("libs/") } + from(project(':connect:runtime').jar) { into("libs/") } + from(project(':connect:runtime').configurations.runtime) { into("libs/") } + from(project(':connect:json').jar) { into("libs/") } + from(project(':connect:json').configurations.runtime) { into("libs/") } + from(project(':connect:file').jar) { into("libs/") } + from(project(':connect:file').configurations.runtime) { into("libs/") } } jar { @@ -638,7 +636,7 @@ project(':log4j-appender') { test.dependsOn('checkstyleMain', 'checkstyleTest') } -project(':connect-api') { +project(':connect:api') { apply plugin: 'checkstyle' archivesBaseName = "connect-api" @@ -695,12 +693,12 @@ project(':connect-api') { test.dependsOn('checkstyleMain', 'checkstyleTest') } -project(':connect-json') { +project(':connect:json') { apply plugin: 'checkstyle' archivesBaseName = "connect-json" dependencies { - compile project(':connect-api') + compile project(':connect:api') compile "$slf4japi" compile "com.fasterxml.jackson.core:jackson-databind:$jackson_version" @@ -756,13 +754,14 @@ project(':connect-json') { test.dependsOn('checkstyleMain', 'checkstyleTest') } -project(':connect-runtime') { +project(':connect:runtime') { apply plugin: 'checkstyle' archivesBaseName = "connect-runtime" dependencies { - compile project(':connect-api') + compile project(':connect:api') compile project(':clients') + compile project(':tools') compile "$slf4japi" compile "org.eclipse.jetty:jetty-server:$jetty_version" @@ -776,7 +775,7 @@ project(':connect-runtime') { testCompile "$powermock_easymock" testCompile project(':clients').sourceSets.test.output testRuntime "$slf4jlog4j" - testRuntime project(":connect-json") + testRuntime project(":connect:json") } task testJar(type: Jar) { @@ -830,75 +829,13 @@ project(':connect-runtime') { } } -project(':connect-file') { +project(':connect:file') { apply plugin: 'checkstyle' archivesBaseName = "connect-file" dependencies { - compile project(':connect-api') - compile "$slf4japi" - - testCompile "$junit" - testCompile "$easymock" - testCompile "$powermock" - testCompile "$powermock_easymock" - testRuntime "$slf4jlog4j" - } - - task testJar(type: Jar) { - classifier = 'test' - from sourceSets.test.output - } - - test { - testLogging { - events "passed", "skipped", "failed" - exceptionFormat = 'full' - } - } - - javadoc { - include "**/org/apache/kafka/connect/*" - } - - tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { - include('slf4j-log4j12*') - } - from (configurations.runtime) { - exclude('kafka-clients*') - exclude('connect-*') - } - into "$buildDir/dependant-libs" - } - - jar { - dependsOn copyDependantLibs - } - - artifacts { - archives testJar - } - - configurations { - archives.extendsFrom(testCompile) - } - - checkstyle { - configFile = new File(rootDir, "checkstyle/checkstyle.xml") - } - test.dependsOn('checkstyleMain', 'checkstyleTest') -} - -project(':connect-tools') { - apply plugin: 'checkstyle' - archivesBaseName = "connect-tools" - - dependencies { - compile project(':connect-api') - compile project(':tools') + compile project(':connect:api') compile "$slf4japi" - compile "com.fasterxml.jackson.core:jackson-databind:$jackson_version" testCompile "$junit" testCompile "$easymock" diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 16a370092f3e0..204bc60bfa5db 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -207,6 +207,7 @@ + diff --git a/connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java similarity index 100% rename from connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java rename to connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java diff --git a/connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkTask.java similarity index 100% rename from connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkTask.java rename to connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkTask.java diff --git a/connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceConnector.java similarity index 100% rename from connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceConnector.java rename to connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceConnector.java diff --git a/connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java similarity index 100% rename from connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java rename to connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java index a85a0e982cb8a..6dcfdc4bda8fd 100644 --- a/connect/tools/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java @@ -19,11 +19,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.tools.ThroughputThrottler; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; -import org.apache.kafka.tools.ThroughputThrottler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/settings.gradle b/settings.gradle index d1543c34737f4..3d69facd64ca8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,4 +15,4 @@ apply from: file('scala.gradle') include 'core', 'examples', 'clients', 'tools', 'streams', 'log4j-appender', - 'connect-api', 'connect-runtime', 'connect-json', 'connect-file', 'connect-tools' + 'connect:api', 'connect:runtime', 'connect:json', 'connect:file' diff --git a/tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java b/tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java index a3bcd2fccf613..a58a535ea60f9 100644 --- a/tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java +++ b/tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - **/ + */ package org.apache.kafka.tools;