From 95f5a6503f77505f839056bf880d970d8bc2c312 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Fri, 24 Feb 2023 16:15:52 -0800 Subject: [PATCH 1/6] KAFKA-14760: Move ThroughputThrottler from tools to clients Signed-off-by: Greg Harris --- .../org/apache/kafka/common/utils}/ThroughputThrottler.java | 2 +- .../java/org/apache/kafka/connect/tools/SchemaSourceTask.java | 2 +- .../org/apache/kafka/connect/tools/VerifiableSourceTask.java | 2 +- .../kafka/connect/integration/MonitorableSourceConnector.java | 2 +- .../main/java/org/apache/kafka/tools/ProducerPerformance.java | 1 + .../main/java/org/apache/kafka/tools/VerifiableProducer.java | 1 + 6 files changed, 6 insertions(+), 4 deletions(-) rename {tools/src/main/java/org/apache/kafka/tools => clients/src/main/java/org/apache/kafka/common/utils}/ThroughputThrottler.java (99%) diff --git a/tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java b/clients/src/main/java/org/apache/kafka/common/utils/ThroughputThrottler.java similarity index 99% rename from tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java rename to clients/src/main/java/org/apache/kafka/common/utils/ThroughputThrottler.java index cfcd04daf79c3..f0ac7433ce3b3 100644 --- a/tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/ThroughputThrottler.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.tools; +package org.apache.kafka.common.utils; /** diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java index bd0e318ff77ca..b5fb3d5fb5754 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java @@ -22,7 +22,7 @@ import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; -import org.apache.kafka.tools.ThroughputThrottler; +import org.apache.kafka.common.utils.ThroughputThrottler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java index f4ae0dee8a3a0..da2ce3b4c4e89 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.tools.ThroughputThrottler; +import org.apache.kafka.common.utils.ThroughputThrottler; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.errors.ConnectException; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java index 33ba1588a7d04..887d775bd3dd2 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java @@ -27,7 +27,7 @@ import org.apache.kafka.connect.source.ExactlyOnceSupport; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; -import org.apache.kafka.tools.ThroughputThrottler; +import org.apache.kafka.common.utils.ThroughputThrottler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tools/src/main/java/org/apache/kafka/tools/ProducerPerformance.java b/tools/src/main/java/org/apache/kafka/tools/ProducerPerformance.java index 6774f23511468..fb00f9a7f2f65 100644 --- a/tools/src/main/java/org/apache/kafka/tools/ProducerPerformance.java +++ b/tools/src/main/java/org/apache/kafka/tools/ProducerPerformance.java @@ -42,6 +42,7 @@ import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; import org.apache.kafka.common.utils.Exit; +import org.apache.kafka.common.utils.ThroughputThrottler; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.server.util.ToolsUtils; diff --git a/tools/src/main/java/org/apache/kafka/tools/VerifiableProducer.java b/tools/src/main/java/org/apache/kafka/tools/VerifiableProducer.java index ee863d4b175d6..ed6fac856d1ff 100644 --- a/tools/src/main/java/org/apache/kafka/tools/VerifiableProducer.java +++ b/tools/src/main/java/org/apache/kafka/tools/VerifiableProducer.java @@ -34,6 +34,7 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.ThroughputThrottler; import java.io.IOException; import java.io.InputStream; From 6e6706f622277742c517503454473c210a789250 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Mon, 27 Feb 2023 11:51:58 -0800 Subject: [PATCH 2/6] MINOR: change v0.8 tools jar injection from dev to v0.9 This will allow the tools implementation to depend on modern features and refactors available after 0.8. Signed-off-by: Greg Harris --- tests/kafkatest/directory_layout/kafka_path.py | 7 ++++++- tests/kafkatest/services/verifiable_client.py | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/kafkatest/directory_layout/kafka_path.py b/tests/kafkatest/directory_layout/kafka_path.py index 40dda22868585..4a36213b45057 100644 --- a/tests/kafkatest/directory_layout/kafka_path.py +++ b/tests/kafkatest/directory_layout/kafka_path.py @@ -16,7 +16,7 @@ import importlib import os -from kafkatest.version import get_version, KafkaVersion, DEV_BRANCH +from kafkatest.version import get_version, KafkaVersion, DEV_BRANCH, LATEST_0_9 """This module serves a few purposes: @@ -49,6 +49,11 @@ CORE_DEPENDANT_TEST_LIBS_JAR_NAME: "core/build/dependant-testlibs/*.jar", TOOLS_JAR_NAME: "tools/build/libs/kafka-tools*.jar", TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME: "tools/build/dependant-libs*/*.jar" + }, + # TODO remove with KAFKA-14762 + LATEST_0_9.__str__(): { + TOOLS_JAR_NAME: "libs/kafka-tools*.jar", + TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME: "libs/{argparse4j,jackson}*.jar" } } diff --git a/tests/kafkatest/services/verifiable_client.py b/tests/kafkatest/services/verifiable_client.py index a2cbf96ef3646..5bd409e7984ae 100644 --- a/tests/kafkatest/services/verifiable_client.py +++ b/tests/kafkatest/services/verifiable_client.py @@ -14,7 +14,7 @@ # limitations under the License. from kafkatest.directory_layout.kafka_path import TOOLS_JAR_NAME, TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME -from kafkatest.version import DEV_BRANCH, LATEST_0_8_2 +from kafkatest.version import DEV_BRANCH, LATEST_0_8_2, LATEST_0_9 from ducktape.cluster.remoteaccount import RemoteCommandError import importlib @@ -252,9 +252,10 @@ def exec_cmd (self, node): cmd = "" if self.java_class_name == 'VerifiableProducer' and node.version <= LATEST_0_8_2: # 0.8.2.X releases do not have VerifiableProducer.java, so cheat and add - # the tools jar from trunk to the classpath - tools_jar = self.parent.path.jar(TOOLS_JAR_NAME, DEV_BRANCH) - tools_dependant_libs_jar = self.parent.path.jar(TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME, DEV_BRANCH) + # the tools jar from 0.9.x to the classpath + # TODO remove with KAFKA-14762 + tools_jar = self.parent.path.jar(TOOLS_JAR_NAME, LATEST_0_9) + tools_dependant_libs_jar = self.parent.path.jar(TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME, LATEST_0_9) cmd += "for file in %s; do CLASSPATH=$CLASSPATH:$file; done; " % tools_jar cmd += "for file in %s; do CLASSPATH=$CLASSPATH:$file; done; " % tools_dependant_libs_jar cmd += "export CLASSPATH; " From 0f9eb9e53a11d3d38027657f897dadc133d35b98 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Mon, 27 Feb 2023 11:59:48 -0800 Subject: [PATCH 3/6] MINOR: remove tools dependency from connect-runtime Signed-off-by: Greg Harris --- build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.gradle b/build.gradle index 67fde2455df90..ed753627bf9d4 100644 --- a/build.gradle +++ b/build.gradle @@ -2651,8 +2651,6 @@ project(':connect:runtime') { api project(':connect:json') api project(':connect:transforms') - implementation project(':tools') - implementation libs.slf4jApi implementation libs.log4j implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation From 3e608b9c48a955646ac632015634352c06a2778d Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Tue, 28 Feb 2023 09:06:04 -0800 Subject: [PATCH 4/6] fixup: disallow importing tools from connect packages Signed-off-by: Greg Harris --- checkstyle/import-control.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 71989a9daa4f6..d1bf812d95563 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -667,7 +667,6 @@ - @@ -690,7 +689,6 @@ - From b5107e8c1cb5b73fbfc8c5bef506e14bb75df6be Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Tue, 6 Jun 2023 10:46:35 -0700 Subject: [PATCH 5/6] fixup: clarify todo comment Signed-off-by: Greg Harris --- tests/kafkatest/directory_layout/kafka_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kafkatest/directory_layout/kafka_path.py b/tests/kafkatest/directory_layout/kafka_path.py index 4a36213b45057..af200905d85f1 100644 --- a/tests/kafkatest/directory_layout/kafka_path.py +++ b/tests/kafkatest/directory_layout/kafka_path.py @@ -50,7 +50,7 @@ TOOLS_JAR_NAME: "tools/build/libs/kafka-tools*.jar", TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME: "tools/build/dependant-libs*/*.jar" }, - # TODO remove with KAFKA-14762 + # TODO: This is only used in 0.8.2.x system tests, remove with KAFKA-14762 LATEST_0_9.__str__(): { TOOLS_JAR_NAME: "libs/kafka-tools*.jar", TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME: "libs/{argparse4j,jackson}*.jar" From d290ce7cc82fc3ea2db8361768b88d3b3df0c917 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Tue, 6 Jun 2023 15:45:26 -0700 Subject: [PATCH 6/6] fixup: change expected version in test_verifiable_producer Signed-off-by: Greg Harris --- tests/kafkatest/sanity_checks/test_verifiable_producer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kafkatest/sanity_checks/test_verifiable_producer.py b/tests/kafkatest/sanity_checks/test_verifiable_producer.py index 1aa2109952194..44982ff19a315 100644 --- a/tests/kafkatest/sanity_checks/test_verifiable_producer.py +++ b/tests/kafkatest/sanity_checks/test_verifiable_producer.py @@ -92,7 +92,7 @@ def test_simple_run(self, producer_version, acks=None, enable_idempotence=False, # Easy fix is to decrease throughput= above, the good fix is to make the producer # not terminate until explicitly killed in this case. if node.version <= LATEST_0_8_2: - assert is_version(node, [node.version.vstring, DEV_BRANCH.vstring], logger=self.logger) + assert is_version(node, [node.version.vstring, LATEST_0_9.vstring], logger=self.logger) else: assert is_version(node, [node.version.vstring], logger=self.logger)