From 28b912b41ed9c4c7c33ff8f4fc3d0a6743be9059 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 27 Jul 2021 21:44:07 +0800 Subject: [PATCH 1/3] Speed up tests and fix flakiness --- .../pulsar/handlers/kop/KopProtocolHandlerTestBase.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KopProtocolHandlerTestBase.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KopProtocolHandlerTestBase.java index 482eca6b38..7edc615f09 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KopProtocolHandlerTestBase.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KopProtocolHandlerTestBase.java @@ -13,6 +13,7 @@ */ package io.streamnative.pulsar.handlers.kop; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; @@ -177,6 +178,9 @@ protected void resetConfig() { + SSL_PREFIX + "localhost:" + kafkaBrokerPortTls); kafkaConfig.setEntryFormat(entryFormat); + // Speed up tests for reducing rebalance time + kafkaConfig.setGroupInitialRebalanceDelayMs(0); + // set protocol related config URL testHandlerUrl = this.getClass().getClassLoader().getResource("test-protocol-handler.nar"); Path handlerPath; @@ -313,6 +317,8 @@ protected void restartBroker() throws Exception { } protected void stopBroker() throws Exception { + // set shutdown timeout to 0 for forceful shutdown + pulsar.getConfiguration().setBrokerShutdownTimeoutMs(0L); pulsar.close(); } @@ -344,6 +350,7 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); doReturn(sameThreadOrderedSafeExecutor).when(pulsar).getOrderedExecutor(); + doAnswer((invocation) -> spy(invocation.callRealMethod())).when(pulsar).newCompactor(); } public static MockZooKeeper createMockZooKeeper(String clusterName, String brokerUrl, String brokerUrlTls, From 2c44e59b1a5f32b821fa3968db0136ac70572ad8 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 27 Jul 2021 21:55:19 +0800 Subject: [PATCH 2/3] Speed up tests that creat their own configs --- .../pulsar/handlers/kop/DistributedClusterTest.java | 2 ++ .../pulsar/handlers/kop/InnerTopicProtectionTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/DistributedClusterTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/DistributedClusterTest.java index 487d6dc7ef..433816391e 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/DistributedClusterTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/DistributedClusterTest.java @@ -87,6 +87,8 @@ protected KafkaServiceConfiguration resetConfig(int brokerPort, int webPort, int kConfig.setAllowAutoTopicCreation(true); kConfig.setAllowAutoTopicCreationType("partitioned"); kConfig.setBrokerDeleteInactiveTopicsEnabled(false); + kConfig.setGroupInitialRebalanceDelayMs(0); + kConfig.setBrokerShutdownTimeoutMs(0); // set protocol related config URL testHandlerUrl = this.getClass().getClassLoader().getResource("test-protocol-handler.nar"); diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/InnerTopicProtectionTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/InnerTopicProtectionTest.java index 92e13ead54..ccb880a115 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/InnerTopicProtectionTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/InnerTopicProtectionTest.java @@ -62,6 +62,8 @@ protected KafkaServiceConfiguration resetConfig(int brokerPort, int webPort, int kConfig.setBrokerDeleteInactiveTopicsEnabled(false); kConfig.setSystemTopicEnabled(true); kConfig.setTopicLevelPoliciesEnabled(true); + kConfig.setGroupInitialRebalanceDelayMs(0); + kConfig.setBrokerShutdownTimeoutMs(0); // set protocol related config URL testHandlerUrl = this.getClass().getClassLoader().getResource("test-protocol-handler.nar"); From c25e1d8156c2e47e7f0bb19f99f1d4399f576924 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 28 Jul 2021 10:29:30 +0800 Subject: [PATCH 3/3] Ignore golang-sarama tests --- .../pulsar/handlers/kop/KafkaIntegrationTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaIntegrationTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaIntegrationTest.java index 1b0097331c..aabd5120ee 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaIntegrationTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaIntegrationTest.java @@ -80,8 +80,11 @@ public KafkaIntegrationTest(final String entryFormat) { @DataProvider public static Object[][] integrations() { return new Object[][]{ - {"golang-sarama", Optional.empty(), true, true}, - {"golang-sarama", Optional.of("persistent://public/default/my-sarama-topic-full-name"), true, true}, + // TODO: golang-sarama works well but it's very likely to complete before testcontainers catch the logs + // so that GenericContainer failed to start. Ignore these two cases first. + // See https://github.com/streamnative/kop/issues/629. + //{"golang-sarama", Optional.empty(), true, true}, + //{"golang-sarama", Optional.of("persistent://public/default/my-sarama-topic-full-name"), true, true}, {"golang-confluent-kafka", Optional.empty(), true, true}, // TODO: rustlang-rdkafka is failing on Github Actions and works locally, we need to investigate // {"rustlang-rdkafka", Optional.empty(), true, true},