From 3035b3082ee1cd20c3869edce93227b25adec66b Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Fri, 15 Oct 2021 11:52:16 -0700 Subject: [PATCH 001/823] [Issue 12040] (#12388) - fix MultipleListenerValidatorTest --- .../MultipleListenerValidatorTest.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/validator/MultipleListenerValidatorTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/validator/MultipleListenerValidatorTest.java index 4cd07438d85d2..d7fd802c17cf7 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/validator/MultipleListenerValidatorTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/validator/MultipleListenerValidatorTest.java @@ -32,15 +32,6 @@ */ public class MultipleListenerValidatorTest { - @Test(expectedExceptions = IllegalArgumentException.class) - public void testAppearTogether() { - ServiceConfiguration config = new ServiceConfiguration(); - config.setAdvertisedAddress("127.0.0.1"); - config.setAdvertisedListeners("internal:pulsar://192.168.1.11:6660,internal:pulsar+ssl://192.168.1.11:6651"); - config.setInternalListenerName("internal"); - MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); - } - @Test public void testGetAppliedAdvertised() throws Exception { ServiceConfiguration config = new ServiceConfiguration(); @@ -67,6 +58,20 @@ public void testGetAppliedAdvertised() throws Exception { ServiceConfigurationUtils.getDefaultOrConfiguredAddress(null)); } + @Test + public void testListenerDefaulting() { + ServiceConfiguration config = new ServiceConfiguration(); + config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660, internal:pulsar+ssl://127.0.0.1:6651"); + MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); + assertEquals("internal", config.getInternalListenerName()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testMalformedListener() { + ServiceConfiguration config = new ServiceConfiguration(); + config.setAdvertisedListeners(":pulsar://127.0.0.1:6660"); + MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); + } @Test(expectedExceptions = IllegalArgumentException.class) public void testListenerDuplicate_1() { @@ -86,18 +91,17 @@ public void testListenerDuplicate_2() { } @Test(expectedExceptions = IllegalArgumentException.class) - public void testDifferentListenerWithSameHostPort() { + public void testListenerDuplicate_3() { ServiceConfiguration config = new ServiceConfiguration(); - config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660," + " external:pulsar://127.0.0.1:6660"); + config.setAdvertisedListeners(" internal:pulsar+ssl://127.0.0.1:6661," + " internal:pulsar+ssl://192.168.1.11:6661"); config.setInternalListenerName("internal"); MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); } - @Test - public void testListenerWithTLSPort() { + @Test(expectedExceptions = IllegalArgumentException.class) + public void testDifferentListenerWithSameHostPort() { ServiceConfiguration config = new ServiceConfiguration(); - config.setBrokerServicePortTls(Optional.of(6651)); - config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660, internal:pulsar+ssl://127.0.0.1:6651"); + config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660," + " external:pulsar://127.0.0.1:6660"); config.setInternalListenerName("internal"); MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); } @@ -105,7 +109,6 @@ public void testListenerWithTLSPort() { @Test(expectedExceptions = IllegalArgumentException.class) public void testWithoutListenerNameInAdvertisedListeners() { ServiceConfiguration config = new ServiceConfiguration(); - config.setBrokerServicePortTls(Optional.of(6651)); config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660, internal:pulsar+ssl://127.0.0.1:6651"); config.setInternalListenerName("external"); MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); From c7157daf9a3ff21ebdf25f2962700bc7623f4804 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 19 Oct 2021 09:04:53 +0200 Subject: [PATCH 002/823] Release 2.9.0 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 126 files changed, 126 insertions(+), 126 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 696b43fafcf6c..3ef689c66bc30 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 07b31ebe0638d..7eb2052850631 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 49f6737d4d325..71c4882316d2b 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index f908c0e3fde36..85b8083e73983 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 7b5b02a510660..4398d50d861ee 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.0-SNAPSHOT + 2.9.0 jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 86a8bbdaeb479..a405b8aaf9e65 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 352ed71e6d868..bb2575ea30e7d 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 381e7194a68df..c5bb62fc9667e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 72aab71e6eb80..3206948baf9eb 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index b91947eb1fe84..77d4d3478c415 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 1d106d77247e2..38f75778618bd 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 11ff264b16abe..20d5120e14d7e 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 06e21e9d46b02..45dae9e5027ab 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 1214656f427ff..773bc6e473221 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 62814c939879d..c4d1da243b32e 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 5bbb0448022cb..5e0f22093cd3e 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pom.xml b/pom.xml index 185b391cf7daa..480ef814c76cb 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index bbec5e0084dcf..bb7c92c9468b6 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 13dd14aa8f58a..a5defa992a603 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 91179a1bd8f6f..3eacc740afeb4 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index bf1e78bba0249..941fa473dd584 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index fd11162d8ea3b..aef0f2e49c6c6 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index d7e6217fd63c0..e155a9d1a979f 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index aa61fb15d152f..b79e18a3b020f 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 1fa512ecc1b4a..eb77cdfef1d4b 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index b9f68f1b6e2a0..1bced59650786 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 406f4b28d0d84..8425a803d91e7 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 2d28b12e5cbde..7e0a3fd46ff24 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index f829386a18ed5..c58ab6cb56807 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 949901d2cdbb8..5c9e615965bae 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 5539f3bc124c2..59544f405bcb5 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 087736b2e11e4..d1321b5a0e2d6 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 45e97a7267b83..90b33e2d1e31a 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index a3df727fa7ed1..181b85570942c 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index efe90ceaaeada..919654bcd77fe 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 4fc38f1336adf..c0724a3da4a4e 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 65aef6af05a5f..8e7d6a8aaff35 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 225c260c4e234..3294452d83111 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 134e3f147ee06..981b065f18bf0 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 68901a3da9cd2..972f98cca1a02 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index c6f5524e76fb9..a4cfd4997ebbb 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 82735a03f0ee8..56156a4dc964a 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index ded9d60bed82b..14bc0108d0987 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index de762ec006453..7bd014023b009 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index d5dcc34ae8475..20d5f67c09276 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index d534b751c7de5..3751fb67f6b2b 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index d7f35e70661be..116dfe0dbbf01 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index f3cbcc21f35f4..20d7aab8c11ea 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 6c56b59c9702c..581a249835d63 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 5611acd316485..e9dfe723445a3 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index f5709031047d1..4731a32492ed2 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index f93329be2644a..d47e54f913f34 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 512f0c3d0aa2c..36da3ce64e448 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 98e6b8c9039fb..246cc67f8297e 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index a29cf663be57f..2dfbc5cdba253 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index de7b1f44ac820..ab5aedf555f64 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 9a18c5fbc4551..ebfa960312953 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 8a6adf9a38a1a..57441f8f121d8 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 250806edda186..4704f7795bded 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 0191c6b671969..50b5e3bbdc8c2 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index e004b0f421206..0714e741d664d 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 7c0f8bcf48f2f..219ab2114c1f6 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 911ddac11edc1..5dee7c221c94c 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index f044b53e3e598..07d779dcda0b4 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 950c46a7ff861..2af6d304a02ef 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index d0fafcb4fa063..07fa2cf79f3bb 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 69f4c93f8594d..fcf9dadb98261 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 03b5114883283..cb9606035fdd0 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 5737b50cab58e..415f77c942e8f 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 43a82c6f07071..f9e264e160b01 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 8b1af8083c1ac..05d0a8c35730c 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 649803a8e0e03..75d4bcdeadd3d 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 0b25207840e19..2e3a418553d23 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 36358c4d5fc53..e76a341e1f73e 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index fe12a54a75b43..dcbd20b2ec623 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 88ef951ded242..a17eff5430036 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index d09feea5499b0..c7f875116225b 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 32a22f43f5a4a..90fe56e07aa82 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index f1ba934af160e..584e923dcf970 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 7935748eeef7d..c91c778a801c6 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 2e76f431571c1..38932936183ae 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 741ecc31e7817..a286f06434f7d 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index bf0a5fc22bb4d..6c093bb9f54b2 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 5f07d24a304e4..8fc6dc2ff3bd0 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 1e6a3869fac31..78d4c0899d1e9 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 30b57475cbee1..9a755f145ec14 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 93f2c1443f43c..4c0b2d416feb1 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 3a41643d65cba..4a5916e578db3 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index ee4dd4150ac52..c8141dbc6a347 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 2e20f7934edaa..07450b03d96f6 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index cef4b1c4a48d0..6d34e9dcf0621 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 30da872435bb9..19a1755f32290 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 6b0c02465533e..d9bf650361581 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index f32034ec829f7..bced6aa1066b7 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.9.0 pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 59052d71ddba6..7ce5a6b778a6a 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 32c18716e6459..d817d8a599fa7 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index f6cfbee94a8fd..5a3396bb9b41d 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 8946793f6135e..b7cbed188e1a1 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 277f63c184e83..a244b58990297 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 296d6f1330de1..0f242adf5c355 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index a0ded27662cc5..d023e705ffdaa 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0-SNAPSHOT + 2.9.0 pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 1cad798fc96e0..54a2a772ab2c7 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0-SNAPSHOT + 2.9.0 pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index d62d947e54a75..7aa74984f2a56 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0-SNAPSHOT + 2.9.0 pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 18f80a04f0be3..de56bf3d22d2f 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index b3b0e3d369a07..aa115fad92887 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.0-SNAPSHOT + 2.9.0 pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index b5bedab65452f..be3c23a45a41a 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.0-SNAPSHOT + 2.9.0 pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 053a494039d4b..a15ee049a8f72 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 35377d7ed3c6e..0c222075594b4 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index f29e10df2ca4e..edd0848a63188 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index cb882523fbafe..b576934402030 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 0d199f1ccb273..20a40bc24543b 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.0-SNAPSHOT + 2.9.0 testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 50d1883cc07dd..f4dde997a0762 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 10830353ee6d4..84022b7c7498f 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 2a81b22e597d5..1608f82ee35fa 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 72dff476c39b0..b82298d9b4bfc 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 7dd26d8040ebb..30d705ba7a8fa 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 8d0aba77ad47f..27567573a65ef 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0-SNAPSHOT + 2.9.0 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index f68e7d4dbbde7..b7fb9cc8fd1c0 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 78382c292984c..5b8d860748e4f 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 integration diff --git a/tests/pom.xml b/tests/pom.xml index df6f2c1b411c2..1122625d6a667 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 75b31765f40a4..869b458c24bf3 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index c5b84a933e83a..597f882ee4d4d 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 438fd758474ee..1b7f3333eb2ce 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.9.0 pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index c261a9f0b7b01..55042c233be11 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 265743c7e376e..2bc10e8370281 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.0-SNAPSHOT + 2.9.0 .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index fb04ef4c4805c..3777d5b492962 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.9.0 .. From fe4bc9af39022cab75fb1677eff231285c6ad5fa Mon Sep 17 00:00:00 2001 From: lipenghui Date: Fri, 22 Oct 2021 14:57:05 +0800 Subject: [PATCH 003/823] Add debug log for WebSocket. (#12458) --- .../pulsar/websocket/ConsumerHandler.java | 20 +++++++++++++++++++ .../pulsar/websocket/ProducerHandler.java | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index a1c76d26fd2c4..d30c0844809de 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -232,6 +232,10 @@ public void onWebSocketText(String message) { // Check and notify consumer if reached end of topic. private void handleEndOfTopic() { + if (log.isDebugEnabled()) { + log.debug("[{}/{}] Received check reach the end of topic request from {} ", consumer.getTopic(), + subscription, getRemote().getInetSocketAddress().toString()); + } try { String msg = ObjectMapperFactory.getThreadLocal().writeValueAsString( new EndOfTopicResponse(consumer.hasReachedEndOfTopic())); @@ -259,6 +263,10 @@ public void writeSuccess() { } private void handleUnsubscribe(ConsumerCommand command) throws PulsarClientException { + if (log.isDebugEnabled()) { + log.debug("[{}/{}] Received unsubscribe request from {} ", consumer.getTopic(), + subscription, getRemote().getInetSocketAddress().toString()); + } consumer.unsubscribe(); } @@ -276,6 +284,10 @@ private void handleAck(ConsumerCommand command) throws IOException { // We should have received an ack MessageId msgId = MessageId.fromByteArrayWithTopic(Base64.getDecoder().decode(command.messageId), topic.toString()); + if (log.isDebugEnabled()) { + log.debug("[{}/{}] Received ack request of message {} from {} ", consumer.getTopic(), + subscription, msgId, getRemote().getInetSocketAddress().toString()); + } consumer.acknowledgeAsync(msgId).thenAccept(consumer -> numMsgsAcked.increment()); checkResumeReceive(); } @@ -283,11 +295,19 @@ private void handleAck(ConsumerCommand command) throws IOException { private void handleNack(ConsumerCommand command) throws IOException { MessageId msgId = MessageId.fromByteArrayWithTopic(Base64.getDecoder().decode(command.messageId), topic.toString()); + if (log.isDebugEnabled()) { + log.debug("[{}/{}] Received negative ack request of message {} from {} ", consumer.getTopic(), + subscription, msgId, getRemote().getInetSocketAddress().toString()); + } consumer.negativeAcknowledge(msgId); checkResumeReceive(); } private void handlePermit(ConsumerCommand command) throws IOException { + if (log.isDebugEnabled()) { + log.debug("[{}/{}] Received {} permits request from {} ", consumer.getTopic(), + subscription, command.permitMessages, getRemote().getInetSocketAddress().toString()); + } if (command.permitMessages == null) { throw new IOException("Missing required permitMessages field for 'permit' command"); } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java index 9552d42462cb6..9b6593d0274fe 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java @@ -131,6 +131,10 @@ public void close() throws IOException { @Override public void onWebSocketText(String message) { + if (log.isDebugEnabled()) { + log.debug("[{}] Received new message from producer {} ", producer.getTopic(), + getRemote().getInetSocketAddress().toString()); + } ProducerMessage sendRequest; byte[] rawPayload = null; String requestContext = null; @@ -188,6 +192,10 @@ public void onWebSocketText(String message) { final long now = System.nanoTime(); builder.sendAsync().thenAccept(msgId -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Success fully write the message to broker with returned message ID {} from producer {}", + producer.getTopic(), msgId, getRemote().getInetSocketAddress().toString()); + } updateSentMsgStats(msgSize, TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - now)); if (isConnected()) { String messageId = Base64.getEncoder().encodeToString(msgId.toByteArray()); From 08d24455a5899145dde3d8438e2760b3959515b9 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sat, 30 Oct 2021 01:53:27 +0800 Subject: [PATCH 004/823] Fix the batch message ack for WebSocket proxy. (#12530) * Fix the batch message ack for WebSocket proxy. The WebSocket proxy uses the consumer instance to process the message acknowledgement. But for the batch message acknowledgement, the batch message instance that deserialized from WebSocket proxy will with the same batch message acker, which will lead to the batch message can't been acked because of the consumer only send the ack request to the broker while the acker becomes empty. The fix is introduce a map to maintain an original which used for getting batch message acker. Added the test for covering the batch message ack. * Simplified the logic a bit * Using time-bound cleanup for ack-timeout cases Co-authored-by: Matteo Merli --- .../proxy/ProxyPublishConsumeTest.java | 43 +++++++++++++++++++ .../pulsar/client/impl/BatchMessageAcker.java | 2 +- .../pulsar/websocket/ConsumerHandler.java | 31 ++++++++++++- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java index 941e410cd30a5..3e3cf9a9e823f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java @@ -34,6 +34,7 @@ import com.google.gson.reflect.TypeToken; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -892,6 +893,48 @@ public void nackRedeliveryDelayTest() throws Exception { } } + @Test(timeOut = 20000) + public void ackBatchMessageTest() throws Exception { + final String subscription = "my-sub"; + final String topic = "my-property/my-ns/ack-batch-message" + UUID.randomUUID(); + final String consumerUri = "ws://localhost:" + proxyServer.getListenPortHTTP().get() + + "/ws/v2/consumer/persistent/" + topic + "/" + subscription; + final int messages = 10; + + WebSocketClient consumerClient = new WebSocketClient(); + SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket(); + Producer producer = pulsarClient.newProducer() + .topic(topic) + .batchingMaxPublishDelay(1, TimeUnit.SECONDS) + .create(); + + try { + consumerClient.start(); + ClientUpgradeRequest consumerRequest = new ClientUpgradeRequest(); + Future consumerFuture = consumerClient.connect(consumeSocket, URI.create(consumerUri), consumerRequest); + + assertTrue(consumerFuture.get().isOpen()); + assertEquals(consumeSocket.getReceivedMessagesCount(), 0); + + for (int i = 0; i < messages; i++) { + producer.sendAsync(String.valueOf(i).getBytes(StandardCharsets.UTF_8)); + } + + producer.flush(); + consumeSocket.sendPermits(messages); + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(consumeSocket.getReceivedMessagesCount(), messages)); + + // The message should not be acked since we only acked 1 message of the batch message + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(admin.topics().getStats(topic).getSubscriptions() + .get(subscription).getMsgBacklog(), 0)); + + } finally { + stopWebSocketClient(consumerClient); + } + } + private void verifyTopicStat(Client client, String baseUrl, String topic) { String statUrl = baseUrl + topic + "/stats"; WebTarget webTarget = client.target(statUrl); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java index 5f9e617ca39a6..f99c54d4ba3d3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java @@ -20,7 +20,7 @@ import java.util.BitSet; -class BatchMessageAcker { +public class BatchMessageAcker { private BatchMessageAcker() { this.bitSet = new BitSet(); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index d30c0844809de..54a5a62a6bbf3 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -24,9 +24,14 @@ import com.google.common.base.Enums; import com.google.common.base.Splitter; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import java.io.IOException; import java.util.Base64; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; @@ -45,6 +50,7 @@ import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.ConsumerBuilderImpl; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; @@ -87,6 +93,12 @@ public class ConsumerHandler extends AbstractWebSocketHandler { private static final AtomicLongFieldUpdater MSG_DELIVERED_COUNTER_UPDATER = AtomicLongFieldUpdater.newUpdater(ConsumerHandler.class, "msgDeliveredCounter"); + // Make sure use the same BatchMessageIdImpl to acknowledge the batch message, otherwise the BatchMessageAcker + // of the BatchMessageIdImpl will not complete. + private Cache messageIdCache = CacheBuilder.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + public ConsumerHandler(WebSocketService service, HttpServletRequest request, ServletUpgradeResponse response) { super(service, request, response); @@ -156,6 +168,8 @@ private void receiveMessage() { } final long msgSize = msg.getData().length; + messageIdCache.put(dm.messageId, msg.getMessageId()); + try { getSession().getRemote() .sendString(ObjectMapperFactory.getThreadLocal().writeValueAsString(dm), new WriteCallback() { @@ -288,7 +302,14 @@ private void handleAck(ConsumerCommand command) throws IOException { log.debug("[{}/{}] Received ack request of message {} from {} ", consumer.getTopic(), subscription, msgId, getRemote().getInetSocketAddress().toString()); } - consumer.acknowledgeAsync(msgId).thenAccept(consumer -> numMsgsAcked.increment()); + + MessageId originalMsgId = messageIdCache.asMap().remove(command.messageId); + if (originalMsgId != null) { + consumer.acknowledgeAsync(originalMsgId).thenAccept(consumer -> numMsgsAcked.increment()); + } else { + consumer.acknowledgeAsync(msgId).thenAccept(consumer -> numMsgsAcked.increment()); + } + checkResumeReceive(); } @@ -299,7 +320,13 @@ private void handleNack(ConsumerCommand command) throws IOException { log.debug("[{}/{}] Received negative ack request of message {} from {} ", consumer.getTopic(), subscription, msgId, getRemote().getInetSocketAddress().toString()); } - consumer.negativeAcknowledge(msgId); + + MessageId originalMsgId = messageIdCache.asMap().remove(command.messageId); + if (originalMsgId != null) { + consumer.negativeAcknowledge(originalMsgId); + } else { + consumer.negativeAcknowledge(msgId); + } checkResumeReceive(); } From 3cbc79d200130b2b114ec98addf4477c3a504a41 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 9 Nov 2021 12:06:38 +0100 Subject: [PATCH 005/823] Functions: add -Dio.netty.tryReflectionSetAccessible=true to Java functions (#12624) (cherry picked from commit d1010faced6608616183e504bcf137a41edd5da8) --- .../apache/pulsar/functions/runtime/RuntimeUtils.java | 3 +++ .../runtime/kubernetes/KubernetesRuntimeTest.java | 6 +++--- .../functions/runtime/process/ProcessRuntimeTest.java | 11 ++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 107d5cff9bf11..4acbd353685b2 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -315,6 +315,9 @@ public static List getCmd(InstanceConfig instanceConfig, "%s-%s", instanceConfig.getFunctionDetails().getName(), shardId)); + + args.add("-Dio.netty.tryReflectionSetAccessible=true"); + if (!isEmpty(instanceConfig.getFunctionDetails().getRuntimeFlags())) { for (String runtimeFlagArg : splitRuntimeArgs(instanceConfig.getFunctionDetails().getRuntimeFlags())) { args.add(runtimeFlagArg); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 9f064f511f050..928497a25cc14 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -396,14 +396,14 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s if (null != depsDir) { extraDepsEnv = " -Dpulsar.functions.extra.dependencies.dir=" + depsDir; classpath = classpath + ":" + depsDir + "/*"; - totalArgs = 39; + totalArgs = 40; portArg = 26; metricsPortArg = 28; } else { extraDepsEnv = ""; portArg = 25; metricsPortArg = 27; - totalArgs = 38; + totalArgs = 39; } if (secretsAttached) { totalArgs += 4; @@ -433,7 +433,7 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s + " -Dlog4j.configurationFile=kubernetes_instance_log4j2.xml " + "-Dpulsar.function.log.dir=" + logDirectory + "/" + FunctionCommon.getFullyQualifiedName(config.getFunctionDetails()) + " -Dpulsar.function.log.file=" + config.getFunctionDetails().getName() + "-$SHARD_ID" - + " -Xmx" + String.valueOf(RESOURCES.getRam()) + + " -Dio.netty.tryReflectionSetAccessible=true -Xmx" + String.valueOf(RESOURCES.getRam()) + " org.apache.pulsar.functions.instance.JavaInstanceMain" + " --jar " + jarLocation + " --instance_id " + "$SHARD_ID" + " --function_id " + config.getFunctionId() diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java index 4751ecd7778d1..6cddf3822d01a 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java @@ -298,7 +298,7 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS String extraDepsEnv; int portArg; int metricsPortArg; - int totalArgCount = 41; + int totalArgCount = 42; if (webServiceUrl != null && config.isExposePulsarAdminClientEnabled()) { totalArgCount += 3; } @@ -306,13 +306,13 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS assertEquals(args.size(), totalArgCount); extraDepsEnv = " -Dpulsar.functions.extra.dependencies.dir=" + depsDir.toString(); classpath = classpath + ":" + depsDir + "/*"; - portArg = 24; - metricsPortArg = 26; + portArg = 25; + metricsPortArg = 27; } else { assertEquals(args.size(), totalArgCount-1); extraDepsEnv = ""; - portArg = 23; - metricsPortArg = 25; + portArg = 24; + metricsPortArg = 26; } if (webServiceUrl != null && config.isExposePulsarAdminClientEnabled()) { portArg += 3; @@ -328,6 +328,7 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS + " -Dlog4j.configurationFile=java_instance_log4j2.xml " + "-Dpulsar.function.log.dir=" + logDirectory + "/functions/" + FunctionCommon.getFullyQualifiedName(config.getFunctionDetails()) + " -Dpulsar.function.log.file=" + config.getFunctionDetails().getName() + "-" + config.getInstanceId() + + " -Dio.netty.tryReflectionSetAccessible=true" + " org.apache.pulsar.functions.instance.JavaInstanceMain" + " --jar " + userJarFile + " --instance_id " + config.getInstanceId() + " --function_id " + config.getFunctionId() From 64a63bb2d198757b95b13bfc72090a30902a231b Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 9 Nov 2021 12:07:29 +0100 Subject: [PATCH 006/823] Pulsar Functions: detect .nar files and prevent spammy logs on functions boot (#13) (#12667) (cherry picked from commit 515a69fc6675322c455d606631cc47490756b453) --- .../apache/pulsar/common/nar/FileUtils.java | 20 +++++++++++++ .../instance/JavaInstanceRunnable.java | 1 + .../runtime/thread/ThreadRuntime.java | 28 +++++++++++++------ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java index cc677302ed802..0bfdb806165e9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java @@ -30,6 +30,9 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; /** @@ -37,6 +40,7 @@ * operations. * */ +@Slf4j public class FileUtils { public static final long MILLIS_BETWEEN_ATTEMPTS = 50L; @@ -221,5 +225,21 @@ public static void sleepQuietly(final long millis) { /* do nothing */ } } + + public static boolean mayBeANarArchive(File jarFile) { + try (ZipFile zipFile = new ZipFile(jarFile);) { + ZipEntry entry = zipFile.getEntry("META-INF/bundled-dependencies"); + if (entry == null || !entry.isDirectory()) { + log.info("Jar file {} does not contain META-INF/bundled-dependencies, it is not a NAR file", jarFile); + return false; + } else { + log.info("Jar file {} contains META-INF/bundled-dependencies, it may be a NAR file", jarFile); + return true; + } + } catch (IOException err) { + log.info("Cannot safely detect if {} is a NAR archive", jarFile, err); + return true; + } + } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index cfdcb08071846..bbc7ced0ca492 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -53,6 +53,7 @@ import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.Reflections; +import org.apache.pulsar.common.nar.FileUtils; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.StateStore; diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index b6dd019140e76..be43bb0f29219 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -19,6 +19,7 @@ package org.apache.pulsar.functions.runtime.thread; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; @@ -33,6 +34,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.nar.FileUtils; import org.apache.pulsar.functions.instance.InstanceConfig; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.instance.stats.FunctionCollectorRegistry; @@ -132,14 +134,24 @@ private static ClassLoader loadJars(String jarFile, return Thread.currentThread().getContextClassLoader(); } ClassLoader fnClassLoader; - try { - log.info("Load JAR: {}", jarFile); - // Let's first try to treat it as a nar archive - fnCache.registerFunctionInstanceWithArchive( - instanceConfig.getFunctionId(), - instanceConfig.getInstanceName(), - jarFile, narExtractionDirectory); - } catch (FileNotFoundException e) { + boolean loadedAsNar = false; + if (FileUtils.mayBeANarArchive(new File(jarFile))) { + try { + log.info("Trying Loading file as NAR file: {}", jarFile); + // Let's first try to treat it as a nar archive + fnCache.registerFunctionInstanceWithArchive( + instanceConfig.getFunctionId(), + instanceConfig.getInstanceName(), + jarFile, narExtractionDirectory); + loadedAsNar = true; + } catch (FileNotFoundException e) { + // this is usually like + // java.io.FileNotFoundException: /tmp/pulsar-nar/xxx.jar-unpacked/xxxxx/META-INF/MANIFEST.MF' + log.error("The file {} does not look like a .nar file", jarFile, e.toString()); + } + } + if (!loadedAsNar) { + log.info("Load file as simple JAR file: {}", jarFile); // create the function class loader fnCache.registerFunctionInstance( instanceConfig.getFunctionId(), From c08373ffda5c506f0cc141fce34c5eaa29669fe6 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Mon, 8 Nov 2021 15:21:43 +0800 Subject: [PATCH 007/823] [Authorization] Support GET_METADATA topic op after enable auth (#12656) ### Motivation Currently, we can get the internal stats of a topic through `bin/pulsar-admin topics stats-internal tn1/ns1/tp1` and also get ledger metadata by specifying flag `--metadata`. However I found that `PulsarAuthorizationProvider` lacks support for topic operation `GET_METADATA` when verifying the role's authorization, code as below: https://github.com/apache/pulsar/blob/08a49c06bff4a52d26319a114961aed6cb6c4791/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java#L1162-L1164 https://github.com/apache/pulsar/blob/08a49c06bff4a52d26319a114961aed6cb6c4791/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java#L567-L596 The purpose of this PR is to support role with `lookup` topic authorization to `GET_METADATA` of ledger. (cherry picked from commit e4767355bc7fa0a2e6bbb09acf1f93e4cd7cbe0b) --- .../PulsarAuthorizationProvider.java | 1 + .../admin/GetMetadataOfTopicWithAuthTest.java | 213 ++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index e355b122fc013..a4d8634083606 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -559,6 +559,7 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, switch (operation) { case LOOKUP: case GET_STATS: + case GET_METADATA: isAuthorizedFuture = canLookupAsync(topicName, role, authData); break; case PRODUCE: diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java new file mode 100644 index 0000000000000..75786290027aa --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.auth.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; +import com.google.common.io.Files; +import java.io.File; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.containers.ZKContainer; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.utils.DockerUtils; +import org.elasticsearch.common.collect.Set; +import org.testcontainers.containers.Network; +import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * GetMetadataOfTopicWithAuthTest will test Getmetadata operation with and without the proper permission. + */ +@Slf4j +public class GetMetadataOfTopicWithAuthTest extends TestRetrySupport { + + private static final String CLUSTER_PREFIX = "get-metadata-auth"; + private static final String PRIVATE_KEY_PATH_INSIDE_CONTAINER = "/tmp/private.key"; + private static final String PUBLIC_KEY_PATH_INSIDE_CONTAINER = "/tmp/public.key"; + + private static final String SUPER_USER_ROLE = "super-user"; + private String superUserAuthToken; + private static final String PROXY_ROLE = "proxy"; + private String proxyAuthToken; + private static final String REGULAR_USER_ROLE = "client"; + private String clientAuthToken; + private File publicKeyFile; + + private PulsarCluster pulsarCluster; + private PulsarContainer cmdContainer; + + @Override + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + incrementSetupNumber(); + // Before starting the cluster, generate the secret key and the token + // Use Zk container to have 1 container available before starting the cluster + final String clusterName = String.format("%s-%s", CLUSTER_PREFIX, RandomStringUtils.randomAlphabetic(6)); + final String cliContainerName = String.format("%s-%s", "cli", RandomStringUtils.randomAlphabetic(6)); + cmdContainer = new ZKContainer<>(cliContainerName); + cmdContainer + .withNetwork(Network.newNetwork()) + .withNetworkAliases(ZKContainer.NAME) + .withEnv("zkServers", ZKContainer.NAME); + cmdContainer.start(); + + createKeysAndTokens(cmdContainer); + + PulsarClusterSpec spec = PulsarClusterSpec.builder() + .numBookies(2) + .numBrokers(2) + .numProxies(1) + .clusterName(clusterName) + .brokerEnvs(getBrokerSettingsEnvs()) + .proxyEnvs(getProxySettingsEnvs()) + .brokerMountFiles(Collections.singletonMap(publicKeyFile.toString(), PUBLIC_KEY_PATH_INSIDE_CONTAINER)) + .proxyMountFiles(Collections.singletonMap(publicKeyFile.toString(), PUBLIC_KEY_PATH_INSIDE_CONTAINER)) + .build(); + + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + } + + @Override + @AfterClass(alwaysRun = true) + public void cleanup() { + markCurrentSetupNumberCleaned(); + if (cmdContainer != null) { + cmdContainer.stop(); + } + if (pulsarCluster != null) { + pulsarCluster.stop(); + } + } + + private Map getBrokerSettingsEnvs() { + Map envs = new HashMap<>(); + envs.put("authenticationEnabled", "true"); + envs.put("authenticationProviders", AuthenticationProviderToken.class.getName()); + envs.put("authorizationEnabled", "true"); + envs.put("superUserRoles", String.format("%s,%s", SUPER_USER_ROLE, PROXY_ROLE)); + envs.put("brokerClientAuthenticationPlugin", AuthenticationToken.class.getName()); + envs.put("brokerClientAuthenticationParameters", String.format("token:%s", superUserAuthToken)); + envs.put("authenticationRefreshCheckSeconds", "1"); + envs.put("authenticateOriginalAuthData", "true"); + envs.put("tokenPublicKey", "file://" + PUBLIC_KEY_PATH_INSIDE_CONTAINER); + return envs; + } + + private Map getProxySettingsEnvs() { + Map envs = new HashMap<>(); + envs.put("authenticationEnabled", "true"); + envs.put("authenticationProviders", AuthenticationProviderToken.class.getName()); + envs.put("authorizationEnabled", "true"); + envs.put("brokerClientAuthenticationPlugin", AuthenticationToken.class.getName()); + envs.put("brokerClientAuthenticationParameters", String.format("token:%s", proxyAuthToken)); + envs.put("authenticationRefreshCheckSeconds", "1"); + envs.put("forwardAuthorizationCredentials", "true"); + envs.put("tokenPublicKey", "file://" + PUBLIC_KEY_PATH_INSIDE_CONTAINER); + return envs; + } + + protected void createKeysAndTokens(PulsarContainer container) throws Exception { + container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create-key-pair", + "--output-private-key", PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--output-public-key", PUBLIC_KEY_PATH_INSIDE_CONTAINER); + + byte[] publicKeyBytes = DockerUtils + .runCommandWithRawOutput(container.getDockerClient(), container.getContainerId(), + "/bin/cat", PUBLIC_KEY_PATH_INSIDE_CONTAINER) + .getStdout(); + + publicKeyFile = File.createTempFile("public-", ".key", new File("/tmp")); + Files.write(publicKeyBytes, publicKeyFile); + + clientAuthToken = container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", + "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--subject", REGULAR_USER_ROLE) + .getStdout().trim(); + log.info("Created client token: {}", clientAuthToken); + + superUserAuthToken = container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", + "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--subject", SUPER_USER_ROLE) + .getStdout().trim(); + log.info("Created super-user token: {}", superUserAuthToken); + + proxyAuthToken = container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", + "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--subject", PROXY_ROLE) + .getStdout().trim(); + log.info("Created proxy token: {}", proxyAuthToken); + } + + @Test + public void testGetMetadataOfTopicWithLookupPermission() throws Exception { + @Cleanup + PulsarAdmin superUserAdmin = PulsarAdmin.builder() + .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) + .authentication(AuthenticationFactory.token(superUserAuthToken)) + .build(); + + @Cleanup + PulsarAdmin clientAdmin = PulsarAdmin.builder() + .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) + .authentication(AuthenticationFactory.token(clientAuthToken)) + .build(); + + // create partitioned topic + superUserAdmin.topics().createPartitionedTopic("public/default/test", 1); + + // do some operation without grant any permissions + try { + clientAdmin.topics().getInternalStats("public/default/test-partition-0", true); + fail("get internal stats and metadata operation should fail because the client hasn't permission to do"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 401); + } + + // grant consume/produce permission to the role + superUserAdmin.topics().grantPermission("public/default/test", + REGULAR_USER_ROLE, Set.of(AuthAction.consume)); + + // then do some get internal stats and metadata operations again, it should success + PersistentTopicInternalStats internalStats = clientAdmin.topics() + .getInternalStats("public/default/test-partition-0", true); + assertNotNull(internalStats); + } +} From 5fdb94eb9eb27178c4a8bca5eb82438b6056e895 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 6 Nov 2021 16:51:49 -0700 Subject: [PATCH 008/823] Allow to configure different implementations for Pulsar functions state store (#12646) (cherry picked from commit 08a49c06bff4a52d26319a114961aed6cb6c4791) --- .../worker/PulsarFunctionLocalRunTest.java | 6 +- .../PulsarFunctionMetadataStoreTest.java | 122 +++++++++++++++ pulsar-functions/instance/pom.xml | 6 + .../instance/JavaInstanceRunnable.java | 15 +- .../state/BKStateStoreProviderImpl.java | 2 - .../state/PulsarMetadataStateStoreImpl.java | 142 ++++++++++++++++++ .../PulsarMetadataStateStoreProviderImpl.java | 67 +++++++++ .../instance/state/StateStoreProvider.java | 2 + .../instance/JavaInstanceRunnableTest.java | 2 +- .../PulsarMetadataStateStoreImplTest.java | 120 +++++++++++++++ .../apache/pulsar/functions/LocalRunner.java | 9 +- .../runtime/JavaInstanceStarter.java | 4 + .../runtime/thread/ThreadRuntime.java | 5 +- .../runtime/thread/ThreadRuntimeFactory.java | 18 ++- .../pulsar/functions/worker/WorkerConfig.java | 8 + .../worker/rest/api/FunctionsImplTest.java | 4 +- 16 files changed, 515 insertions(+), 17 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionMetadataStoreTest.java create mode 100644 pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java create mode 100644 pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java create mode 100644 pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index ed6c4aa8912ac..13ac623aa1de5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -304,7 +304,7 @@ void shutdown() throws Exception { } } - private WorkerConfig createWorkerConfig(ServiceConfiguration config) { + protected WorkerConfig createWorkerConfig(ServiceConfiguration config) { System.setProperty(JAVA_INSTANCE_JAR_PROPERTY, FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath()); @@ -560,7 +560,7 @@ private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelis } } - private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl) throws Exception { + protected void testE2EPulsarFunctionLocalRun(String jarFilePathUrl) throws Exception { testE2EPulsarFunctionLocalRun(jarFilePathUrl, 1); } @@ -1133,7 +1133,7 @@ private void runWithNarClassLoader(Assert.ThrowingRunnable throwingRunnable) thr } } - private void runWithPulsarFunctionsClassLoader(Assert.ThrowingRunnable throwingRunnable) throws Throwable { + protected void runWithPulsarFunctionsClassLoader(Assert.ThrowingRunnable throwingRunnable) throws Throwable { ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(pulsarApiExamplesClassLoader); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionMetadataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionMetadataStoreTest.java new file mode 100644 index 0000000000000..715837f4e2603 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionMetadataStoreTest.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; +import static org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.ServiceConfigurationUtils; +import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; +import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; +import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; +import org.apache.pulsar.client.admin.BrokerStats; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; +import org.apache.pulsar.common.functions.ConsumerConfig; +import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.Utils; +import org.apache.pulsar.common.io.SinkConfig; +import org.apache.pulsar.common.io.SourceConfig; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.ConsumerStats; +import org.apache.pulsar.common.policies.data.PublisherStats; +import org.apache.pulsar.common.policies.data.SubscriptionStats; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.functions.LocalRunner; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.functions.instance.state.PulsarMetadataStateStoreProviderImpl; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig; +import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Test Pulsar sink on function + */ +@Slf4j +@Test +public class PulsarFunctionMetadataStoreTest extends PulsarFunctionLocalRunTest { + + + protected WorkerConfig createWorkerConfig(ServiceConfiguration config) { + WorkerConfig wc = super.createWorkerConfig(config); + wc.setStateStorageProviderImplementation(PulsarMetadataStateStoreProviderImpl.class.getName()); + wc.setStateStorageServiceUrl("memory://local"); + return wc; + } + + @Test + public void testE2EPulsarFunctionLocalRun() throws Throwable { + runWithPulsarFunctionsClassLoader(() -> testE2EPulsarFunctionLocalRun(null)); + } +} diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index a4cfd4997ebbb..67318a8f23053 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -53,6 +53,12 @@ ${project.version} + + ${project.groupId} + pulsar-metadata + ${project.version} + + ${project.groupId} pulsar-io-core diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index bbc7ced0ca492..ce8c9fc9296fe 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -104,6 +104,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { private LogAppender logAppender; // provide tables for storing states + private final String stateStorageImplClass; private final String stateStorageServiceUrl; private StateStoreProvider stateStoreProvider; private StateManager stateManager; @@ -145,6 +146,7 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, ClientBuilder clientBuilder, PulsarClient pulsarClient, PulsarAdmin pulsarAdmin, + String stateStorageImplClass, String stateStorageServiceUrl, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, @@ -153,6 +155,7 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, this.clientBuilder = clientBuilder; this.client = (PulsarClientImpl) pulsarClient; this.pulsarAdmin = pulsarAdmin; + this.stateStorageImplClass = stateStorageImplClass; this.stateStorageServiceUrl = stateStorageServiceUrl; this.secretsProvider = secretsProvider; this.functionClassLoader = functionClassLoader; @@ -323,8 +326,8 @@ private void setupStateStore() throws Exception { if (null == stateStorageServiceUrl) { stateStoreProvider = StateStoreProvider.NULL; } else { - stateStoreProvider = new BKStateStoreProviderImpl(); - Map stateStoreProviderConfig = new HashMap(); + stateStoreProvider = getStateStoreProvider(); + Map stateStoreProviderConfig = new HashMap<>(); stateStoreProviderConfig.put(BKStateStoreProviderImpl.STATE_STORAGE_SERVICE_URL, stateStorageServiceUrl); stateStoreProvider.init(stateStoreProviderConfig, instanceConfig.getFunctionDetails()); @@ -340,6 +343,14 @@ private void setupStateStore() throws Exception { } } + private StateStoreProvider getStateStoreProvider() throws Exception { + if (stateStorageImplClass == null) { + return new BKStateStoreProviderImpl(); + } else { + return (StateStoreProvider) Class.forName(stateStorageImplClass).getConstructor().newInstance(); + } + } + private void handleResult(Record srcRecord, JavaExecutionResult result) throws Exception { if (result.getUserException() != null) { Exception t = result.getUserException(); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java index fd4228a1adfa9..32901bd478aed 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java @@ -56,8 +56,6 @@ @Slf4j public class BKStateStoreProviderImpl implements StateStoreProvider { - public static final String STATE_STORAGE_SERVICE_URL = "stateStorageServiceUrl"; - private String stateStorageServiceUrl; private Map clients; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java new file mode 100644 index 0000000000000..1c1df7f6a6f20 --- /dev/null +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.instance.state; + +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.functions.api.StateStoreContext; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreException; + +public class PulsarMetadataStateStoreImpl implements DefaultStateStore { + + private final MetadataStore store; + private final String prefixPath; + private final MetadataCache countersCache; + + private final String namespace; + private final String tenant; + private final String name; + private final String fqsn; + + PulsarMetadataStateStoreImpl(MetadataStore store, String prefix, String tenant, String namespace, String name) { + this.store = store; + this.tenant = tenant; + this.namespace = namespace; + this.name = name; + this.fqsn = tenant + '/' + namespace + '/' + name; + + this.prefixPath = prefix + '/' + fqsn + '/'; + this.countersCache = store.getMetadataCache(Long.class); + } + + @Override + public String tenant() { + return tenant; + } + + @Override + public String namespace() { + return namespace; + } + + @Override + public String name() { + return name; + } + + @Override + public String fqsn() { + return fqsn; + } + + @Override + public void init(StateStoreContext ctx) { + } + + @Override + public void close() { + } + + @Override + public void put(String key, ByteBuffer value) { + putAsync(key, value).join(); + } + + @Override + public CompletableFuture putAsync(String key, ByteBuffer value) { + byte[] bytes = new byte[value.remaining()]; + value.get(bytes); + return store.put(getPath(key), bytes, Optional.empty()) + .thenApply(__ -> null); + } + + @Override + public void delete(String key) { + deleteAsync(key).join(); + } + + @Override + public CompletableFuture deleteAsync(String key) { + return store.delete(getPath(key), Optional.empty()); + } + + @Override + public ByteBuffer get(String key) { + return getAsync(key).join(); + } + + @Override + public CompletableFuture getAsync(String key) { + return store.get(getPath(key)) + .thenApply(optRes -> + optRes.map(x -> ByteBuffer.wrap(x.getValue())) + .orElse(null)); + } + + @Override + public void incrCounter(String key, long amount) { + incrCounterAsync(key, amount); + } + + @Override + public CompletableFuture incrCounterAsync(String key, long amount) { + return countersCache.readModifyUpdateOrCreate(getPath(key), optValue -> + optValue.orElse(0L) + amount + ).thenApply(__ -> null); + } + + @Override + public long getCounter(String key) { + return getCounterAsync(key).join(); + } + + @Override + public CompletableFuture getCounterAsync(String key) { + return countersCache.get(getPath(key)) + .thenApply(optValue -> optValue.orElse(0L)); + } + + private String getPath(String key) { + return prefixPath + key; + } +} diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java new file mode 100644 index 0000000000000..819bfd94cb7a5 --- /dev/null +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.instance.state; + +import java.util.Map; +import lombok.SneakyThrows; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreFactory; + +public class PulsarMetadataStateStoreProviderImpl implements StateStoreProvider { + + private static final String METADATA_URL = "METADATA_URL"; + private static final String METADATA_STORE_INSTANCE = "METADATA_STORE_INSTANCE"; + + private static final String METADATA_PREFIX = "METADATA_PREFIX"; + private static final String METADATA_DEFAULT_PREFIX = "/state-store"; + + private MetadataStore store; + private String prefix; + private boolean shouldCloseStore; + + @Override + public void init(Map config, Function.FunctionDetails functionDetails) throws Exception { + + prefix = (String) config.getOrDefault(METADATA_PREFIX, METADATA_DEFAULT_PREFIX); + + if (config.containsKey(METADATA_STORE_INSTANCE)) { + store = (MetadataStore) config.get(METADATA_STORE_INSTANCE); + shouldCloseStore = false; + } else { + String metadataUrl = (String) config.get(METADATA_URL); + store = MetadataStoreFactory.create(metadataUrl, MetadataStoreConfig.builder().build()); + shouldCloseStore = true; + } + } + + @Override + public DefaultStateStore getStateStore(String tenant, String namespace, String name) throws Exception { + return new PulsarMetadataStateStoreImpl(store, prefix, tenant, namespace, name); + } + + @SneakyThrows + @Override + public void close() { + if (shouldCloseStore) { + store.close(); + } + } +} diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java index db3c6b3aedda9..4c01e9db3d438 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java @@ -27,6 +27,8 @@ */ public interface StateStoreProvider extends AutoCloseable { + String STATE_STORAGE_SERVICE_URL = "stateStorageServiceUrl"; + /** * The state store provider returns `null` state stores. */ diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java index 59a4f5b50a618..b96e8134bf9d7 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java @@ -71,7 +71,7 @@ private JavaInstanceRunnable createRunnable(String outputSerde) throws Exception ClientBuilder clientBuilder = mock(ClientBuilder.class); when(clientBuilder.build()).thenReturn(null); JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable( - config, clientBuilder, null, null, null, null, null, null); + config, clientBuilder, null, null, null,null, null, null, null); return javaInstanceRunnable; } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java new file mode 100644 index 0000000000000..a6fb3eb365a06 --- /dev/null +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.instance.state; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.*; +import io.kubernetes.client.proto.Meta; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import lombok.SneakyThrows; +import org.apache.bookkeeper.api.kv.Table; +import org.apache.bookkeeper.api.kv.options.Options; +import org.apache.bookkeeper.api.kv.result.DeleteResult; +import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link BKStateStoreImpl}. + */ +public class PulsarMetadataStateStoreImplTest { + + private static final String TENANT = "test-tenant"; + private static final String NS = "test-ns"; + private static final String NAME = "test-name"; + private static final String FQSN = "test-tenant/test-ns/test-name"; + private static final String PREFIX = "/prefix"; + private static final String PREFIX_PATH = PREFIX + '/' + FQSN + '/'; + + private MetadataStore store; + private MetadataCache countersCache; + private DefaultStateStore stateContext; + + @BeforeMethod + public void setup() throws Exception { + this.store = MetadataStoreFactory.create("memory://local", MetadataStoreConfig.builder().build()); + this.countersCache = store.getMetadataCache(Long.class); + this.stateContext = new PulsarMetadataStateStoreImpl(store, "/prefix", TENANT, NS, NAME); + } + + @AfterMethod + public void cleanup() throws Exception { + this.store.close(); + } + + @Test + public void testGetter() { + assertEquals(stateContext.tenant(), TENANT); + assertEquals(stateContext.namespace(), NS); + assertEquals(stateContext.name(), NAME); + assertEquals(stateContext.fqsn(), FQSN); + } + + @Test + public void testIncr() throws Exception { + stateContext.incrCounter("test-key", 10L); + assertEquals(countersCache.get(PREFIX_PATH + "test-key").join().get().longValue(), 10); + } + + @Test + public void testPut() throws Exception { + stateContext.put("test-key", ByteBuffer.wrap("test-value".getBytes(UTF_8))); + assertEquals(store.get(PREFIX_PATH + "test-key").join().get().getValue(), "test-value".getBytes(UTF_8)); + } + + @Test + public void testDelete() throws Exception { + stateContext.put("test-key", ByteBuffer.wrap("test-value".getBytes(UTF_8))); + assertEquals("test-value".getBytes(UTF_8), store.get(PREFIX_PATH + "test-key").join().get().getValue()); + stateContext.delete("test-key"); + assertFalse(store.get(PREFIX_PATH + "test-key").join().isPresent()); + } + + @Test + public void testGetAmount() throws Exception { + assertEquals(stateContext.getCounter("test-key"), 0); + stateContext.incrCounter("test-key", 10L); + assertEquals(countersCache.get(PREFIX_PATH + "test-key").join().get().longValue(), 10); + assertEquals(stateContext.getCounter("test-key"), 10); + } + + @Test + public void testGetKeyNotPresent() throws Exception { + CompletableFuture result = stateContext.getAsync("test-key"); + assertTrue(result != null); + assertEquals(result.get(), null); + } + +} diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 0bedb74409665..295199c3b393b 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -151,6 +151,8 @@ public RuntimeEnv convert(String value) { protected SourceConfig sourceConfig; @Parameter(names = "--sinkConfig", description = "The json representation of SinkConfig", hidden = true, converter = SinkConfigConverter.class) protected SinkConfig sinkConfig; + @Parameter(names = "--stateStorageImplClass", description = "The implemenatation class state storage service (by default Apache BookKeeper)", hidden = true, required = false) + protected String stateStorageImplClass; @Parameter(names = "--stateStorageServiceUrl", description = "The URL for the state storage service (by default Apache BookKeeper)", hidden = true) protected String stateStorageServiceUrl; @Parameter(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) @@ -201,8 +203,9 @@ public static void main(String[] args) throws Exception { } @Builder - public LocalRunner(FunctionConfig functionConfig, SourceConfig sourceConfig, SinkConfig sinkConfig, String - stateStorageServiceUrl, String brokerServiceUrl, String clientAuthPlugin, String clientAuthParams, + public LocalRunner(FunctionConfig functionConfig, SourceConfig sourceConfig, SinkConfig sinkConfig, + String stateStorageImplClass, String stateStorageServiceUrl, String brokerServiceUrl, + String clientAuthPlugin, String clientAuthParams, boolean useTls, boolean tlsAllowInsecureConnection, boolean tlsHostNameVerificationEnabled, String tlsTrustCertFilePath, int instanceIdOffset, RuntimeEnv runtimeEnv, String secretsProviderClassName, String secretsProviderConfig, String narExtractionDirectory, @@ -210,6 +213,7 @@ public LocalRunner(FunctionConfig functionConfig, SourceConfig sourceConfig, Sin this.functionConfig = functionConfig; this.sourceConfig = sourceConfig; this.sinkConfig = sinkConfig; + this.stateStorageImplClass = stateStorageImplClass; this.stateStorageServiceUrl = stateStorageServiceUrl; this.brokerServiceUrl = brokerServiceUrl; this.clientAuthPlugin = clientAuthPlugin; @@ -614,6 +618,7 @@ private void startThreadedMode(org.apache.pulsar.functions.proto.Function.Functi } runtimeFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup", serviceUrl, + stateStorageImplClass, stateStorageServiceUrl, authConfig, secretsProvider, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index 1881b5556008d..5995b87b705e3 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -94,6 +94,9 @@ public class JavaInstanceStarter implements AutoCloseable { @Parameter(names = "--tls_trust_cert_path", description = "tls trust cert file path") public String tlsTrustCertFilePath; + @Parameter(names = "--state_storage_impl_class", description = "State Storage Service Implementation class\n", required= false) + public String stateStorageImplClass; + @Parameter(names = "--state_storage_serviceurl", description = "State Storage Service Url\n", required= false) public String stateStorageServiceUrl; @@ -196,6 +199,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL RuntimeUtils.registerDefaultCollectors(collectorRegistry); containerFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup", pulsarServiceUrl, + stateStorageImplClass, stateStorageServiceUrl, AuthenticationConfig.builder().clientAuthenticationPlugin(clientAuthenticationPlugin) .clientAuthenticationParameters(clientAuthenticationParameters).useTls(isTrue(useTls)) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index be43bb0f29219..950f48bdb0e7c 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -27,7 +27,6 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; -import io.prometheus.client.CollectorRegistry; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarServerException; @@ -68,6 +67,7 @@ public class ThreadRuntime implements Runtime { private ClientBuilder clientBuilder; private PulsarClient pulsarClient; private PulsarAdmin pulsarAdmin; + private String stateStorageImplClass; private String stateStorageServiceUrl; private SecretsProvider secretsProvider; private FunctionCollectorRegistry collectorRegistry; @@ -81,6 +81,7 @@ public class ThreadRuntime implements Runtime { PulsarClient client, ClientBuilder clientBuilder, PulsarAdmin pulsarAdmin, + String stateStorageImplClass, String stateStorageServiceUrl, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, @@ -97,6 +98,7 @@ public class ThreadRuntime implements Runtime { this.clientBuilder = clientBuilder; this.pulsarClient = client; this.pulsarAdmin = pulsarAdmin; + this.stateStorageImplClass = stateStorageImplClass; this.stateStorageServiceUrl = stateStorageServiceUrl; this.secretsProvider = secretsProvider; this.collectorRegistry = collectorRegistry; @@ -186,6 +188,7 @@ public void start() throws Exception { clientBuilder, pulsarClient, pulsarAdmin, + stateStorageImplClass, stateStorageServiceUrl, secretsProvider, collectorRegistry, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java index 864a067c1be2c..1e8c96a7ab66d 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java @@ -63,6 +63,7 @@ public class ThreadRuntimeFactory implements RuntimeFactory { private ClientBuilder clientBuilder; private PulsarClient pulsarClient; private PulsarAdmin pulsarAdmin; + private String stateStorageImplClass; private String storageServiceUrl; private SecretsProvider defaultSecretsProvider; private FunctionCollectorRegistry collectorRegistry; @@ -76,21 +77,27 @@ public class ThreadRuntimeFactory implements RuntimeFactory { * This constructor is used by other runtimes (e.g. ProcessRuntime and KubernetesRuntime) that rely on ThreadRuntime to actually run an instance of the function. * When used by other runtimes, the arguments such as secretsProvider and rootClassLoader will be provided. */ - public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl, String storageServiceUrl, + public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl, + String stateStorageImplClass, + String storageServiceUrl, AuthenticationConfig authConfig, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory, ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled, String pulsarWebServiceUrl) throws Exception { initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig, - storageServiceUrl, null, secretsProvider, collectorRegistry, narExtractionDirectory, + stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry, + narExtractionDirectory, rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty()); } - private void initialize(String threadGroupName, Optional memoryLimit, String pulsarServiceUrl, AuthenticationConfig authConfig, String storageServiceUrl, + private void initialize(String threadGroupName, Optional memoryLimit, + String pulsarServiceUrl, AuthenticationConfig authConfig, String stateStorageImplClass, + String storageServiceUrl, SecretsProviderConfigurator secretsProviderConfigurator, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory, ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled, - String pulsarWebServiceUrl, Optional connectorsManager) throws PulsarClientException { + String pulsarWebServiceUrl, Optional connectorsManager) + throws PulsarClientException { if (rootClassLoader == null) { rootClassLoader = Thread.currentThread().getContextClassLoader(); @@ -104,6 +111,7 @@ private void initialize(String threadGroupName, Optional metricsDataCompletableFuture = new CompletableFuture(); metricsDataCompletableFuture.complete(javaInstanceRunnable.getMetrics()); Runtime runtime = mock(Runtime.class); @@ -222,7 +222,7 @@ public void testMetricsEmpty() throws PulsarClientException { instanceConfig.setMaxBufferedTuples(1024); JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable( - instanceConfig, null, null, null, null, null, null, null); + instanceConfig, null, null, null, null, null, null, null, null); CompletableFuture completableFuture = new CompletableFuture(); completableFuture.complete(javaInstanceRunnable.getMetrics()); Runtime runtime = mock(Runtime.class); From f0dc3443b91167fddcc0aaace1e912a5093242fb Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Sat, 6 Nov 2021 13:17:04 -0700 Subject: [PATCH 009/823] k8s runtime: force deletion to avoid hung function worker during connector restart (#12504) (cherry picked from commit a3f6aba81a7bbd55a8429cd724694cdcee7d3f2e) --- conf/functions_worker.yml | 4 ++++ .../runtime/kubernetes/KubernetesRuntime.java | 11 +++++++---- .../runtime/kubernetes/KubernetesRuntimeFactory.java | 3 +++ .../kubernetes/KubernetesRuntimeFactoryConfig.java | 6 ++++++ site2/docs/functions-runtime.md | 4 ++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 58dfc692617d3..9ca5f7bc923e5 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -216,6 +216,10 @@ functionRuntimeFactoryConfigs: # extraFunctionDependenciesDir: # # Additional memory padding added on top of the memory requested by the function per on a per instance basis # percentMemoryPadding: 10 +# # The duration in seconds before the StatefulSet deleted on function stop/restart. +# # Value must be non-negative integer. The value zero indicates delete immediately. +# # Default is 5 seconds. +# gracePeriodSeconds: 5 ## A set of the minimum amount of resources functions must request. ## Support for this depends on function runtime. diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index a483bd07fbd00..1301596348d70 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -149,6 +149,7 @@ public class KubernetesRuntime implements Runtime { private int percentMemoryPadding; private double cpuOverCommitRatio; private double memoryOverCommitRatio; + private int gracePeriodSeconds; private final Optional functionAuthDataCacheProvider; private final AuthenticationConfig authConfig; private Integer grpcPort; @@ -186,6 +187,7 @@ public class KubernetesRuntime implements Runtime { int percentMemoryPadding, double cpuOverCommitRatio, double memoryOverCommitRatio, + int gracePeriodSeconds, Optional functionAuthDataCacheProvider, boolean authenticationEnabled, Integer grpcPort, @@ -212,6 +214,7 @@ public class KubernetesRuntime implements Runtime { this.percentMemoryPadding = percentMemoryPadding; this.cpuOverCommitRatio = cpuOverCommitRatio; this.memoryOverCommitRatio = memoryOverCommitRatio; + this.gracePeriodSeconds = gracePeriodSeconds; this.authenticationEnabled = authenticationEnabled; this.manifestCustomizer = manifestCustomizer; this.functionInstanceClassPath = functionInstanceClassPath; @@ -567,7 +570,7 @@ private void submitStatefulSet() throws Exception { public void deleteStatefulSet() throws InterruptedException { String statefulSetName = createJobName(instanceConfig.getFunctionDetails(), this.jobName); final V1DeleteOptions options = new V1DeleteOptions(); - options.setGracePeriodSeconds(5L); + options.setGracePeriodSeconds((long)gracePeriodSeconds); options.setPropagationPolicy("Foreground"); String fqfn = FunctionCommon.getFullyQualifiedName(instanceConfig.getFunctionDetails()); @@ -583,8 +586,8 @@ public void deleteStatefulSet() throws InterruptedException { response = appsClient.deleteNamespacedStatefulSetCall( statefulSetName, jobNamespace, null, null, - 5, null, "Foreground", - null, null) + gracePeriodSeconds, null, "Foreground", + options, null) .execute(); } catch (ApiException e) { // if already deleted @@ -735,7 +738,7 @@ public void deleteService() throws InterruptedException { serviceName, jobNamespace, null, null, 0, null, - "Foreground", null, null).execute(); + "Foreground", options, null).execute(); } catch (ApiException e) { // if already deleted if (e.getCode() == HTTP_NOT_FOUND) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 2a47da66ae742..d98a161c2e02e 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -103,6 +103,7 @@ public class KubernetesRuntimeFactory implements RuntimeFactory { private String narExtractionDirectory; private String functionInstanceClassPath; private String downloadDirectory; + private int gracePeriodSeconds; @ToString.Exclude @EqualsAndHashCode.Exclude @@ -200,6 +201,7 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic this.percentMemoryPadding = factoryConfig.getPercentMemoryPadding(); this.cpuOverCommitRatio = factoryConfig.getCpuOverCommitRatio(); this.memoryOverCommitRatio = factoryConfig.getMemoryOverCommitRatio(); + this.gracePeriodSeconds = factoryConfig.getGracePeriodSeconds(); this.pulsarServiceUrl = StringUtils.isEmpty(factoryConfig.getPulsarServiceUrl()) ? workerConfig.getPulsarServiceUrl() : factoryConfig.getPulsarServiceUrl(); this.pulsarAdminUrl = StringUtils.isEmpty(factoryConfig.getPulsarAdminUrl()) @@ -318,6 +320,7 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c percentMemoryPadding, cpuOverCommitRatio, memoryOverCommitRatio, + gracePeriodSeconds, authProvider, authenticationEnabled, grpcPort, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java index a3ef41895aa86..e3b758f479c1c 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java @@ -161,5 +161,11 @@ public class KubernetesRuntimeFactoryConfig { doc = "The classpath where function instance files stored" ) private String functionInstanceClassPath = ""; + @FieldContext( + doc = "The duration in seconds before the StatefulSet deleted on function stop/restart. " + + "Value must be non-negative integer. The value zero indicates delete immediately. " + + "Default is 5 seconds." + ) + protected int gracePeriodSeconds = 5; } diff --git a/site2/docs/functions-runtime.md b/site2/docs/functions-runtime.md index 0155f17bf6695..09eb1ef7c23b3 100644 --- a/site2/docs/functions-runtime.md +++ b/site2/docs/functions-runtime.md @@ -136,6 +136,10 @@ functionRuntimeFactoryConfigs: extraFunctionDependenciesDir: # Additional memory padding added on top of the memory requested by the function per on a per instance basis percentMemoryPadding: 10 + # The duration (in seconds) before the StatefulSet is deleted after a function stops or restarts. + # Value must be a non-negative integer. 0 indicates the StatefulSet is deleted immediately. + # Default is 5 seconds. + gracePeriodSeconds: 5 ``` If you run functions worker embedded in a broker on Kubernetes, you can use the default settings. From 3a31716d422468c187c44a174ea2546222679404 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 5 Nov 2021 09:30:57 +0800 Subject: [PATCH 010/823] allow consume permission to do GetTopics op (#12600) Fixes #12423 ### Motivation Regex subscription requires to get the topics list of given namespace with GetTopicsOfNamespace request, but this request requires tenant admin permission which will block the regex consumers who only have consume permission. ### Modifications This PR added the consume permission check for GetTopicsOfNamespace, which allows consumers get the topics list with consume permission. (cherry picked from commit 7e078aad5cb0b07f5e0d609025cf13b934cf28eb) --- .../authorization/AuthorizationProvider.java | 10 + .../MultiRolesTokenAuthorizationProvider.java | 5 + .../PulsarAuthorizationProvider.java | 8 + .../auth/MockAuthorizationProvider.java | 6 + .../AuthorizationProducerConsumerTest.java | 5 + .../PatternTopicsConsumerImplAuthTest.java | 5 + .../GetTopicsOfNamespaceWithAuthTest.java | 208 ++++++++++++++++++ 7 files changed, 247 insertions(+) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetTopicsOfNamespaceWithAuthTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java index c83ae4ce44d60..ca1bfbf250592 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java @@ -184,6 +184,16 @@ CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, Stri CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData); + /** + * Allow consume operations with in this namespace + * @param namespaceName The namespace that the consume operations can be executed in + * @param role The role to check + * @param authenticationData authentication data related to the role + * @return a boolean to determine whether authorized or not + */ + CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, + AuthenticationDataSource authenticationData); + /** * * Grant authorization-action permission on a namespace to the given client diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index dcdf779780ef6..89ed8341b6ee3 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -210,6 +210,11 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return authorize(authenticationData, r -> super.allowSinkOpsAsync(namespaceName, r, authenticationData)); } + @Override + public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { + return authorize(authenticationData, r -> super.allowConsumeOpsAsync(namespaceName, r, authenticationData)); + } + @Override public CompletableFuture allowTenantOperationAsync(String tenantName, String role, diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index a4d8634083606..641591c03a791 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -227,6 +227,11 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return allowTheSpecifiedActionOpsAsync(namespaceName, role, authenticationData, AuthAction.sinks); } + @Override + public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { + return allowTheSpecifiedActionOpsAsync(namespaceName, role, authenticationData, AuthAction.consume); + } + private CompletableFuture allowTheSpecifiedActionOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData, AuthAction authAction) { @@ -525,6 +530,9 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam case PACKAGES: isAuthorizedFuture = allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.packages); break; + case GET_TOPICS: + isAuthorizedFuture = allowConsumeOpsAsync(namespaceName, role, authData); + break; default: isAuthorizedFuture = CompletableFuture.completedFuture(false); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java index 3af2568c95465..74ba55eeb4fed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java @@ -96,6 +96,12 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return roleAuthorizedAsync(role); } + @Override + public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, + AuthenticationDataSource authenticationData) { + return roleAuthorizedAsync(role); + } + @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authDataJson) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0ad79ff816cef..0a799fbb51dfe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -498,6 +498,11 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return null; } + @Override + public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { + return null; + } + @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authenticationData) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java index b1328a263011c..40c31dbf7373d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java @@ -297,6 +297,11 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return null; } + @Override + public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { + return null; + } + @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authenticationData) { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetTopicsOfNamespaceWithAuthTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetTopicsOfNamespaceWithAuthTest.java new file mode 100644 index 0000000000000..68de70d069273 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetTopicsOfNamespaceWithAuthTest.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.auth.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; +import com.google.common.io.Files; +import java.io.File; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.containers.ZKContainer; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.utils.DockerUtils; +import org.elasticsearch.common.collect.Set; +import org.testcontainers.containers.Network; +import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * GetTopicsOfNamespaceWithAuthTest will test GetTopics operation with and without the proper permission. + */ +@Slf4j +public class GetTopicsOfNamespaceWithAuthTest extends TestRetrySupport { + + private static final String CLUSTER_PREFIX = "get-topics-auth"; + private static final String PRIVATE_KEY_PATH_INSIDE_CONTAINER = "/tmp/private.key"; + private static final String PUBLIC_KEY_PATH_INSIDE_CONTAINER = "/tmp/public.key"; + + private static final String SUPER_USER_ROLE = "super-user"; + private String superUserAuthToken; + private static final String PROXY_ROLE = "proxy"; + private String proxyAuthToken; + private static final String REGULAR_USER_ROLE = "client"; + private String clientAuthToken; + private File publicKeyFile; + + private PulsarCluster pulsarCluster; + private PulsarContainer cmdContainer; + + @Override + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + incrementSetupNumber(); + // Before starting the cluster, generate the secret key and the token + // Use Zk container to have 1 container available before starting the cluster + final String clusterName = String.format("%s-%s", CLUSTER_PREFIX, RandomStringUtils.randomAlphabetic(6)); + final String cliContainerName = String.format("%s-%s", "cli", RandomStringUtils.randomAlphabetic(6)); + cmdContainer = new ZKContainer<>(cliContainerName); + cmdContainer + .withNetwork(Network.newNetwork()) + .withNetworkAliases(ZKContainer.NAME) + .withEnv("zkServers", ZKContainer.NAME); + cmdContainer.start(); + + createKeysAndTokens(cmdContainer); + + PulsarClusterSpec spec = PulsarClusterSpec.builder() + .numBookies(2) + .numBrokers(2) + .numProxies(1) + .clusterName(clusterName) + .brokerEnvs(getBrokerSettingsEnvs()) + .proxyEnvs(getProxySettingsEnvs()) + .brokerMountFiles(Collections.singletonMap(publicKeyFile.toString(), PUBLIC_KEY_PATH_INSIDE_CONTAINER)) + .proxyMountFiles(Collections.singletonMap(publicKeyFile.toString(), PUBLIC_KEY_PATH_INSIDE_CONTAINER)) + .build(); + + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + } + + @Override + @AfterClass(alwaysRun = true) + public void cleanup() { + markCurrentSetupNumberCleaned(); + if (cmdContainer != null) { + cmdContainer.stop(); + } + if (pulsarCluster != null) { + pulsarCluster.stop(); + } + } + + private Map getBrokerSettingsEnvs() { + Map envs = new HashMap<>(); + envs.put("authenticationEnabled", "true"); + envs.put("authenticationProviders", AuthenticationProviderToken.class.getName()); + envs.put("authorizationEnabled", "true"); + envs.put("superUserRoles", String.format("%s,%s", SUPER_USER_ROLE, PROXY_ROLE)); + envs.put("brokerClientAuthenticationPlugin", AuthenticationToken.class.getName()); + envs.put("brokerClientAuthenticationParameters", String.format("token:%s", superUserAuthToken)); + envs.put("authenticationRefreshCheckSeconds", "1"); + envs.put("authenticateOriginalAuthData", "true"); + envs.put("tokenPublicKey", "file://" + PUBLIC_KEY_PATH_INSIDE_CONTAINER); + return envs; + } + + private Map getProxySettingsEnvs() { + Map envs = new HashMap<>(); + envs.put("authenticationEnabled", "true"); + envs.put("authenticationProviders", AuthenticationProviderToken.class.getName()); + envs.put("authorizationEnabled", "true"); + envs.put("brokerClientAuthenticationPlugin", AuthenticationToken.class.getName()); + envs.put("brokerClientAuthenticationParameters", String.format("token:%s", proxyAuthToken)); + envs.put("authenticationRefreshCheckSeconds", "1"); + envs.put("forwardAuthorizationCredentials", "true"); + envs.put("tokenPublicKey", "file://" + PUBLIC_KEY_PATH_INSIDE_CONTAINER); + return envs; + } + + protected void createKeysAndTokens(PulsarContainer container) throws Exception { + container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create-key-pair", + "--output-private-key", PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--output-public-key", PUBLIC_KEY_PATH_INSIDE_CONTAINER); + + byte[] publicKeyBytes = DockerUtils + .runCommandWithRawOutput(container.getDockerClient(), container.getContainerId(), + "/bin/cat", PUBLIC_KEY_PATH_INSIDE_CONTAINER) + .getStdout(); + + publicKeyFile = File.createTempFile("public-", ".key", new File("/tmp")); + Files.write(publicKeyBytes, publicKeyFile); + + clientAuthToken = container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", + "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--subject", REGULAR_USER_ROLE) + .getStdout().trim(); + log.info("Created client token: {}", clientAuthToken); + + superUserAuthToken = container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", + "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--subject", SUPER_USER_ROLE) + .getStdout().trim(); + log.info("Created super-user token: {}", superUserAuthToken); + + proxyAuthToken = container + .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", + "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, + "--subject", PROXY_ROLE) + .getStdout().trim(); + log.info("Created proxy token: {}", proxyAuthToken); + } + + @Test + public void testGetTopicsOfNamespaceOpsWithConsumePermission() throws Exception { + @Cleanup + PulsarAdmin superUserAdmin = PulsarAdmin.builder() + .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) + .authentication(AuthenticationFactory.token(superUserAuthToken)) + .build(); + + @Cleanup + PulsarAdmin clientAdmin = PulsarAdmin.builder() + .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) + .authentication(AuthenticationFactory.token(clientAuthToken)) + .build(); + + // do some operation without grant any permissions + try { + clientAdmin.namespaces().getTopics("public/default"); + fail("list topics operation should fail because the client hasn't permission to do"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 401); + } + + // grant consume permission to the role + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + REGULAR_USER_ROLE, Set.of(AuthAction.consume)); + + // then do some get topics operations again, it should success + List topics = clientAdmin.namespaces().getTopics("public/default"); + assertEquals(topics.size(), 0); + } +} From f99ff9cffbebe58f828a32acdf085a7bf12ebe34 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Fri, 5 Nov 2021 09:36:44 +0000 Subject: [PATCH 011/823] [pulsar-perf] Write histogram files for consume command (#12569) * [pulsar-perf] Write histogram files for consume command * [pulsar-perf] Disable writing to histogram files by default Most users don't use the histogram files and instead opt for sending metrics to prometheus, etc, so there's no need to have this enabled by default. Instead, add a new --histogram-file parameter to pulsar-perf produce/consume which, when specified, dumps the contents of the internal histogram to the given filename. Previous behaviour can be achieved with the following options: $ pulsar-perf produce --histogram-file perf-producer-$(date +%s).hgrm * [pulsar-perf] Update docs with --histogram-file param (cherry picked from commit 48de2e251ee3ea38449d696402d0cc57c88ee3c7) --- .../testclient/PerformanceConsumer.java | 22 ++++++++++++++++ .../testclient/PerformanceProducer.java | 25 +++++++++++++------ site2/docs/performance-pulsar-perf.md | 4 ++- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 52ba73e6781b8..db69778dc6e57 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -26,6 +26,8 @@ import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.PrintStream; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.util.Collections; @@ -36,6 +38,7 @@ import java.util.concurrent.atomic.LongAdder; import org.HdrHistogram.Histogram; +import org.HdrHistogram.HistogramLogWriter; import org.HdrHistogram.Recorder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; @@ -185,6 +188,9 @@ static class Arguments { @Parameter(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client") public boolean enableBusyWait = false; + + @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") + public String histogramFile = null; } public static void main(String[] args) throws Exception { @@ -406,7 +412,19 @@ public static void main(String[] args) throws Exception { long oldTime = System.nanoTime(); Histogram reportHistogram = null; + HistogramLogWriter histogramLogWriter = null; + + if (arguments.histogramFile != null) { + String statsFileName = arguments.histogramFile; + log.info("Dumping latency stats to {}", statsFileName); + + PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); + histogramLogWriter = new HistogramLogWriter(histogramLog); + // Some log header bits + histogramLogWriter.outputLogFormatVersion(); + histogramLogWriter.outputLegend(); + } while (true) { try { @@ -431,6 +449,10 @@ public static void main(String[] args) throws Exception { reportHistogram.getValueAtPercentile(99), reportHistogram.getValueAtPercentile(99.9), reportHistogram.getValueAtPercentile(99.99), reportHistogram.getMaxValue()); + if (histogramLogWriter != null) { + histogramLogWriter.outputIntervalHistogram(reportHistogram); + } + reportHistogram.reset(); oldTime = now; } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 44b3c20d67d9e..fee96a8a8ee67 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -253,6 +253,9 @@ static class Arguments { @Parameter(names = {"-fc", "--format-class"}, description="Custom Formatter class name") public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; + + @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") + public String histogramFile = null; } public static void main(String[] args) throws Exception { @@ -436,16 +439,19 @@ public static void main(String[] args) throws Exception { long oldTime = System.nanoTime(); Histogram reportHistogram = null; + HistogramLogWriter histogramLogWriter = null; - String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; - log.info("Dumping latency stats to {}", statsFileName); + if (arguments.histogramFile != null) { + String statsFileName = arguments.histogramFile; + log.info("Dumping latency stats to {}", statsFileName); - PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); - HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); + PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); + histogramLogWriter = new HistogramLogWriter(histogramLog); - // Some log header bits - histogramLogWriter.outputLogFormatVersion(); - histogramLogWriter.outputLegend(); + // Some log header bits + histogramLogWriter.outputLogFormatVersion(); + histogramLogWriter.outputLegend(); + } while (true) { try { @@ -480,7 +486,10 @@ public static void main(String[] args) throws Exception { dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); - histogramLogWriter.outputIntervalHistogram(reportHistogram); + if (histogramLogWriter != null) { + histogramLogWriter.outputIntervalHistogram(reportHistogram); + } + reportHistogram.reset(); oldTime = now; diff --git a/site2/docs/performance-pulsar-perf.md b/site2/docs/performance-pulsar-perf.md index bb831154b80de..c5aafafe4b5c4 100644 --- a/site2/docs/performance-pulsar-perf.md +++ b/site2/docs/performance-pulsar-perf.md @@ -32,7 +32,7 @@ After the command is executed, the test data is continuously output on the Conso 19:54:44.336 [Thread-1] INFO org.apache.pulsar.testclient.PerformanceProducer - Aggregated latency stats --- Latency: mean: 3.383 ms - med: 3.293 - 95pct: 4.610 - 99pct: 5.059 - 99.9pct: 5.588 - 99.99pct: 5.837 - 99.999pct: 6.609 - Max: 6.609 ``` -From the above test data, you can get the throughput statistics and the write latency statistics. The aggregated statistics is printed when the Pulsar Perf is stopped. You can press **Ctrl**+**C** to stop the Pulsar Perf. After the Pulsar Perf is stopped, the [HdrHistogram](http://hdrhistogram.github.io/HdrHistogram/) formatted test result appears under your directory. The document looks like `perf-producer-1589370810837.hgrm`. You can also check the test result through [HdrHistogram Plotter](https://hdrhistogram.github.io/HdrHistogram/plotFiles.html). For details about how to check the test result through [HdrHistogram Plotter](https://hdrhistogram.github.io/HdrHistogram/plotFiles.html), see [HdrHistogram Plotter](#hdrhistogram-plotter). +From the above test data, you can get the throughput statistics and the write latency statistics. The aggregated statistics is printed when the Pulsar Perf is stopped. You can press **Ctrl**+**C** to stop the Pulsar Perf. If you specify a filename with the `--histogram-file` parameter, a file with the [HdrHistogram](http://hdrhistogram.github.io/HdrHistogram/) formatted test result appears under your directory after Pulsar Perf is stopped. You can also check the test result through [HdrHistogram Plotter](https://hdrhistogram.github.io/HdrHistogram/plotFiles.html). For details about how to check the test result through [HdrHistogram Plotter](https://hdrhistogram.github.io/HdrHistogram/plotFiles.html), see [HdrHistogram Plotter](#hdrhistogram-plotter). ### Configuration options for `pulsar-perf produce` @@ -61,6 +61,7 @@ The following table lists configuration options available for the `pulsar-perf p | format-class | Set the custom formatter class name. | org.apache.pulsar.testclient.DefaultMessageFormatter | | format-payload | Configure whether to format %i as a message index in the stream from producer and/or %t as the timestamp nanoseconds. | false | | help | Configure the help message. | false | +| histogram-file | HdrHistogram output file | N/A | | max-connections | Set the maximum number of TCP connections to a single broker. | 100 | | max-outstanding | Set the maximum number of outstanding messages. | 1000 | | max-outstanding-across-partitions | Set the maximum number of outstanding messages across partitions. | 50000 | @@ -131,6 +132,7 @@ The following table lists configuration options available for the `pulsar-perf c | encryption-key-name | Set the name of the public key used to encrypt the payload. | N/A | | encryption-key-value-file | Set the file which contains the public key used to encrypt the payload. | N/A | | help | Configure the help message. | false | +| histogram-file | HdrHistogram output file | N/A | | expire_time_incomplete_chunked_messages | Set the expiration time for incomplete chunk messages (in milliseconds). | 0 | | max-connections | Set the maximum number of TCP connections to a single broker. | 100 | | max_chunked_msg | Set the max pending chunk messages. | 0 | From c89e1c6a840cc799087685cdd5e2332254386638 Mon Sep 17 00:00:00 2001 From: Boyang Jerry Peng Date: Thu, 4 Nov 2021 08:24:45 -0700 Subject: [PATCH 012/823] add additional error handling in auto partition update task MultiTopicsConsumerImpl (#12620) Co-authored-by: Jerry Peng (cherry picked from commit 11cfbe4f5cf66ac93ec0fb6dd2ea160a0a8bf6cf) --- .../client/impl/MultiTopicsConsumerImpl.java | 33 +++++++++++-------- .../client/impl/PartitionedProducerImpl.java | 30 +++++++++-------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 520e7f3ab099c..fa7c4a2eb2334 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -1324,28 +1324,35 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa topicName, oldPartitionNumber, currentPartitionNumber); return FutureUtil.failedFuture(new NotSupportedException("not support shrink topic partitions")); } + }).exceptionally(throwable -> { + log.warn("[{}] Failed to get partitions for topic to determine if new partitions are added", throwable); + return null; }); } private TimerTask partitionsAutoUpdateTimerTask = new TimerTask() { @Override public void run(Timeout timeout) throws Exception { - if (timeout.isCancelled() || getState() != State.Ready) { - return; - } + try { + if (timeout.isCancelled() || getState() != State.Ready) { + return; + } - if (log.isDebugEnabled()) { - log.debug("[{}] run partitionsAutoUpdateTimerTask", topic); - } + if (log.isDebugEnabled()) { + log.debug("[{}] run partitionsAutoUpdateTimerTask", topic); + } - // if last auto update not completed yet, do nothing. - if (partitionsAutoUpdateFuture == null || partitionsAutoUpdateFuture.isDone()) { - partitionsAutoUpdateFuture = topicsPartitionChangedListener.onTopicsExtended(partitionedTopics.keySet()); + // if last auto update not completed yet, do nothing. + if (partitionsAutoUpdateFuture == null || partitionsAutoUpdateFuture.isDone()) { + partitionsAutoUpdateFuture = topicsPartitionChangedListener.onTopicsExtended(partitionedTopics.keySet()); + } + } catch (Throwable th) { + log.warn("Encountered error in partition auto update timer task for multi-topic consumer. Another task will be scheduled.", th); + } finally { + // schedule the next re-check task + partitionsAutoUpdateTimeout = client.timer() + .newTimeout(partitionsAutoUpdateTimerTask, conf.getAutoUpdatePartitionsIntervalSeconds(), TimeUnit.SECONDS); } - - // schedule the next re-check task - partitionsAutoUpdateTimeout = client.timer() - .newTimeout(partitionsAutoUpdateTimerTask, conf.getAutoUpdatePartitionsIntervalSeconds(), TimeUnit.SECONDS); } }; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index 452553181ef2d..33110503612e2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -410,22 +410,26 @@ public CompletableFuture onTopicsExtended(Collection topicsExtende private TimerTask partitionsAutoUpdateTimerTask = new TimerTask() { @Override public void run(Timeout timeout) throws Exception { - if (timeout.isCancelled() || getState() != State.Ready) { - return; - } + try { + if (timeout.isCancelled() || getState() != State.Ready) { + return; + } - if (log.isDebugEnabled()) { - log.debug("[{}] run partitionsAutoUpdateTimerTask for partitioned producer", topic); - } + if (log.isDebugEnabled()) { + log.debug("[{}] run partitionsAutoUpdateTimerTask for partitioned producer", topic); + } - // if last auto update not completed yet, do nothing. - if (partitionsAutoUpdateFuture == null || partitionsAutoUpdateFuture.isDone()) { - partitionsAutoUpdateFuture = topicsPartitionChangedListener.onTopicsExtended(ImmutableList.of(topic)); + // if last auto update not completed yet, do nothing. + if (partitionsAutoUpdateFuture == null || partitionsAutoUpdateFuture.isDone()) { + partitionsAutoUpdateFuture = topicsPartitionChangedListener.onTopicsExtended(ImmutableList.of(topic)); + } + } catch (Throwable th) { + log.warn("Encountered error in partition auto update timer task for partition producer. Another task will be scheduled.", th); + } finally { + // schedule the next re-check task + partitionsAutoUpdateTimeout = client.timer() + .newTimeout(partitionsAutoUpdateTimerTask, conf.getAutoUpdatePartitionsIntervalSeconds(), TimeUnit.SECONDS); } - - // schedule the next re-check task - partitionsAutoUpdateTimeout = client.timer() - .newTimeout(partitionsAutoUpdateTimerTask, conf.getAutoUpdatePartitionsIntervalSeconds(), TimeUnit.SECONDS); } }; From ce8f4505cfc799a74d15f786542cf9639ef33393 Mon Sep 17 00:00:00 2001 From: Jason918 Date: Thu, 4 Nov 2021 23:18:10 +0800 Subject: [PATCH 013/823] [ISSUE-12291][Client] 'StartMessageId' and 'RollbackDuration' not working in MultiTopicsReader for non-partitioned topics (#12308) Fixes #12291 ### Motivation Bug fix. 'StartMessageId' and 'RollbackDuration' is not working in MultiTopicsReader for non-partitioned topics. ### Modifications This fix is quite simple. Just add `startMessageId` and `startMessageRollbackDurationInSec` when creating underlying consumer with `ConsumerImpl.newConsumerImpl` (cherry picked from commit cb48152254b8c16596e7251ef9a7229d918d2e90) --- .../client/impl/MultiTopicsReaderTest.java | 102 +++++++++++++++++- .../pulsar/client/impl/ConsumerImpl.java | 2 +- .../client/impl/MultiTopicsConsumerImpl.java | 8 +- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java index 31a426e9f9dc8..a8a6ced5a325e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java @@ -48,7 +48,6 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; @@ -378,6 +377,107 @@ public void testMultiTopic() throws Exception { Awaitility.await().untilAsserted(() -> assertEquals(client.consumersCount(), 0)); } + @Test(timeOut = 20000) + public void testMultiNonPartitionedTopicWithStartMessageId() throws Exception { + final String topic1 = "persistent://my-property/my-ns/topic1" + UUID.randomUUID(); + final String topic2 = "persistent://my-property/my-ns/topic2" + UUID.randomUUID(); + List topics = Arrays.asList(topic1, topic2); + PulsarClientImpl client = (PulsarClientImpl) pulsarClient; + + // create producer and send msg + List> producerList = new ArrayList<>(); + for (String topicName : topics) { + producerList.add(pulsarClient.newProducer(Schema.STRING).topic(topicName).create()); + } + int msgNum = 10; + Set messages = new HashSet<>(); + for (int i = 0; i < producerList.size(); i++) { + Producer producer = producerList.get(i); + for (int j = 0; j < msgNum; j++) { + String msg = i + "msg" + j; + producer.send(msg); + messages.add(msg); + } + } + Reader reader = pulsarClient.newReader(Schema.STRING) + .startMessageId(MessageId.earliest) + .topics(topics).readerName("my-reader").create(); + // receive messages + while (reader.hasMessageAvailable()) { + messages.remove(reader.readNext(5, TimeUnit.SECONDS).getValue()); + } + assertEquals(messages.size(), 0); + assertEquals(client.consumersCount(), 1); + // clean up + for (Producer producer : producerList) { + producer.close(); + } + reader.close(); + Awaitility.await().untilAsserted(() -> assertEquals(client.consumersCount(), 0)); + } + + @Test(timeOut = 20000) + public void testMultiNonPartitionedTopicWithRollbackDuration() throws Exception { + final String topic1 = "persistent://my-property/my-ns/topic1" + UUID.randomUUID(); + final String topic2 = "persistent://my-property/my-ns/topic2" + UUID.randomUUID(); + List topics = Arrays.asList(topic1, topic2); + PulsarClientImpl client = (PulsarClientImpl) pulsarClient; + + // create producer and send msg + List> producerList = new ArrayList<>(); + for (String topicName : topics) { + producerList.add(pulsarClient.newProducer(Schema.STRING).topic(topicName).create()); + } + int totalMsg = 10; + Set messages = new HashSet<>(); + long oldMsgPublishTime = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(5); // 5 hours old + long newMsgPublishTime = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); // 5 hours old + for (int i = 0; i < producerList.size(); i++) { + Producer producer = producerList.get(i); + // (1) Publish 10 messages with publish-time 5 HOUR back + for (int j = 0; j < totalMsg; j++) { + TypedMessageBuilderImpl msg = (TypedMessageBuilderImpl) producer.newMessage() + .value(i + "-old-msg-" + j); + msg.getMetadataBuilder() + .setPublishTime(oldMsgPublishTime) + .setProducerName(producer.getProducerName()) + .setReplicatedFrom("us-west1"); + msg.send(); + messages.add(msg.getMessage().getValue()); + } + // (2) Publish 10 messages with publish-time 1 HOUR back + for (int j = 0; j < totalMsg; j++) { + TypedMessageBuilderImpl msg = (TypedMessageBuilderImpl) producer.newMessage() + .value(i + "-new-msg-" + j); + msg.getMetadataBuilder() + .setPublishTime(newMsgPublishTime) + .setProducerName(producer.getProducerName()) + .setReplicatedFrom("us-west1"); + msg.send(); + messages.add(msg.getMessage().getValue()); + } + } + + Reader reader = pulsarClient.newReader(Schema.STRING) + .startMessageFromRollbackDuration(2, TimeUnit.HOURS) + .topics(topics).readerName("my-reader").create(); + // receive messages + while (reader.hasMessageAvailable()) { + messages.remove(reader.readNext(5, TimeUnit.SECONDS).getValue()); + } + assertEquals(messages.size(), 2 * totalMsg); + for (String message : messages) { + assertTrue(message.contains("old-msg")); + } + assertEquals(client.consumersCount(), 1); + // clean up + for (Producer producer : producerList) { + producer.close(); + } + reader.close(); + Awaitility.await().untilAsserted(() -> assertEquals(client.consumersCount(), 0)); + } + @Test(timeOut = 10000) public void testKeyHashRangeReader() throws Exception { final List keys = Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index b29d6c73deff6..88b6df114118d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2000,7 +2000,7 @@ public CompletableFuture hasMessageAvailableAsync() { if (lastDequeuedMessageId == MessageId.earliest) { // if we are starting from latest, we should seek to the actual last message first. // allow the last one to be read when read head inclusively. - if (startMessageId.equals(MessageId.latest)) { + if (MessageId.latest.equals(startMessageId)) { CompletableFuture future = internalGetLastMessageIdAsync(); // if the consumer is configured to read inclusive then we need to seek to the last message diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index fa7c4a2eb2334..18bd7bc45b8e5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -998,8 +998,8 @@ private void doSubscribeTopicPartitions(Schema schema, } else { ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, topicName, internalConfig, client.externalExecutorProvider(), -1, - true, subFuture, null, schema, interceptors, - createIfDoesNotExist); + true, subFuture, startMessageId, schema, interceptors, + createIfDoesNotExist, startMessageRollbackDurationInSec); synchronized (pauseMutex) { if (paused) { @@ -1294,8 +1294,8 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl( client, partitionName, configurationData, client.externalExecutorProvider(), - partitionIndex, true, subFuture, null, schema, interceptors, - true /* createTopicIfDoesNotExist */); + partitionIndex, true, subFuture, startMessageId, schema, interceptors, + true /* createTopicIfDoesNotExist */, startMessageRollbackDurationInSec); synchronized (pauseMutex) { if (paused) { newConsumer.pause(); From a8a207b1ae11564b98e3332204c0b20cb6c91118 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sat, 6 Nov 2021 22:06:18 +0800 Subject: [PATCH 014/823] Fix call sync method in an async callback when enabling geo replicator. (#12590) After enabled geo-replication, we are not able to produce messages to a partitoned non-persistent topic. Reproduce step: 1. Start a geo-replication instance with 2 clusters, I have 3 brokers for each cluster 2. Create a non-persistent partitioned topic such as 10 partitions 3. Use pulsar-perf to publish messages to the partitioned topic 4. Verify if the message produce throughput is 0, such as `bin/pulsar-perf produce -s 2048 -r 100 -bm 1 non-persistent://public/default/test` The root cause is there are 2 places calling a sync method in the async method callback. So the fix is: 1. Async the `validatePartitionedTopicAsync` method 2. Avoid call get cluster sync method when getting the replication client Integration tests added. (cherry picked from commit a2b7cae3cfe8d3776483b1ecaf69af47949b70e1) --- .../broker/resources/BaseResources.java | 4 + .../broker/resources/NamespaceResources.java | 11 +- .../pulsar/broker/admin/AdminResource.java | 31 +++-- .../admin/impl/PersistentTopicsBase.java | 11 +- .../broker/service/AbstractReplicator.java | 31 +++-- .../pulsar/broker/service/BrokerService.java | 56 +++++---- .../NonPersistentReplicator.java | 7 +- .../nonpersistent/NonPersistentTopic.java | 53 +++++---- .../persistent/PersistentReplicator.java | 8 +- .../service/persistent/PersistentTopic.java | 85 ++++++------- .../broker/service/PersistentTopicTest.java | 10 +- .../service/ReplicatorRemoveClusterTest.java | 4 +- .../pulsar/broker/service/ReplicatorTest.java | 4 +- .../messaging/GeoReplicationTest.java | 112 ++++++++++++++++++ .../integration/topologies/PulsarCluster.java | 59 +++++---- .../topologies/PulsarGeoCluster.java | 82 +++++++++++++ .../topologies/PulsarGeoClusterTestBase.java | 92 ++++++++++++++ .../topologies/PulsarTestBase.java | 9 ++ .../src/test/resources/pulsar-messaging.xml | 1 + 19 files changed, 516 insertions(+), 154 deletions(-) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/GeoReplicationTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoCluster.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoClusterTestBase.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java index 9061dd76b47fd..1e463fad1c954 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java @@ -167,6 +167,10 @@ protected boolean exists(String path) throws MetadataStoreException { } } + protected CompletableFuture existsAsync(String path) { + return cache.exists(path); + } + public int getOperationTimeoutSec() { return operationTimeoutSec; } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index dcba2ac854c3c..2beeab8c9510b 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -18,14 +18,11 @@ */ package org.apache.pulsar.broker.resources; -import static org.apache.pulsar.common.policies.path.PolicyPath.path; import com.fasterxml.jackson.core.type.TypeReference; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import lombok.Getter; @@ -39,7 +36,6 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies; import org.apache.pulsar.common.util.Codec; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -122,7 +118,7 @@ public CompletableFuture setPoliciesAsync(NamespaceName ns, Function partitionedTopicExistsAsync(TopicName tn) { + return existsAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(), + tn.getEncodedLocalName())); + } + public CompletableFuture deletePartitionedTopicAsync(TopicName tn) { return deleteAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(), tn.getEncodedLocalName())); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 625f4191d6c04..f3a94d222fdca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -632,10 +632,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n return; } - List> createFutureList = new ArrayList<>(); - CompletableFuture createLocalFuture = new CompletableFuture<>(); - createFutureList.add(createLocalFuture); checkTopicExistsAsync(topicName).thenAccept(exists -> { if (exists) { log.warn("[{}] Failed to create already existing topic {}", clientAppId(), topicName); @@ -658,7 +655,13 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n return null; }); - FutureUtil.waitForAll(createFutureList).whenComplete((ignored, ex) -> { + List replicatedClusters = new ArrayList<>(); + if (!createLocalTopicOnly && topicName.isGlobal() && isNamespaceReplicated(namespaceName)) { + getNamespaceReplicatedClusters(namespaceName) + .stream().filter(cluster -> !cluster.equals(pulsar().getConfiguration().getClusterName())) + .forEach(replicatedClusters::add); + } + createLocalFuture.whenComplete((ignored, ex) -> { if (ex != null) { log.error("[{}] Failed to create partitions for topic {}", clientAppId(), topicName, ex.getCause()); if (ex.getCause() instanceof RestException) { @@ -669,14 +672,20 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n return; } - if (!createLocalTopicOnly && topicName.isGlobal() && isNamespaceReplicated(namespaceName)) { - getNamespaceReplicatedClusters(namespaceName) - .stream() - .filter(cluster -> !cluster.equals(pulsar().getConfiguration().getClusterName())) - .forEach(cluster -> createFutureList.add( - ((TopicsImpl) pulsar().getBrokerService().getClusterPulsarAdmin(cluster).topics()) + if (!replicatedClusters.isEmpty()) { + replicatedClusters.forEach(cluster -> { + pulsar().getPulsarResources().getClusterResources().getClusterAsync(cluster) + .thenAccept(clusterDataOp -> { + ((TopicsImpl) pulsar().getBrokerService() + .getClusterPulsarAdmin(cluster, clusterDataOp).topics()) .createPartitionedTopicAsync( - topicName.getPartitionedTopicName(), numPartitions, true))); + topicName.getPartitionedTopicName(), numPartitions, true); + }) + .exceptionally(throwable -> { + log.error("Failed to create partition topic in cluster {}.", cluster, throwable); + return null; + }); + }); } log.info("[{}] Successfully created partitions for topic {} in cluster {}", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 0271abdc05e42..a6d8f2cb028c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -508,9 +508,14 @@ private CompletableFuture updatePartitionInOtherCluster(int numPartitions, if (cluster.equals(pulsar().getConfig().getClusterName())) { return; } - results.add(pulsar().getBrokerService().getClusterPulsarAdmin(cluster).topics() - .updatePartitionedTopicAsync(topicName.toString(), - numPartitions, true, false)); + CompletableFuture updatePartitionTopicFuture = + pulsar().getPulsarResources().getClusterResources().getClusterAsync(cluster) + .thenApply(clusterDataOp -> + pulsar().getBrokerService().getClusterPulsarAdmin(cluster, clusterDataOp)) + .thenCompose(pulsarAdmin -> + pulsarAdmin.topics().updatePartitionedTopicAsync( + topicName.toString(), numPartitions, true, false)); + results.add(updatePartitionTopicFuture); }); return FutureUtil.waitForAll(results); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index b7e85dadb2464..59ec74fee29a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -32,6 +32,7 @@ import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,14 +65,14 @@ protected enum State { } public AbstractReplicator(String topicName, String replicatorPrefix, String localCluster, String remoteCluster, - BrokerService brokerService) throws NamingException, PulsarServerException { - validatePartitionedTopic(topicName, brokerService); + BrokerService brokerService, PulsarClientImpl replicationClient) + throws PulsarServerException { this.brokerService = brokerService; this.topicName = topicName; this.replicatorPrefix = replicatorPrefix; this.localCluster = localCluster.intern(); this.remoteCluster = remoteCluster.intern(); - this.replicationClient = (PulsarClientImpl) brokerService.getReplicationClient(remoteCluster); + this.replicationClient = replicationClient; this.client = (PulsarClientImpl) brokerService.pulsar().getClient(); this.producer = null; this.producerQueueSize = brokerService.pulsar().getConfiguration().getReplicationProducerQueueSize(); @@ -242,20 +243,18 @@ public static String getReplicatorName(String replicatorPrefix, String cluster) * @param topic * @param brokerService */ - private void validatePartitionedTopic(String topic, BrokerService brokerService) throws NamingException { + public static CompletableFuture validatePartitionedTopicAsync(String topic, BrokerService brokerService) { TopicName topicName = TopicName.get(topic); - boolean isPartitionedTopic = false; - try { - isPartitionedTopic = - brokerService.pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .partitionedTopicExists(topicName); - } catch (Exception e) { - log.warn("Failed to verify partitioned topic {}-{}", topicName, e.getMessage()); - } - if (isPartitionedTopic) { - throw new NamingException( - topicName + " is a partitioned-topic and replication can't be started for partitioned-producer "); - } + return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExistsAsync(topicName).thenCompose(isPartitionedTopic -> { + if (isPartitionedTopic) { + String s = topicName + + " is a partitioned-topic and replication can't be started for partitioned-producer "; + log.error(s); + return FutureUtil.failedFuture(new NamingException(s)); + } + return CompletableFuture.completedFuture(null); + }); } private static final Logger log = LoggerFactory.getLogger(AbstractReplicator.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index c3a241d6077ee..456af1f0c1783 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -994,6 +994,7 @@ public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { } private CompletableFuture> createNonPersistentTopic(String topic) { + CompletableFuture> topicFuture = new CompletableFuture<>(); if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) { if (log.isDebugEnabled()) { log.debug("Broker is unable to load non-persistent topic {}", topic); @@ -1003,27 +1004,40 @@ private CompletableFuture> createNonPersistentTopic(String topic } final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); NonPersistentTopic nonPersistentTopic = new NonPersistentTopic(topic, this); - - CompletableFuture> future = nonPersistentTopic.initialize() - .thenCompose(__ -> nonPersistentTopic.checkReplication()) - .thenApply(__ -> { - log.info("Created topic {}", nonPersistentTopic); - long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs; - pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); - addTopicToStatsMaps(TopicName.get(topic), nonPersistentTopic); - return Optional.of(nonPersistentTopic); - }); - - future.exceptionally((ex) -> { - log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex); - nonPersistentTopic.stopReplProducers().whenComplete((v, exception) -> { - pulsar.getExecutor().execute(() -> topics.remove(topic, future)); + CompletableFuture isOwner = checkTopicNsOwnership(topic); + isOwner.thenRun(() -> { + nonPersistentTopic.initialize() + .thenCompose(__ -> nonPersistentTopic.checkReplication()) + .thenRun(() -> { + log.info("Created topic {}", nonPersistentTopic); + long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs; + pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); + addTopicToStatsMaps(TopicName.get(topic), nonPersistentTopic); + topicFuture.complete(Optional.of(nonPersistentTopic)); + }).exceptionally(ex -> { + log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex.getCause()); + nonPersistentTopic.stopReplProducers().whenComplete((v, exception) -> { + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + topicFuture.completeExceptionally(ex); + }); + return null; }); - + }).exceptionally(e -> { + log.warn("CheckTopicNsOwnership fail when createNonPersistentTopic! {}", topic, e.getCause()); + // CheckTopicNsOwnership fail dont create nonPersistentTopic, when topic do lookup will find the correct + // broker. When client get non-persistent-partitioned topic + // metadata will the non-persistent-topic will be created. + // so we should add checkTopicNsOwnership logic otherwise the topic will be created + // if it dont own by this broker,we should return success + // otherwise it will keep retrying getPartitionedTopicMetadata + topicFuture.complete(Optional.of(nonPersistentTopic)); + // after get metadata return success, we should delete this topic from this broker, because this topic not + // owner by this broker and it don't initialize and checkReplication + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); return null; }); - return future; + return topicFuture; } private CompletableFuture futureWithDeadline() { @@ -1031,7 +1045,7 @@ private CompletableFuture futureWithDeadline() { () -> FUTURE_DEADLINE_TIMEOUT_EXCEPTION); } - public PulsarClient getReplicationClient(String cluster) { + public PulsarClient getReplicationClient(String cluster, Optional clusterDataOp) { PulsarClient client = replicationClients.get(cluster); if (client != null) { return client; @@ -1039,7 +1053,7 @@ public PulsarClient getReplicationClient(String cluster) { return replicationClients.computeIfAbsent(cluster, key -> { try { - ClusterData data = pulsar.getPulsarResources().getClusterResources().getCluster(cluster) + ClusterData data = clusterDataOp .orElseThrow(() -> new MetadataStoreException.NotFoundException(cluster)); ClientBuilder clientBuilder = PulsarClient.builder() .enableTcpNoDelay(false) @@ -1106,14 +1120,14 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, } } - public PulsarAdmin getClusterPulsarAdmin(String cluster) { + public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { PulsarAdmin admin = clusterAdmins.get(cluster); if (admin != null) { return admin; } return clusterAdmins.computeIfAbsent(cluster, key -> { try { - ClusterData data = pulsar.getPulsarResources().getClusterResources().getCluster(cluster) + ClusterData data = clusterDataOp .orElseThrow(() -> new MetadataStoreException.NotFoundException(cluster)); ServiceConfiguration conf = pulsar.getConfig(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index dd57fd91337cc..ce1fe3443f5f3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -28,13 +28,13 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.BrokerServiceException.NamingException; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.SendCallback; import org.apache.pulsar.common.policies.data.stats.NonPersistentReplicatorStatsImpl; import org.apache.pulsar.common.stats.Rate; @@ -49,8 +49,9 @@ public class NonPersistentReplicator extends AbstractReplicator implements Repli private final NonPersistentReplicatorStatsImpl stats = new NonPersistentReplicatorStatsImpl(); public NonPersistentReplicator(NonPersistentTopic topic, String localCluster, String remoteCluster, - BrokerService brokerService) throws NamingException, PulsarServerException { - super(topic.getName(), topic.getReplicatorPrefix(), localCluster, remoteCluster, brokerService); + BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { + super(topic.getName(), topic.getReplicatorPrefix(), localCluster, remoteCluster, brokerService, + replicationClient); producerBuilder.blockIfQueueFull(false); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index c1687a35f50e3..2c62f2ed92a59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -32,13 +32,14 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -61,6 +62,7 @@ import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.KeySharedMeta; @@ -528,14 +530,7 @@ public CompletableFuture checkReplication() { } if (!replicators.containsKey(cluster)) { - if (!startReplicator(cluster)) { - // it happens when global topic is a partitioned topic and replicator can't start on - // original - // non partitioned-topic (topic without partition prefix) - return FutureUtil - .failedFuture(new NamingException( - topic + " failed to start replicator for " + cluster)); - } + futures.add(startReplicator(cluster)); } } @@ -553,29 +548,35 @@ public CompletableFuture checkReplication() { } - boolean startReplicator(String remoteCluster) { + CompletableFuture startReplicator(String remoteCluster) { log.info("[{}] Starting replicator to remote: {}", topic, remoteCluster); String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); return addReplicationCluster(remoteCluster, NonPersistentTopic.this, localCluster); } - protected boolean addReplicationCluster(String remoteCluster, NonPersistentTopic nonPersistentTopic, + protected CompletableFuture addReplicationCluster(String remoteCluster, NonPersistentTopic nonPersistentTopic, String localCluster) { - AtomicBoolean isReplicatorStarted = new AtomicBoolean(true); - replicators.computeIfAbsent(remoteCluster, r -> { - try { - return new NonPersistentReplicator(NonPersistentTopic.this, localCluster, remoteCluster, brokerService); - } catch (NamingException | PulsarServerException e) { - isReplicatorStarted.set(false); - log.error("[{}] Replicator startup failed due to partitioned-topic {}", topic, remoteCluster); - } - return null; - }); - // clean up replicator if startup is failed - if (!isReplicatorStarted.get()) { - replicators.remove(remoteCluster); - } - return isReplicatorStarted.get(); + return AbstractReplicator.validatePartitionedTopicAsync(nonPersistentTopic.getName(), brokerService) + .thenCompose(__ -> brokerService.pulsar().getPulsarResources().getClusterResources() + .getClusterAsync(remoteCluster) + .thenApply(clusterData -> + brokerService.getReplicationClient(remoteCluster, clusterData))) + .thenAccept(replicationClient -> { + replicators.computeIfAbsent(remoteCluster, r -> { + try { + return new NonPersistentReplicator(NonPersistentTopic.this, localCluster, + remoteCluster, brokerService, (PulsarClientImpl) replicationClient); + } catch (PulsarServerException e) { + log.error("[{}] Replicator startup failed {}", topic, remoteCluster, e); + } + return null; + }); + + // clean up replicator if startup is failed + if (replicators.containsKey(remoteCluster) && replicators.get(remoteCluster) == null) { + replicators.remove(remoteCluster); + } + }); } CompletableFuture removeReplicator(String remoteCluster) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index aa53e150672ea..4d79c9a37cf72 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -45,7 +45,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.BrokerServiceException.NamingException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; @@ -54,6 +53,7 @@ import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.SendCallback; import org.apache.pulsar.common.api.proto.MarkerType; import org.apache.pulsar.common.policies.data.Policies; @@ -105,8 +105,10 @@ public class PersistentReplicator extends AbstractReplicator private final ReplicatorStatsImpl stats = new ReplicatorStatsImpl(); public PersistentReplicator(PersistentTopic topic, ManagedCursor cursor, String localCluster, String remoteCluster, - BrokerService brokerService) throws NamingException, PulsarServerException { - super(topic.getName(), topic.getReplicatorPrefix(), localCluster, remoteCluster, brokerService); + BrokerService brokerService, PulsarClientImpl replicationClient) + throws PulsarServerException { + super(topic.getName(), topic.getReplicatorPrefix(), localCluster, remoteCluster, brokerService, + replicationClient); this.topic = topic; this.cursor = cursor; this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 64e4b7ff4ed49..f74060c93f4c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -76,6 +76,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources; +import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -116,6 +117,7 @@ import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; @@ -265,20 +267,8 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS this.compactedTopic = new CompactedTopicImpl(brokerService.pulsar().getBookKeeperClient()); for (ManagedCursor cursor : ledger.getCursors()) { - if (cursor.getName().startsWith(replicatorPrefix)) { - String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); - String remoteCluster = PersistentReplicator.getRemoteCluster(cursor.getName()); - boolean isReplicatorStarted = false; - try { - isReplicatorStarted = addReplicationCluster(remoteCluster, cursor, localCluster); - } catch (Exception e) { - log.warn("[{}] failed to start replication", topic, e); - } - if (!isReplicatorStarted) { - throw new NamingException( - PersistentTopic.this.getName() + " Failed to start replicator " + remoteCluster); - } - } else if (cursor.getName().equals(DEDUPLICATION_CURSOR_NAME)) { + if (cursor.getName().equals(DEDUPLICATION_CURSOR_NAME) + || cursor.getName().startsWith(replicatorPrefix)) { // This is not a regular subscription, we are going to // ignore it for now and let the message dedup logic to take care of it } else { @@ -309,7 +299,16 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS @Override public CompletableFuture initialize() { - return brokerService.pulsar().getPulsarResources().getNamespaceResources() + List> futures = new ArrayList<>(); + for (ManagedCursor cursor : ledger.getCursors()) { + if (cursor.getName().startsWith(replicatorPrefix)) { + String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); + String remoteCluster = PersistentReplicator.getRemoteCluster(cursor.getName()); + futures.add(addReplicationCluster(remoteCluster, cursor, localCluster)); + } + } + return FutureUtil.waitForAll(futures).thenCompose(__ -> + brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) .thenAccept(optPolicies -> { if (!optPolicies.isPresent()) { @@ -338,7 +337,7 @@ public CompletableFuture initialize() { updateUnackedMessagesAppliedOnSubscription(null); updateUnackedMessagesExceededOnConsumer(null); return null; - }); + })); } // for testing purposes @@ -1519,13 +1518,13 @@ CompletableFuture startReplicator(String remoteCluster) { @Override public void openCursorComplete(ManagedCursor cursor, Object ctx) { String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); - boolean isReplicatorStarted = addReplicationCluster(remoteCluster, cursor, localCluster); - if (isReplicatorStarted) { - future.complete(null); - } else { - future.completeExceptionally(new NamingException( - PersistentTopic.this.getName() + " Failed to start replicator " + remoteCluster)); - } + addReplicationCluster(remoteCluster, cursor, localCluster).whenComplete((__, ex) -> { + if (ex == null) { + future.complete(null); + } else { + future.completeExceptionally(ex); + } + }); } @Override @@ -1538,23 +1537,29 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { return future; } - protected boolean addReplicationCluster(String remoteCluster, ManagedCursor cursor, String localCluster) { - AtomicBoolean isReplicatorStarted = new AtomicBoolean(true); - replicators.computeIfAbsent(remoteCluster, r -> { - try { - return new PersistentReplicator(PersistentTopic.this, cursor, localCluster, remoteCluster, - brokerService); - } catch (NamingException | PulsarServerException e) { - isReplicatorStarted.set(false); - log.error("[{}] Replicator startup failed due to partitioned-topic {}", topic, remoteCluster); - } - return null; - }); - // clean up replicator if startup is failed - if (!isReplicatorStarted.get()) { - replicators.remove(remoteCluster); - } - return isReplicatorStarted.get(); + protected CompletableFuture addReplicationCluster(String remoteCluster, ManagedCursor cursor, + String localCluster) { + return AbstractReplicator.validatePartitionedTopicAsync(PersistentTopic.this.getName(), brokerService) + .thenCompose(__ -> brokerService.pulsar().getPulsarResources().getClusterResources() + .getClusterAsync(remoteCluster) + .thenApply(clusterData -> + brokerService.getReplicationClient(remoteCluster, clusterData))) + .thenAccept(replicationClient -> { + replicators.computeIfAbsent(remoteCluster, r -> { + try { + return new PersistentReplicator(PersistentTopic.this, cursor, localCluster, + remoteCluster, brokerService, (PulsarClientImpl) replicationClient); + } catch (PulsarServerException e) { + log.error("[{}] Replicator startup failed {}", topic, remoteCluster, e); + } + return null; + }); + + // clean up replicator if startup is failed + if (replicators.containsKey(remoteCluster) && replicators.get(remoteCluster) == null) { + replicators.remove(remoteCluster); + } + }); } CompletableFuture removeReplicator(String remoteCluster) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 8603404a86a31..543d897575756 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -1768,7 +1768,10 @@ public void testAtomicReplicationRemoval() throws Exception { doReturn(remoteCluster).when(cursor).getName(); brokerService.getReplicationClients().put(remoteCluster, client); PersistentReplicator replicator = spy( - new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService)); + new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService, + (PulsarClientImpl) brokerService.getReplicationClient(remoteCluster, + brokerService.pulsar().getPulsarResources().getClusterResources() + .getCluster(remoteCluster)))); replicatorMap.put(remoteReplicatorName, replicator); // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed @@ -1813,7 +1816,10 @@ public void testClosingReplicationProducerTwice() throws Exception { ManagedCursor cursor = mock(ManagedCursorImpl.class); doReturn(remoteCluster).when(cursor).getName(); brokerService.getReplicationClients().put(remoteCluster, client); - PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService); + PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, + brokerService, (PulsarClientImpl) brokerService.getReplicationClient(remoteCluster, + brokerService.pulsar().getPulsarResources().getClusterResources() + .getCluster(remoteCluster))); // PersistentReplicator constructor calls startProducer() verify(clientImpl) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRemoveClusterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRemoveClusterTest.java index 701ab47fa7f69..65e90966ea541 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRemoveClusterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRemoveClusterTest.java @@ -75,7 +75,9 @@ public void testRemoveClusterFromNamespace() throws Exception { admin1.namespaces().createNamespace("pulsar1/ns1", Sets.newHashSet("r1", "r2", "r3")); - PulsarClient repClient1 = pulsar1.getBrokerService().getReplicationClient("r3"); + PulsarClient repClient1 = pulsar1.getBrokerService().getReplicationClient("r3", + pulsar1.getBrokerService().pulsar().getPulsarResources().getClusterResources() + .getCluster("r3")); Assert.assertNotNull(repClient1); Assert.assertFalse(repClient1.isClosed()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 9b4fcc87aa186..11c8e197f8df9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -262,7 +262,9 @@ public void testConcurrentReplicator() throws Exception { .getOrCreateTopic(topicName.toString()).get(); PulsarClientImpl pulsarClient = spy((PulsarClientImpl) pulsar1.getBrokerService() - .getReplicationClient("r3")); + .getReplicationClient("r3", + pulsar1.getBrokerService().pulsar().getPulsarResources().getClusterResources() + .getCluster("r3"))); final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class); startRepl.setAccessible(true); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/GeoReplicationTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/GeoReplicationTest.java new file mode 100644 index 0000000000000..75ce3630b80e7 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/GeoReplicationTest.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.messaging; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.tests.integration.topologies.PulsarGeoClusterTestBase; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +/** + * Geo replication test. + */ +@Slf4j +public class GeoReplicationTest extends PulsarGeoClusterTestBase { + + @BeforeClass(alwaysRun = true) + public final void setupBeforeClass() throws Exception { + setup(); + } + + @AfterClass(alwaysRun = true) + public final void tearDownAfterClass() throws Exception { + cleanup(); + } + + @Test(timeOut = 1000 * 30, dataProvider = "TopicDomain") + public void testTopicReplication(String domain) throws Exception { + String cluster1 = getGeoCluster().getClusters()[0].getClusterName(); + String cluster2 = getGeoCluster().getClusters()[1].getClusterName(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder() + .serviceHttpUrl(getGeoCluster().getClusters()[0].getHttpServiceUrl()) + .requestTimeout(30, TimeUnit.SECONDS) + .build(); + + String topic = domain + "://public/default/testTopicReplication-" + UUID.randomUUID(); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + try { + admin.topics().createPartitionedTopic(topic, 10); + } catch (Exception e) { + log.error("Failed to create partitioned topic {}.", topic, e); + Assert.fail("Failed to create partitioned topic " + topic); + } + Assert.assertEquals(admin.topics().getPartitionedTopicMetadata(topic).partitions, 10); + }); + log.info("Test geo-replication produce and consume for topic {}.", topic); + + @Cleanup + PulsarClient client1 = PulsarClient.builder() + .serviceUrl(getGeoCluster().getClusters()[0].getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarClient client2 = PulsarClient.builder() + .serviceUrl(getGeoCluster().getClusters()[1].getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer p = client1.newProducer() + .topic(topic) + .create(); + log.info("Successfully create producer in cluster {} for topic {}.", cluster1, topic); + + @Cleanup + Consumer c = client2.newConsumer() + .topic(topic) + .subscriptionName("geo-sub") + .subscribe(); + log.info("Successfully create consumer in cluster {} for topic {}.", cluster2, topic); + + for (int i = 0; i < 10; i++) { + p.send(String.format("Message [%d]", i).getBytes(StandardCharsets.UTF_8)); + } + log.info("Successfully produce message to cluster {} for topic {}.", cluster1, topic); + + for (int i = 0; i < 10; i++) { + Message message = c.receive(10, TimeUnit.SECONDS); + Assert.assertNotNull(message); + } + log.info("Successfully consume message from cluster {} for topic {}.", cluster2, topic); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index 043762ce1f761..2191799b68622 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -70,7 +70,14 @@ public class PulsarCluster { * @return the built pulsar cluster */ public static PulsarCluster forSpec(PulsarClusterSpec spec) { - return new PulsarCluster(spec); + CSContainer csContainer = new CSContainer(spec.clusterName) + .withNetwork(Network.newNetwork()) + .withNetworkAliases(CSContainer.NAME); + return new PulsarCluster(spec, csContainer, false); + } + + public static PulsarCluster forSpec(PulsarClusterSpec spec, CSContainer csContainer) { + return new PulsarCluster(spec, csContainer, true); } private final PulsarClusterSpec spec; @@ -80,6 +87,7 @@ public static PulsarCluster forSpec(PulsarClusterSpec spec) { private final Network network; private final ZKContainer zkContainer; private final CSContainer csContainer; + private final boolean sharedCsContainer; private final Map bookieContainers; private final Map brokerContainers; private final Map workerContainers; @@ -90,11 +98,12 @@ public static PulsarCluster forSpec(PulsarClusterSpec spec) { private Map> externalServices = Collections.emptyMap(); private final boolean enablePrestoWorker; - private PulsarCluster(PulsarClusterSpec spec) { + private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean sharedCsContainer) { this.spec = spec; + this.sharedCsContainer = sharedCsContainer; this.clusterName = spec.clusterName(); - this.network = Network.newNetwork(); + this.network = csContainer.getNetwork(); this.enablePrestoWorker = spec.enablePrestoWorker(); this.sqlFollowWorkerContainers = Maps.newTreeMap(); @@ -109,26 +118,24 @@ private PulsarCluster(PulsarClusterSpec spec) { this.zkContainer = new ZKContainer(clusterName); this.zkContainer .withNetwork(network) - .withNetworkAliases(ZKContainer.NAME) + .withNetworkAliases(appendClusterName(ZKContainer.NAME)) .withEnv("clusterName", clusterName) - .withEnv("zkServers", ZKContainer.NAME) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) .withEnv("configurationStore", CSContainer.NAME + ":" + CS_PORT) .withEnv("forceSync", "no") - .withEnv("pulsarNode", "pulsar-broker-0"); + .withEnv("pulsarNode", appendClusterName("pulsar-broker-0")); - this.csContainer = new CSContainer(clusterName) - .withNetwork(network) - .withNetworkAliases(CSContainer.NAME); + this.csContainer = csContainer; this.bookieContainers = Maps.newTreeMap(); this.brokerContainers = Maps.newTreeMap(); this.workerContainers = Maps.newTreeMap(); - this.proxyContainer = new ProxyContainer(clusterName, ProxyContainer.NAME) + this.proxyContainer = new ProxyContainer(appendClusterName("pulsar-proxy"), ProxyContainer.NAME) .withNetwork(network) - .withNetworkAliases("pulsar-proxy") - .withEnv("zkServers", ZKContainer.NAME) - .withEnv("zookeeperServers", ZKContainer.NAME) + .withNetworkAliases(appendClusterName("pulsar-proxy")) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) + .withEnv("zookeeperServers", appendClusterName(ZKContainer.NAME)) .withEnv("configurationStoreServers", CSContainer.NAME + ":" + CS_PORT) .withEnv("clusterName", clusterName); if (spec.proxyEnvs != null) { @@ -142,8 +149,8 @@ private PulsarCluster(PulsarClusterSpec spec) { bookieContainers.putAll( runNumContainers("bookie", spec.numBookies(), (name) -> new BKContainer(clusterName, name) .withNetwork(network) - .withNetworkAliases(name) - .withEnv("zkServers", ZKContainer.NAME) + .withNetworkAliases(appendClusterName(name)) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) .withEnv("useHostNameAsBookieID", "true") // Disable fsyncs for tests since they're slow within the containers .withEnv("journalSyncData", "false") @@ -156,11 +163,11 @@ private PulsarCluster(PulsarClusterSpec spec) { // create brokers brokerContainers.putAll( runNumContainers("broker", spec.numBrokers(), (name) -> { - BrokerContainer brokerContainer = new BrokerContainer(clusterName, name) + BrokerContainer brokerContainer = new BrokerContainer(clusterName, appendClusterName(name)) .withNetwork(network) - .withNetworkAliases(name) - .withEnv("zkServers", ZKContainer.NAME) - .withEnv("zookeeperServers", ZKContainer.NAME) + .withNetworkAliases(appendClusterName(name)) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) + .withEnv("zookeeperServers", appendClusterName(ZKContainer.NAME)) .withEnv("configurationStoreServers", CSContainer.NAME + ":" + CS_PORT) .withEnv("clusterName", clusterName) .withEnv("brokerServiceCompactionMonitorIntervalInSeconds", "1") @@ -235,8 +242,10 @@ public void start() throws Exception { log.info("Successfully started local zookeeper container."); // start the configuration store - csContainer.start(); - log.info("Successfully started configuration store container."); + if (!sharedCsContainer) { + csContainer.start(); + log.info("Successfully started configuration store container."); + } // init the cluster zkContainer.execCmd( @@ -335,9 +344,11 @@ public synchronized void stop() { if (null != proxyContainer) { containers.add(proxyContainer); } - if (null != csContainer) { + + if (!sharedCsContainer && null != csContainer) { containers.add(csContainer); } + if (null != zkContainer) { containers.add(zkContainer); } @@ -668,4 +679,8 @@ public void dumpFunctionLogs(String name) { } } } + + private String appendClusterName(String name) { + return sharedCsContainer ? clusterName + "-" + name : name; + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoCluster.java new file mode 100644 index 0000000000000..9be3c382b7035 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoCluster.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.topologies; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.containers.CSContainer; +import org.testcontainers.containers.Network; + +@Slf4j +public class PulsarGeoCluster { + + @Getter + private final PulsarClusterSpec[] clusterSpecs; + + @Getter + private final CSContainer csContainer; + + @Getter + private final PulsarCluster[] clusters; + + /** + * Pulsar Cluster Spec + * + * @param specs each pulsar cluster spec. + * @return the built a pulsar cluster with geo replication + */ + public static PulsarGeoCluster forSpec(PulsarClusterSpec... specs) { + return new PulsarGeoCluster(specs); + } + + public PulsarGeoCluster(PulsarClusterSpec... clusterSpecs) { + this.clusterSpecs = clusterSpecs; + this.clusters = new PulsarCluster[clusterSpecs.length]; + + this.csContainer = new CSContainer("geo-cluster") + .withNetwork(Network.newNetwork()) + .withNetworkAliases(CSContainer.NAME); + + for (int i = 0; i < this.clusters.length; i++) { + clusters[i] = PulsarCluster.forSpec(this.clusterSpecs[i], this.csContainer); + } + } + + public void start() throws Exception { + // start the configuration store + this.csContainer.start(); + log.info("Successfully started configuration store container."); + + for (PulsarCluster cluster : clusters) { + cluster.start(); + log.info("Successfully started all components for cluster {}.", cluster.getClusterName()); + } + } + + public void stop() throws Exception { + for (PulsarCluster cluster : clusters) { + cluster.stop(); + log.info("Successfully stopped all components for cluster {}.", cluster.getClusterName()); + } + // stop the configuration store + this.csContainer.stop(); + log.info("Successfully stopped configuration store container."); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoClusterTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoClusterTestBase.java new file mode 100644 index 0000000000000..51c74eee50b18 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarGeoClusterTestBase.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.topologies; + +import static java.util.stream.Collectors.joining; +import java.util.stream.Stream; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class PulsarGeoClusterTestBase extends PulsarTestBase { + + @Override + protected final void setup() throws Exception { + setupCluster(); + } + + @Override + protected final void cleanup() throws Exception { + tearDownCluster(); + } + + protected void setupCluster() throws Exception { + this.setupCluster(""); + } + + @Getter + private PulsarGeoCluster geoCluster; + + public void setupCluster(String namePrefix) throws Exception { + PulsarClusterSpec.PulsarClusterSpecBuilder[] specBuilders = new PulsarClusterSpec.PulsarClusterSpecBuilder[2]; + for (int i = 0; i < 2; i++) { + String clusterName = Stream.of(this.getClass().getSimpleName(), namePrefix, String.valueOf(i), + randomName(5)) + .filter(s -> s != null && !s.isEmpty()) + .collect(joining("-")); + specBuilders[i] = PulsarClusterSpec.builder().clusterName(clusterName); + } + specBuilders = beforeSetupCluster(specBuilders); + PulsarClusterSpec[] specs = new PulsarClusterSpec[2]; + for (int i = 0; i < specBuilders.length; i++) { + specs[i] = specBuilders[i].build(); + } + setupCluster0(specs); + } + + protected PulsarClusterSpec.PulsarClusterSpecBuilder[] beforeSetupCluster ( + PulsarClusterSpec.PulsarClusterSpecBuilder... specBuilder) { + return specBuilder; + } + + protected void setupCluster0(PulsarClusterSpec... specs) throws Exception { + incrementSetupNumber(); + log.info("Setting up geo cluster with {} local clusters}", specs.length); + + this.geoCluster = PulsarGeoCluster.forSpec(specs); + + beforeStartCluster(); + + this.geoCluster.start(); + + log.info("Geo Cluster is setup!"); + } + + protected void beforeStartCluster() throws Exception { + // no-op + } + + public void tearDownCluster() throws Exception { + markCurrentSetupNumberCleaned(); + if (null != this.geoCluster) { + this.geoCluster.stop(); + } + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java index 9989b15faa95e..ebdfbe84de401 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java @@ -34,9 +34,18 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.tests.TestRetrySupport; import org.testng.Assert; +import org.testng.annotations.DataProvider; public abstract class PulsarTestBase extends TestRetrySupport { + @DataProvider(name = "TopicDomain") + public Object[][] topicDomain() { + return new Object[][] { + {"persistent"}, + {"non-persistent"} + }; + } + public static String randomName(int numChars) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < numChars; i++) { diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index aa31852fd6f62..6421561fa8a75 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -24,6 +24,7 @@ + From 4939204baad8241c3fb1a92d35d3ac7650754e04 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 9 Nov 2021 14:31:33 +0100 Subject: [PATCH 015/823] Pulsar Client: restore SchemaInfo.builder() API (#12673) (cherry picked from commit 849e4dc5fa59b774287436a054efb17d198054b4) --- .../AdminApiSchemaValidationEnforced.java | 8 ++-- .../JsonSchemaCompatibilityCheckTest.java | 3 +- .../SchemaCompatibilityCheckTest.java | 5 +- .../client/admin/internal/SchemasImpl.java | 3 +- .../PulsarClientImplementationBinding.java | 2 + .../pulsar/common/schema/SchemaInfo.java | 48 +++++++++++++++++++ ...PulsarClientImplementationBindingImpl.java | 34 ++----------- .../client/impl/schema/SchemaInfoTest.java | 6 +-- .../common/protocol/schema/SchemaData.java | 3 +- .../schema/KafkaSchemaWrappedSchema.java | 3 +- .../pulsar/io/kafka/AvroSchemaCache.java | 3 +- .../apache/pulsar/sql/presto/PulsarSplit.java | 3 +- .../pulsar/sql/presto/TestPulsarMetadata.java | 4 +- .../primitive/TestPrimitiveDecoder.java | 24 +++++----- 14 files changed, 82 insertions(+), 67 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java index 3daf920c975bc..b7747de67595e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java @@ -30,9 +30,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.schema.SchemaInfo; @@ -98,7 +96,7 @@ public void testDisableSchemaValidationEnforcedHasSchema() throws Exception { assertTrue(e.getMessage().contains("HTTP 404 Not Found")); } Map properties = Maps.newHashMap(); - SchemaInfo schemaInfo = SchemaInfoImpl.builder() + SchemaInfo schemaInfo = SchemaInfo.builder() .type(SchemaType.STRING) .properties(properties) .name("test") @@ -147,7 +145,7 @@ public void testEnableSchemaValidationEnforcedHasSchemaMismatch() throws Excepti } Map properties = Maps.newHashMap(); properties.put("key1", "value1"); - SchemaInfo schemaInfo = SchemaInfoImpl.builder() + SchemaInfo schemaInfo = SchemaInfo.builder() .type(SchemaType.STRING) .properties(properties) .name("test") @@ -177,7 +175,7 @@ public void testEnableSchemaValidationEnforcedHasSchemaMatch() throws Exception } admin.namespaces().setSchemaValidationEnforced(namespace,true); Map properties = Maps.newHashMap(); - SchemaInfo schemaInfo = SchemaInfoImpl.builder() + SchemaInfo schemaInfo = SchemaInfo.builder() .type(SchemaType.STRING) .properties(properties) .name("test") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java index 32a9f9e78a874..9bf0189037a3a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java @@ -33,7 +33,6 @@ import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaInfo; @@ -119,7 +118,7 @@ public static OldJSONSchema of(Class pojo, Map propert JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper); JsonSchema schema = schemaGen.generateSchema(pojo); - SchemaInfo info = SchemaInfoImpl.builder() + SchemaInfo info = SchemaInfo.builder() .name("") .properties(properties) .type(SchemaType.JSON) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 8def5dc3f06e6..02913c628f27f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -35,15 +35,12 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfo; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.schema.Schemas; @@ -324,7 +321,7 @@ public void testSchemaComparison() throws Exception { SchemaCompatibilityStrategy.FULL); byte[] changeSchemaBytes = (new String(Schema.AVRO(Schemas.PersonOne.class) .getSchemaInfo().getSchema(), UTF_8) + "/n /n /n").getBytes(); - SchemaInfo schemaInfo = SchemaInfoImpl.builder().type(SchemaType.AVRO).schema(changeSchemaBytes).build(); + SchemaInfo schemaInfo = SchemaInfo.builder().type(SchemaType.AVRO).schema(changeSchemaBytes).build(); admin.schemas().createSchema(fqtn, schemaInfo); admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java index a072acd8b73a4..54e731e1b979b 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java @@ -32,7 +32,6 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Schemas; import org.apache.pulsar.client.api.Authentication; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.schema.DeleteSchemaResponse; @@ -449,7 +448,7 @@ static SchemaInfo convertGetSchemaResponseToSchemaInfo(TopicName tn, schema = response.getData().getBytes(UTF_8); } - return SchemaInfoImpl.builder() + return SchemaInfo.builder() .schema(schema) .type(response.getType()) .properties(response.getProperties()) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java index f7bcf05230e82..c8c300cc2eb1b 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java @@ -251,4 +251,6 @@ static byte[] getBytes(ByteBuffer byteBuffer) { byteBuffer.get(array); return array; } + + SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, Map propertiesValue); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java index 01ba7465c112b..e05b0d24cb946 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.common.schema; +import java.util.Collections; import java.util.Map; +import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; @@ -48,4 +50,50 @@ public interface SchemaInfo { Map getProperties(); String getSchemaDefinition(); + + static SchemaInfoBuilder builder() { + return new SchemaInfoBuilder(); + } + + class SchemaInfoBuilder { + private String name; + private byte[] schema; + private SchemaType type; + private Map properties; + private boolean propertiesSet; + + SchemaInfoBuilder() { + } + + public SchemaInfoBuilder name(String name) { + this.name = name; + return this; + } + + public SchemaInfoBuilder schema(byte[] schema) { + this.schema = schema; + return this; + } + + public SchemaInfoBuilder type(SchemaType type) { + this.type = type; + return this; + } + + public SchemaInfoBuilder properties(Map properties) { + this.properties = properties; + this.propertiesSet = true; + return this; + } + + public SchemaInfo build() { + Map propertiesValue = this.properties; + if (!this.propertiesSet) { + propertiesValue = Collections.emptyMap(); + } + return DefaultImplementation + .getDefaultImplementation() + .newSchemaInfoImpl(name, schema, type, propertiesValue); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java index c146f238d55fc..c7f3af9227907 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java @@ -45,35 +45,7 @@ import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.client.impl.auth.AuthenticationToken; -import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; -import org.apache.pulsar.client.impl.schema.AutoProduceBytesSchema; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.client.impl.schema.BooleanSchema; -import org.apache.pulsar.client.impl.schema.ByteBufferSchema; -import org.apache.pulsar.client.impl.schema.ByteSchema; -import org.apache.pulsar.client.impl.schema.BytesSchema; -import org.apache.pulsar.client.impl.schema.DateSchema; -import org.apache.pulsar.client.impl.schema.DoubleSchema; -import org.apache.pulsar.client.impl.schema.FloatSchema; -import org.apache.pulsar.client.impl.schema.InstantSchema; -import org.apache.pulsar.client.impl.schema.IntSchema; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; -import org.apache.pulsar.client.impl.schema.LocalDateSchema; -import org.apache.pulsar.client.impl.schema.LocalDateTimeSchema; -import org.apache.pulsar.client.impl.schema.LocalTimeSchema; -import org.apache.pulsar.client.impl.schema.LongSchema; -import org.apache.pulsar.client.impl.schema.NativeAvroBytesSchema; -import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema; -import org.apache.pulsar.client.impl.schema.ProtobufSchema; -import org.apache.pulsar.client.impl.schema.RecordSchemaBuilderImpl; -import org.apache.pulsar.client.impl.schema.SchemaDefinitionBuilderImpl; -import org.apache.pulsar.client.impl.schema.SchemaUtils; -import org.apache.pulsar.client.impl.schema.ShortSchema; -import org.apache.pulsar.client.impl.schema.StringSchema; -import org.apache.pulsar.client.impl.schema.TimeSchema; -import org.apache.pulsar.client.impl.schema.TimestampSchema; +import org.apache.pulsar.client.impl.schema.*; import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; import org.apache.pulsar.client.impl.schema.generic.GenericSchemaImpl; import org.apache.pulsar.client.internal.PulsarClientImplementationBinding; @@ -383,4 +355,8 @@ public BatcherBuilder newKeyBasedBatcherBuilder() { public MessagePayloadFactory newDefaultMessagePayloadFactory() { return new MessagePayloadFactoryImpl(); } + + public SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, Map propertiesValue) { + return new SchemaInfoImpl(name, schema, type, propertiesValue); + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java index f96e84e158345..719704c5c6941 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java @@ -289,7 +289,7 @@ public static class SchemaInfoBuilderTest { @Test public void testUnsetProperties() { - final SchemaInfo schemaInfo = SchemaInfoImpl.builder() + final SchemaInfo schemaInfo = SchemaInfo.builder() .type(SchemaType.STRING) .schema(new byte[0]) .name("string") @@ -305,7 +305,7 @@ public void testUnsetProperties() { public void testSetProperties() { final Map map = Maps.newHashMap(); map.put("test", "value"); - final SchemaInfo schemaInfo = SchemaInfoImpl.builder() + final SchemaInfo schemaInfo = SchemaInfo.builder() .type(SchemaType.STRING) .schema(new byte[0]) .name("string") @@ -323,7 +323,7 @@ public void testNullPropertyValue() { final Map map = new HashMap<>(); map.put("key", null); - SchemaInfo si = SchemaInfoImpl.builder() + SchemaInfo si = SchemaInfo.builder() .name("INT32") .schema(new byte[0]) .type(SchemaType.INT32) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaData.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaData.java index d5b440589288e..5c00f06d9698c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaData.java @@ -22,7 +22,6 @@ import java.util.Map; import lombok.Builder; import lombok.Data; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -46,7 +45,7 @@ public class SchemaData { * @return the converted schema info. */ public SchemaInfo toSchemaInfo() { - return SchemaInfoImpl.builder() + return SchemaInfo.builder() .name("") .type(type) .schema(data) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index ba57692edf17d..2db9d6cd93bc6 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -27,7 +27,6 @@ import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.storage.Converter; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -45,7 +44,7 @@ public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema Map props = new HashMap<>(); boolean isJsonConverter = converter instanceof JsonConverter; props.put(GenericAvroSchema.OFFSET_PROP, isJsonConverter ? "0" : "5"); - this.schemaInfo = SchemaInfoImpl.builder() + this.schemaInfo = SchemaInfo.builder() .name(isJsonConverter? "KafKaJson" : "KafkaAvro") .type(isJsonConverter ? SchemaType.JSON : SchemaType.AVRO) .schema(schema.toString().getBytes(UTF_8)) diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java index 2a9e1c44af456..eda8c96adc84a 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java @@ -25,7 +25,6 @@ import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -66,7 +65,7 @@ private Schema fetchSchema(int schemaId) { org.apache.avro.Schema schema = schemaRegistryClient.getById(schemaId); String definition = schema.toString(false); log.info("Schema {} definition {}", schemaId, definition); - SchemaInfo schemaInfo = SchemaInfoImpl.builder() + SchemaInfo schemaInfo = SchemaInfo.builder() .type(SchemaType.AVRO) .name(schema.getName()) .properties(Collections.emptyMap()) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java index 645edbd84f2d0..03a6b771bbaa1 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Map; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -102,7 +101,7 @@ public PulsarSplit( this.offloadPolicies = offloadPolicies; ObjectMapper objectMapper = new ObjectMapper(); - this.schemaInfo = SchemaInfoImpl.builder() + this.schemaInfo = SchemaInfo.builder() .name(originSchemaName) .type(schemaType) .schema(schema.getBytes("ISO8859-1")) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java index 79fb7893203c2..26199ba31debc 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java @@ -189,7 +189,7 @@ public void testGetTableMetadataTableNoSchema(String delimiter) throws PulsarAdm @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) public void testGetTableMetadataTableBlankSchema(String delimiter) throws PulsarAdminException { updateRewriteNamespaceDelimiterIfNeeded(delimiter); - SchemaInfo badSchemaInfo = SchemaInfoImpl.builder() + SchemaInfo badSchemaInfo = SchemaInfo.builder() .schema(new byte[0]) .type(SchemaType.AVRO) .build(); @@ -216,7 +216,7 @@ public void testGetTableMetadataTableBlankSchema(String delimiter) throws Pulsar @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) public void testGetTableMetadataTableInvalidSchema(String delimiter) throws PulsarAdminException { updateRewriteNamespaceDelimiterIfNeeded(delimiter); - SchemaInfo badSchemaInfo = SchemaInfoImpl.builder() + SchemaInfo badSchemaInfo = SchemaInfo.builder() .schema("foo".getBytes()) .type(SchemaType.AVRO) .build(); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java index c1b97d3a32b22..d01210b47adc5 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java @@ -66,7 +66,7 @@ public void init() { public void testPrimitiveType() { byte int8Value = 1; - SchemaInfo schemaInfoInt8 = SchemaInfoImpl.builder().type(SchemaType.INT8).build(); + SchemaInfo schemaInfoInt8 = SchemaInfo.builder().type(SchemaType.INT8).build(); Schema schemaInt8 = Schema.getSchema(schemaInfoInt8); List pulsarColumnHandleInt8 = getColumnColumnHandles(topicName, schemaInfoInt8, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); PulsarRowDecoder pulsarRowDecoderInt8 = decoderFactory.createRowDecoder(topicName, schemaInfoInt8, @@ -78,7 +78,7 @@ public void testPrimitiveType() { PRIMITIVE_COLUMN_NAME, TINYINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int8Value); short int16Value = 2; - SchemaInfo schemaInfoInt16 = SchemaInfoImpl.builder().type(SchemaType.INT16).build(); + SchemaInfo schemaInfoInt16 = SchemaInfo.builder().type(SchemaType.INT16).build(); Schema schemaInt16 = Schema.getSchema(schemaInfoInt16); List pulsarColumnHandleInt16 = getColumnColumnHandles(topicName, schemaInfoInt16, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); PulsarRowDecoder pulsarRowDecoderInt16 = decoderFactory.createRowDecoder(topicName, schemaInfoInt16, @@ -90,7 +90,7 @@ public void testPrimitiveType() { PRIMITIVE_COLUMN_NAME, SMALLINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int16Value); int int32Value = 2; - SchemaInfo schemaInfoInt32 = SchemaInfoImpl.builder().type(SchemaType.INT32).build(); + SchemaInfo schemaInfoInt32 = SchemaInfo.builder().type(SchemaType.INT32).build(); Schema schemaInt32 = Schema.getSchema(schemaInfoInt32); List pulsarColumnHandleInt32 = getColumnColumnHandles(topicName, schemaInfoInt32, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -103,7 +103,7 @@ public void testPrimitiveType() { PRIMITIVE_COLUMN_NAME, INTEGER, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int32Value); long int64Value = 2; - SchemaInfo schemaInfoInt64 = SchemaInfoImpl.builder().type(SchemaType.INT64).build(); + SchemaInfo schemaInfoInt64 = SchemaInfo.builder().type(SchemaType.INT64).build(); Schema schemaInt64 = Schema.getSchema(schemaInfoInt64); List pulsarColumnHandleInt64 = getColumnColumnHandles(topicName, schemaInfoInt64, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -117,7 +117,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), int64Value); String stringValue = "test"; - SchemaInfo schemaInfoString = SchemaInfoImpl.builder().type(SchemaType.STRING).build(); + SchemaInfo schemaInfoString = SchemaInfo.builder().type(SchemaType.STRING).build(); Schema schemaString = Schema.getSchema(schemaInfoString); List pulsarColumnHandleString = getColumnColumnHandles(topicName, schemaInfoString, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -131,7 +131,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), stringValue); float floatValue = 0.2f; - SchemaInfo schemaInfoFloat = SchemaInfoImpl.builder().type(SchemaType.FLOAT).build(); + SchemaInfo schemaInfoFloat = SchemaInfo.builder().type(SchemaType.FLOAT).build(); Schema schemaFloat = Schema.getSchema(schemaInfoFloat); List pulsarColumnHandleFloat = getColumnColumnHandles(topicName, schemaInfoFloat, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -145,7 +145,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), Long.valueOf(Float.floatToIntBits(floatValue))); double doubleValue = 0.22d; - SchemaInfo schemaInfoDouble = SchemaInfoImpl.builder().type(SchemaType.DOUBLE).build(); + SchemaInfo schemaInfoDouble = SchemaInfo.builder().type(SchemaType.DOUBLE).build(); Schema schemaDouble = Schema.getSchema(schemaInfoDouble); List pulsarColumnHandleDouble = getColumnColumnHandles(topicName, schemaInfoDouble, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -159,7 +159,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), doubleValue); boolean booleanValue = true; - SchemaInfo schemaInfoBoolean = SchemaInfoImpl.builder().type(SchemaType.BOOLEAN).build(); + SchemaInfo schemaInfoBoolean = SchemaInfo.builder().type(SchemaType.BOOLEAN).build(); Schema schemaBoolean = Schema.getSchema(schemaInfoBoolean); List pulsarColumnHandleBoolean = getColumnColumnHandles(topicName, schemaInfoBoolean, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -174,7 +174,7 @@ public void testPrimitiveType() { byte[] bytesValue = new byte[1]; bytesValue[0] = 1; - SchemaInfo schemaInfoBytes = SchemaInfoImpl.builder().type(SchemaType.BYTES).build(); + SchemaInfo schemaInfoBytes = SchemaInfo.builder().type(SchemaType.BYTES).build(); Schema schemaBytes = Schema.getSchema(schemaInfoBytes); List pulsarColumnHandleBytes = getColumnColumnHandles(topicName, schemaInfoBytes, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -188,7 +188,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), Slices.wrappedBuffer(bytesValue)); Date dateValue = new Date(System.currentTimeMillis()); - SchemaInfo schemaInfoDate = SchemaInfoImpl.builder().type(SchemaType.DATE).build(); + SchemaInfo schemaInfoDate = SchemaInfo.builder().type(SchemaType.DATE).build(); Schema schemaDate = Schema.getSchema(schemaInfoDate); List pulsarColumnHandleDate = getColumnColumnHandles(topicName, schemaInfoDate, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -202,7 +202,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), dateValue.getTime()); Time timeValue = new Time(System.currentTimeMillis()); - SchemaInfo schemaInfoTime = SchemaInfoImpl.builder().type(SchemaType.TIME).build(); + SchemaInfo schemaInfoTime = SchemaInfo.builder().type(SchemaType.TIME).build(); Schema schemaTime = Schema.getSchema(schemaInfoTime); List pulsarColumnHandleTime = getColumnColumnHandles(topicName, schemaInfoTime, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); @@ -216,7 +216,7 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE), timeValue.getTime()); Timestamp timestampValue = new Timestamp(System.currentTimeMillis()); - SchemaInfo schemaInfoTimestamp = SchemaInfoImpl.builder().type(SchemaType.TIMESTAMP).build(); + SchemaInfo schemaInfoTimestamp = SchemaInfo.builder().type(SchemaType.TIMESTAMP).build(); Schema schemaTimestamp = Schema.getSchema(schemaInfoTimestamp); List pulsarColumnHandleTimestamp = getColumnColumnHandles(topicName, schemaInfoTimestamp, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); From fff54d5593471bf38de915d9f72bae5e988226a7 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:34:14 +0800 Subject: [PATCH 016/823] [Transaction] Fix close pulsarClient then close transaction client connection (#12689) (cherry picked from commit 6162ccf34aa29e9495bf2f0cdc7e88e9e8c1d067) --- ...java => TransactionClientConnectTest.java} | 29 ++++++++++++++++++- .../transaction/TransactionTestBase.java | 2 +- .../pulsar/client/impl/ConnectionHandler.java | 16 ++++++---- .../pulsar/client/impl/HandlerState.java | 7 +++++ .../pulsar/client/impl/PulsarClientImpl.java | 16 +++++----- .../impl/TransactionMetaStoreHandler.java | 7 +++++ 6 files changed, 61 insertions(+), 16 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/{TransactionClientReconnectTest.java => TransactionClientConnectTest.java} (89%) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientReconnectTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java similarity index 89% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientReconnectTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java index 1f5ab15cf42be..42eadfe98d87c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientReconnectTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java @@ -26,6 +26,7 @@ import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.TransactionMetaStoreHandler; import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -39,6 +40,7 @@ import org.testng.annotations.Test; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.Collections; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -47,7 +49,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.FileAssert.fail; -public class TransactionClientReconnectTest extends TransactionTestBase { +public class TransactionClientConnectTest extends TransactionTestBase { private static final String RECONNECT_TOPIC = "persistent://public/txn/txn-client-reconnect-test"; private static final int NUM_PARTITIONS = 1; @@ -223,6 +225,31 @@ public void testTransactionAddPublishPartitionToTxnReconnect() throws Exception reconnect(); } + @Test + public void testPulsarClientCloseThenCloseTcClient() throws Exception { + TransactionCoordinatorClientImpl transactionCoordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient(); + Field field = TransactionCoordinatorClientImpl.class.getDeclaredField("handlers"); + field.setAccessible(true); + TransactionMetaStoreHandler[] handlers = + (TransactionMetaStoreHandler[]) field.get(transactionCoordinatorClient); + + for (TransactionMetaStoreHandler handler : handlers) { + handler.newTransactionAsync(10, TimeUnit.SECONDS).get(); + } + pulsarClient.close(); + for (TransactionMetaStoreHandler handler : handlers) { + Method method = TransactionMetaStoreHandler.class.getMethod("getConnectHandleState"); + method.setAccessible(true); + assertEquals(method.invoke(handler).toString(), "Closed"); + try { + handler.newTransactionAsync(10, TimeUnit.SECONDS).get(); + } catch (ExecutionException | InterruptedException e) { + assertTrue(e.getCause() + instanceof TransactionCoordinatorClientException.MetaStoreHandlerNotReadyException); + } + } + } + public void start() throws Exception { // wait transaction coordinator init success Awaitility.await().until(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 622421b16c22b..1dba73a0378a5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -246,7 +246,7 @@ protected final void internalCleanup() { admin = null; } if (pulsarClient != null) { - pulsarClient.shutdown(); + pulsarClient.close(); pulsarClient = null; } if (pulsarServiceList.size() > 0) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 8fb7ab41d8867..9babd2ad419ec 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -103,12 +103,16 @@ protected void reconnectLater(Throwable exception) { long delayMs = backoff.next(); log.warn("[{}] [{}] Could not get connection to broker: {} -- Will try again in {} s", state.topic, state.getHandlerName(), exception.getMessage(), delayMs / 1000.0); - state.setState(State.Connecting); - state.client.timer().newTimeout(timeout -> { - log.info("[{}] [{}] Reconnecting after connection was closed", state.topic, state.getHandlerName()); - incrementEpoch(); - grabCnx(); - }, delayMs, TimeUnit.MILLISECONDS); + if (state.changeToConnecting()) { + state.client.timer().newTimeout(timeout -> { + log.info("[{}] [{}] Reconnecting after connection was closed", state.topic, state.getHandlerName()); + incrementEpoch(); + grabCnx(); + }, delayMs, TimeUnit.MILLISECONDS); + } else { + log.info("[{}] [{}] Ignoring reconnection request (state: {})", + state.topic, state.getHandlerName(), state.getState()); + } } protected long incrementEpoch() { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HandlerState.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HandlerState.java index e72c97fadadcc..582df8c112bc3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HandlerState.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HandlerState.java @@ -64,6 +64,13 @@ protected State getState() { return STATE_UPDATER.get(this); } + protected boolean changeToConnecting() { + return (STATE_UPDATER.compareAndSet(this, State.Uninitialized, State.Connecting) + || STATE_UPDATER.compareAndSet(this, State.Ready, State.Connecting) + || STATE_UPDATER.compareAndSet(this, State.RegisteringSchema, State.Connecting) + || STATE_UPDATER.compareAndSet(this, State.Connecting, State.Connecting)); + } + protected void setState(State s) { STATE_UPDATER.set(this, s); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 1234b8bbf3791..7703afccb96e1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -712,6 +712,14 @@ public void shutdown() throws PulsarClientException { throwable = t; } } + if (tcClient != null) { + try { + tcClient.close(); + } catch (Throwable t) { + log.warn("Failed to close tcClient"); + throwable = t; + } + } try { // Shutting down eventLoopGroup separately because in some cases, cnxPool might be using different // eventLoopGroup. @@ -747,14 +755,6 @@ public void shutdown() throws PulsarClientException { throwable = t; } } - if (tcClient != null) { - try { - tcClient.close(); - } catch (Throwable t) { - log.warn("Failed to close tcClient"); - throwable = t; - } - } if (throwable != null) { throw throwable; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index f96cf57867a70..ba6ee50d99fd2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; import io.netty.util.ReferenceCountUtil; @@ -540,6 +541,12 @@ void connectionClosed(ClientCnx cnx) { @Override public void close() throws IOException { this.requestTimeout.cancel(); + this.setState(State.Closed); + } + + @VisibleForTesting + public State getConnectHandleState() { + return getState(); } @Override From f849d467689ed6e3e30438722e3ebc6779d059ae Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 7 Nov 2021 01:50:13 +0200 Subject: [PATCH 017/823] [Java Client] Remove invalid call to Thread.currentThread().interrupt(); (#12652) - Thread.currentThread().interrupt() shouldn't be called here. - it must only be called when handling an InterruptedException. - this looks like a copy-paste bug introduced in https://github.com/apache/pulsar/pull/731/files#diff-d6fcf8aa2d0035cc386dca0942a452343d6854763c7fd397efa4e660c0069767R1183 (cherry picked from commit b5f78f8ae5b4f2419e97044541722062fe388d9d) --- .../main/java/org/apache/pulsar/client/impl/ProducerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 0f27a63ee8e5b..34e4b6c2d54af 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1775,7 +1775,6 @@ private void batchMessageAndSend() { processOpSendMsg(opSendMsg); } } catch (PulsarClientException e) { - Thread.currentThread().interrupt(); semaphore.ifPresent(s -> s.release(batchMessageContainer.getNumMessagesInBatch())); } catch (Throwable t) { semaphore.ifPresent(s -> s.release(batchMessageContainer.getNumMessagesInBatch())); From f5730b734031cc2381eef973b3047206209d5f61 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Sun, 31 Oct 2021 17:14:00 +0800 Subject: [PATCH 018/823] Add close method in the class of NegativeAcksTracker (#12469) (cherry picked from commit 3694aa1554e7f408a90eda2ba46eae17b425140a) --- .../apache/pulsar/client/impl/ConsumerImpl.java | 1 + .../pulsar/client/impl/NegativeAcksTracker.java | 16 +++++++++++++++- .../pulsar/client/impl/ConsumerImplTest.java | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 88b6df114118d..acc7cea61d4e2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -979,6 +979,7 @@ private void closeConsumerTasks() { if (batchReceiveTimeout != null) { batchReceiveTimeout.cancel(); } + negativeAcksTracker.close(); stats.getStatTimeout().ifPresent(Timeout::cancel); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java index bbdd7864987bf..b5e6fd1325163 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java @@ -21,6 +21,7 @@ import io.netty.util.Timeout; import io.netty.util.Timer; +import java.io.Closeable; import java.util.HashMap; import java.util.HashSet; import java.util.Set; @@ -30,7 +31,7 @@ import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import static org.apache.pulsar.client.impl.UnAckedMessageTracker.addChunkedMessageIdsAndRemoveFromSequnceMap; -class NegativeAcksTracker { +class NegativeAcksTracker implements Closeable { private HashMap nackedMessages = null; @@ -93,4 +94,17 @@ public synchronized void add(MessageId messageId) { this.timeout = timer.newTimeout(this::triggerRedelivery, timerIntervalNanos, TimeUnit.NANOSECONDS); } } + + @Override + public synchronized void close() { + if (timeout != null && !timeout.isCancelled()) { + timeout.cancel(); + timeout = null; + } + + if (nackedMessages != null) { + nackedMessages.clear(); + nackedMessages = null; + } + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index 37c9e0cdeb377..8a9e665f9143a 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -187,4 +187,18 @@ public void testBatchReceiveAsyncCanBeCancelled() { // then Assert.assertFalse(consumer.hasPendingBatchReceive()); } + + @Test + public void testClose() { + Exception checkException = null; + try { + if (consumer != null) { + consumer.negativeAcknowledge(new MessageIdImpl(-1, -1, -1)); + consumer.close(); + } + } catch (Exception e) { + checkException = e; + } + Assert.assertNull(checkException); + } } From 9941edbaa1212ec7447d2c1f9571c93cfeed15ac Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Sat, 23 Oct 2021 21:34:38 -0700 Subject: [PATCH 019/823] Stop OffsetStore when stopping the connector (#12457) ### Motivation Source connectors based on KCA (all debezium ones) don't stop properly on error / don't restart. https://github.com/apache/pulsar/pull/12441 fixes one problem, this PR fixes another: ofsetStore is not closed on connector stop() and producer/consumer aren't closed too, preventing the connector from shutting down. ### Modifications Closing offset store on connector stop. (cherry picked from commit 63454e9b2573b8f7ba6a023402c92a17b033ee56) --- .../io/kafka/connect/AbstractKafkaConnectSource.java | 6 ++++++ .../io/kafka/connect/PulsarOffsetBackingStore.java | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java index 3901c5f3d176b..5bd85a00f1762 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java @@ -180,6 +180,12 @@ public synchronized Record read() throws Exception { public void close() { if (sourceTask != null) { sourceTask.stop(); + sourceTask = null; + } + + if (offsetStore != null) { + offsetStore.stop(); + offsetStore = null; } } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java index b1338f837bb49..495c8b9c194c0 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java @@ -158,12 +158,19 @@ public void start() { @Override public void stop() { + log.info("Stopping PulsarOffsetBackingStore"); if (null != producer) { + try { + producer.flush(); + } catch (PulsarClientException pce) { + log.warn("Failed to flush the producer", pce); + } try { producer.close(); } catch (PulsarClientException e) { log.warn("Failed to close producer", e); } + producer = null; } if (null != reader) { try { @@ -171,7 +178,10 @@ public void stop() { } catch (IOException e) { log.warn("Failed to close reader", e); } + reader = null; } + + // do not close the client, it is provided by the sink context } @Override From 13d935b94f3259c22620b2cdb8abe01c9ec2b651 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Fri, 22 Oct 2021 03:01:42 -0700 Subject: [PATCH 020/823] KCA doesn't handle unchecked unchecked ConnectException/KafkaException for the task, it may lead to the connector hanging (#12441) (cherry picked from commit 34f237b3d6ba7c51902fd9574e62f242431db313) --- .../connect/AbstractKafkaConnectSource.java | 13 +- .../connect/ErrFileStreamSourceTask.java | 31 ++++ .../connect/KafkaConnectSourceErrTest.java | 145 ++++++++++++++++++ 3 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrFileStreamSourceTask.java create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrTest.java diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java index 5bd85a00f1762..4612633677b1a 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java @@ -27,8 +27,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.connect.runtime.TaskConfig; @@ -168,6 +166,7 @@ public synchronized Record read() throws Exception { } catch (ExecutionException ex) { // log the error, continue execution log.error("execution exception while get flushFuture", ex); + throw new Exception("Flush failed", ex.getCause()); } finally { flushFuture = null; currentBatch = null; @@ -193,7 +192,6 @@ public void close() { private static Map PROPERTIES = Collections.emptyMap(); private static Optional RECORD_SEQUENCE = Optional.empty(); - private static long FLUSH_TIMEOUT_MS = 60000; public abstract class AbstractKafkaSourceRecord implements Record { @Getter @@ -254,8 +252,15 @@ private void completedFlushOffset(Throwable error, Void result) { flushFuture.complete(null); } catch (InterruptedException exception) { log.warn("Flush of {} offsets interrupted, cancelling", this); + Thread.currentThread().interrupt(); offsetWriter.cancelFlush(); - flushFuture.completeExceptionally(new Exception("Failed to commit offsets")); + flushFuture.completeExceptionally(new Exception("Failed to commit offsets", exception)); + } catch (Throwable t) { + // SourceTask can throw unchecked ConnectException/KafkaException. + // Make sure the future is cancelled in that case + log.warn("Flush of {} offsets failed, cancelling", this); + offsetWriter.cancelFlush(); + flushFuture.completeExceptionally(new Exception("Failed to commit offsets", t)); } } } diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrFileStreamSourceTask.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrFileStreamSourceTask.java new file mode 100644 index 0000000000000..d17f32cfc47a0 --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrFileStreamSourceTask.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.io.kafka.connect; + +import org.apache.kafka.connect.file.FileStreamSourceTask; + +public class ErrFileStreamSourceTask extends FileStreamSourceTask { + + @Override + public void commit() throws InterruptedException { + throw new org.apache.kafka.connect.errors.ConnectException("blah"); + } + +} diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrTest.java new file mode 100644 index 0000000000000..cc04706f3ee45 --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrTest.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.kafka.connect; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.file.FileStreamSourceConnector; +import org.apache.kafka.connect.runtime.TaskConfig; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.SourceContext; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * Test the implementation of {@link KafkaConnectSource}. + */ +@Slf4j +public class KafkaConnectSourceErrTest extends ProducerConsumerBase { + + private Map config = new HashMap<>(); + private String offsetTopicName; + // The topic to publish data to, for kafkaSource + private String topicName; + private KafkaConnectSource kafkaConnectSource; + private File tempFile; + private SourceContext context; + private PulsarClient client; + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + + config.put(TaskConfig.TASK_CLASS_CONFIG, "org.apache.pulsar.io.kafka.connect.ErrFileStreamSourceTask"); + config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); + config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); + + this.offsetTopicName = "persistent://my-property/my-ns/kafka-connect-source-offset"; + config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); + + this.topicName = "persistent://my-property/my-ns/kafka-connect-source"; + config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); + tempFile = File.createTempFile("some-file-name", null); + config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); + config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + + this.context = mock(SourceContext.class); + this.client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .build(); + when(context.getPulsarClient()).thenReturn(this.client); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + if (this.client != null) { + this.client.close(); + } + tempFile.delete(); + super.internalCleanup(); + } + + @Test + public void testOpenAndRead() throws Exception { + kafkaConnectSource = new KafkaConnectSource(); + kafkaConnectSource.open(config, context); + + // use FileStreamSourceConnector, each line is a record, need "\n" and end of each record. + OutputStream os = Files.newOutputStream(tempFile.toPath()); + + String line1 = "This is the first line\n"; + os.write(line1.getBytes()); + os.flush(); + log.info("write 2 lines."); + + String line2 = "This is the second line\n"; + os.write(line2.getBytes()); + os.flush(); + + log.info("finish write, will read 2 lines"); + + // Note: FileStreamSourceTask read the whole line as Value, and set Key as null. + Record> record = kafkaConnectSource.read(); + String readBack1 = new String(record.getValue().getValue()); + assertTrue(line1.contains(readBack1)); + assertNull(record.getValue().getKey()); + log.info("read line1: {}", readBack1); + record.ack(); + + record = kafkaConnectSource.read(); + String readBack2 = new String(record.getValue().getValue()); + assertTrue(line2.contains(readBack2)); + assertNull(record.getValue().getKey()); + assertTrue(record.getPartitionId().isPresent()); + assertFalse(record.getPartitionIndex().isPresent()); + log.info("read line2: {}", readBack2); + record.ack(); + + String line3 = "This is the 3rd line\n"; + os.write(line3.getBytes()); + os.flush(); + + try { + kafkaConnectSource.read(); + fail("expected exception"); + } catch (Exception e) { + log.info("got exception", e); + assertTrue(e.getCause().getCause() instanceof org.apache.kafka.connect.errors.ConnectException); + } + } +} From edbf15c2059ed92181730bdacbe803eb1d456f8c Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 9 Nov 2021 15:49:49 +0100 Subject: [PATCH 021/823] Fix checkstyle after resolving conflicts --- .../pulsar/broker/service/nonpersistent/NonPersistentTopic.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 2c62f2ed92a59..916aa53db8e8e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -38,7 +38,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; From 28e3d79db10f43d0b8a152e73e1daa7f5a577884 Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Thu, 11 Nov 2021 13:00:56 -0800 Subject: [PATCH 022/823] [issue-12764] Revert "BookkeeperClientFactory should pass in the ZK instance (#12192)" (#12766) This reverts commit 33580ca6c28f612d3eeb250341a0cb8883961afc. --- .../org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java index 996bf225892af..9a09deb166b0f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java @@ -81,7 +81,6 @@ public BookKeeper create(ServiceConfiguration conf, ZooKeeper zkClient, EventLoo try { return BookKeeper.forConfig(bkConf) .allocator(PulsarByteBufAllocator.DEFAULT) - .setZookeeper(zkClient) .eventLoopGroup(eventLoopGroup) .statsLogger(statsLogger) .build(); From 94db24a42120786186043c1ae5dc1dca6ef58d18 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:11:42 +0800 Subject: [PATCH 023/823] [Transaction] Fix topicTransactionBuffer handle null snapshot (#12758) fix https://github.com/apache/pulsar/issues/12754 Now when delete topic, we will write a null value to Transaction buffer snapshot topic, other topic recover by this transaction buffer snapshot system topic, will produce NPE judge NPE logic (cherry picked from commit c90c89b07ce544b92becedfaa7a0090b4b73edd2) --- .../buffer/impl/TopicTransactionBuffer.java | 16 +++++++++------- .../broker/transaction/TransactionTest.java | 2 +- .../client/impl/TransactionEndToEndTest.java | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 79f7f35732754..735927cc99b3f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -532,14 +532,16 @@ public void run() { try { boolean hasSnapshot = false; while (reader.hasMoreEvents()) { - hasSnapshot = true; Message message = reader.readNext(); - TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); - if (topic.getName().equals(transactionBufferSnapshot.getTopicName())) { - callBack.handleSnapshot(transactionBufferSnapshot); - this.startReadCursorPosition = PositionImpl.get( - transactionBufferSnapshot.getMaxReadPositionLedgerId(), - transactionBufferSnapshot.getMaxReadPositionEntryId()); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); + if (transactionBufferSnapshot != null) { + hasSnapshot = true; + callBack.handleSnapshot(transactionBufferSnapshot); + this.startReadCursorPosition = PositionImpl.get( + transactionBufferSnapshot.getMaxReadPositionLedgerId(), + transactionBufferSnapshot.getMaxReadPositionEntryId()); + } } } if (!hasSnapshot) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 971f8b24bce8b..e14c7777e8e2a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -295,4 +295,4 @@ public void testTakeSnapshotBeforeBuildTxnProducer() throws Exception { Assert.assertEquals(snapshot1.getMaxReadPositionEntryId(), 1); }); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index d7cb6c9cc3220..9c30f4a651251 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -376,6 +376,23 @@ private void txnAckTest(boolean batchEnable, int maxBatchSize, } } + @Test + public void testAfterDeleteTopicOtherTopicCanRecover() throws Exception { + String topicOne = "persistent://" + NAMESPACE1 + "/topic-one"; + String topicTwo = "persistent://" + NAMESPACE1 + "/topic-two"; + String sub = "test"; + admin.topics().createNonPartitionedTopic(topicOne); + admin.topics().createSubscription(topicOne, "test", MessageId.earliest); + admin.topics().delete(topicOne); + + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicTwo).create(); + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicTwo).subscriptionName(sub).subscribe(); + String content = "test"; + producer.send(content); + assertEquals(consumer.receive().getValue(), content); + } + @Test public void txnMessageAckTest() throws Exception { String topic = TOPIC_MESSAGE_ACK_TEST; From 38b05dc6f201c6b1c32dc54ddabc23349210a247 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Sat, 13 Nov 2021 09:15:30 +0800 Subject: [PATCH 024/823] [Transaction] Fix transaction system topic create in loop. (#12749) fix https://github.com/apache/pulsar/issues/12727 Now transaction system topic can be created. we should not allow broker or user create by transaction system format topic. 1. checkout topic auto create. 2. admin create topic. add some test for it (cherry picked from commit 2c4d913c4b3fb1c6d924efaa0a24c93a2d2de7d0) --- .../apache/pulsar/broker/PulsarService.java | 14 ++-- .../admin/impl/PersistentTopicsBase.java | 17 ++++- .../broker/admin/v2/PersistentTopics.java | 2 + .../pulsar/broker/service/BrokerService.java | 9 +++ .../broker/transaction/TransactionTest.java | 73 +++++++++++++++++++ .../common/events/EventsTopicNames.java | 3 +- .../pulsar/common/naming/TopicName.java | 3 + 7 files changed, 113 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 14f5003f379b6..c897f1585a2d3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -22,7 +22,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.broker.resourcegroup.ResourceUsageTransportManager.DISABLE_RESOURCE_USAGE_TRANSPORT_MANAGER; -import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; +import static org.apache.pulsar.common.naming.TopicName.TRANSACTION_COORDINATOR_LOG; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -132,7 +132,6 @@ import org.apache.pulsar.common.configuration.VipStatus; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; @@ -1648,11 +1647,16 @@ public void shutdownNow() { } - private static boolean isTransactionSystemTopic(TopicName topicName) { + public static boolean isTransactionSystemTopic(TopicName topicName) { String topic = topicName.toString(); return topic.startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()) - || topic.startsWith(TopicName.get(TopicDomain.persistent.value(), - NamespaceName.SYSTEM_NAMESPACE, TRANSACTION_LOG_PREFIX).toString()) + || topic.startsWith(TRANSACTION_COORDINATOR_LOG.toString()) + || topic.endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX); + } + + public static boolean isTransactionInternalName(TopicName topicName) { + String topic = topicName.toString(); + return topic.startsWith(TRANSACTION_COORDINATOR_LOG.toString()) || topic.endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index a6d8f2cb028c2..f223869fe7f93 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static org.apache.pulsar.broker.PulsarService.isTransactionInternalName; import static org.apache.pulsar.broker.resources.PulsarResources.DEFAULT_OPERATION_TIMEOUT_SEC; import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsTransactionCoordinatorAssign; import com.fasterxml.jackson.core.JsonProcessingException; @@ -160,7 +161,9 @@ protected List internalGetList() { } try { - return topicResources().listPersistentTopicsAsync(namespaceName).join(); + return topicResources().listPersistentTopicsAsync(namespaceName).thenApply(topics -> + topics.stream().filter(topic -> + !isTransactionInternalName(TopicName.get(topic))).collect(Collectors.toList())).join(); } catch (Exception e) { log.error("[{}] Failed to get topics list for namespace {}", clientAppId(), namespaceName, e); throw new RestException(e); @@ -243,6 +246,13 @@ protected void validateAdminAndClientPermission() { } } + protected void validateCreateTopic(TopicName topicName) { + if (isTransactionInternalName(topicName)) { + log.warn("Try to create a topic in the system topic format! {}", topicName); + throw new RestException(Status.CONFLICT, "Cannot create topic in system topic format!"); + } + } + public void validateAdminOperationOnTopic(boolean authoritative) { validateAdminAccessForTenant(topicName.getTenant()); validateTopicOwnership(topicName, authoritative); @@ -3598,7 +3608,10 @@ private Topic getTopicReference(TopicName topicName) { } catch (RestException e) { throw e; } catch (Exception e) { - throw new RestException(e); + if (e.getCause() instanceof NotAllowedException) { + throw new RestException(Status.CONFLICT, e.getCause()); + } + throw new RestException(e.getCause()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 09b694cc34e02..1aeaccf8114fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -234,6 +234,7 @@ public void createPartitionedTopic( validateGlobalNamespaceOwnership(); validatePartitionedTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); + validateCreateTopic(topicName); internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); @@ -267,6 +268,7 @@ public void createNonPartitionedTopic( validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); validateTopicName(tenant, namespace, encodedTopic); + validateCreateTopic(topicName); internalCreateNonPartitionedTopic(authoritative); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 456af1f0c1783..e592336867f4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -23,6 +23,7 @@ import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.apache.commons.collections.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.PulsarService.isTransactionSystemTopic; import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -1226,6 +1227,14 @@ private void createPersistentTopic(final String topic, boolean createIfMissing, return; } + if (isTransactionSystemTopic(topicName)) { + String msg = String.format("Can not create transaction system topic %s", topic); + log.warn(msg); + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + topicFuture.completeExceptionally(new NotAllowedException(msg)); + return; + } + CompletableFuture maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index e14c7777e8e2a..67da2d11a4c00 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -19,11 +19,27 @@ package org.apache.pulsar.broker.transaction; import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore.PENDING_ACK_STORE_SUFFIX; +import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + import com.google.common.collect.Sets; import java.lang.reflect.Field; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -105,6 +121,63 @@ protected void setup() throws Exception { waitForCoordinatorToBeAvailable(NUM_PARTITIONS); } + @Test + public void testCreateTransactionSystemTopic() throws Exception { + String subName = "test"; + String topicName = TopicName.get(NAMESPACE1 + "/" + "testCreateTransactionSystemTopic").toString(); + + try { + // init pending ack + @Cleanup + Consumer consumer = getConsumer(topicName, subName); + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); + + consumer.acknowledgeAsync(new MessageIdImpl(10, 10, 10), transaction).get(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + topicName = MLPendingAckStore.getTransactionPendingAckStoreSuffix(topicName, subName); + + // getList does not include transaction system topic + List list = admin.topics().getList(NAMESPACE1); + assertEquals(list.size(), 4); + list.forEach(topic -> assertFalse(topic.contains(PENDING_ACK_STORE_SUFFIX))); + + try { + // can't create transaction system topic + @Cleanup + Consumer consumer = getConsumer(topicName, subName); + fail(); + } catch (PulsarClientException.NotAllowedException e) { + assertTrue(e.getMessage().contains("Can not create transaction system topic")); + } + + // can't create transaction system topic + try { + admin.topics().getSubscriptions(topicName); + fail(); + } catch (PulsarAdminException.ConflictException e) { + assertEquals(e.getMessage(), "Can not create transaction system topic " + topicName); + } + + // can't create transaction system topic + try { + admin.topics().createPartitionedTopic(topicName, 3); + fail(); + } catch (PulsarAdminException.ConflictException e) { + assertEquals(e.getMessage(), "Cannot create topic in system topic format!"); + } + + // can't create transaction system topic + try { + admin.topics().createNonPartitionedTopic(topicName); + fail(); + } catch (PulsarAdminException.ConflictException e) { + assertEquals(e.getMessage(), "Cannot create topic in system topic format!"); + } + } + @Test public void brokerNotInitTxnManagedLedgerTopic() throws Exception { String subName = "test"; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java b/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java index 2aa9e122d63c5..f82c9ae8519a4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java @@ -49,6 +49,7 @@ public static boolean checkTopicIsEventsNames(TopicName topicName) { } public static boolean checkTopicIsTransactionCoordinatorAssign(TopicName topicName) { - return TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString().equals(topicName.toString()); + return topicName != null && topicName.toString() + .startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index 2cb9f23153bf3..3729697c9e36b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -63,6 +63,9 @@ public TopicName load(String name) throws Exception { public static final TopicName TRANSACTION_COORDINATOR_ASSIGN = TopicName.get(TopicDomain.persistent.value(), NamespaceName.SYSTEM_NAMESPACE, "transaction_coordinator_assign"); + public static final TopicName TRANSACTION_COORDINATOR_LOG = TopicName.get(TopicDomain.persistent.value(), + NamespaceName.SYSTEM_NAMESPACE, "__transaction_log_"); + public static TopicName get(String domain, NamespaceName namespaceName, String topic) { String name = domain + "://" + namespaceName.toString() + '/' + topic; return TopicName.get(name); From f047b5a109d117e8b62d3a2d3e37929187d101ff Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 15 Nov 2021 01:39:45 -0600 Subject: [PATCH 025/823] [Java Client] Let producer reconnect for state RegisteringSchema (#12781) Motivation In the Java Client, if a producer is in the RegisteringSchema state, it is a valid state for it to reconnect. Fix the ConnectionHandler to align with this behavior. Modifications Update the isValidStateForReconnection method to return true for state RegisteringSchema. Verifying this change This change is a trivial rework / code cleanup without any test coverage. Does this pull request potentially affect one of the following parts: This update does not contain breaking changes. (cherry picked from commit 8d75eedbf6a37c1e77f6c5b6eac080ae00b77768) --- .../java/org/apache/pulsar/client/impl/ConnectionHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 9babd2ad419ec..02429d2791855 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -156,6 +156,7 @@ private boolean isValidStateForReconnection() { switch (state) { case Uninitialized: case Connecting: + case RegisteringSchema: case Ready: // Ok return true; From cad93713f7267b420784446a775c2caaaa036619 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Sat, 13 Nov 2021 17:09:39 -0600 Subject: [PATCH 026/823] [ServerCnx] Close connection after receiving unexpected SendCommand (#12780) (cherry picked from commit ba5809553344f074c5dce15618a70f3b20d368c7) --- .../pulsar/broker/service/ServerCnx.java | 4 +++- .../pulsar/broker/service/ServerCnxTest.java | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 5be21cc585f04..7762c225910a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1352,7 +1352,9 @@ protected void handleSend(CommandSend send, ByteBuf headersAndPayload) { CompletableFuture producerFuture = producers.get(send.getProducerId()); if (producerFuture == null || !producerFuture.isDone() || producerFuture.isCompletedExceptionally()) { - log.warn("[{}] Producer had already been closed: {}", remoteAddress, send.getProducerId()); + log.warn("[{}] Received message, but the producer is not ready : {}. Closing the connection.", + remoteAddress, send.getProducerId()); + close(); return; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 2dc57b54e2c36..3d3a30e89516c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -648,6 +648,28 @@ public void testSendCommand() throws Exception { channel.finish(); } + @Test(timeOut = 30000) + public void testSendCommandBeforeCreatingProducer() throws Exception { + resetChannel(); + setChannelConnected(); + + // test SEND before producer is created + MessageMetadata messageMetadata = new MessageMetadata() + .setPublishTime(System.currentTimeMillis()) + .setProducerName("prod-name") + .setSequenceId(0); + ByteBuf data = Unpooled.buffer(1024); + + ByteBuf clientCommand = ByteBufPair.coalesce(Commands.newSend(1, 0, 1, + ChecksumType.None, messageMetadata, data)); + channel.writeInbound(Unpooled.copiedBuffer(clientCommand)); + clientCommand.release(); + + // Then expect channel to close + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> !channel.isActive()); + channel.finish(); + } + @Test(timeOut = 30000) public void testUseSameProducerName() throws Exception { resetChannel(); From 4711db91f2ce6816af9d98b9ce7ee8d353e92f00 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Mon, 15 Nov 2021 10:28:15 +0100 Subject: [PATCH 027/823] Fix build - TransactionTest --- .../org/apache/pulsar/broker/transaction/TransactionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 67da2d11a4c00..34ea362130501 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -67,6 +67,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; From bdd57b21a66b81aab72c4ec39d516ffd2a769c35 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 15 Nov 2021 14:04:17 -0600 Subject: [PATCH 028/823] [Java Client] Use info, not warn, log level when successfully registering schema (#12819) (cherry picked from commit e993f32cfe84a16079805eb0fb9cf75881df2829) --- .../main/java/org/apache/pulsar/client/impl/ProducerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 34e4b6c2d54af..7ee00e7ed61d1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -630,7 +630,7 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call callback.sendComplete((PulsarClientException.IncompatibleSchemaException) t); } } else { - log.warn("[{}] [{}] GetOrCreateSchema succeed", topic, producerName); + log.info("[{}] [{}] GetOrCreateSchema succeed", topic, producerName); SchemaHash schemaHash = SchemaHash.of(msg.getSchemaInternal()); schemaCache.putIfAbsent(schemaHash, v); msg.getMessageBuilder().setSchemaVersion(v); From 5099869bbbd15b53cb7fc5488ac7532e514551d7 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Mon, 8 Nov 2021 13:07:01 +0800 Subject: [PATCH 029/823] Fix StringIndexOutOfBoundsException in org.apache.pulsar.broker.resources.NamespaceResources#pathIsFromNamespace (#12659) Co-authored-by: Jiang Haiting (cherry picked from commit 213f14c1b74f5c56bcbdca51d4ee61254b47e31f) --- .../broker/resources/NamespaceResources.java | 3 +- .../resources/NamespaceResourcesTest.java | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index 2beeab8c9510b..dd33ee831ab73 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -118,7 +118,8 @@ public CompletableFuture setPoliciesAsync(NamespaceName ns, Function Date: Thu, 2 Dec 2021 20:34:35 +0200 Subject: [PATCH 030/823] [Broker] Fix LeaderElectionService.getCurrentLeader and add support for empheralOwner in MockZooKeeper (#13066) * Add leader election unit test that uses multiple brokers * Support empheralOwner in MockZooKeeper * Use unique sessions for all brokers * Add failing test case for leader broker information not available on other brokers * Add test for reading the current leader * Fix issue in leader election * Address review feedback: make methods static * Use unique-session only in MultiBrokerBaseTests * Move tenant and namespace creation to it's own method * Improve cleanup * Add alwaysRun to BeforeClass * Fix leaks in locking in MockZooKeeper * Reduce code duplication * Fix NPE when CreateMode is null (cherry picked from commit 36a45ee89639bf66a12de23e415d80451deaa4e0) --- .../pulsar/broker/MultiBrokerBaseTest.java | 153 ++++ .../auth/MockedPulsarServiceBaseTest.java | 12 +- .../MultiBrokerLeaderElectionTest.java | 71 ++ .../coordination/impl/LeaderElectionImpl.java | 5 +- .../org/apache/zookeeper/MockZooKeeper.java | 860 ++++++++++-------- .../zookeeper/MockZooKeeperSession.java | 16 +- 6 files changed, 728 insertions(+), 389 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java new file mode 100644 index 0000000000000..f4d106da67f0b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.MockZooKeeperSession; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; + +public abstract class MultiBrokerBaseTest extends MockedPulsarServiceBaseTest { + protected List additionalBrokers; + protected List additionalBrokerAdmins; + protected List additionalBrokerClients; + + protected int numberOfAdditionalBrokers() { + return 2; + } + + @BeforeClass(alwaysRun = true) + @Override + public final void setup() throws Exception { + super.internalSetup(); + additionalBrokersSetup(); + pulsarResourcesSetup(); + } + + protected void pulsarResourcesSetup() throws PulsarAdminException { + admin.tenants().createTenant("public", createDefaultTenantInfo()); + admin.namespaces() + .createNamespace("public/default", getPulsar().getConfiguration().getDefaultNumberOfNamespaceBundles()); + } + + protected void additionalBrokersSetup() throws Exception { + int numberOfAdditionalBrokers = numberOfAdditionalBrokers(); + additionalBrokers = new ArrayList<>(numberOfAdditionalBrokers); + additionalBrokerAdmins = new ArrayList<>(numberOfAdditionalBrokers); + additionalBrokerClients = new ArrayList<>(numberOfAdditionalBrokers); + for (int i = 0; i < numberOfAdditionalBrokers; i++) { + PulsarService pulsarService = createAdditionalBroker(i); + additionalBrokers.add(i, pulsarService); + PulsarAdminBuilder pulsarAdminBuilder = + PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress() != null + ? pulsarService.getWebServiceAddress() + : pulsarService.getWebServiceAddressTls()); + customizeNewPulsarAdminBuilder(pulsarAdminBuilder); + additionalBrokerAdmins.add(i, pulsarAdminBuilder.build()); + additionalBrokerClients.add(i, newPulsarClient(pulsarService.getBrokerServiceUrl(), 0)); + } + } + + protected ServiceConfiguration createConfForAdditionalBroker(int additionalBrokerIndex) { + return getDefaultConf(); + } + + protected PulsarService createAdditionalBroker(int additionalBrokerIndex) throws Exception { + return startBroker(createConfForAdditionalBroker(additionalBrokerIndex)); + } + + @Override + protected ZKMetadataStore createLocalMetadataStore() { + // use MockZooKeeperSession to provide a unique session id for each instance + return new ZKMetadataStore(MockZooKeeperSession.newInstance(mockZooKeeper)); + } + + @Override + protected ZKMetadataStore createConfigurationMetadataStore() { + // use MockZooKeeperSession to provide a unique session id for each instance + return new ZKMetadataStore(MockZooKeeperSession.newInstance(mockZooKeeperGlobal)); + } + + @AfterClass(alwaysRun = true) + @Override + public final void cleanup() throws Exception { + additionalBrokersCleanup(); + super.internalCleanup(); + } + + protected void additionalBrokersCleanup() { + if (additionalBrokerAdmins != null) { + for (PulsarAdmin additionalBrokerAdmin : additionalBrokerAdmins) { + additionalBrokerAdmin.close(); + } + additionalBrokerAdmins = null; + } + if (additionalBrokerClients != null) { + for (PulsarClient additionalBrokerClient : additionalBrokerClients) { + try { + additionalBrokerClient.shutdown(); + } catch (PulsarClientException e) { + // ignore + } + } + additionalBrokerClients = null; + } + if (additionalBrokers != null) { + for (PulsarService pulsarService : additionalBrokers) { + try { + pulsarService.getConfiguration().setBrokerShutdownTimeoutMs(0L); + pulsarService.close(); + } catch (PulsarServerException e) { + // ignore + } + } + additionalBrokers = null; + } + } + + public final List getAllBrokers() { + List brokers = new ArrayList<>(numberOfAdditionalBrokers() + 1); + brokers.add(getPulsar()); + brokers.addAll(additionalBrokers); + return Collections.unmodifiableList(brokers); + } + + public final List getAllAdmins() { + List admins = new ArrayList<>(numberOfAdditionalBrokers() + 1); + admins.add(admin); + admins.addAll(additionalBrokerAdmins); + return Collections.unmodifiableList(admins); + } + + public final List getAllClients() { + List clients = new ArrayList<>(numberOfAdditionalBrokers() + 1); + clients.add(pulsarClient); + clients.addAll(additionalBrokerClients); + return Collections.unmodifiableList(clients); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 49e7ef30fa70c..a3f7166cace8e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -317,8 +317,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { // Override default providers with mocked ones doReturn(mockZooKeeperClientFactory).when(pulsar).getZooKeeperClientFactory(); doReturn(mockBookKeeperClientFactory).when(pulsar).newBookKeeperClientFactory(); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(pulsar).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(pulsar).createConfigurationMetadataStore(); + doReturn(createLocalMetadataStore()).when(pulsar).createLocalMetadataStore(); + doReturn(createConfigurationMetadataStore()).when(pulsar).createConfigurationMetadataStore(); Supplier namespaceServiceSupplier = () -> spy(new NamespaceService(pulsar)); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); @@ -332,6 +332,14 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { } } + protected ZKMetadataStore createLocalMetadataStore() { + return new ZKMetadataStore(mockZooKeeper); + } + + protected ZKMetadataStore createConfigurationMetadataStore() { + return new ZKMetadataStore(mockZooKeeperGlobal); + } + private void mockConfigBrokerInterceptors(PulsarService pulsarService) { ServiceConfiguration configuration = spy(conf); Set mockBrokerInterceptors = mock(Set.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java new file mode 100644 index 0000000000000..0045dddeb8161 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.broker.PulsarService; +import org.awaitility.Awaitility; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class MultiBrokerLeaderElectionTest extends MultiBrokerBaseTest { + + @Test + public void shouldElectOneLeader() { + int leaders = 0; + for (PulsarService broker : getAllBrokers()) { + if (broker.getLeaderElectionService().isLeader()) { + leaders++; + } + } + assertEquals(leaders, 1); + } + + @Test + public void shouldAllBrokersKnowTheLeader() { + Awaitility.await().untilAsserted(() -> { + for (PulsarService broker : getAllBrokers()) { + Optional currentLeader = broker.getLeaderElectionService().getCurrentLeader(); + assertTrue(currentLeader.isPresent(), "Leader wasn't known on broker " + broker.getBrokerServiceUrl()); + } + }); + } + + @Test + public void shouldAllBrokersBeAbleToGetTheLeader() { + Awaitility.await().untilAsserted(() -> { + LeaderBroker leader = null; + for (PulsarService broker : getAllBrokers()) { + Optional currentLeader = + broker.getLeaderElectionService().readCurrentLeader().get(1, TimeUnit.SECONDS); + assertTrue(currentLeader.isPresent(), "Leader wasn't known on broker " + broker.getBrokerServiceUrl()); + if (leader != null) { + assertEquals(currentLeader.get(), leader, + "Different leader on broker " + broker.getBrokerServiceUrl()); + } else { + leader = currentLeader.get(); + } + } + }); + } +} diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index 29cb4f97fe0e3..6599c625f788b 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -111,7 +111,10 @@ private synchronized CompletableFuture elect() { } else { return tryToBecomeLeader(); } - }); + }).thenCompose(leaderElectionState -> + // make sure that the cache contains the current leader + // so that getLeaderValueIfPresent works on all brokers + cache.get(path).thenApply(__ -> leaderElectionState)); } private synchronized CompletableFuture handleExistingLeaderValue(GetResult res) { diff --git a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java index c7f677ddec699..cd0c60c0087df 100644 --- a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java +++ b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java @@ -19,32 +19,33 @@ package org.apache.zookeeper; import com.google.common.collect.HashMultimap; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimaps; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import io.netty.util.concurrent.DefaultThreadFactory; - -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiPredicate; - import lombok.AllArgsConstructor; import lombok.Data; -import org.apache.commons.lang3.tuple.Pair; +import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.AsyncCallback.Children2Callback; import org.apache.zookeeper.AsyncCallback.ChildrenCallback; import org.apache.zookeeper.AsyncCallback.DataCallback; @@ -53,7 +54,6 @@ import org.apache.zookeeper.AsyncCallback.VoidCallback; import org.apache.zookeeper.Watcher.Event.EventType; import org.apache.zookeeper.Watcher.Event.KeeperState; -import org.apache.zookeeper.client.HostProvider; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.objenesis.Objenesis; @@ -64,7 +64,19 @@ import org.slf4j.LoggerFactory; public class MockZooKeeper extends ZooKeeper { - private TreeMap> tree; + @Data + @AllArgsConstructor + private static class MockZNode { + byte[] content; + int version; + long ephemeralOwner; + + static MockZNode of(byte[] content, int version, long ephemeralOwner) { + return new MockZNode(content, version, ephemeralOwner); + } + } + + private TreeMap tree; private SetMultimap watchers; private volatile boolean stopped; private AtomicReference alwaysFail; @@ -78,6 +90,7 @@ public class MockZooKeeper extends ZooKeeper { private ReentrantLock mutex; private AtomicLong sequentialIdGenerator; + private ThreadLocal epheralOwnerThreadLocal; //see details of Objenesis caching - http://objenesis.org/details.html //see supported jvms - https://github.com/easymock/objenesis/blob/master/SupportedJVMs.md @@ -85,9 +98,9 @@ public class MockZooKeeper extends ZooKeeper { public enum Op { CREATE, GET, SET, GET_CHILDREN, DELETE, EXISTS, SYNC, - }; + } - private class Failure { + private static class Failure { final KeeperException.Code failReturnCode; final BiPredicate predicate; @@ -121,14 +134,7 @@ public static MockZooKeeper newInstanceForGlobalZK(ExecutorService executor) { public static MockZooKeeper newInstanceForGlobalZK(ExecutorService executor, int readOpDelayMs) { try { - ObjectInstantiator mockZooKeeperInstantiator = - new ObjenesisStd().getInstantiatorOf(MockZooKeeper.class); - MockZooKeeper zk = (MockZooKeeper) mockZooKeeperInstantiator.newInstance(); - zk.init(executor); - zk.readOpDelayMs = readOpDelayMs; - zk.mutex = new ReentrantLock(); - zk.sequentialIdGenerator = new AtomicLong(); - return zk; + return createMockZooKeeperInstance(executor, readOpDelayMs); } catch (RuntimeException e) { throw e; } catch (Exception e) { @@ -138,14 +144,9 @@ public static MockZooKeeper newInstanceForGlobalZK(ExecutorService executor, int public static MockZooKeeper newInstance(ExecutorService executor, int readOpDelayMs) { try { - ObjectInstantiator mockZooKeeperInstantiator = objenesis.getInstantiatorOf(MockZooKeeper.class); - MockZooKeeper zk = (MockZooKeeper) mockZooKeeperInstantiator.newInstance(); - zk.init(executor); - zk.readOpDelayMs = readOpDelayMs; - zk.mutex = new ReentrantLock(); + MockZooKeeper zk = createMockZooKeeperInstance(executor, readOpDelayMs); ObjectInstantiator clientCnxnObjectInstantiator = objenesis.getInstantiatorOf(ClientCnxn.class); Whitebox.setInternalState(zk, "cnxn", clientCnxnObjectInstantiator.newInstance()); - zk.sequentialIdGenerator = new AtomicLong(); return zk; } catch (RuntimeException e) { throw e; @@ -154,6 +155,19 @@ public static MockZooKeeper newInstance(ExecutorService executor, int readOpDela } } + private static MockZooKeeper createMockZooKeeperInstance(ExecutorService executor, int readOpDelayMs) { + ObjectInstantiator mockZooKeeperInstantiator = + objenesis.getInstantiatorOf(MockZooKeeper.class); + MockZooKeeper zk = mockZooKeeperInstantiator.newInstance(); + zk.epheralOwnerThreadLocal = new ThreadLocal<>(); + zk.init(executor); + zk.readOpDelayMs = readOpDelayMs; + zk.mutex = new ReentrantLock(); + zk.lockInstance = ThreadLocal.withInitial(zk::createLock); + zk.sequentialIdGenerator = new AtomicLong(); + return zk; + } + private void init(ExecutorService executor) { tree = Maps.newTreeMap(); if (executor != null) { @@ -176,7 +190,8 @@ public int getSessionTimeout() { private MockZooKeeper(String quorum) throws Exception { // This constructor is never called - super(quorum, 1, event -> {}); + super(quorum, 1, event -> { + }); assert false; } @@ -185,27 +200,68 @@ public States getState() { return States.CONNECTED; } + + @Slf4j + private static class SingleAcquireAndReleaseLock { + private final AtomicBoolean acquired = new AtomicBoolean(false); + private final Lock lock; + + SingleAcquireAndReleaseLock(Lock lock) { + this.lock = lock; + } + + public void lock() { + if (acquired.compareAndSet(false, true)) { + lock.lock(); + } else { + throw new IllegalStateException("Lock was already acquired!"); + } + } + + public void unlockIfNeeded() { + if (acquired.compareAndSet(true, false)) { + lock.unlock(); + } + } + } + + private ThreadLocal lockInstance; + + private SingleAcquireAndReleaseLock createLock() { + return new SingleAcquireAndReleaseLock(mutex); + } + + private void lock() { + lockInstance.get().lock(); + } + + private void unlockIfLocked() { + lockInstance.get().unlockIfNeeded(); + } + @Override public void register(Watcher watcher) { - mutex.lock(); + lock(); sessionWatcher = watcher; - mutex.unlock(); + unlockIfLocked(); } @Override public String create(String path, byte[] data, List acl, CreateMode createMode) throws KeeperException, InterruptedException { - mutex.lock(); - final Set toNotifyCreate = Sets.newHashSet(); final Set toNotifyParent = Sets.newHashSet(); final String parent = path.substring(0, path.lastIndexOf("/")); + lock(); try { + + maybeThrowProgrammedFailure(Op.CREATE, path); - if (stopped) + if (stopped) { throw new KeeperException.ConnectionLossException(); + } if (tree.containsKey(path)) { throw new KeeperException.NodeExistsException(path); @@ -215,16 +271,17 @@ public String create(String path, byte[] data, List acl, CreateMode createM throw new KeeperException.NoNodeException(); } - if (createMode == CreateMode.EPHEMERAL_SEQUENTIAL || createMode == CreateMode.PERSISTENT_SEQUENTIAL) { - byte[] parentData = tree.get(parent).getLeft(); - int parentVersion = tree.get(parent).getRight(); + if (createMode.isSequential()) { + MockZNode parentNode = tree.get(parent); + int parentVersion = tree.get(parent).getVersion(); path = path + parentVersion; // Update parent version - tree.put(parent, Pair.of(parentData, parentVersion + 1)); + tree.put(parent, + MockZNode.of(parentNode.getContent(), parentVersion + 1, parentNode.getEphemeralOwner())); } - tree.put(path, Pair.of(data, 0)); + tree.put(path, MockZNode.of(data, 0, createMode.isEphemeral() ? getEphemeralOwner() : -1L)); toNotifyCreate.addAll(watchers.get(path)); @@ -233,8 +290,7 @@ public String create(String path, byte[] data, List acl, CreateMode createM } watchers.removeAll(path); } finally { - - mutex.unlock(); + unlockIfLocked(); } final String finalPath = path; @@ -256,68 +312,90 @@ public String create(String path, byte[] data, List acl, CreateMode createM return path; } + protected long getEphemeralOwner() { + Long epheralOwner = epheralOwnerThreadLocal.get(); + if (epheralOwner != null) { + return epheralOwner; + } + return getSessionId(); + } + + public void overrideEpheralOwner(long epheralOwner) { + epheralOwnerThreadLocal.set(epheralOwner); + } + + public void removeEpheralOwnerOverride() { + epheralOwnerThreadLocal.remove(); + } + @Override public void create(final String path, final byte[] data, final List acl, CreateMode createMode, - final StringCallback cb, final Object ctx) { + final StringCallback cb, final Object ctx) { executor.execute(() -> { - mutex.lock(); + lock(); + try { - if (stopped) { - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); - return; - } + if (stopped) { + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); + return; + } - final Set toNotifyCreate = Sets.newHashSet(); - toNotifyCreate.addAll(watchers.get(path)); + final Set toNotifyCreate = Sets.newHashSet(); + toNotifyCreate.addAll(watchers.get(path)); - final Set toNotifyParent = Sets.newHashSet(); - final String parent = path.substring(0, path.lastIndexOf("/")); - if (!parent.isEmpty()) { - toNotifyParent.addAll(watchers.get(parent)); - } + final Set toNotifyParent = Sets.newHashSet(); + final String parent = path.substring(0, path.lastIndexOf("/")); + if (!parent.isEmpty()) { + toNotifyParent.addAll(watchers.get(parent)); + } - final String name; - if (createMode != null && createMode.isSequential()) { - name = path + Long.toString(sequentialIdGenerator.getAndIncrement()); - } else { - name = path; - } + final String name; + if (createMode != null && createMode.isSequential()) { + name = path + sequentialIdGenerator.getAndIncrement(); + } else { + name = path; + } - Optional failure = programmedFailure(Op.CREATE, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null); - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); - } else if (tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NODEEXISTS.intValue(), path, ctx, null); - } else if (!parent.isEmpty() && !tree.containsKey(parent)) { - mutex.unlock(); - toNotifyParent.forEach(watcher -> watcher - .process(new WatchedEvent(EventType.NodeChildrenChanged, KeeperState.SyncConnected, parent))); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); - } else { - tree.put(name, Pair.of(data, 0)); - watchers.removeAll(name); - mutex.unlock(); - cb.processResult(0, path, ctx, name); - - triggerPersistentWatches(path, parent, EventType.NodeCreated); - - toNotifyCreate.forEach( - watcher -> watcher.process( - new WatchedEvent(EventType.NodeCreated, - KeeperState.SyncConnected, - name))); - toNotifyParent.forEach( - watcher -> watcher.process( - new WatchedEvent(EventType.NodeChildrenChanged, - KeeperState.SyncConnected, - parent))); + Optional failure = programmedFailure(Op.CREATE, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx, null); + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); + } else if (tree.containsKey(path)) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NODEEXISTS.intValue(), path, ctx, null); + } else if (!parent.isEmpty() && !tree.containsKey(parent)) { + unlockIfLocked(); + toNotifyParent.forEach(watcher -> watcher + .process(new WatchedEvent(EventType.NodeChildrenChanged, KeeperState.SyncConnected, + parent))); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); + } else { + tree.put(name, MockZNode.of(data, 0, + createMode != null && createMode.isEphemeral() ? getEphemeralOwner() : -1L)); + watchers.removeAll(name); + unlockIfLocked(); + cb.processResult(0, path, ctx, name); + + triggerPersistentWatches(path, parent, EventType.NodeCreated); + + toNotifyCreate.forEach( + watcher -> watcher.process( + new WatchedEvent(EventType.NodeCreated, + KeeperState.SyncConnected, + name))); + toNotifyParent.forEach( + watcher -> watcher.process( + new WatchedEvent(EventType.NodeChildrenChanged, + KeeperState.SyncConnected, + parent))); + } + } finally { + unlockIfLocked(); } }); @@ -325,10 +403,10 @@ public void create(final String path, final byte[] data, final List acl, Cr @Override public byte[] getData(String path, Watcher watcher, Stat stat) throws KeeperException { - mutex.lock(); + lock(); try { maybeThrowProgrammedFailure(Op.GET, path); - Pair value = tree.get(path); + MockZNode value = tree.get(path); if (value == null) { throw new KeeperException.NoNodeException(path); } else { @@ -336,12 +414,12 @@ public byte[] getData(String path, Watcher watcher, Stat stat) throws KeeperExce watchers.put(path, watcher); } if (stat != null) { - stat.setVersion(value.getRight()); + applyToStat(value, stat); } - return value.getLeft(); + return value.getContent(); } } finally { - mutex.unlock(); + unlockIfLocked(); } } @@ -358,20 +436,18 @@ public void getData(final String path, boolean watch, final DataCallback cb, fin return; } - Pair value; - mutex.lock(); + MockZNode value; + lock(); try { value = tree.get(path); } finally { - mutex.unlock(); + unlockIfLocked(); } if (value == null) { cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null, null); } else { - Stat stat = new Stat(); - stat.setVersion(value.getRight()); - cb.processResult(0, path, ctx, value.getLeft(), stat); + cb.processResult(0, path, ctx, value.getContent(), createStatForZNode(value)); } }); } @@ -380,31 +456,34 @@ public void getData(final String path, boolean watch, final DataCallback cb, fin public void getData(final String path, final Watcher watcher, final DataCallback cb, final Object ctx) { executor.execute(() -> { checkReadOpDelay(); - mutex.lock(); - Optional failure = programmedFailure(Op.GET, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null, null); - return; - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null, null); - return; - } - - Pair value = tree.get(path); - if (value == null) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null, null); - } else { - if (watcher != null) { - watchers.put(path, watcher); + lock(); + try { + Optional failure = programmedFailure(Op.GET, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx, null, null); + return; + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null, null); + return; } - Stat stat = new Stat(); - stat.setVersion(value.getRight()); - mutex.unlock(); - cb.processResult(0, path, ctx, value.getLeft(), stat); + MockZNode value = tree.get(path); + if (value == null) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null, null); + } else { + if (watcher != null) { + watchers.put(path, watcher); + } + + Stat stat = createStatForZNode(value); + unlockIfLocked(); + cb.processResult(0, path, ctx, value.getContent(), stat); + } + } finally { + unlockIfLocked(); } }); } @@ -412,44 +491,47 @@ public void getData(final String path, final Watcher watcher, final DataCallback @Override public void getChildren(final String path, final Watcher watcher, final ChildrenCallback cb, final Object ctx) { executor.execute(() -> { - mutex.lock(); - Optional failure = programmedFailure(Op.GET_CHILDREN, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null); - return; - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); - return; - } + lock(); + List children = Lists.newArrayList(); + try { + Optional failure = programmedFailure(Op.GET_CHILDREN, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx, null); + return; + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); + return; + } - if (!tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); - return; - } + if (!tree.containsKey(path)) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); + return; + } - List children = Lists.newArrayList(); - for (String item : tree.tailMap(path).keySet()) { - if (!item.startsWith(path)) { - break; - } else { - if (path.length() >= item.length()) { - continue; - } + for (String item : tree.tailMap(path).keySet()) { + if (!item.startsWith(path)) { + break; + } else { + if (path.length() >= item.length()) { + continue; + } - String child = item.substring(path.length() + 1); - if (item.charAt(path.length()) == '/' && !child.contains("/")) { - children.add(child); + String child = item.substring(path.length() + 1); + if (item.charAt(path.length()) == '/' && !child.contains("/")) { + children.add(child); + } } } - } - if (watcher != null) { - watchers.put(path, watcher); + if (watcher != null) { + watchers.put(path, watcher); + } + } finally { + unlockIfLocked(); } - mutex.unlock(); cb.processResult(0, path, ctx, children); }); @@ -457,7 +539,7 @@ public void getChildren(final String path, final Watcher watcher, final Children @Override public List getChildren(String path, Watcher watcher) throws KeeperException { - mutex.lock(); + lock(); try { maybeThrowProgrammedFailure(Op.GET_CHILDREN, path); @@ -465,35 +547,31 @@ public List getChildren(String path, Watcher watcher) throws KeeperExcep throw new KeeperException.NoNodeException(); } - List children = Lists.newArrayList(); - for (String item : tree.tailMap(path).keySet()) { - if (!item.startsWith(path)) { - break; - } else { - if (path.length() >= item.length()) { - continue; - } + String firstKey = path.equals("/") ? path : path + "/"; + String lastKey = path.equals("/") ? "0" : path + "0"; // '0' is lexicographically just after '/' - String child = item.substring(path.length() + 1); - if (!child.contains("/")) { - children.add(child); - } - } - } + Set children = new TreeSet<>(); + tree.subMap(firstKey, false, lastKey, false).forEach((key, value) -> { + String relativePath = key.replace(firstKey, ""); + + // Only return first-level children + String child = relativePath.split("/", 2)[0]; + children.add(child); + }); if (watcher != null) { watchers.put(path, watcher); } - return children; + return new ArrayList<>(children); } finally { - mutex.unlock(); + unlockIfLocked(); } } @Override public List getChildren(String path, boolean watch) throws KeeperException, InterruptedException { - mutex.lock(); + lock(); try { maybeThrowProgrammedFailure(Op.GET_CHILDREN, path); @@ -503,173 +581,153 @@ public List getChildren(String path, boolean watch) throws KeeperExcepti throw new KeeperException.NoNodeException(); } - List children = Lists.newArrayList(); - for (String item : tree.tailMap(path).keySet()) { - if (!item.startsWith(path)) { - break; - } else { - if (path.length() >= item.length()) { - continue; - } - String child = item.substring(path.length()); - if (child.indexOf("/") == 0) { - child = child.substring(1); - log.debug("child: '{}'", child); - if (!child.contains("/")) { - children.add(child); - } - } - } - } - return children; + String firstKey = path.equals("/") ? path : path + "/"; + String lastKey = path.equals("/") ? "0" : path + "0"; // '0' is lexicographically just after '/' + + Set children = new TreeSet<>(); + tree.subMap(firstKey, false, lastKey, false).forEach((key, value) -> { + String relativePath = key.replace(firstKey, ""); + + // Only return first-level children + String child = relativePath.split("/", 2)[0]; + children.add(child); + }); + + return new ArrayList<>(children); } finally { - mutex.unlock(); + unlockIfLocked(); } } @Override public void getChildren(final String path, boolean watcher, final Children2Callback cb, final Object ctx) { executor.execute(() -> { - mutex.lock(); + Set children = new TreeSet<>(); + lock(); + try { + Optional failure = programmedFailure(Op.GET_CHILDREN, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx, null, null); + return; + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null, null); + return; + } else if (!tree.containsKey(path)) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null, null); + return; + } - Optional failure = programmedFailure(Op.GET_CHILDREN, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null, null); - return; - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null, null); - return; - } else if (!tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null, null); - return; - } + String firstKey = path.equals("/") ? path : path + "/"; + String lastKey = path.equals("/") ? "0" : path + "0"; // '0' is lexicographically just after '/' - log.debug("getChildren path={}", path); - List children = Lists.newArrayList(); - for (String item : tree.tailMap(path).keySet()) { - log.debug("Checking path {}", item); - if (!item.startsWith(path)) { - break; - } else if (item.equals(path)) { - continue; - } else { - String child = item.substring(path.length()); - if (child.indexOf("/") == 0) { - child = child.substring(1); - log.debug("child: '{}'", child); - if (!child.contains("/")) { - children.add(child); - } - } - } - } + tree.subMap(firstKey, false, lastKey, false).forEach((key, value) -> { + String relativePath = key.replace(firstKey, ""); - log.debug("getChildren done path={} result={}", path, children); - mutex.unlock(); - cb.processResult(0, path, ctx, children, new Stat()); + // Only return first-level children + String child = relativePath.split("/", 2)[0]; + children.add(child); + }); + } finally { + unlockIfLocked(); + } + cb.processResult(0, path, ctx, new ArrayList<>(children), new Stat()); }); } @Override public Stat exists(String path, boolean watch) throws KeeperException, InterruptedException { - mutex.lock(); + lock(); try { maybeThrowProgrammedFailure(Op.EXISTS, path); - if (stopped) + if (stopped) { throw new KeeperException.ConnectionLossException(); + } if (tree.containsKey(path)) { - Stat stat = new Stat(); - stat.setVersion(tree.get(path).getRight()); - return stat; + return createStatForZNode(tree.get(path)); } else { return null; } } finally { - mutex.unlock(); + unlockIfLocked(); + } + } + + private static Stat createStatForZNode(MockZNode zNode) { + return applyToStat(zNode, new Stat()); + } + + private static Stat applyToStat(MockZNode zNode, Stat stat) { + stat.setVersion(zNode.getVersion()); + if (zNode.getEphemeralOwner() != -1L) { + stat.setEphemeralOwner(zNode.getEphemeralOwner()); } + return stat; } @Override public Stat exists(String path, Watcher watcher) throws KeeperException, InterruptedException { - mutex.lock(); + lock(); try { maybeThrowProgrammedFailure(Op.EXISTS, path); - if (stopped) + if (stopped) { throw new KeeperException.ConnectionLossException(); + } if (watcher != null) { watchers.put(path, watcher); } if (tree.containsKey(path)) { - Stat stat = new Stat(); - stat.setVersion(tree.get(path).getRight()); - return stat; + return createStatForZNode(tree.get(path)); } else { return null; } } finally { - mutex.unlock(); + unlockIfLocked(); } } @Override public void exists(String path, boolean watch, StatCallback cb, Object ctx) { - executor.execute(() -> { - mutex.lock(); - Optional failure = programmedFailure(Op.EXISTS, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null); - return; - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); - return; - } - - if (tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(0, path, ctx, new Stat()); - } else { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); - } - }); + exists(path, null, cb, ctx); } @Override public void exists(String path, Watcher watcher, StatCallback cb, Object ctx) { executor.execute(() -> { - mutex.lock(); - Optional failure = programmedFailure(Op.EXISTS, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null); - return; - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); - return; - } + lock(); + try { + Optional failure = programmedFailure(Op.EXISTS, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx, null); + return; + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); + return; + } - if (watcher != null) { - watchers.put(path, watcher); - } + if (watcher != null) { + watchers.put(path, watcher); + } - if (tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(0, path, ctx, new Stat()); - } else { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); + if (tree.containsKey(path)) { + unlockIfLocked(); + cb.processResult(0, path, ctx, new Stat()); + } else { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); + } + } finally { + unlockIfLocked(); } }); } @@ -693,11 +751,10 @@ public void sync(String path, VoidCallback cb, Object ctx) { @Override public Stat setData(final String path, byte[] data, int version) throws KeeperException, InterruptedException { - mutex.lock(); - final Set toNotify = Sets.newHashSet(); - int newVersion; + MockZNode newZNode; + lock(); try { maybeThrowProgrammedFailure(Op.SET, path); @@ -709,21 +766,22 @@ public Stat setData(final String path, byte[] data, int version) throws KeeperEx throw new KeeperException.NoNodeException(); } - int currentVersion = tree.get(path).getRight(); + MockZNode mockZNode = tree.get(path); + int currentVersion = mockZNode.getVersion(); // Check version if (version != -1 && version != currentVersion) { throw new KeeperException.BadVersionException(path); } - newVersion = currentVersion + 1; log.debug("[{}] Updating -- current version: {}", path, currentVersion); - tree.put(path, Pair.of(data, newVersion)); + newZNode = MockZNode.of(data, currentVersion + 1, mockZNode.getEphemeralOwner()); + tree.put(path, newZNode); toNotify.addAll(watchers.get(path)); watchers.removeAll(path); } finally { - mutex.unlock(); + unlockIfLocked(); } executor.execute(() -> { @@ -733,9 +791,7 @@ public Stat setData(final String path, byte[] data, int version) throws KeeperEx .process(new WatchedEvent(EventType.NodeDataChanged, KeeperState.SyncConnected, path))); }); - Stat stat = new Stat(); - stat.setVersion(newVersion); - return stat; + return createStatForZNode(newZNode); } @Override @@ -747,43 +803,45 @@ public void setData(final String path, final byte[] data, int version, final Sta executor.execute(() -> { final Set toNotify = Sets.newHashSet(); + Stat stat; + lock(); + try { - mutex.lock(); + Optional failure = programmedFailure(Op.SET, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx, null); + return; + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); + return; + } - Optional failure = programmedFailure(Op.SET, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx, null); - return; - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx, null); - return; - } + if (!tree.containsKey(path)) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); + return; + } - if (!tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx, null); - return; - } + MockZNode mockZNode = tree.get(path); + int currentVersion = mockZNode.getVersion(); - int currentVersion = tree.get(path).getRight(); + // Check version + if (version != -1 && version != currentVersion) { + log.debug("[{}] Current version: {} -- Expected: {}", path, currentVersion, version); + unlockIfLocked(); + cb.processResult(KeeperException.Code.BADVERSION.intValue(), path, ctx, null); + return; + } - // Check version - if (version != -1 && version != currentVersion) { - log.debug("[{}] Current version: {} -- Expected: {}", path, currentVersion, version); - mutex.unlock(); - cb.processResult(KeeperException.Code.BADVERSION.intValue(), path, ctx, null); - return; + log.debug("[{}] Updating -- current version: {}", path, currentVersion); + MockZNode newZNode = MockZNode.of(data, currentVersion + 1, mockZNode.getEphemeralOwner()); + tree.put(path, newZNode); + stat = createStatForZNode(newZNode); + } finally { + unlockIfLocked(); } - - int newVersion = currentVersion + 1; - log.debug("[{}] Updating -- current version: {}", path, currentVersion); - tree.put(path, Pair.of(data, newVersion)); - Stat stat = new Stat(); - stat.setVersion(newVersion); - - mutex.unlock(); cb.processResult(0, path, ctx, stat); toNotify.addAll(watchers.get(path)); @@ -805,7 +863,7 @@ public void delete(final String path, int version) throws InterruptedException, final Set toNotifyParent; final String parent; - mutex.lock(); + lock(); try { if (stopped) { throw new KeeperException.ConnectionLossException(); @@ -816,7 +874,7 @@ public void delete(final String path, int version) throws InterruptedException, } if (version != -1) { - int currentVersion = tree.get(path).getRight(); + int currentVersion = tree.get(path).getVersion(); if (version != currentVersion) { throw new KeeperException.BadVersionException(path); } @@ -835,7 +893,7 @@ public void delete(final String path, int version) throws InterruptedException, watchers.removeAll(path); } finally { - mutex.unlock(); + unlockIfLocked(); } executor.execute(() -> { @@ -857,50 +915,55 @@ public void delete(final String path, int version) throws InterruptedException, @Override public void delete(final String path, int version, final VoidCallback cb, final Object ctx) { Runnable r = () -> { - mutex.lock(); - final Set toNotifyDelete = Sets.newHashSet(); - toNotifyDelete.addAll(watchers.get(path)); - - final Set toNotifyParent = Sets.newHashSet(); - final String parent = path.substring(0, path.lastIndexOf("/")); - if (!parent.isEmpty()) { - toNotifyParent.addAll(watchers.get(parent)); - } - watchers.removeAll(path); + lock(); + try { + final Set toNotifyDelete = Sets.newHashSet(); + toNotifyDelete.addAll(watchers.get(path)); - Optional failure = programmedFailure(Op.DELETE, path); - if (failure.isPresent()) { - mutex.unlock(); - cb.processResult(failure.get().intValue(), path, ctx); - } else if (stopped) { - mutex.unlock(); - cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx); - } else if (!tree.containsKey(path)) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx); - } else if (hasChildren(path)) { - mutex.unlock(); - cb.processResult(KeeperException.Code.NOTEMPTY.intValue(), path, ctx); - } else { - if (version != -1) { - int currentVersion = tree.get(path).getRight(); - if (version != currentVersion) { - mutex.unlock(); - cb.processResult(KeeperException.Code.BADVERSION.intValue(), path, ctx); - return; - } + final Set toNotifyParent = Sets.newHashSet(); + final String parent = path.substring(0, path.lastIndexOf("/")); + if (!parent.isEmpty()) { + toNotifyParent.addAll(watchers.get(parent)); } + watchers.removeAll(path); + + Optional failure = programmedFailure(Op.DELETE, path); + if (failure.isPresent()) { + unlockIfLocked(); + cb.processResult(failure.get().intValue(), path, ctx); + } else if (stopped) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.CONNECTIONLOSS.intValue(), path, ctx); + } else if (!tree.containsKey(path)) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NONODE.intValue(), path, ctx); + } else if (hasChildren(path)) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.NOTEMPTY.intValue(), path, ctx); + } else { + if (version != -1) { + int currentVersion = tree.get(path).getVersion(); + if (version != currentVersion) { + unlockIfLocked(); + cb.processResult(KeeperException.Code.BADVERSION.intValue(), path, ctx); + return; + } + } - tree.remove(path); + tree.remove(path); - mutex.unlock(); - cb.processResult(0, path, ctx); + unlockIfLocked(); + cb.processResult(0, path, ctx); - toNotifyDelete.forEach(watcher -> watcher - .process(new WatchedEvent(EventType.NodeDeleted, KeeperState.SyncConnected, path))); - toNotifyParent.forEach(watcher -> watcher - .process(new WatchedEvent(EventType.NodeChildrenChanged, KeeperState.SyncConnected, parent))); - triggerPersistentWatches(path, parent, EventType.NodeDeleted); + toNotifyDelete.forEach(watcher -> watcher + .process(new WatchedEvent(EventType.NodeDeleted, KeeperState.SyncConnected, path))); + toNotifyParent.forEach(watcher -> watcher + .process(new WatchedEvent(EventType.NodeChildrenChanged, KeeperState.SyncConnected, + parent))); + triggerPersistentWatches(path, parent, EventType.NodeDeleted); + } + } finally { + unlockIfLocked(); } }; @@ -908,7 +971,6 @@ public void delete(final String path, int version, final VoidCallback cb, final executor.execute(r); } catch (RejectedExecutionException ree) { cb.processResult(KeeperException.Code.SESSIONEXPIRED.intValue(), path, ctx); - return; } } @@ -917,30 +979,62 @@ public void delete(final String path, int version, final VoidCallback cb, final public void multi(Iterable ops, AsyncCallback.MultiCallback cb, Object ctx) { try { List res = multi(ops); - cb.processResult(KeeperException.Code.OK.intValue(), (String)null, ctx, res); + cb.processResult(KeeperException.Code.OK.intValue(), null, ctx, res); } catch (Exception e) { - cb.processResult(KeeperException.Code.APIERROR.intValue(), (String)null, ctx, null); + cb.processResult(KeeperException.Code.APIERROR.intValue(), null, ctx, null); } } @Override public List multi(Iterable ops) throws InterruptedException, KeeperException { List res = new ArrayList<>(); - for (org.apache.zookeeper.Op op : ops) { - switch (op.getType()) { - case ZooDefs.OpCode.create: - this.create(op.getPath(), ((org.apache.zookeeper.Op.Create)op).data, null, null); - res.add(new OpResult.CreateResult(op.getPath())); - break; - case ZooDefs.OpCode.delete: - this.delete(op.getPath(), -1); - res.add(new OpResult.DeleteResult()); - break; - case ZooDefs.OpCode.setData: - this.create(op.getPath(), ((org.apache.zookeeper.Op.Create)op).data, null, null); - res.add(new OpResult.SetDataResult(null)); - break; - default: + try { + for (org.apache.zookeeper.Op op : ops) { + switch (op.getType()) { + case ZooDefs.OpCode.create: { + org.apache.zookeeper.Op.Create opc = ((org.apache.zookeeper.Op.Create) op); + CreateMode cm = CreateMode.fromFlag(opc.flags); + String path = this.create(op.getPath(), opc.data, null, cm); + res.add(new OpResult.CreateResult(path)); + break; + } + case ZooDefs.OpCode.delete: + this.delete(op.getPath(), Whitebox.getInternalState(op, "version")); + res.add(new OpResult.DeleteResult()); + break; + case ZooDefs.OpCode.setData: { + Stat stat = this.setData(op.getPath(), Whitebox.getInternalState(op, "data"), + Whitebox.getInternalState(op, "version")); + res.add(new OpResult.SetDataResult(stat)); + break; + } + case ZooDefs.OpCode.getChildren: { + try { + List children = this.getChildren(op.getPath(), null); + res.add(new OpResult.GetChildrenResult(children)); + } catch (KeeperException e) { + res.add(new OpResult.ErrorResult(e.code().intValue())); + } + break; + } + case ZooDefs.OpCode.getData: { + Stat stat = new Stat(); + try { + byte[] payload = this.getData(op.getPath(), null, stat); + res.add(new OpResult.GetDataResult(payload, stat)); + } catch (KeeperException e) { + res.add(new OpResult.ErrorResult(e.code().intValue())); + } + break; + } + default: + } + } + } catch (KeeperException e) { + res.add(new OpResult.ErrorResult(e.code().intValue())); + int total = Iterables.size(ops); + for (int i = res.size(); i < total; i++) { + res.add(new OpResult.ErrorResult(KeeperException.Code.RUNTIMEINCONSISTENCY.intValue())); } } return res; @@ -972,7 +1066,7 @@ public void close() throws InterruptedException { } public void shutdown() throws InterruptedException { - mutex.lock(); + lock(); try { stopped = true; tree.clear(); @@ -984,7 +1078,7 @@ public void shutdown() throws InterruptedException { log.error("MockZooKeeper shutdown had error", ex); } } finally { - mutex.unlock(); + unlockIfLocked(); } } diff --git a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeperSession.java b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeperSession.java index 499da0e345d44..a33d4483c89f3 100644 --- a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeperSession.java +++ b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeperSession.java @@ -44,7 +44,7 @@ public class MockZooKeeperSession extends ZooKeeper { private static final Objenesis objenesis = new ObjenesisStd(); - private static final AtomicInteger sessionIdGenerator = new AtomicInteger(0); + private static final AtomicInteger sessionIdGenerator = new AtomicInteger(1000); public static MockZooKeeperSession newInstance(MockZooKeeper mockZooKeeper) { ObjectInstantiator instantiator = objenesis.getInstantiatorOf(MockZooKeeperSession.class); @@ -80,13 +80,23 @@ public void register(Watcher watcher) { @Override public String create(String path, byte[] data, List acl, CreateMode createMode) throws KeeperException, InterruptedException { - return mockZooKeeper.create(path, data, acl, createMode); + try { + mockZooKeeper.overrideEpheralOwner(getSessionId()); + return mockZooKeeper.create(path, data, acl, createMode); + } finally { + mockZooKeeper.removeEpheralOwnerOverride(); + } } @Override public void create(final String path, final byte[] data, final List acl, CreateMode createMode, final AsyncCallback.StringCallback cb, final Object ctx) { - mockZooKeeper.create(path, data, acl, createMode, cb, ctx); + try { + mockZooKeeper.overrideEpheralOwner(getSessionId()); + mockZooKeeper.create(path, data, acl, createMode, cb, ctx); + } finally { + mockZooKeeper.removeEpheralOwnerOverride(); + } } @Override From da03d897a61f18fa4a5bbfcb1b656f626977cf8d Mon Sep 17 00:00:00 2001 From: Peter Tinti Date: Thu, 2 Dec 2021 22:22:29 -0800 Subject: [PATCH 031/823] [Functions][k8s] Expose function container metrics port (#12065) (cherry picked from commit 4dd4bd66db68283fc8486178df6f71a803654e29) --- .../runtime/kubernetes/KubernetesRuntime.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 1301596348d70..70839fc7bda06 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -1130,20 +1130,23 @@ V1Container getFunctionContainer(List instanceCommand, Function.Resource private List getFunctionContainerPorts() { List ports = new ArrayList<>(); + ports.add(getGRPCPort()); + ports.add(getPrometheusPort()); + return ports; + } + + private V1ContainerPort getGRPCPort() { final V1ContainerPort port = new V1ContainerPort(); port.setName("grpc"); port.setContainerPort(grpcPort); - ports.add(port); - return ports; + return port; } - private List getPrometheusContainerPorts() { - List ports = new ArrayList<>(); + private V1ContainerPort getPrometheusPort() { final V1ContainerPort port = new V1ContainerPort(); port.setName("prometheus"); port.setContainerPort(metricsPort); - ports.add(port); - return ports; + return port; } public static String createJobName(Function.FunctionDetails functionDetails, String jobName) { From 06c344f2a60ffa995be81669059be1cd6ef6851e Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 2 Dec 2021 22:55:44 -0600 Subject: [PATCH 032/823] [Java Client] Make Audience Field Optional in OAuth2 Client Credentials (#11988) * [Java Client] Make Audience Field in Client Credentials Optional * Update site2/website-next/docs * Remove update to 2.8.1 docs * Update more docs based on code review (cherry picked from commit b2b5463a574806b611c4065a0a27c154e7cbe0bd) --- .../oauth2/AuthenticationFactoryOAuth2.java | 6 ++--- .../auth/oauth2/ClientCredentialsFlow.java | 4 +-- .../pulsar/client/impl/auth/oauth2/README.md | 4 +-- .../auth/oauth2/protocol/TokenClient.java | 25 +++++++++++-------- .../auth/oauth2/AuthenticationOAuth2Test.java | 13 ++++++++++ .../auth/oauth2/protocol/TokenClientTest.java | 18 +++---------- site2/docs/security-oauth2.md | 4 +-- site2/website-next/docs/security-oauth2.md | 4 +-- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java index 707fcaf99c6d3..cf567747567e4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java @@ -33,7 +33,7 @@ public final class AuthenticationFactoryOAuth2 { * * @param issuerUrl the issuer URL * @param credentialsUrl the credentials URL - * @param audience the audience identifier + * @param audience An optional field. The audience identifier used by some Identity Providers, like Auth0. * @return an Authentication object */ public static Authentication clientCredentials(URL issuerUrl, URL credentialsUrl, String audience) { @@ -45,9 +45,9 @@ public static Authentication clientCredentials(URL issuerUrl, URL credentialsUrl * * @param issuerUrl the issuer URL * @param credentialsUrl the credentials URL - * @param audience the audience identifier + * @param audience An optional field. The audience identifier used by some Identity Providers, like Auth0. * @param scope An optional field. The value of the scope parameter is expressed as a list of space-delimited, - * case-sensitive strings. The strings are defined by the authorization server. + * case-sensitive strings. The strings are defined by the authorization server. * If the value contains multiple space-delimited strings, their order does not matter, * and each string adds an additional access range to the requested scope. * From here: https://datatracker.ietf.org/doc/html/rfc6749#section-4.4.2 diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java index b011e85dde0f5..bf0c289c0df60 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java @@ -118,10 +118,10 @@ public void close() throws Exception { */ public static ClientCredentialsFlow fromParameters(Map params) { URL issuerUrl = parseParameterUrl(params, CONFIG_PARAM_ISSUER_URL); - String audience = parseParameterString(params, CONFIG_PARAM_AUDIENCE); String privateKeyUrl = parseParameterString(params, CONFIG_PARAM_KEY_FILE); - // This is an optional parameter + // These are optional parameters, so we only perform a get String scope = params.get(CONFIG_PARAM_SCOPE); + String audience = params.get(CONFIG_PARAM_AUDIENCE); return ClientCredentialsFlow.builder() .issuerUrl(issuerUrl) .audience(audience) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/README.md b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/README.md index 55ffe58cf0503..b8b1237aa153b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/README.md +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/README.md @@ -46,7 +46,7 @@ The following parameters are supported: | `type` | Oauth 2.0 auth type. Optional. | default: `client_credentials` | | `issuerUrl` | URL of the provider which allows Pulsar to obtain an access token. Required. | `https://accounts.google.com` | | `privateKey` | URL to a JSON credentials file (in JSON format; see below). Required. | See "Supported Pattern Formats" | -| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster. Required. | `https://broker.example.com` | +| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster. Required by some Identity Providers. Optional for client. | `https://broker.example.com` | ### Supported Pattern Formats of `privateKey` The `privateKey` parameter supports the following three pattern formats, and contains client Credentials: @@ -88,7 +88,7 @@ curl --request POST \ In which, - `issuerUrl` parameter in this plugin is mapped to `--url https://dev-kt-aa9ne.us.auth0.com` - `privateKey` file parameter in this plugin should at least contains fields `client_id` and `client_secret`. -- `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"` +- `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. This field is only used by some identity providers. ## Pulsar Client Config You can use the provider with the following Pulsar clients. diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java index f8667e8625a77..c2b97793e3512 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java @@ -73,10 +73,21 @@ public void close() throws Exception { /** * Constructing http request parameters. - * @param bodyMap List of parameters to be requested. + * @param req object with relevant request parameters * @return Generate the final request body from a map. */ - String buildClientCredentialsBody(Map bodyMap) { + String buildClientCredentialsBody(ClientCredentialsExchangeRequest req) { + Map bodyMap = new TreeMap<>(); + bodyMap.put("grant_type", "client_credentials"); + bodyMap.put("client_id", req.getClientId()); + bodyMap.put("client_secret", req.getClientSecret()); + // Only set audience and scope if they are non-empty. + if (!StringUtils.isBlank(req.getAudience())) { + bodyMap.put("audience", req.getAudience()); + } + if (!StringUtils.isBlank(req.getScope())) { + bodyMap.put("scope", req.getScope()); + } return bodyMap.entrySet().stream() .map(e -> { try { @@ -96,15 +107,7 @@ String buildClientCredentialsBody(Map bodyMap) { */ public TokenResult exchangeClientCredentials(ClientCredentialsExchangeRequest req) throws TokenExchangeException, IOException { - Map bodyMap = new TreeMap<>(); - bodyMap.put("grant_type", "client_credentials"); - bodyMap.put("client_id", req.getClientId()); - bodyMap.put("client_secret", req.getClientSecret()); - bodyMap.put("audience", req.getAudience()); - if (!StringUtils.isBlank(req.getScope())) { - bodyMap.put("scope", req.getScope()); - } - String body = buildClientCredentialsBody(bodyMap); + String body = buildClientCredentialsBody(req); try { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java index ac14dd2aee105..3ae578c34845c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java @@ -86,6 +86,19 @@ public void testConfigure() throws Exception { params.put("privateKey", "data:base64,e30="); params.put("issuerUrl", "http://localhost"); params.put("audience", "http://localhost"); + params.put("scope", "http://localhost"); + ObjectMapper mapper = new ObjectMapper(); + String authParams = mapper.writeValueAsString(params); + this.auth.configure(authParams); + assertNotNull(this.auth.flow); + } + + @Test + public void testConfigureWithoutOptionalParams() throws Exception { + Map params = new HashMap<>(); + params.put("type", "client_credentials"); + params.put("privateKey", "data:base64,e30="); + params.put("issuerUrl", "http://localhost"); ObjectMapper mapper = new ObjectMapper(); String authParams = mapper.writeValueAsString(params); this.auth.configure(authParams); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java index 1617359ad08a9..da70d6cd58510 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java @@ -47,19 +47,13 @@ public void exchangeClientCredentialsSuccessByScopeTest() throws DefaultAsyncHttpClient defaultAsyncHttpClient = mock(DefaultAsyncHttpClient.class); URL url = new URL("http://localhost"); TokenClient tokenClient = new TokenClient(url, defaultAsyncHttpClient); - Map bodyMap = new TreeMap<>(); ClientCredentialsExchangeRequest request = ClientCredentialsExchangeRequest.builder() .audience("test-audience") .clientId("test-client-id") .clientSecret("test-client-secret") .scope("test-scope") .build(); - bodyMap.put("grant_type", "client_credentials"); - bodyMap.put("client_id", request.getClientId()); - bodyMap.put("client_secret", request.getClientSecret()); - bodyMap.put("audience", request.getAudience()); - bodyMap.put("scope", request.getScope()); - String body = tokenClient.buildClientCredentialsBody(bodyMap); + String body = tokenClient.buildClientCredentialsBody(request); BoundRequestBuilder boundRequestBuilder = mock(BoundRequestBuilder.class); Response response = mock(Response.class); ListenableFuture listenableFuture = mock(ListenableFuture.class); @@ -80,22 +74,16 @@ public void exchangeClientCredentialsSuccessByScopeTest() throws @Test @SuppressWarnings("unchecked") - public void exchangeClientCredentialsSuccessByNoScopeTest() throws + public void exchangeClientCredentialsSuccessWithoutOptionalClientCredentialsTest() throws IOException, TokenExchangeException, ExecutionException, InterruptedException { DefaultAsyncHttpClient defaultAsyncHttpClient = mock(DefaultAsyncHttpClient.class); URL url = new URL("http://localhost"); TokenClient tokenClient = new TokenClient(url, defaultAsyncHttpClient); - Map bodyMap = new TreeMap<>(); ClientCredentialsExchangeRequest request = ClientCredentialsExchangeRequest.builder() - .audience("test-audience") .clientId("test-client-id") .clientSecret("test-client-secret") .build(); - bodyMap.put("grant_type", "client_credentials"); - bodyMap.put("client_id", request.getClientId()); - bodyMap.put("client_secret", request.getClientSecret()); - bodyMap.put("audience", request.getAudience()); - String body = tokenClient.buildClientCredentialsBody(bodyMap); + String body = tokenClient.buildClientCredentialsBody(request); BoundRequestBuilder boundRequestBuilder = mock(BoundRequestBuilder.class); Response response = mock(Response.class); ListenableFuture listenableFuture = mock(ListenableFuture.class); diff --git a/site2/docs/security-oauth2.md b/site2/docs/security-oauth2.md index 7ea9f35969397..35d0b5f4ccc40 100644 --- a/site2/docs/security-oauth2.md +++ b/site2/docs/security-oauth2.md @@ -28,7 +28,7 @@ The following table lists parameters supported for the `client credentials` auth | `type` | Oauth 2.0 authentication type. | `client_credentials` (default) | Optional | | `issuerUrl` | URL of the authentication provider which allows the Pulsar client to obtain an access token | `https://accounts.google.com` | Required | | `privateKey` | URL to a JSON credentials file | Support the following pattern formats:
  • `file:///path/to/file`
  • `file:/path/to/file`
  • `data:application/json;base64,` | Required | -| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster | `https://broker.example.com` | Required | +| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster | `https://broker.example.com` | Optional | The credentials file contains service account credentials used with the client authentication type. The following shows an example of a credentials file `credentials_file.json`. @@ -63,7 +63,7 @@ In the above example, the mapping relationship is shown as below. - The `issuerUrl` parameter in this plugin is mapped to `--url https://dev-kt-aa9ne.us.auth0.com`. - The `privateKey` file parameter in this plugin should at least contains the `client_id` and `client_secret` fields. -- The `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. +- The `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. This field is only used by some identity providers. ## Client Configuration diff --git a/site2/website-next/docs/security-oauth2.md b/site2/website-next/docs/security-oauth2.md index 19c37893f9972..820a6968604eb 100644 --- a/site2/website-next/docs/security-oauth2.md +++ b/site2/website-next/docs/security-oauth2.md @@ -32,7 +32,7 @@ The following table lists parameters supported for the `client credentials` auth | `type` | Oauth 2.0 authentication type. | `client_credentials` (default) | Optional | | `issuerUrl` | URL of the authentication provider which allows the Pulsar client to obtain an access token | `https://accounts.google.com` | Required | | `privateKey` | URL to a JSON credentials file | Support the following pattern formats:
  • `file:///path/to/file`
  • `file:/path/to/file`
  • `data:application/json;base64,`
    • | Required | -| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster | `https://broker.example.com` | Required | +| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster | `https://broker.example.com` | Optional | The credentials file contains service account credentials used with the client authentication type. The following shows an example of a credentials file `credentials_file.json`. @@ -71,7 +71,7 @@ In the above example, the mapping relationship is shown as below. - The `issuerUrl` parameter in this plugin is mapped to `--url https://dev-kt-aa9ne.us.auth0.com`. - The `privateKey` file parameter in this plugin should at least contains the `client_id` and `client_secret` fields. -- The `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. +- The `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. This field is only used by some identity providers. ## Client Configuration From a40eda0262822a7c4c9b556b72822dedcd1179eb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 3 Dec 2021 13:37:21 +0200 Subject: [PATCH 033/823] [Functions] Fix classloader leaks (#12973) * Fix classloader leak in FunctionCommon.getClassLoaderFromPackage * Fix classloader leak in SinksImpl and SourcesImpl * Fix logic for shouldCloseClassLoader (cherry picked from commit cab946b4ca68e1ffc6dee3932bc4c0fc7e7da66e) --- .../pulsar/common/util/ClassLoaderUtils.java | 14 ++ .../functions/utils/FunctionCommon.java | 151 ++++++++++-------- .../functions/worker/rest/api/SinksImpl.java | 32 ++-- .../worker/rest/api/SourcesImpl.java | 33 ++-- 4 files changed, 140 insertions(+), 90 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ClassLoaderUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ClassLoaderUtils.java index 0e1e1884aaa8e..69e4c6319c0a6 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ClassLoaderUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ClassLoaderUtils.java @@ -18,16 +18,20 @@ */ package org.apache.pulsar.common.util; +import java.io.Closeable; import java.io.File; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.security.AccessController; import java.security.PrivilegedAction; +import lombok.extern.slf4j.Slf4j; /** * Helper methods wrt Classloading. */ +@Slf4j public class ClassLoaderUtils { /** * Load a jar. @@ -76,4 +80,14 @@ public static void implementsClass(String className, Class klass, ClassLoader String.format("%s does not implement %s", className, klass.getName())); } } + + public static void closeClassLoader(ClassLoader classLoader) { + if (classLoader instanceof Closeable) { + try { + ((Closeable) classLoader).close(); + } catch (IOException e) { + log.error("Error closing classloader {}", classLoader, e); + } + } + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index a13695e6fbcc6..f72814df9905c 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -382,97 +382,114 @@ public static ClassLoader getClassLoaderFromPackage( String narExtractionDirectory) { String connectorClassName = className; ClassLoader jarClassLoader = null; + boolean keepJarClassLoader = false; ClassLoader narClassLoader = null; + boolean keepNarClassLoader = false; Exception jarClassLoaderException = null; Exception narClassLoaderException = null; try { - jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile); - } catch (Exception e) { - jarClassLoaderException = e; - } - try { - narClassLoader = FunctionCommon.extractNarClassLoader(packageFile, narExtractionDirectory); - } catch (Exception e) { - narClassLoaderException = e; - } - - // if connector class name is not provided, we can only try to load archive as a NAR - if (isEmpty(connectorClassName)) { - if (narClassLoader == null) { - throw new IllegalArgumentException(String.format("%s package does not have the correct format. " + - "Pulsar cannot determine if the package is a NAR package or JAR package. " + - "%s classname is not provided and attempts to load it as a NAR package produced the following error.", - capFirstLetter(componentType), capFirstLetter(componentType)), - narClassLoaderException); - } try { - if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { - connectorClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) narClassLoader); - } else { - connectorClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) narClassLoader); - } - } catch (IOException e) { - throw new IllegalArgumentException(String.format("Failed to extract %s class from archive", - componentType.toString().toLowerCase()), e); + jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile); + } catch (Exception e) { + jarClassLoaderException = e; } - try { - narClassLoader.loadClass(connectorClassName); - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", capFirstLetter(componentType), connectorClassName), e); + narClassLoader = FunctionCommon.extractNarClassLoader(packageFile, narExtractionDirectory); + } catch (Exception e) { + narClassLoaderException = e; } - } else { - // if connector class name is provided, we need to try to load it as a JAR and as a NAR. - if (jarClassLoader != null) { + // if connector class name is not provided, we can only try to load archive as a NAR + if (isEmpty(connectorClassName)) { + if (narClassLoader == null) { + throw new IllegalArgumentException(String.format("%s package does not have the correct format. " + + "Pulsar cannot determine if the package is a NAR package or JAR package. " + + "%s classname is not provided and attempts to load it as a NAR package produced " + + "the following error.", + capFirstLetter(componentType), capFirstLetter(componentType)), + narClassLoaderException); + } try { - jarClassLoader.loadClass(connectorClassName); - return jarClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e) { - // class not found in JAR try loading as a NAR and searching for the class - if (narClassLoader != null) { - - try { - narClassLoader.loadClass(connectorClassName); - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e1) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", - capFirstLetter(componentType), connectorClassName), e1); - } + if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { + connectorClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) narClassLoader); } else { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", capFirstLetter(componentType), - connectorClassName), e); + connectorClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) narClassLoader); } + } catch (IOException e) { + throw new IllegalArgumentException(String.format("Failed to extract %s class from archive", + componentType.toString().toLowerCase()), e); } - } else if (narClassLoader != null) { + try { narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e1) { + } catch (ClassNotFoundException | NoClassDefFoundError e) { throw new IllegalArgumentException( - String.format("%s class %s must be in class path", - capFirstLetter(componentType), connectorClassName), e1); + String.format("%s class %s must be in class path", capFirstLetter(componentType), + connectorClassName), e); } + } else { - StringBuilder errorMsg = new StringBuilder(capFirstLetter(componentType) - + " package does not have the correct format." - + " Pulsar cannot determine if the package is a NAR package or JAR package."); + // if connector class name is provided, we need to try to load it as a JAR and as a NAR. + if (jarClassLoader != null) { + try { + jarClassLoader.loadClass(connectorClassName); + keepJarClassLoader = true; + return jarClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e) { + // class not found in JAR try loading as a NAR and searching for the class + if (narClassLoader != null) { + + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e1) { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", + capFirstLetter(componentType), connectorClassName), e1); + } + } else { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", capFirstLetter(componentType), + connectorClassName), e); + } + } + } else if (narClassLoader != null) { + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e1) { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", + capFirstLetter(componentType), connectorClassName), e1); + } + } else { + StringBuilder errorMsg = new StringBuilder(capFirstLetter(componentType) + + " package does not have the correct format." + + " Pulsar cannot determine if the package is a NAR package or JAR package."); - if (jarClassLoaderException != null) { - errorMsg.append(" Attempts to load it as a JAR package produced error: " + jarClassLoaderException.getMessage()); - } + if (jarClassLoaderException != null) { + errorMsg.append(" Attempts to load it as a JAR package produced error: " + jarClassLoaderException.getMessage()); + } - if (narClassLoaderException != null) { - errorMsg.append(" Attempts to load it as a NAR package produced error: " + narClassLoaderException.getMessage()); - } + if (narClassLoaderException != null) { + errorMsg.append(" Attempts to load it as a NAR package produced error: " + narClassLoaderException.getMessage()); + } - throw new IllegalArgumentException(errorMsg.toString()); + throw new IllegalArgumentException(errorMsg.toString()); + } + } + } finally { + if (!keepJarClassLoader) { + ClassLoaderUtils.closeClassLoader(jarClassLoader); + } + if (!keepNarClassLoader) { + ClassLoaderUtils.closeClassLoader(narClassLoader); } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 31a72347de04c..bbb16806b5539 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -50,6 +50,7 @@ import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.SinkStatus; +import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; @@ -734,19 +735,28 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant } } - // if sink is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && sinkPackageFile != null) { - classLoader = getClassLoaderFromPackage(sinkConfig.getClassName(), - sinkPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - } + boolean shouldCloseClassLoader = false; + try { - if (classLoader == null) { - throw new IllegalArgumentException("Sink package is not provided"); - } + // if sink is not builtin, attempt to extract classloader from package file if it exists + if (classLoader == null && sinkPackageFile != null) { + classLoader = getClassLoaderFromPackage(sinkConfig.getClassName(), + sinkPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); + shouldCloseClassLoader = true; + } - SinkConfigUtils.ExtractedSinkDetails sinkDetails = SinkConfigUtils.validateAndExtractDetails( - sinkConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); - return SinkConfigUtils.convert(sinkConfig, sinkDetails); + if (classLoader == null) { + throw new IllegalArgumentException("Sink package is not provided"); + } + + SinkConfigUtils.ExtractedSinkDetails sinkDetails = SinkConfigUtils.validateAndExtractDetails( + sinkConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + return SinkConfigUtils.convert(sinkConfig, sinkDetails); + } finally { + if (shouldCloseClassLoader) { + ClassLoaderUtils.closeClassLoader(classLoader); + } + } } private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index 1e9148bd658ae..82a818d773d97 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -50,6 +50,7 @@ import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.SourceStatus; +import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; @@ -730,20 +731,28 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant } } - // if source is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && sourcePackageFile != null) { - classLoader = getClassLoaderFromPackage(sourceConfig.getClassName(), - sourcePackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - } + boolean shouldCloseClassLoader = false; + try { + // if source is not builtin, attempt to extract classloader from package file if it exists + if (classLoader == null && sourcePackageFile != null) { + classLoader = getClassLoaderFromPackage(sourceConfig.getClassName(), + sourcePackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); + shouldCloseClassLoader = true; + } - if (classLoader == null) { - throw new IllegalArgumentException("Source package is not provided"); - } + if (classLoader == null) { + throw new IllegalArgumentException("Source package is not provided"); + } - SourceConfigUtils.ExtractedSourceDetails sourceDetails - = SourceConfigUtils.validateAndExtractDetails( - sourceConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); - return SourceConfigUtils.convert(sourceConfig, sourceDetails); + SourceConfigUtils.ExtractedSourceDetails sourceDetails + = SourceConfigUtils.validateAndExtractDetails( + sourceConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + return SourceConfigUtils.convert(sourceConfig, sourceDetails); + } finally { + if (shouldCloseClassLoader) { + ClassLoaderUtils.closeClassLoader(classLoader); + } + } } private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { From b95c6f3113bd1b927a98de2c0641323431527ab8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 3 Dec 2021 13:38:55 +0200 Subject: [PATCH 034/823] [Broker] Fix and improve topic ownership assignment (#13069) * Add warning log message when leader broker isn't available * Add more logging about load manager decisions * Use cached information for available brokers * Reproduce lookup race issue * Use java.util.concurrent.Phaser to increase the chances of a race * Address review feedback * Increase concurrency of test case to reproduce race conditions * Use real Zookeeper server in MultiBrokerLeaderElectionTest * Add retry with backoff to loading namespace bundles * Add more topics to test * Address review comment * Fix checkstyle * Improve logging * Address review comments (cherry picked from commit 537dee113ccb1e405a8fd1d5698b8f066f52bb53) --- pulsar-broker/pom.xml | 8 ++ .../broker/namespace/NamespaceService.java | 104 ++++++++++++------ .../common/naming/NamespaceBundleFactory.java | 32 +++++- .../pulsar/broker/MultiBrokerBaseTest.java | 6 +- .../auth/MockedPulsarServiceBaseTest.java | 11 +- .../MultiBrokerLeaderElectionTest.java | 97 ++++++++++++++++ pulsar-metadata/pom.xml | 11 ++ 7 files changed, 227 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index aef0f2e49c6c6..bb7bee7e04722 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -568,6 +568,14 @@ test-jar test     + + + ${project.groupId} + pulsar-metadata + ${project.version} + test-jar + test + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index adcb504c699a5..dc8f38390a4fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -454,6 +454,9 @@ private void searchForCandidateBroker(NamespaceBundle bundle, // The leader election service was not initialized yet. This can happen because the broker service is // initialized first and it might start receiving lookup requests before the leader election service is // fully initialized. + LOG.warn("Leader election service isn't initialized yet. " + + "Returning empty result to lookup. NamespaceBundle[{}]", + bundle); lookupFuture.complete(Optional.empty()); return; } @@ -480,23 +483,45 @@ private void searchForCandidateBroker(NamespaceBundle bundle, if (options.isAuthoritative()) { // leader broker already assigned the current broker as owner candidateBroker = pulsar.getSafeWebServiceAddress(); - } else if (!this.loadManager.get().isCentralized() - || pulsar.getLeaderElectionService().isLeader() - || !currentLeader.isPresent() - + } else { + LoadManager loadManager = this.loadManager.get(); + boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader(); + if (!makeLoadManagerDecisionOnThisBroker) { // If leader is not active, fallback to pick the least loaded from current broker loadmanager - || !isBrokerActive(currentLeader.get().getServiceUrl()) - ) { - Optional availableBroker = getLeastLoadedFromLoadManager(bundle); - if (!availableBroker.isPresent()) { - lookupFuture.complete(Optional.empty()); - return; + boolean leaderBrokerActive = currentLeader.isPresent() + && isBrokerActive(currentLeader.get().getServiceUrl()); + if (!leaderBrokerActive) { + makeLoadManagerDecisionOnThisBroker = true; + if (!currentLeader.isPresent()) { + LOG.warn( + "The information about the current leader broker wasn't available. " + + "Handling load manager decisions in a decentralized way. " + + "NamespaceBundle[{}]", + bundle); + } else { + LOG.warn( + "The current leader broker {} isn't active. " + + "Handling load manager decisions in a decentralized way. " + + "NamespaceBundle[{}]", + currentLeader.get(), bundle); + } + } + } + if (makeLoadManagerDecisionOnThisBroker) { + Optional availableBroker = getLeastLoadedFromLoadManager(bundle); + if (!availableBroker.isPresent()) { + LOG.warn("Load manager didn't return any available broker. " + + "Returning empty result to lookup. NamespaceBundle[{}]", + bundle); + lookupFuture.complete(Optional.empty()); + return; + } + candidateBroker = availableBroker.get(); + authoritativeRedirect = true; + } else { + // forward to leader broker to make assignment + candidateBroker = currentLeader.get().getServiceUrl(); } - candidateBroker = availableBroker.get(); - authoritativeRedirect = true; - } else { - // forward to leader broker to make assignment - candidateBroker = currentLeader.get().getServiceUrl(); } } } catch (Exception e) { @@ -577,19 +602,16 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } protected CompletableFuture createLookupResult(String candidateBroker, boolean authoritativeRedirect, - final String advertisedListenerName) - throws Exception { + final String advertisedListenerName) { CompletableFuture lookupFuture = new CompletableFuture<>(); try { - checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null " + candidateBroker); - URI uri = new URI(candidateBroker); - String path = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri.getHost(), - uri.getPort()); + checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null %s", candidateBroker); + String path = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + parseHostAndPort(candidateBroker); localBrokerDataCache.get(path).thenAccept(reportData -> { if (reportData.isPresent()) { - LocalBrokerData lookupData = (LocalBrokerData) reportData.get(); + LocalBrokerData lookupData = reportData.get(); if (StringUtils.isNotBlank(advertisedListenerName)) { AdvertisedListener listener = lookupData.getAdvertisedListeners().get(advertisedListenerName); if (listener == null) { @@ -622,22 +644,36 @@ protected CompletableFuture createLookupResult(String candidateBro } private boolean isBrokerActive(String candidateBroker) { - List brokers = pulsar.getLocalMetadataStore().getChildren(LoadManager.LOADBALANCE_BROKERS_ROOT).join(); - - for (String brokerHostPort : brokers) { - if (candidateBroker.equals("http://" + brokerHostPort)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Broker {} found for SLA Monitoring Namespace", brokerHostPort); - } - return true; + String candidateBrokerHostAndPort = parseHostAndPort(candidateBroker); + Set availableBrokers = getAvailableBrokers(); + if (availableBrokers.contains(candidateBrokerHostAndPort)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Broker {} ({}) is available for.", candidateBroker, candidateBrokerHostAndPort); } + return true; + } else { + LOG.warn("Broker {} ({}) couldn't be found in available brokers {}", + candidateBroker, candidateBrokerHostAndPort, + availableBrokers.stream().collect(Collectors.joining(","))); + return false; } + } - if (LOG.isDebugEnabled()) { - LOG.debug("Broker not found for SLA Monitoring Namespace {}", - candidateBroker + ":" + config.getWebServicePort()); + private static String parseHostAndPort(String candidateBroker) { + int uriSeparatorPos = candidateBroker.indexOf("://"); + if (uriSeparatorPos == -1) { + throw new IllegalArgumentException("'" + candidateBroker + "' isn't an URI."); + } + String candidateBrokerHostAndPort = candidateBroker.substring(uriSeparatorPos + 3); + return candidateBrokerHostAndPort; + } + + private Set getAvailableBrokers() { + try { + return loadManager.get().getAvailableBrokers(); + } catch (Exception e) { + throw new RuntimeException(e); } - return false; } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java index 9fbcf17a78b8c..586e3b39aec86 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java @@ -31,6 +31,7 @@ import com.google.common.collect.Range; import com.google.common.hash.HashFunction; import java.io.IOException; +import java.time.Duration; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -46,6 +47,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.LocalPolicies; import org.apache.pulsar.common.policies.data.Policies; @@ -64,6 +66,7 @@ public class NamespaceBundleFactory { private final PulsarService pulsar; private final MetadataCache policiesCache; + private final Duration maxRetryDuration = Duration.ofSeconds(10); public NamespaceBundleFactory(PulsarService pulsar, HashFunction hashFunc) { this.hashFunc = hashFunc; @@ -90,22 +93,27 @@ private CompletableFuture loadBundles(NamespaceName namespace, } CompletableFuture future = new CompletableFuture<>(); + doLoadBundles(namespace, future, createBackoff(), System.nanoTime() + maxRetryDuration.toNanos()); + return future; + } + + private void doLoadBundles(NamespaceName namespace, CompletableFuture future, + Backoff backoff, long retryDeadline) { // Read the static bundle data from the policies pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesWithVersion(namespace).thenAccept(result -> { - if (result.isPresent()) { try { future.complete(readBundles(namespace, result.get().getValue(), result.get().getStat().getVersion())); } catch (IOException e) { - future.completeExceptionally(e); + handleLoadBundlesRetry(namespace, future, backoff, retryDeadline, e); } } else { // If no local policies defined for namespace, copy from global config copyToLocalPolicies(namespace) .thenAccept(b -> future.complete(b)) .exceptionally(ex -> { - future.completeExceptionally(ex); + handleLoadBundlesRetry(namespace, future, backoff, retryDeadline, ex); return null; }); } @@ -113,7 +121,23 @@ private CompletableFuture loadBundles(NamespaceName namespace, future.completeExceptionally(ex); return null; }); - return future; + } + + private void handleLoadBundlesRetry(NamespaceName namespace, + CompletableFuture future, + Backoff backoff, long retryDeadline, Throwable e) { + if (e instanceof Error || System.nanoTime() > retryDeadline) { + future.completeExceptionally(e); + } else { + LOG.warn("Error loading bundle for {}. Retrying exception", namespace, e); + long retryDelay = backoff.next(); + pulsar.getExecutor().schedule(() -> + doLoadBundles(namespace, future, backoff, retryDeadline), retryDelay, TimeUnit.MILLISECONDS); + } + } + + private static Backoff createBackoff() { + return new Backoff(100, TimeUnit.MILLISECONDS, 5, TimeUnit.SECONDS, 0, TimeUnit.MILLISECONDS); } private NamespaceBundles readBundles(NamespaceName namespace, LocalPolicies localPolicies, long version) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java index f4d106da67f0b..c00ae8cd0d39d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java @@ -27,6 +27,8 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.zookeeper.MockZooKeeperSession; import org.testng.annotations.AfterClass; @@ -82,13 +84,13 @@ protected PulsarService createAdditionalBroker(int additionalBrokerIndex) throws } @Override - protected ZKMetadataStore createLocalMetadataStore() { + protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException { // use MockZooKeeperSession to provide a unique session id for each instance return new ZKMetadataStore(MockZooKeeperSession.newInstance(mockZooKeeper)); } @Override - protected ZKMetadataStore createConfigurationMetadataStore() { + protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { // use MockZooKeeperSession to provide a unique session id for each instance return new ZKMetadataStore(MockZooKeeperSession.newInstance(mockZooKeeperGlobal)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index a3f7166cace8e..c50b477b5d313 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -60,6 +60,8 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.tests.TestRetrySupport; import org.apache.pulsar.zookeeper.ZooKeeperClientFactory; @@ -245,6 +247,11 @@ protected final void internalCleanup() throws Exception { } bkExecutor = null; } + onCleanup(); + } + + protected void onCleanup() { + } protected abstract void setup() throws Exception; @@ -332,11 +339,11 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { } } - protected ZKMetadataStore createLocalMetadataStore() { + protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException { return new ZKMetadataStore(mockZooKeeper); } - protected ZKMetadataStore createConfigurationMetadataStore() { + protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { return new ZKMetadataStore(mockZooKeeperGlobal); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java index 0045dddeb8161..462b640c17511 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java @@ -20,15 +20,68 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.MultiBrokerBaseTest; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.metadata.TestZKServer; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.awaitility.Awaitility; import org.testng.annotations.Test; +@Slf4j @Test(groups = "broker") public class MultiBrokerLeaderElectionTest extends MultiBrokerBaseTest { + @Override + protected int numberOfAdditionalBrokers() { + return 9; + } + + TestZKServer testZKServer; + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + testZKServer = new TestZKServer(); + } + + @Override + protected void onCleanup() { + super.onCleanup(); + if (testZKServer != null) { + try { + testZKServer.close(); + } catch (Exception e) { + log.error("Error in stopping ZK server", e); + } + } + } + + @Override + protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException { + return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build()); + } + + @Override + protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { + return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build()); + } @Test public void shouldElectOneLeader() { @@ -68,4 +121,48 @@ public void shouldAllBrokersBeAbleToGetTheLeader() { } }); } + + @Test + public void shouldProvideConsistentAnswerToTopicLookups() + throws PulsarAdminException, ExecutionException, InterruptedException { + String topicNameBase = "persistent://public/default/lookuptest" + UUID.randomUUID() + "-"; + List topicNames = IntStream.range(0, 500).mapToObj(i -> topicNameBase + i) + .collect(Collectors.toList()); + List allAdmins = getAllAdmins(); + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(allAdmins.size()); + List>> resultFutures = new ArrayList<>(); + String leaderBrokerUrl = admin.brokers().getLeaderBroker().getServiceUrl(); + log.info("LEADER is {}", leaderBrokerUrl); + // use Phaser to increase the chances of a race condition by triggering all threads once + // they are waiting just before the lookupTopic call + final Phaser phaser = new Phaser(1); + for (PulsarAdmin brokerAdmin : allAdmins) { + if (!leaderBrokerUrl.equals(brokerAdmin.getServiceUrl())) { + phaser.register(); + log.info("Doing lookup to broker {}", brokerAdmin.getServiceUrl()); + resultFutures.add(executorService.submit(() -> { + phaser.arriveAndAwaitAdvance(); + return topicNames.stream().map(topicName -> { + try { + return brokerAdmin.lookups().lookupTopic(topicName); + } catch (PulsarAdminException e) { + log.error("Error looking up topic {} in {}", topicName, brokerAdmin.getServiceUrl()); + throw new RuntimeException(e); + } + }).collect(Collectors.toList()); + })); + } + } + phaser.arriveAndAwaitAdvance(); + List firstResult = null; + for (Future> resultFuture : resultFutures) { + List result = resultFuture.get(); + if (firstResult == null) { + firstResult = result; + } else { + assertEquals(result, firstResult, "The lookup results weren't consistent."); + } + } + } } diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 7ce5a6b778a6a..c98d9514797a7 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -77,6 +77,17 @@ + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + com.github.spotbugs spotbugs-maven-plugin From c33b53ada29c7d18ed7d11456c34554d061a0a1f Mon Sep 17 00:00:00 2001 From: Ali Ahmed Date: Fri, 22 Oct 2021 09:12:56 -0700 Subject: [PATCH 035/823] Update lombok to 1.18.22 (#12466) Co-authored-by: Ali Ahmed (cherry picked from commit 9ee6656fbd662d1d6f36b8e3b71e01f07c93a2fa) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 480ef814c76cb..09c5c6e60b102 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ flexible messaging model and an intuitive client API.     0.7.3 2.1.0 3.18.1 - 1.18.20 + 1.18.22 1.3.2 2.3.1 1.2.0 From 8afa570f1cdf4a4acfa34de330e855f819c2ae63 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Fri, 12 Nov 2021 10:29:56 +0800 Subject: [PATCH 036/823] [Authorization] Support UNSUBSCRIBE namespace op after enable auth (#12742) ### Motivation Currently, we can `unsubscribe` the given subscription on all topics on a namespace through `bin/pulsar-admin namespaces unsubscribe -s sub tn1/ns1`. However, role(not super-user or administrator) with `consume` auth action for namespace cannot perform `unsubscribe` operation when enable auth. The root of the problem is that `PulsarAuthorizationProvider` lacks support for namespace operation `UNSUBSCRIBE` when verifying the role's authorization, code as below: https://github.com/apache/pulsar/blob/8cae63557a318240e95697f382b4f61c22b70d64/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java#L1667-L1669 https://github.com/apache/pulsar/blob/8cae63557a318240e95697f382b4f61c22b70d64/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java#L522-L536 The purpose of this PR is to support that role with `consume` namespace authorization could `unsubscribe` subscriptions on a namespace. (cherry picked from commit 8926631db9c9a78341726b53ca119ad4c69720e8) --- .../PulsarAuthorizationProvider.java | 1 + .../AuthorizationProducerConsumerTest.java | 42 +++- .../admin/GetMetadataOfTopicWithAuthTest.java | 213 ------------------ 3 files changed, 42 insertions(+), 214 deletions(-) delete mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index 641591c03a791..d7d4531edf00d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -531,6 +531,7 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam isAuthorizedFuture = allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.packages); break; case GET_TOPICS: + case UNSUBSCRIBE: isAuthorizedFuture = allowConsumeOpsAsync(namespaceName, role, authData); break; default: diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0a799fbb51dfe..0987a33314114 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -20,6 +20,8 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Maps; @@ -27,6 +29,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -52,6 +55,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TenantOperation; @@ -175,6 +179,7 @@ public void testSubscriberPermission() throws Exception { final String tenantRole = "tenant-role"; final String subscriptionRole = "sub1-role"; final String subscriptionName = "sub1"; + final String subscriptionName2 = "sub2"; final String namespace = "my-property/my-ns-sub-auth"; final String topicName = "persistent://" + namespace + "/my-topic"; Authentication adminAuthentication = new ClientAuthentication("superUser"); @@ -202,7 +207,18 @@ public void testSubscriberPermission() throws Exception { superAdmin.tenants().createTenant("my-property", new TenantInfoImpl(Sets.newHashSet(tenantRole), Sets.newHashSet("test"))); superAdmin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); - tenantAdmin.namespaces().grantPermissionOnNamespace(namespace, subscriptionRole, + + // subscriptionRole doesn't have topic-level authorization, so it will fail to get topic stats-internal info + try { + sub1Admin.topics().getInternalStats(topicName, true); + fail("should have failed with authorization exception"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith( + "Unauthorized to validateTopicOperation for operation [GET_STATS]")); + } + + // grant topic consume authorization to the subscriptionRole + tenantAdmin.topics().grantPermission(topicName, subscriptionRole, Collections.singleton(AuthAction.consume)); replacePulsarClient(PulsarClient.builder() @@ -212,7 +228,17 @@ public void testSubscriberPermission() throws Exception { // (1) Create subscription name Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) .subscribe(); + Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName2) + .subscribe(); consumer.close(); + consumer2.close(); + + List subscriptions = sub1Admin.topics().getSubscriptions(topicName); + assertEquals(subscriptions.size(), 2); + + // now, subscriptionRole have consume authorization on topic, so it will successfully get topic internal stats + PersistentTopicInternalStats internalStats = superAdmin.topics().getInternalStats(topicName, true); + assertNotNull(internalStats); // verify tenant is able to perform all subscription-admin api tenantAdmin.topics().skipAllMessages(topicName, subscriptionName); @@ -228,10 +254,24 @@ public void testSubscriberPermission() throws Exception { tenantAdmin.topics().resetCursor(topicName, subscriptionName, 10); tenantAdmin.topics().resetCursor(topicName, subscriptionName, MessageId.earliest); + // subscriptionRole doesn't have namespace-level authorization, so it will fail to unsubscribe namespace + try { + sub1Admin.namespaces().unsubscribeNamespace(namespace, subscriptionName2); + fail("should have failed with authorization exception"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith( + "Unauthorized to validateNamespaceOperation for operation [UNSUBSCRIBE]")); + } + // grant namespace-level authorization to the subscriptionRole tenantAdmin.namespaces().grantPermissionOnNamespace(namespace, subscriptionRole, Collections.singleton(AuthAction.consume)); + // now, subscriptionRole have consume authorization on namespace, so it will successfully unsubscribe namespace + superAdmin.namespaces().unsubscribeNamespaceBundle(namespace, "0x00000000_0xffffffff", subscriptionName2); + subscriptions = sub1Admin.topics().getSubscriptions(topicName); + assertEquals(subscriptions.size(), 1); + // subscriptionRole has namespace-level authorization sub1Admin.topics().resetCursor(topicName, subscriptionName, 10); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java deleted file mode 100644 index 75786290027aa..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/admin/GetMetadataOfTopicWithAuthTest.java +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.auth.admin; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; -import com.google.common.io.Files; -import java.io.File; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.client.impl.auth.AuthenticationToken; -import org.apache.pulsar.common.policies.data.AuthAction; -import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; -import org.apache.pulsar.tests.TestRetrySupport; -import org.apache.pulsar.tests.integration.containers.PulsarContainer; -import org.apache.pulsar.tests.integration.containers.ZKContainer; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; -import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; -import org.apache.pulsar.tests.integration.utils.DockerUtils; -import org.elasticsearch.common.collect.Set; -import org.testcontainers.containers.Network; -import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -/** - * GetMetadataOfTopicWithAuthTest will test Getmetadata operation with and without the proper permission. - */ -@Slf4j -public class GetMetadataOfTopicWithAuthTest extends TestRetrySupport { - - private static final String CLUSTER_PREFIX = "get-metadata-auth"; - private static final String PRIVATE_KEY_PATH_INSIDE_CONTAINER = "/tmp/private.key"; - private static final String PUBLIC_KEY_PATH_INSIDE_CONTAINER = "/tmp/public.key"; - - private static final String SUPER_USER_ROLE = "super-user"; - private String superUserAuthToken; - private static final String PROXY_ROLE = "proxy"; - private String proxyAuthToken; - private static final String REGULAR_USER_ROLE = "client"; - private String clientAuthToken; - private File publicKeyFile; - - private PulsarCluster pulsarCluster; - private PulsarContainer cmdContainer; - - @Override - @BeforeClass(alwaysRun = true) - protected void setup() throws Exception { - incrementSetupNumber(); - // Before starting the cluster, generate the secret key and the token - // Use Zk container to have 1 container available before starting the cluster - final String clusterName = String.format("%s-%s", CLUSTER_PREFIX, RandomStringUtils.randomAlphabetic(6)); - final String cliContainerName = String.format("%s-%s", "cli", RandomStringUtils.randomAlphabetic(6)); - cmdContainer = new ZKContainer<>(cliContainerName); - cmdContainer - .withNetwork(Network.newNetwork()) - .withNetworkAliases(ZKContainer.NAME) - .withEnv("zkServers", ZKContainer.NAME); - cmdContainer.start(); - - createKeysAndTokens(cmdContainer); - - PulsarClusterSpec spec = PulsarClusterSpec.builder() - .numBookies(2) - .numBrokers(2) - .numProxies(1) - .clusterName(clusterName) - .brokerEnvs(getBrokerSettingsEnvs()) - .proxyEnvs(getProxySettingsEnvs()) - .brokerMountFiles(Collections.singletonMap(publicKeyFile.toString(), PUBLIC_KEY_PATH_INSIDE_CONTAINER)) - .proxyMountFiles(Collections.singletonMap(publicKeyFile.toString(), PUBLIC_KEY_PATH_INSIDE_CONTAINER)) - .build(); - - pulsarCluster = PulsarCluster.forSpec(spec); - pulsarCluster.start(); - } - - @Override - @AfterClass(alwaysRun = true) - public void cleanup() { - markCurrentSetupNumberCleaned(); - if (cmdContainer != null) { - cmdContainer.stop(); - } - if (pulsarCluster != null) { - pulsarCluster.stop(); - } - } - - private Map getBrokerSettingsEnvs() { - Map envs = new HashMap<>(); - envs.put("authenticationEnabled", "true"); - envs.put("authenticationProviders", AuthenticationProviderToken.class.getName()); - envs.put("authorizationEnabled", "true"); - envs.put("superUserRoles", String.format("%s,%s", SUPER_USER_ROLE, PROXY_ROLE)); - envs.put("brokerClientAuthenticationPlugin", AuthenticationToken.class.getName()); - envs.put("brokerClientAuthenticationParameters", String.format("token:%s", superUserAuthToken)); - envs.put("authenticationRefreshCheckSeconds", "1"); - envs.put("authenticateOriginalAuthData", "true"); - envs.put("tokenPublicKey", "file://" + PUBLIC_KEY_PATH_INSIDE_CONTAINER); - return envs; - } - - private Map getProxySettingsEnvs() { - Map envs = new HashMap<>(); - envs.put("authenticationEnabled", "true"); - envs.put("authenticationProviders", AuthenticationProviderToken.class.getName()); - envs.put("authorizationEnabled", "true"); - envs.put("brokerClientAuthenticationPlugin", AuthenticationToken.class.getName()); - envs.put("brokerClientAuthenticationParameters", String.format("token:%s", proxyAuthToken)); - envs.put("authenticationRefreshCheckSeconds", "1"); - envs.put("forwardAuthorizationCredentials", "true"); - envs.put("tokenPublicKey", "file://" + PUBLIC_KEY_PATH_INSIDE_CONTAINER); - return envs; - } - - protected void createKeysAndTokens(PulsarContainer container) throws Exception { - container - .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create-key-pair", - "--output-private-key", PRIVATE_KEY_PATH_INSIDE_CONTAINER, - "--output-public-key", PUBLIC_KEY_PATH_INSIDE_CONTAINER); - - byte[] publicKeyBytes = DockerUtils - .runCommandWithRawOutput(container.getDockerClient(), container.getContainerId(), - "/bin/cat", PUBLIC_KEY_PATH_INSIDE_CONTAINER) - .getStdout(); - - publicKeyFile = File.createTempFile("public-", ".key", new File("/tmp")); - Files.write(publicKeyBytes, publicKeyFile); - - clientAuthToken = container - .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", - "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, - "--subject", REGULAR_USER_ROLE) - .getStdout().trim(); - log.info("Created client token: {}", clientAuthToken); - - superUserAuthToken = container - .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", - "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, - "--subject", SUPER_USER_ROLE) - .getStdout().trim(); - log.info("Created super-user token: {}", superUserAuthToken); - - proxyAuthToken = container - .execCmd(PulsarCluster.PULSAR_COMMAND_SCRIPT, "tokens", "create", - "--private-key", "file://" + PRIVATE_KEY_PATH_INSIDE_CONTAINER, - "--subject", PROXY_ROLE) - .getStdout().trim(); - log.info("Created proxy token: {}", proxyAuthToken); - } - - @Test - public void testGetMetadataOfTopicWithLookupPermission() throws Exception { - @Cleanup - PulsarAdmin superUserAdmin = PulsarAdmin.builder() - .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) - .authentication(AuthenticationFactory.token(superUserAuthToken)) - .build(); - - @Cleanup - PulsarAdmin clientAdmin = PulsarAdmin.builder() - .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) - .authentication(AuthenticationFactory.token(clientAuthToken)) - .build(); - - // create partitioned topic - superUserAdmin.topics().createPartitionedTopic("public/default/test", 1); - - // do some operation without grant any permissions - try { - clientAdmin.topics().getInternalStats("public/default/test-partition-0", true); - fail("get internal stats and metadata operation should fail because the client hasn't permission to do"); - } catch (PulsarAdminException e) { - assertEquals(e.getStatusCode(), 401); - } - - // grant consume/produce permission to the role - superUserAdmin.topics().grantPermission("public/default/test", - REGULAR_USER_ROLE, Set.of(AuthAction.consume)); - - // then do some get internal stats and metadata operations again, it should success - PersistentTopicInternalStats internalStats = clientAdmin.topics() - .getInternalStats("public/default/test-partition-0", true); - assertNotNull(internalStats); - } -} From da9a5acba373f5fb5dfcae3decc57e5d8d2b8846 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Fri, 26 Nov 2021 18:42:31 +0800 Subject: [PATCH 037/823] [Authorization] Support CLEAR_BACKLOG namespace op after enable auth (#12963) (cherry picked from commit 64af8df83b7463d7e9231ddabc603705f15d30d6) --- .../PulsarAuthorizationProvider.java | 1 + .../AuthorizationProducerConsumerTest.java | 78 ++++++++++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index d7d4531edf00d..774aa38eddb25 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -532,6 +532,7 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam break; case GET_TOPICS: case UNSUBSCRIBE: + case CLEAR_BACKLOG: isAuthorizedFuture = allowConsumeOpsAsync(namespaceName, role, authData); break; default: diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0987a33314114..d8526401cc400 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -268,7 +268,7 @@ public void testSubscriberPermission() throws Exception { Collections.singleton(AuthAction.consume)); // now, subscriptionRole have consume authorization on namespace, so it will successfully unsubscribe namespace - superAdmin.namespaces().unsubscribeNamespaceBundle(namespace, "0x00000000_0xffffffff", subscriptionName2); + sub1Admin.namespaces().unsubscribeNamespaceBundle(namespace, "0x00000000_0xffffffff", subscriptionName2); subscriptions = sub1Admin.topics().getSubscriptions(topicName); assertEquals(subscriptions.size(), 1); @@ -323,6 +323,82 @@ public void testSubscriberPermission() throws Exception { log.info("-- Exiting {} test --", methodName); } + @Test + public void testClearBacklogPermission() throws Exception { + log.info("-- Starting {} test --", methodName); + + conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); + setup(); + + final String subscriptionRole = "sub-role"; + final String subscriptionName = "sub1"; + final String namespace = "my-property/my-ns-sub-auth"; + final String topicName = "persistent://" + namespace + "/my-topic"; + Authentication adminAuthentication = new ClientAuthentication("superUser"); + + clientAuthProviderSupportedRoles.add(subscriptionRole); + + @Cleanup + PulsarAdmin superAdmin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) + .authentication(adminAuthentication).build()); + + Authentication subAdminAuthentication = new ClientAuthentication(subscriptionRole); + @Cleanup + PulsarAdmin sub1Admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) + .authentication(subAdminAuthentication).build()); + + superAdmin.clusters().createCluster("test", + ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + superAdmin.tenants().createTenant("my-property", + new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet("test"))); + superAdmin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); + superAdmin.topics().createPartitionedTopic(topicName, 1); + + // grant topic consume&produce authorization to the subscriptionRole + superAdmin.topics().grantPermission(topicName, subscriptionRole, + Sets.newHashSet(AuthAction.produce, AuthAction.consume)); + replacePulsarClient(PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .authentication(subAdminAuthentication)); + + @Cleanup + Producer batchProducer = pulsarClient.newProducer().topic(topicName) + .enableBatching(false) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName(subscriptionName) + .subscribe(); + + CompletableFuture completableFuture = new CompletableFuture<>(); + for (int i = 0; i < 10; i++) { + completableFuture = batchProducer.sendAsync("a".getBytes()); + } + completableFuture.get(); + assertEquals(sub1Admin.topics().getStats(topicName + "-partition-0").getSubscriptions() + .get(subscriptionName).getMsgBacklog(), 10); + + // subscriptionRole doesn't have namespace-level authorization, so it will fail to clear backlog + try { + sub1Admin.namespaces().clearNamespaceBundleBacklog(namespace, "0x00000000_0xffffffff"); + fail("should have failed with authorization exception"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith( + "Unauthorized to validateNamespaceOperation for operation [CLEAR_BACKLOG]")); + } + + superAdmin.namespaces().grantPermissionOnNamespace(namespace, subscriptionRole, + Sets.newHashSet(AuthAction.consume)); + // now, subscriptionRole have consume authorization on namespace, so it will successfully clear backlog + sub1Admin.namespaces().clearNamespaceBundleBacklog(namespace, "0x00000000_0xffffffff"); + assertEquals(sub1Admin.topics().getStats(topicName + "-partition-0").getSubscriptions() + .get(subscriptionName).getMsgBacklog(), 0); + + log.info("-- Exiting {} test --", methodName); + } + @Test public void testSubscriptionPrefixAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); From 9b861b2b995835a217faeff6cc12c6fc04cb9b18 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 6 Dec 2021 08:22:00 -0600 Subject: [PATCH 038/823] [Authorization] Revert new AuthorizationProvider method (#13133) (cherry picked from commit d1156caf0dbe8c04a604a28a32800281d7ee2d2a) --- .../broker/authorization/AuthorizationProvider.java | 10 ---------- .../MultiRolesTokenAuthorizationProvider.java | 5 ----- .../authorization/PulsarAuthorizationProvider.java | 7 +------ .../pulsar/broker/auth/MockAuthorizationProvider.java | 6 ------ .../client/api/AuthorizationProducerConsumerTest.java | 5 ----- .../client/impl/PatternTopicsConsumerImplAuthTest.java | 5 ----- 6 files changed, 1 insertion(+), 37 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java index ca1bfbf250592..c83ae4ce44d60 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java @@ -184,16 +184,6 @@ CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, Stri CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData); - /** - * Allow consume operations with in this namespace - * @param namespaceName The namespace that the consume operations can be executed in - * @param role The role to check - * @param authenticationData authentication data related to the role - * @return a boolean to determine whether authorized or not - */ - CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, - AuthenticationDataSource authenticationData); - /** * * Grant authorization-action permission on a namespace to the given client diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index 89ed8341b6ee3..dcdf779780ef6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -210,11 +210,6 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return authorize(authenticationData, r -> super.allowSinkOpsAsync(namespaceName, r, authenticationData)); } - @Override - public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowConsumeOpsAsync(namespaceName, r, authenticationData)); - } - @Override public CompletableFuture allowTenantOperationAsync(String tenantName, String role, diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index 774aa38eddb25..b6b1bafd4c377 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -227,11 +227,6 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return allowTheSpecifiedActionOpsAsync(namespaceName, role, authenticationData, AuthAction.sinks); } - @Override - public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return allowTheSpecifiedActionOpsAsync(namespaceName, role, authenticationData, AuthAction.consume); - } - private CompletableFuture allowTheSpecifiedActionOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData, AuthAction authAction) { @@ -533,7 +528,7 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam case GET_TOPICS: case UNSUBSCRIBE: case CLEAR_BACKLOG: - isAuthorizedFuture = allowConsumeOpsAsync(namespaceName, role, authData); + isAuthorizedFuture = allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.consume); break; default: isAuthorizedFuture = CompletableFuture.completedFuture(false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java index 74ba55eeb4fed..3af2568c95465 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java @@ -96,12 +96,6 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return roleAuthorizedAsync(role); } - @Override - public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, - AuthenticationDataSource authenticationData) { - return roleAuthorizedAsync(role); - } - @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authDataJson) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index d8526401cc400..d852a9bafd92d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -614,11 +614,6 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return null; } - @Override - public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return null; - } - @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authenticationData) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java index 40c31dbf7373d..b1328a263011c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java @@ -297,11 +297,6 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return null; } - @Override - public CompletableFuture allowConsumeOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return null; - } - @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authenticationData) { From b33c33c682a846fbd91d88b39f4c3ef4ae4cedd8 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 7 Dec 2021 11:18:04 -0600 Subject: [PATCH 039/823] =?UTF-8?q?[Java=20Client]=20Use=20epoch=20to=20ve?= =?UTF-8?q?rsion=20producer's=20cnx=20to=20prevent=20early=20de=E2=80=A6?= =?UTF-8?q?=20(#12779)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Java Client] Use epoch to version producer's cnx to prevent early delivery of messages * Update initial epoch value: it now gets incremented before the first connection (cherry picked from commit ab652b8dbaecbb59e4a20e5b1f5ce93147502b24) --- .../pulsar/client/impl/ConnectionHandler.java | 20 ++++++--- .../pulsar/client/impl/ProducerImpl.java | 43 +++++++++++++------ 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 02429d2791855..8cd812256ac7f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -37,7 +37,8 @@ public class ConnectionHandler { protected final Backoff backoff; private static final AtomicLongFieldUpdater EPOCH_UPDATER = AtomicLongFieldUpdater .newUpdater(ConnectionHandler.class, "epoch"); - private volatile long epoch = 0L; + // Start with -1L because it gets incremented before sending on the first connection + private volatile long epoch = -1L; protected volatile long lastConnectionClosedTimestamp = 0L; interface Connection { @@ -106,7 +107,6 @@ protected void reconnectLater(Throwable exception) { if (state.changeToConnecting()) { state.client.timer().newTimeout(timeout -> { log.info("[{}] [{}] Reconnecting after connection was closed", state.topic, state.getHandlerName()); - incrementEpoch(); grabCnx(); }, delayMs, TimeUnit.MILLISECONDS); } else { @@ -115,10 +115,6 @@ protected void reconnectLater(Throwable exception) { } } - protected long incrementEpoch() { - return EPOCH_UPDATER.incrementAndGet(this); - } - public void connectionClosed(ClientCnx cnx) { lastConnectionClosedTimestamp = System.currentTimeMillis(); state.client.getCnxPool().releaseConnection(cnx); @@ -133,7 +129,6 @@ public void connectionClosed(ClientCnx cnx) { delayMs / 1000.0); state.client.timer().newTimeout(timeout -> { log.info("[{}] [{}] Reconnecting after timeout", state.topic, state.getHandlerName()); - incrementEpoch(); grabCnx(); }, delayMs, TimeUnit.MILLISECONDS); } @@ -151,6 +146,17 @@ protected void setClientCnx(ClientCnx clientCnx) { CLIENT_CNX_UPDATER.set(this, clientCnx); } + /** + * Update the {@link ClientCnx} for the class, then increment and get the epoch value. Note that the epoch value is + * currently only used by the {@link ProducerImpl}. + * @param clientCnx - the new {@link ClientCnx} + * @return the epoch value to use for this pair of {@link ClientCnx} and {@link ProducerImpl} + */ + protected long switchClientCnx(ClientCnx clientCnx) { + setClientCnx(clientCnx); + return EPOCH_UPDATER.incrementAndGet(this); + } + private boolean isValidStateForReconnection() { State state = this.state.getState(); switch (state) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 7ee00e7ed61d1..60d05e017b506 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -613,7 +613,7 @@ private boolean rePopulateMessageSchema(MessageImpl msg) { return true; } - private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback callback) { + private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback callback, long expectedCnxEpoch) { if (!changeToRegisteringSchemaState()) { return; } @@ -638,7 +638,7 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call } cnx.ctx().channel().eventLoop().execute(() -> { synchronized (ProducerImpl.this) { - recoverProcessOpSendMsgFrom(cnx, msg); + recoverProcessOpSendMsgFrom(cnx, msg, expectedCnxEpoch); } }); return null; @@ -1070,7 +1070,7 @@ protected synchronized void recoverChecksumError(ClientCnx cnx, long sequenceId) } } // as msg is not corrupted : let producer resend pending-messages again including checksum failed message - resendMessages(cnx); + resendMessages(cnx, this.connectionHandler.getEpoch()); } protected synchronized void recoverNotAllowedError(long sequenceId) { @@ -1345,9 +1345,17 @@ public Iterator iterator() { public void connectionOpened(final ClientCnx cnx) { previousExceptions.clear(); - // we set the cnx reference before registering the producer on the cnx, so if the cnx breaks before creating the - // producer, it will try to grab a new cnx - connectionHandler.setClientCnx(cnx); + final long epoch; + synchronized (this) { + // Because the state could have been updated while retrieving the connection, we set it back to connecting, + // as long as the change from current state to connecting is a valid state change. + if (!changeToConnecting()) { + return; + } + // We set the cnx reference before registering the producer on the cnx, so if the cnx breaks before creating + // the producer, it will try to grab a new cnx. We also increment and get the epoch value for the producer. + epoch = connectionHandler.switchClientCnx(cnx); + } cnx.registerProducer(producerId, this); log.info("[{}] [{}] Creating producer on cnx {}", topic, producerName, cnx.ctx().channel()); @@ -1384,7 +1392,7 @@ public void connectionOpened(final ClientCnx cnx) { cnx.sendRequestWithId( Commands.newProducer(topic, producerId, requestId, producerName, conf.isEncryptionEnabled(), metadata, - schemaInfo, connectionHandler.getEpoch(), userProvidedProducerName, + schemaInfo, epoch, userProvidedProducerName, conf.getAccessMode(), topicEpoch, client.conf.isEnableTransaction()), requestId).thenAccept(response -> { String producerName = response.getProducerName(); @@ -1446,7 +1454,7 @@ public void connectionOpened(final ClientCnx cnx) { } }, 0, conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS); } - resendMessages(cnx); + resendMessages(cnx, epoch); } }).exceptionally((e) -> { Throwable cause = e.getCause(); @@ -1543,7 +1551,7 @@ public void connectionFailed(PulsarClientException exception) { } } - private void resendMessages(ClientCnx cnx) { + private void resendMessages(ClientCnx cnx, long expectedEpoch) { cnx.ctx().channel().eventLoop().execute(() -> { synchronized (this) { if (getState() == State.Closing || getState() == State.Closed) { @@ -1570,7 +1578,7 @@ private void resendMessages(ClientCnx cnx) { } log.info("[{}] [{}] Re-Sending {} messages to server", topic, producerName, messagesToResend); - recoverProcessOpSendMsgFrom(cnx, null); + recoverProcessOpSendMsgFrom(cnx, null, expectedEpoch); } }); } @@ -1799,7 +1807,7 @@ protected void processOpSendMsg(OpSendMsg op) { if (shouldWriteOpSendMsg()) { ClientCnx cnx = cnx(); if (op.msg != null && op.msg.getSchemaState() == None) { - tryRegisterSchema(cnx, op.msg, op.callback); + tryRegisterSchema(cnx, op.msg, op.callback, this.connectionHandler.getEpoch()); return; } // If we do have a connection, the message is sent immediately, otherwise we'll try again once a new @@ -1830,7 +1838,16 @@ protected boolean shouldWriteOpSendMsg() { return isConnected(); } - private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from) { + // Must acquire a lock on ProducerImpl.this before calling method. + private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from, long expectedEpoch) { + if (expectedEpoch != this.connectionHandler.getEpoch() || cnx() == null) { + // In this case, the cnx passed to this method is no longer the active connection. This method will get + // called again once the new connection registers the producer with the broker. + log.info("[{}][{}] Producer epoch mismatch or the current connection is null. Skip re-sending the " + + " {} pending messages since they will deliver using another connection.", topic, producerName, + pendingMessages.size()); + return; + } final boolean stripChecksum = cnx.getRemoteEndpointProtocolVersion() < brokerChecksumSupportedVersion(); Iterator msgIterator = pendingMessages.iterator(); OpSendMsg pendingRegisteringOp = null; @@ -1879,7 +1896,7 @@ private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from) { return; } if (pendingRegisteringOp != null) { - tryRegisterSchema(cnx, pendingRegisteringOp.msg, pendingRegisteringOp.callback); + tryRegisterSchema(cnx, pendingRegisteringOp.msg, pendingRegisteringOp.callback, expectedEpoch); } } From b55e7409817ecb78fa2a9ff9cf9ff6a3c0f92991 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 7 Dec 2021 15:34:42 -0600 Subject: [PATCH 040/823] [Java Client] Fix producer data race to get cnx (#13176) (cherry picked from commit 42469de02172324f974947c4b5186e897d430e09) --- .../pulsar/client/impl/PulsarTestClient.java | 6 ++-- .../pulsar/client/impl/ProducerImpl.java | 32 +++++++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java index eebcf5b68c9d4..8136cf07c345a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java @@ -151,11 +151,11 @@ public boolean add(OpSendMsg opSendMsg) { } @Override - protected boolean shouldWriteOpSendMsg() { + protected ClientCnx getCnxIfReady() { if (dropOpSendMessages) { - return false; + return null; } else { - return super.shouldWriteOpSendMsg(); + return super.getCnxIfReady(); } } }; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 60d05e017b506..c2227c4ffe8be 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -913,7 +913,22 @@ private synchronized void closeAndClearPendingMessages() { @Override public boolean isConnected() { - return connectionHandler.cnx() != null && (getState() == State.Ready); + return getCnxIfReady() != null; + } + + /** + * Hook method for testing. By returning null, it's possible to prevent messages + * being delivered to the broker. + * + * @return cnx if OpSend messages should be written to open connection. Caller must + * verify that the returned cnx is not null before using reference. + */ + protected ClientCnx getCnxIfReady() { + if (getState() == State.Ready) { + return connectionHandler.cnx(); + } else { + return null; + } } @Override @@ -1804,8 +1819,9 @@ protected void processOpSendMsg(OpSendMsg op) { LAST_SEQ_ID_PUSHED_UPDATER.getAndUpdate(this, last -> Math.max(last, getHighestSequenceId(op))); } - if (shouldWriteOpSendMsg()) { - ClientCnx cnx = cnx(); + + final ClientCnx cnx = getCnxIfReady(); + if (cnx != null) { if (op.msg != null && op.msg.getSchemaState() == None) { tryRegisterSchema(cnx, op.msg, op.callback, this.connectionHandler.getEpoch()); return; @@ -1828,16 +1844,6 @@ protected void processOpSendMsg(OpSendMsg op) { } } - /** - * Hook method for testing. By returning false, it's possible to prevent messages - * being delivered to the broker. - * - * @return true if OpSend messages should be written to open connection - */ - protected boolean shouldWriteOpSendMsg() { - return isConnected(); - } - // Must acquire a lock on ProducerImpl.this before calling method. private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from, long expectedEpoch) { if (expectedEpoch != this.connectionHandler.getEpoch() || cnx() == null) { From 87933e9bed91adbf06b47471e51a8bf7e4b3a127 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Wed, 27 Oct 2021 10:01:39 +0800 Subject: [PATCH 041/823] [Broker] Fix messageDedup delete inactive producer name (#12493) ## Issue Now, remove inactive producerName in MessageDeduplication when producer close. But the producer has been closed before topic unload, this producerName will not be remove if producer don't connect broker with the same producerName. ## implement When topic recover `MessageDeduplication`, we should put every producerName into inactive producerNameMap. When producer with the same name, we will remove it from the inactive map, if this producerName can not connect within brokerDeduplicationProducerInactivityTimeoutMinutes, we can remove it. (cherry picked from commit 928924b5a37dbab8eea791200d328739d72da016) --- .../persistent/MessageDeduplication.java | 2 + .../persistent/MessageDuplicationTest.java | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 0df50cb623f7a..b2c42b0dd3d03 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -138,6 +138,7 @@ public MessageDeduplication(PulsarService pulsar, PersistentTopic topic, Managed private CompletableFuture recoverSequenceIdsMap() { // Load the sequence ids from the snapshot in the cursor properties managedCursor.getProperties().forEach((k, v) -> { + inactiveProducers.put(k, System.currentTimeMillis()); highestSequencedPushed.put(k, v); highestSequencedPersisted.put(k, v); }); @@ -168,6 +169,7 @@ public void readEntriesComplete(List entries, Object ctx) { long sequenceId = Math.max(md.getHighestSequenceId(), md.getSequenceId()); highestSequencedPushed.put(producerName, sequenceId); highestSequencedPersisted.put(producerName, sequenceId); + inactiveProducers.put(producerName, System.currentTimeMillis()); entry.release(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index bfef2b705d4ba..4dc7f7f42320d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -30,7 +30,13 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.awaitility.Awaitility; import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.util.Map; + import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -40,8 +46,11 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; @Slf4j @Test(groups = "broker") @@ -142,6 +151,64 @@ public void testIsDuplicate() { assertEquals(lastSequenceIdPushed.longValue(), 5); } + @Test + public void testInactiveProducerRemove() throws Exception { + PulsarService pulsarService = mock(PulsarService.class); + PersistentTopic topic = mock(PersistentTopic.class); + ManagedLedger managedLedger = mock(ManagedLedger.class); + + ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + serviceConfiguration.setBrokerDeduplicationEntriesInterval(BROKER_DEDUPLICATION_ENTRIES_INTERVAL); + serviceConfiguration.setBrokerDeduplicationMaxNumberOfProducers(BROKER_DEDUPLICATION_MAX_NUMBER_PRODUCERS); + serviceConfiguration.setReplicatorPrefix(REPLICATOR_PREFIX); + serviceConfiguration.setBrokerDeduplicationProducerInactivityTimeoutMinutes(1); + + doReturn(serviceConfiguration).when(pulsarService).getConfiguration(); + MessageDeduplication messageDeduplication = spy(new MessageDeduplication(pulsarService, topic, managedLedger)); + doReturn(true).when(messageDeduplication).isEnabled(); + + Topic.PublishContext publishContext = mock(Topic.PublishContext.class); + + Field field = MessageDeduplication.class.getDeclaredField("inactiveProducers"); + field.setAccessible(true); + Map map = (Map) field.get(messageDeduplication); + + String producerName1 = "test1"; + when(publishContext.getHighestSequenceId()).thenReturn(2L); + when(publishContext.getSequenceId()).thenReturn(1L); + when(publishContext.getProducerName()).thenReturn(producerName1); + messageDeduplication.isDuplicate(publishContext, null); + + String producerName2 = "test2"; + when(publishContext.getProducerName()).thenReturn(producerName2); + messageDeduplication.isDuplicate(publishContext, null); + + String producerName3 = "test3"; + when(publishContext.getProducerName()).thenReturn(producerName3); + messageDeduplication.isDuplicate(publishContext, null); + + messageDeduplication.producerRemoved(producerName1); + assertTrue(map.containsKey(producerName1)); + messageDeduplication.producerAdded(producerName1); + assertFalse(map.containsKey(producerName1)); + messageDeduplication.purgeInactiveProducers(); + // messageDeduplication.purgeInactiveProducers() will remove producer2 and producer3 + map.put(producerName2, System.currentTimeMillis() - 70000); + map.put(producerName3, System.currentTimeMillis() - 70000); + messageDeduplication.purgeInactiveProducers(); + assertFalse(map.containsKey(producerName2)); + assertFalse(map.containsKey(producerName3)); + + field = MessageDeduplication.class.getDeclaredField("highestSequencedPushed"); + field.setAccessible(true); + ConcurrentOpenHashMap highestSequencedPushed = (ConcurrentOpenHashMap) field.get(messageDeduplication); + + assertEquals((long) highestSequencedPushed.get(producerName1), 2L); + assertFalse(highestSequencedPushed.containsKey(producerName2)); + assertFalse(highestSequencedPushed.containsKey(producerName3)); + + } + @Test public void testIsDuplicateWithFailure() { From b87bd78683c5bdba1c430151592fcbb710b5bdbe Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 15 Nov 2021 15:02:36 -0600 Subject: [PATCH 042/823] [Broker] Synchronize updates to the inactiveProducers map in MessageDeduplication (#12820) (cherry picked from commit 9994614173205abd075fcc670396cebd71227047) --- .../broker/service/persistent/MessageDeduplication.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index b2c42b0dd3d03..e2436bb2a10c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -138,7 +138,7 @@ public MessageDeduplication(PulsarService pulsar, PersistentTopic topic, Managed private CompletableFuture recoverSequenceIdsMap() { // Load the sequence ids from the snapshot in the cursor properties managedCursor.getProperties().forEach((k, v) -> { - inactiveProducers.put(k, System.currentTimeMillis()); + producerRemoved(k); highestSequencedPushed.put(k, v); highestSequencedPersisted.put(k, v); }); @@ -169,7 +169,7 @@ public void readEntriesComplete(List entries, Object ctx) { long sequenceId = Math.max(md.getHighestSequenceId(), md.getSequenceId()); highestSequencedPushed.put(producerName, sequenceId); highestSequencedPersisted.put(producerName, sequenceId); - inactiveProducers.put(producerName, System.currentTimeMillis()); + producerRemoved(producerName); entry.release(); } From 15b2831c33f74b38a430c80ca9b2825b7aa0da26 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 25 Oct 2021 16:31:59 -0500 Subject: [PATCH 043/823] [Java Client] Remove data race in MultiTopicsConsumerImpl to ensure correct message order (#12456) * [Java Client] Remove data race in MultiTopicsConsumerImpl to ensure correct message order * Fix test * Return the checkState method call to keep original behavior * Reproduce out-of-order delivery issue in PR 12456 * Remove unnecessary scheduling of receiveMessageFromConsumer Co-authored-by: Lari Hotari (cherry picked from commit 6a2e3a1ad735465154dc3fa12988c3068eae7da5) --- .../client/api/MultiTopicsConsumerTest.java | 75 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 3 +- .../client/impl/MultiTopicsConsumerImpl.java | 37 ++++----- .../impl/MultiTopicsConsumerImplTest.java | 3 +- 4 files changed, 98 insertions(+), 20 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java index 715f3adb7aaf6..d8c8bd657f8cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.client.api; import static org.mockito.ArgumentMatchers.any; @@ -24,9 +25,16 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import com.google.common.collect.Lists; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; @@ -34,6 +42,7 @@ import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -70,6 +79,7 @@ protected PulsarClient createNewPulsarClient(ClientBuilder clientBuilder) throws // method calls on the interface. Mockito.withSettings().defaultAnswer(AdditionalAnswers.delegatesTo(internalExecutorService))); } + @Override public ExecutorService getInternalExecutorService() { return internalExecutorServiceDelegate; @@ -119,4 +129,69 @@ public void testMultiTopicsConsumerCloses() throws Exception { verify(internalExecutorServiceDelegate, times(0)) .schedule(any(Runnable.class), anyLong(), any()); } + + // test that reproduces the issue that PR https://github.com/apache/pulsar/pull/12456 fixes + // where MultiTopicsConsumerImpl has a data race that causes out-of-order delivery of messages + @Test + public void testShouldMaintainOrderForIndividualTopicInMultiTopicsConsumer() + throws PulsarAdminException, PulsarClientException, ExecutionException, InterruptedException, + TimeoutException { + String topicName = newTopicName(); + int numPartitions = 2; + int numMessages = 100000; + admin.topics().createPartitionedTopic(topicName, numPartitions); + + Producer[] producers = new Producer[numPartitions]; + + for (int i = 0; i < numPartitions; i++) { + producers[i] = pulsarClient.newProducer(Schema.INT64) + // produce to each partition directly so that order can be maintained in sending + .topic(topicName + "-partition-" + i) + .enableBatching(true) + .maxPendingMessages(30000) + .maxPendingMessagesAcrossPartitions(60000) + .batchingMaxMessages(10000) + .batchingMaxPublishDelay(5, TimeUnit.SECONDS) + .batchingMaxBytes(4 * 1024 * 1024) + .blockIfQueueFull(true) + .create(); + } + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.INT64) + // consume on the partitioned topic + .topic(topicName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .receiverQueueSize(numMessages) + .subscriptionName(methodName) + .subscribe(); + + // produce sequence numbers to each partition topic + long sequenceNumber = 1L; + for (int i = 0; i < numMessages; i++) { + for (Producer producer : producers) { + producer.newMessage() + .value(sequenceNumber) + .sendAsync(); + } + sequenceNumber++; + } + for (Producer producer : producers) { + producer.close(); + } + + // receive and validate sequences in the partitioned topic + Map receivedSequences = new HashMap<>(); + int receivedCount = 0; + while (receivedCount < numPartitions * numMessages) { + Message message = consumer.receiveAsync().get(5, TimeUnit.SECONDS); + consumer.acknowledge(message); + receivedCount++; + AtomicLong receivedSequenceCounter = + receivedSequences.computeIfAbsent(message.getTopicName(), k -> new AtomicLong(1L)); + Assert.assertEquals(message.getValue().longValue(), receivedSequenceCounter.getAndIncrement()); + } + Assert.assertEquals(numPartitions * numMessages, receivedCount); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index acc7cea61d4e2..7e42d0dd39616 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -427,8 +427,7 @@ protected CompletableFuture> internalReceiveAsync() { if (message == null) { pendingReceives.add(result); cancellationHandler.setCancelAction(() -> pendingReceives.remove(result)); - } - if (message != null) { + } else { messageProcessed(message); result.complete(beforeConsume(message)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 18bd7bc45b8e5..2c356fb8e28f5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -245,7 +245,7 @@ private void startReceivingMessages(List> newConsumers) { } private void receiveMessageFromConsumer(ConsumerImpl consumer) { - consumer.receiveAsync().thenAccept(message -> { + consumer.receiveAsync().thenAcceptAsync(message -> { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Receive message from sub consumer:{}", topic, subscription, consumer.getTopic()); @@ -260,16 +260,16 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { // or if any consumer is already paused (to create fair chance for already paused consumers) pausedConsumers.add(consumer); - // Since we din't get a mutex, the condition on the incoming queue might have changed after + // Since we didn't get a mutex, the condition on the incoming queue might have changed after // we have paused the current consumer. We need to re-check in order to avoid this consumer // from getting stalled. resumeReceivingFromPausedConsumersIfNeeded(); } else { - // Schedule next receiveAsync() if the incoming queue is not full. Use a different thread to avoid - // recursion and stack overflow - internalPinnedExecutor.execute(() -> receiveMessageFromConsumer(consumer)); + // Call receiveAsync() if the incoming queue is not full. Because this block is run with + // thenAcceptAsync, there is no chance for recursion that would lead to stack overflow. + receiveMessageFromConsumer(consumer); } - }).exceptionally(ex -> { + }, internalPinnedExecutor).exceptionally(ex -> { if (ex instanceof PulsarClientException.AlreadyClosedException || ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { // ignore the exception that happens when the consumer is closed @@ -281,6 +281,7 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { }); } + // Must be called from the internalPinnedExecutor thread private void messageReceived(ConsumerImpl consumer, Message message) { checkArgument(message instanceof MessageImpl); TopicMessageImpl topicMessage = new TopicMessageImpl<>(consumer.getTopic(), @@ -409,17 +410,19 @@ protected CompletableFuture> internalBatchReceiveAsync() { protected CompletableFuture> internalReceiveAsync() { CompletableFutureCancellationHandler cancellationHandler = new CompletableFutureCancellationHandler(); CompletableFuture> result = cancellationHandler.createFuture(); - Message message = incomingMessages.poll(); - if (message == null) { - pendingReceives.add(result); - cancellationHandler.setCancelAction(() -> pendingReceives.remove(result)); - } else { - decreaseIncomingMessageSize(message); - checkState(message instanceof TopicMessageImpl); - unAckedMessageTracker.add(message.getMessageId()); - resumeReceivingFromPausedConsumersIfNeeded(); - result.complete(message); - } + internalPinnedExecutor.execute(() -> { + Message message = incomingMessages.poll(); + if (message == null) { + pendingReceives.add(result); + cancellationHandler.setCancelAction(() -> pendingReceives.remove(result)); + } else { + decreaseIncomingMessageSize(message); + checkState(message instanceof TopicMessageImpl); + unAckedMessageTracker.add(message.getMessageId()); + resumeReceivingFromPausedConsumersIfNeeded(); + result.complete(message); + } + }); return result; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index 6af8914d6943d..faa621cae2193 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -36,6 +36,7 @@ import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Before; import org.testng.annotations.AfterMethod; @@ -165,7 +166,7 @@ public void testReceiveAsyncCanBeCancelled() { // given MultiTopicsConsumerImpl consumer = createMultiTopicsConsumer(); CompletableFuture> future = consumer.receiveAsync(); - assertTrue(consumer.hasNextPendingReceive()); + Awaitility.await().untilAsserted(() -> assertTrue(consumer.hasNextPendingReceive())); // when future.cancel(true); // then From 593c2da8fd4bb8b8838ba9bb1d93c978baccd9ad Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 18 Nov 2021 02:42:56 -0600 Subject: [PATCH 044/823] [Broker] Fix producer getting incorrectly removed from topic's producers map (#12846) (cherry picked from commit e33687d3f202ab104d41ad086c48b66b6f0d5ff5) --- .../pulsar/broker/service/AbstractTopic.java | 8 +-- .../pulsar/broker/service/Producer.java | 27 ++++---- .../broker/service/PersistentTopicTest.java | 14 +++- .../pulsar/broker/service/ServerCnxTest.java | 66 +++++++++++++++++++ 4 files changed, 91 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 951d6025eb51b..9463be9f8fefc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -631,13 +631,9 @@ protected void internalAddProducer(Producer producer) throws BrokerServiceExcept private void tryOverwriteOldProducer(Producer oldProducer, Producer newProducer) throws BrokerServiceException { - boolean canOverwrite = false; - if (oldProducer.equals(newProducer) && !isUserProvidedProducerName(oldProducer) - && !isUserProvidedProducerName(newProducer) && newProducer.getEpoch() > oldProducer.getEpoch()) { + if (newProducer.isSuccessorTo(oldProducer) && !isUserProvidedProducerName(oldProducer) + && !isUserProvidedProducerName(newProducer)) { oldProducer.close(false); - canOverwrite = true; - } - if (canOverwrite) { if (!producers.replace(newProducer.getProducerName(), oldProducer, newProducer)) { // Met concurrent update, throw exception here so that client can try reconnect later. throw new BrokerServiceException.NamingException("Producer with name '" + newProducer.getProducerName() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index d13e2f93632fb..64e0fd64fd4e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -138,22 +138,17 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN this.clientAddress = cnx.clientSourceAddress(); } - @Override - public int hashCode() { - return Objects.hash(producerName); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof Producer) { - Producer other = (Producer) obj; - return Objects.equals(producerName, other.producerName) - && Objects.equals(topic, other.topic) - && producerId == other.producerId - && Objects.equals(cnx, other.cnx); - } - - return false; + /** + * Method to determine if this producer can replace another producer. + * @param other - producer to compare to this one + * @return true if this producer is a subsequent instantiation of the same logical producer. Otherwise, false. + */ + public boolean isSuccessorTo(Producer other) { + return Objects.equals(producerName, other.producerName) + && Objects.equals(topic, other.topic) + && producerId == other.producerId + && Objects.equals(cnx, other.cnx) + && other.getEpoch() < epoch; } public void publishMessage(long producerId, long sequenceId, ByteBuf headersAndPayload, long batchSize, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 543d897575756..cb10d72fe1764 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -36,6 +36,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -440,11 +441,20 @@ public void testAddRemoveProducer() throws Exception { // OK } - // 4. simple remove producer + // 4. Try to remove with unequal producer + Producer producerCopy = new Producer(topic, serverCnx, 1 /* producer id */, "prod-name", + role, false, null, SchemaVersion.Latest, 0, false, + ProducerAccessMode.Shared, Optional.empty()); + topic.removeProducer(producerCopy); + // Expect producer to be in map + assertEquals(topic.getProducers().size(), 1); + assertSame(topic.getProducers().get(producer.getProducerName()), producer); + + // 5. simple remove producer topic.removeProducer(producer); assertEquals(topic.getProducers().size(), 0); - // 5. duplicate remove + // 6. duplicate remove topic.removeProducer(producer); /* noop */ } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 3d3a30e89516c..8881ce90812a0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -838,6 +838,72 @@ public void testCreateProducerTimeout() throws Exception { channel.finish(); } + @Test(timeOut = 30000) + public void testCreateProducerTimeoutThenCreateSameNamedProducerShouldFail() throws Exception { + resetChannel(); + setChannelConnected(); + + // Delay the topic creation in a deterministic way + CompletableFuture openTopicFuture = new CompletableFuture<>(); + doAnswer(invocationOnMock -> { + openTopicFuture.complete(() -> { + ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerComplete(ledgerMock, null); + }); + return null; + }).when(mlFactoryMock).asyncOpen(matches(".*success.*"), any(ManagedLedgerConfig.class), + any(OpenLedgerCallback.class), any(Supplier.class), any()); + + // In a create producer timeout from client side we expect to see this sequence of commands : + // 1. create producer + // 2. close producer (when the timeout is triggered, which may be before the producer was created on the broker + // 3. create producer (triggered by reconnection logic) + // Then, when another producer is created with the same name, it should fail. Because we only have one + // channel here, we just use a different producer id + + // These operations need to be serialized, to allow the last create producer to finally succeed + // (There can be more create/close pairs in the sequence, depending on the client timeout + + String producerName = "my-producer"; + + ByteBuf createProducer1 = Commands.newProducer(successTopicName, 1 /* producer id */, 1 /* request id */, + producerName, Collections.emptyMap(), false); + channel.writeInbound(createProducer1); + + ByteBuf closeProducer = Commands.newCloseProducer(1 /* producer id */, 2 /* request id */ ); + channel.writeInbound(closeProducer); + + ByteBuf createProducer2 = Commands.newProducer(successTopicName, 1 /* producer id */, 3 /* request id */, + producerName, Collections.emptyMap(), false); + channel.writeInbound(createProducer2); + + // Complete the topic opening: It will make 2nd producer creation successful + openTopicFuture.get().run(); + + // Close succeeds + Object response = getResponse(); + assertEquals(response.getClass(), CommandSuccess.class); + assertEquals(((CommandSuccess) response).getRequestId(), 2); + + // 2nd producer will be successfully created as topic is open by then + response = getResponse(); + assertEquals(response.getClass(), CommandProducerSuccess.class); + assertEquals(((CommandProducerSuccess) response).getRequestId(), 3); + + // Send create command after getting the CommandProducerSuccess to ensure correct ordering + ByteBuf createProducer3 = Commands.newProducer(successTopicName, 2 /* producer id */, 4 /* request id */, + producerName, Collections.emptyMap(), false); + channel.writeInbound(createProducer3); + + // 3nd producer will fail + response = getResponse(); + assertEquals(response.getClass(), CommandError.class); + assertEquals(((CommandError) response).getRequestId(), 4); + + assertTrue(channel.isActive()); + + channel.finish(); + } + @Test(timeOut = 30000, enabled = false) public void testCreateProducerMultipleTimeouts() throws Exception { resetChannel(); From 5a00c02f19adaae9ef27394aa3cdf20a3b1bc2d1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 19 Nov 2021 08:23:27 +0200 Subject: [PATCH 045/823] Catch exceptions in scheduled tasks to prevent unintended cancellation (#12853) * Catch exceptions in scheduled tasks to prevent unintended cancellation - ScheduledExecutorService#scheduleAtFixedRate won't schedule the next execution if running the task throws an exception. This can lead to unintended cancellation of the scheduled task in failure scenarios. * Address review feedback: use private constructor * Address review feedback: Use private inner class instead of Lambda (cherry picked from commit afdfe199be73aad7b061e0d5b625787d249f1530) --- .../impl/ManagedLedgerFactoryImpl.java | 5 +- .../impl/GenericBrokerHostUsageImpl.java | 6 +- .../impl/LinuxBrokerHostUsageImpl.java | 4 +- .../resourcegroup/ResourceGroupService.java | 9 +-- .../ResourceUsageTopicTransportManager.java | 3 +- .../pulsar/broker/service/BrokerService.java | 2 +- .../ReplicatedSubscriptionsController.java | 3 +- .../persistent/SubscribeRateLimiter.java | 3 +- .../metrics/PrometheusMetricsProvider.java | 6 +- .../apache/pulsar/client/impl/ClientCnx.java | 6 +- .../pulsar/client/impl/ConsumerImpl.java | 9 +-- ...sistentAcknowledgmentsGroupingTracker.java | 3 +- .../pulsar/client/impl/ProducerImpl.java | 43 +++++++------- .../pulsar/common/protocol/PulsarHandler.java | 6 +- .../pulsar/common/stats/JvmMetrics.java | 3 +- .../pulsar/common/util/RateLimiter.java | 4 +- .../apache/pulsar/common/util/Runnables.java | 59 +++++++++++++++++++ .../pulsar/common/util/RunnablesTest.java | 32 ++++++++++ .../windowing/WaterMarkEventGenerator.java | 5 +- .../windowing/triggers/TimeTriggerPolicy.java | 5 +- .../functions/runtime/RuntimeSpawner.java | 37 ++++++------ .../runtime/process/ProcessRuntime.java | 22 +++---- .../worker/ClusterServiceCoordinator.java | 5 +- .../functions/worker/SchedulerManager.java | 9 +-- .../metadata/impl/ZKSessionWatcher.java | 7 ++- .../pulsar/websocket/stats/JvmMetrics.java | 4 +- .../pulsar/websocket/stats/ProxyStats.java | 4 +- 27 files changed, 218 insertions(+), 86 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/Runnables.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/RunnablesTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index 94dee65fafd59..e1761818b2ff6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.ManagedLedgerException.getManagedLedgerException; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.Predicates; import com.google.common.collect.Maps; import io.netty.util.concurrent.DefaultThreadFactory; @@ -187,9 +188,9 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, this.config = config; this.mbean = new ManagedLedgerFactoryMBeanImpl(this); this.entryCacheManager = new EntryCacheManager(this); - this.statsTask = scheduledExecutor.scheduleAtFixedRate(this::refreshStats, + this.statsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::refreshStats), 0, config.getStatsPeriodSeconds(), TimeUnit.SECONDS); - this.flushCursorsTask = scheduledExecutor.scheduleAtFixedRate(this::flushCursors, + this.flushCursorsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::flushCursors), config.getCursorPositionFlushSeconds(), config.getCursorPositionFlushSeconds(), TimeUnit.SECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java index f55d75a4ce16d..72d36abdfb004 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.sun.management.OperatingSystemMXBean; import java.lang.management.ManagementFactory; import java.util.concurrent.ScheduledExecutorService; @@ -53,9 +54,10 @@ public GenericBrokerHostUsageImpl(int hostUsageCheckIntervalMin, this.totalCpuLimit = getTotalCpuLimit(); // Call now to initialize values before the constructor returns calculateBrokerHostUsage(); - executorService.scheduleAtFixedRate(this::checkCpuLoad, CPU_CHECK_MILLIS, + executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::checkCpuLoad), CPU_CHECK_MILLIS, CPU_CHECK_MILLIS, TimeUnit.MILLISECONDS); - executorService.scheduleAtFixedRate(this::doCalculateBrokerHostUsage, hostUsageCheckIntervalMin, + executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::doCalculateBrokerHostUsage), + hostUsageCheckIntervalMin, hostUsageCheckIntervalMin, TimeUnit.MINUTES); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 8e4e3c57f1de7..ccbe8e3e413ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.Charsets; import com.sun.management.OperatingSystemMXBean; import java.io.IOException; @@ -86,7 +87,8 @@ public LinuxBrokerHostUsageImpl(int hostUsageCheckIntervalMin, // Call now to initialize values before the constructor returns calculateBrokerHostUsage(); - executorService.scheduleAtFixedRate(this::calculateBrokerHostUsage, hostUsageCheckIntervalMin, + executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::calculateBrokerHostUsage), + hostUsageCheckIntervalMin, hostUsageCheckIntervalMin, TimeUnit.MINUTES); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index 677c04ad9a6f4..4900027d21810 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.resourcegroup; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import io.prometheus.client.Counter; import io.prometheus.client.Summary; import java.util.Map; @@ -575,7 +576,7 @@ protected void aggregateResourceGroupLocalUsages() { cancelStatus, this.aggregateLocalUsagePeriodInSeconds, newPeriodInSeconds, timeUnitScale); } this.aggreagteLocalUsagePeriodicTask = pulsar.getExecutor().scheduleAtFixedRate( - this::aggregateResourceGroupLocalUsages, + catchingAndLoggingThrowables(this::aggregateResourceGroupLocalUsages), newPeriodInSeconds, newPeriodInSeconds, timeUnitScale); @@ -665,7 +666,7 @@ protected void calculateQuotaForAllResourceGroups() { cancelStatus, this.resourceUsagePublishPeriodInSeconds, newPeriodInSeconds, timeUnitScale); } this.calculateQuotaPeriodicTask = pulsar.getExecutor().scheduleAtFixedRate( - this::calculateQuotaForAllResourceGroups, + catchingAndLoggingThrowables(this::calculateQuotaForAllResourceGroups), newPeriodInSeconds, newPeriodInSeconds, timeUnitScale); @@ -680,12 +681,12 @@ private void initialize() { long periodInSecs = config.getResourceUsageTransportPublishIntervalInSecs(); this.aggregateLocalUsagePeriodInSeconds = this.resourceUsagePublishPeriodInSeconds = periodInSecs; this.aggreagteLocalUsagePeriodicTask = this.pulsar.getExecutor().scheduleAtFixedRate( - this::aggregateResourceGroupLocalUsages, + catchingAndLoggingThrowables(this::aggregateResourceGroupLocalUsages), periodInSecs, periodInSecs, this.timeUnitScale); this.calculateQuotaPeriodicTask = this.pulsar.getExecutor().scheduleAtFixedRate( - this::calculateQuotaForAllResourceGroups, + catchingAndLoggingThrowables(this::calculateQuotaForAllResourceGroups), periodInSecs, periodInSecs, this.timeUnitScale); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTopicTransportManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTopicTransportManager.java index d3584a81cafde..98269910fec3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTopicTransportManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTopicTransportManager.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.resourcegroup; import static org.apache.pulsar.client.api.CompressionType.LZ4; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -79,7 +80,7 @@ private Producer createProducer() throws PulsarClientException { public ResourceUsageWriterTask() throws PulsarClientException { producer = createProducer(); resourceUsagePublishTask = pulsarService.getExecutor().scheduleAtFixedRate( - this, + catchingAndLoggingThrowables(this), pulsarService.getConfig().getResourceUsageTransportPublishIntervalInSecs(), pulsarService.getConfig().getResourceUsageTransportPublishIntervalInSecs(), TimeUnit.SECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index e592336867f4a..0835eb0939497 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -331,7 +331,7 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws log.info("Enabling per-broker unack-message limit {} and dispatcher-limit {} on blocked-broker", maxUnackedMessages, maxUnackedMsgsPerDispatcher); // block misbehaving dispatcher by checking periodically - pulsar.getExecutor().scheduleAtFixedRate(() -> checkUnAckMessageDispatching(), + pulsar.getExecutor().scheduleAtFixedRate(safeRun(this::checkUnAckMessageDispatching), 600, 30, TimeUnit.SECONDS); } else { this.maxUnackedMessages = 0; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index e0501ea187e39..2b1ae4ba19331 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import io.netty.buffer.ByteBuf; import io.prometheus.client.Gauge; import java.io.IOException; @@ -78,7 +79,7 @@ public ReplicatedSubscriptionsController(PersistentTopic topic, String localClus this.topic = topic; this.localCluster = localCluster; timer = topic.getBrokerService().pulsar().getExecutor() - .scheduleAtFixedRate(this::startNewSnapshot, 0, + .scheduleAtFixedRate(catchingAndLoggingThrowables(this::startNewSnapshot), 0, topic.getBrokerService().pulsar().getConfiguration() .getReplicatedSubscriptionsSnapshotFrequencyMillis(), TimeUnit.MILLISECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java index e1d39aee2f3cb..93707b4977b68 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.MoreObjects; import java.util.Objects; import java.util.Optional; @@ -271,7 +272,7 @@ public void close() { } private ScheduledFuture createTask() { - return executorService.scheduleAtFixedRate(this::closeAndClearRateLimiters, + return executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::closeAndClearRateLimiters), this.subscribeRate.ratePeriodInSecond, this.subscribeRate.ratePeriodInSecond, TimeUnit.SECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java index dfb27f4c92c27..0e59286861a40 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.DefaultThreadFactory; import io.prometheus.client.Collector; @@ -95,9 +96,8 @@ public void start(Configuration conf) { DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS); cluster = conf.getString(CLUSTER_NAME, DEFAULT_CLUSTER_NAME); - executor.scheduleAtFixedRate(() -> { - rotateLatencyCollection(); - }, 1, latencyRolloverSeconds, TimeUnit.SECONDS); + executor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::rotateLatencyCollection), + 1, latencyRolloverSeconds, TimeUnit.SECONDS); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index b0895f482d50e..e2386a91436dc 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static org.apache.pulsar.client.impl.TransactionMetaStoreHandler.getExceptionByServerError; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.collect.Queues; import io.netty.buffer.ByteBuf; @@ -217,8 +218,9 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { this.localAddress = ctx.channel().localAddress(); this.remoteAddress = ctx.channel().remoteAddress(); - this.timeoutTask = this.eventLoopGroup.scheduleAtFixedRate(() -> checkRequestTimeout(), operationTimeoutMs, - operationTimeoutMs, TimeUnit.MILLISECONDS); + this.timeoutTask = this.eventLoopGroup + .scheduleAtFixedRate(catchingAndLoggingThrowables(this::checkRequestTimeout), operationTimeoutMs, + operationTimeoutMs, TimeUnit.MILLISECONDS); if (proxyToTargetBrokerAddress == null) { if (log.isDebugEnabled()) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 7e42d0dd39616..11b1def52e0df 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.common.protocol.Commands.hasChecksum; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Iterables; import com.scurrilous.circe.checksum.Crc32cIntChecksum; @@ -1248,10 +1249,10 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // Lazy task scheduling to expire incomplete chunk message if (!expireChunkMessageTaskScheduled && expireTimeOfIncompleteChunkedMessageMillis > 0) { - internalPinnedExecutor.scheduleAtFixedRate(() -> { - removeExpireIncompleteChunkedMessages(); - }, expireTimeOfIncompleteChunkedMessageMillis, expireTimeOfIncompleteChunkedMessageMillis, - TimeUnit.MILLISECONDS); + internalPinnedExecutor + .scheduleAtFixedRate(catchingAndLoggingThrowables(this::removeExpireIncompleteChunkedMessages), + expireTimeOfIncompleteChunkedMessageMillis, expireTimeOfIncompleteChunkedMessageMillis, + TimeUnit.MILLISECONDS); expireChunkMessageTaskScheduled = true; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index 21ef9b38dbed9..aa65c6119a38c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; import java.util.ArrayList; @@ -102,7 +103,7 @@ public PersistentAcknowledgmentsGroupingTracker(ConsumerImpl consumer, Consum this.currentCumulativeAckFuture = new TimedCompletableFuture<>(); if (acknowledgementGroupTimeMicros > 0) { - scheduledTask = eventLoopGroup.next().scheduleWithFixedDelay(this::flush, acknowledgementGroupTimeMicros, + scheduledTask = eventLoopGroup.next().scheduleWithFixedDelay(catchingAndLoggingThrowables(this::flush), acknowledgementGroupTimeMicros, acknowledgementGroupTimeMicros, TimeUnit.MICROSECONDS); } else { scheduledTask = null; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index c2227c4ffe8be..e33fb5f699463 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -29,6 +29,7 @@ import static org.apache.pulsar.client.impl.ProducerBase.MultiSchemaMode.Enabled; import static org.apache.pulsar.common.protocol.Commands.hasChecksum; import static org.apache.pulsar.common.protocol.Commands.readChecksum; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; @@ -204,7 +205,7 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration if (this.msgCrypto != null) { // Regenerate data key cipher at fixed interval - keyGeneratorTask = client.eventLoopGroup().scheduleWithFixedDelay(() -> { + keyGeneratorTask = client.eventLoopGroup().scheduleWithFixedDelay(catchingAndLoggingThrowables(() -> { try { msgCrypto.addPublicKeyCipher(conf.getEncryptionKeys(), conf.getCryptoKeyReader()); } catch (CryptoException e) { @@ -217,7 +218,7 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration producerName, topic))); } } - }, 0L, 4L, TimeUnit.HOURS); + }), 0L, 4L, TimeUnit.HOURS); } if (conf.getSendTimeoutMs() > 0) { @@ -1450,24 +1451,26 @@ public void connectionOpened(final ClientCnx cnx) { if (!producerCreatedFuture.isDone() && isBatchMessagingEnabled()) { // schedule the first batch message task - batchTimerTask = cnx.ctx().executor().scheduleAtFixedRate(() -> { - if (log.isTraceEnabled()) { - log.trace( - "[{}] [{}] Batching the messages from the batch container from timer thread", - topic, - producerName); - } - // semaphore acquired when message was enqueued to container - synchronized (ProducerImpl.this) { - // If it's closing/closed we need to ignore the send batch timer and not - // schedule next timeout. - if (getState() == State.Closing || getState() == State.Closed) { - return; - } - - batchMessageAndSend(); - } - }, 0, conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS); + batchTimerTask = cnx.ctx().executor() + .scheduleAtFixedRate(catchingAndLoggingThrowables(() -> { + if (log.isTraceEnabled()) { + log.trace( + "[{}] [{}] Batching the messages from the batch container from " + + "timer thread", + topic, + producerName); + } + // semaphore acquired when message was enqueued to container + synchronized (ProducerImpl.this) { + // If it's closing/closed we need to ignore the send batch timer and not + // schedule next timeout. + if (getState() == State.Closing || getState() == State.Closed) { + return; + } + + batchMessageAndSend(); + } + }), 0, conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS); } resendMessages(cnx, epoch); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index 481517c794ab9..557dbfa9a92c0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.common.protocol; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import io.netty.channel.ChannelHandlerContext; import io.netty.util.concurrent.ScheduledFuture; import java.net.SocketAddress; @@ -65,8 +66,9 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { log.debug("[{}] Scheduling keep-alive task every {} s", ctx.channel(), keepAliveIntervalSeconds); } if (keepAliveIntervalSeconds > 0) { - this.keepAliveTask = ctx.executor().scheduleAtFixedRate(this::handleKeepAliveTimeout, - keepAliveIntervalSeconds, keepAliveIntervalSeconds, TimeUnit.SECONDS); + this.keepAliveTask = ctx.executor() + .scheduleAtFixedRate(catchingAndLoggingThrowables(this::handleKeepAliveTimeout), + keepAliveIntervalSeconds, keepAliveIntervalSeconds, TimeUnit.SECONDS); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java index 6780563959b3b..83b14060a99a2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.common.stats; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.netty.buffer.PoolArenaMetric; @@ -96,7 +97,7 @@ private static String detectGCType() { public JvmMetrics(ScheduledExecutorService executor, String componentName, JvmGCMetricsLogger gcLogger) { this.gcLogger = gcLogger; if (executor != null) { - executor.scheduleAtFixedRate(gcLogger::refresh, 0, 1, TimeUnit.MINUTES); + executor.scheduleAtFixedRate(catchingAndLoggingThrowables(gcLogger::refresh), 0, 1, TimeUnit.MINUTES); } this.componentName = componentName; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java index 6f2d899808434..20ca181c40080 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.common.util; import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.MoreObjects; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -256,7 +257,8 @@ public synchronized TimeUnit getRateTimeUnit() { } protected ScheduledFuture createTask() { - return executorService.scheduleAtFixedRate(this::renew, this.rateTime, this.rateTime, this.timeUnit); + return executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::renew), this.rateTime, + this.rateTime, this.timeUnit); } synchronized void renew() { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/Runnables.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/Runnables.java new file mode 100644 index 0000000000000..b720c4fb6af77 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/Runnables.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class Runnables { + private static final Logger LOGGER = LoggerFactory.getLogger(Runnables.class); + + private Runnables() {} + + /** + * Wraps a Runnable so that throwables are caught and logged when a Runnable is run. + * + * The main usecase for this method is to be used in {@link java.util.concurrent.ScheduledExecutorService#scheduleAtFixedRate(Runnable, long, long, TimeUnit)} + * calls to ensure that the scheduled task doesn't get cancelled as a result of an uncaught exception. + * + * @param runnable The runnable to wrap + * @return a wrapped Runnable + */ + public static Runnable catchingAndLoggingThrowables(Runnable runnable) { + return new CatchingAndLoggingRunnable(runnable); + } + + private static final class CatchingAndLoggingRunnable implements Runnable { + private final Runnable runnable; + + private CatchingAndLoggingRunnable(Runnable runnable) { + this.runnable = runnable; + } + + @Override + public void run() { + try { + runnable.run(); + } catch (Throwable t) { + LOGGER.error("Unexpected throwable caught", t); + } + } + } +} \ No newline at end of file diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/RunnablesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/RunnablesTest.java new file mode 100644 index 0000000000000..6dfbe1beececf --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/RunnablesTest.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.common.util; + +import org.testng.annotations.Test; + +public class RunnablesTest { + + @Test + public void shouldCatchAndLogException() { + Runnables.catchingAndLoggingThrowables(() -> { + throw new RuntimeException(); + }).run(); + } +} \ No newline at end of file diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WaterMarkEventGenerator.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WaterMarkEventGenerator.java index d9042022b293c..0024586941d48 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WaterMarkEventGenerator.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WaterMarkEventGenerator.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.windowing; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.ThreadContext; @@ -134,7 +135,9 @@ private void checkFailures() { } public void start() { - this.executorFuture = executorService.scheduleAtFixedRate(this, intervalMs, intervalMs, TimeUnit.MILLISECONDS); + this.executorFuture = + executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this), intervalMs, intervalMs, + TimeUnit.MILLISECONDS); } public void shutdown() { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java index bc40f5329e460..3ae0c23901193 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.windowing.triggers; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.ThreadContext; @@ -75,7 +76,9 @@ public void reset() { @Override public void start() { - executorFuture = executor.scheduleAtFixedRate(newTriggerTask(), duration, duration, TimeUnit.MILLISECONDS); + executorFuture = + executor.scheduleAtFixedRate(catchingAndLoggingThrowables(newTriggerTask()), duration, duration, + TimeUnit.MILLISECONDS); } @Override diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java index 471af68c1652c..60c869795079e 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java @@ -23,6 +23,7 @@ */ package org.apache.pulsar.functions.runtime; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledFuture; @@ -81,23 +82,25 @@ public void start() throws Exception { // monitor function runtime to make sure it is running. If not, restart the function runtime if (!runtimeFactory.externallyManaged() && instanceLivenessCheckFreqMs > 0) { - processLivenessCheckTimer = InstanceCache.getInstanceCache().getScheduledExecutorService().scheduleAtFixedRate(() -> { - Runtime runtime = RuntimeSpawner.this.runtime; - if (runtime != null && !runtime.isAlive()) { - log.error("{}/{}/{} Function Container is dead with following exception. Restarting.", details.getTenant(), - details.getNamespace(), details.getName(), runtime.getDeathException()); - // Just for the sake of sanity, just destroy the runtime - try { - runtime.stop(); - runtimeDeathException = runtime.getDeathException(); - runtime.start(); - } catch (Exception e) { - log.error("{}/{}/{}-{} Function Restart failed", details.getTenant(), - details.getNamespace(), details.getName(), e, e); - } - numRestarts++; - } - }, instanceLivenessCheckFreqMs, instanceLivenessCheckFreqMs, TimeUnit.MILLISECONDS); + processLivenessCheckTimer = InstanceCache.getInstanceCache().getScheduledExecutorService() + .scheduleAtFixedRate(catchingAndLoggingThrowables(() -> { + Runtime runtime = RuntimeSpawner.this.runtime; + if (runtime != null && !runtime.isAlive()) { + log.error("{}/{}/{} Function Container is dead with following exception. Restarting.", + details.getTenant(), + details.getNamespace(), details.getName(), runtime.getDeathException()); + // Just for the sake of sanity, just destroy the runtime + try { + runtime.stop(); + runtimeDeathException = runtime.getDeathException(); + runtime.start(); + } catch (Exception e) { + log.error("{}/{}/{}-{} Function Restart failed", details.getTenant(), + details.getNamespace(), details.getName(), e, e); + } + numRestarts++; + } + }), instanceLivenessCheckFreqMs, instanceLivenessCheckFreqMs, TimeUnit.MILLISECONDS); } } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java index 7d585bcd77b22..6135b6b680688 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java @@ -19,6 +19,7 @@ package org.apache.pulsar.functions.runtime.process; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -180,16 +181,17 @@ public void start() { .build(); stub = InstanceControlGrpc.newFutureStub(channel); - timer = InstanceCache.getInstanceCache().getScheduledExecutorService().scheduleAtFixedRate(() -> { - CompletableFuture result = healthCheck(); - try { - result.get(); - } catch (Exception e) { - log.error("Health check failed for {}-{}", - instanceConfig.getFunctionDetails().getName(), - instanceConfig.getInstanceId(), e); - } - }, expectedHealthCheckInterval, expectedHealthCheckInterval, TimeUnit.SECONDS); + timer = InstanceCache.getInstanceCache().getScheduledExecutorService() + .scheduleAtFixedRate(catchingAndLoggingThrowables(() -> { + CompletableFuture result = healthCheck(); + try { + result.get(); + } catch (Exception e) { + log.error("Health check failed for {}-{}", + instanceConfig.getFunctionDetails().getName(), + instanceConfig.getInstanceId(), e); + } + }), expectedHealthCheckInterval, expectedHealthCheckInterval, TimeUnit.SECONDS); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java index c1682a8607225..570408e975f34 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java @@ -19,6 +19,7 @@ package org.apache.pulsar.functions.worker; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -69,7 +70,7 @@ public void start() { for (Map.Entry entry : this.tasks.entrySet()) { TimerTaskInfo timerTaskInfo = entry.getValue(); String taskName = entry.getKey(); - this.executor.scheduleAtFixedRate(() -> { + this.executor.scheduleAtFixedRate(catchingAndLoggingThrowables(() -> { if (isLeader.get()) { try { timerTaskInfo.getTask().run(); @@ -77,7 +78,7 @@ public void start() { log.error("Cluster timer task {} failed with exception.", taskName, e); } } - }, timerTaskInfo.getInterval(), timerTaskInfo.getInterval(), TimeUnit.MILLISECONDS); + }), timerTaskInfo.getInterval(), timerTaskInfo.getInterval(), TimeUnit.MILLISECONDS); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java index b7fd0a4690740..130e839618fba 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -532,19 +533,19 @@ private void invokeRebalance() { private void scheduleCompaction(ScheduledExecutorService executor, long scheduleFrequencySec) { if (executor != null) { - executor.scheduleWithFixedDelay(() -> { + executor.scheduleWithFixedDelay(catchingAndLoggingThrowables(() -> { if (leaderService.isLeader() && isCompactionNeeded.get()) { compactAssignmentTopic(); isCompactionNeeded.set(false); } - }, scheduleFrequencySec, scheduleFrequencySec, TimeUnit.SECONDS); + }), scheduleFrequencySec, scheduleFrequencySec, TimeUnit.SECONDS); - executor.scheduleWithFixedDelay(() -> { + executor.scheduleWithFixedDelay(catchingAndLoggingThrowables(() -> { if (leaderService.isLeader() && metadataTopicLastMessage.compareTo(functionMetaDataManager.getLastMessageSeen()) != 0) { metadataTopicLastMessage = functionMetaDataManager.getLastMessageSeen(); compactFunctionMetadataTopic(); } - }, scheduleFrequencySec, scheduleFrequencySec, TimeUnit.SECONDS); + }), scheduleFrequencySec, scheduleFrequencySec, TimeUnit.SECONDS); } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index 9ba6be57b4e71..51b3df4805ead 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.metadata.impl; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.concurrent.CompletableFuture; @@ -67,8 +68,10 @@ public ZKSessionWatcher(ZooKeeper zk, Consumer sessionListener) { this.scheduler = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("metadata-store-zk-session-watcher")); - this.task = scheduler.scheduleAtFixedRate(this::checkConnectionStatus, tickTimeMillis, tickTimeMillis, - TimeUnit.MILLISECONDS); + this.task = + scheduler.scheduleAtFixedRate(catchingAndLoggingThrowables(this::checkConnectionStatus), tickTimeMillis, + tickTimeMillis, + TimeUnit.MILLISECONDS); this.currentStatus = SessionEvent.SessionReestablished; } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/JvmMetrics.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/JvmMetrics.java index 905c49bf56205..112fc205f08cb 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/JvmMetrics.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/JvmMetrics.java @@ -19,6 +19,7 @@ package org.apache.pulsar.websocket.stats; import static org.apache.pulsar.common.stats.Metrics.create; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import java.lang.management.ManagementFactory; import java.util.Map; @@ -52,7 +53,8 @@ public class JvmMetrics { private static final Logger log = LoggerFactory.getLogger(JvmMetrics.class); public JvmMetrics(WebSocketService service) { - service.getExecutor().scheduleAtFixedRate(this::updateGcStats, 0, 1, TimeUnit.MINUTES); + service.getExecutor() + .scheduleAtFixedRate(catchingAndLoggingThrowables(this::updateGcStats), 0, 1, TimeUnit.MINUTES); } public Metrics generate() { diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java index ef7523b84fffc..9bf5f4a68f6e5 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.websocket.stats; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import static org.apache.pulsar.websocket.ProducerHandler.ENTRY_LATENCY_BUCKETS_USEC; import java.util.List; @@ -55,7 +56,8 @@ public ProxyStats(WebSocketService service) { this.metricsCollection = Lists.newArrayList(); this.tempMetricsCollection = Lists.newArrayList(); // schedule stat generation task every 1 minute - service.getExecutor().scheduleAtFixedRate(() -> generate(), 120, 60, TimeUnit.SECONDS); + service.getExecutor() + .scheduleAtFixedRate(catchingAndLoggingThrowables(this::generate), 120, 60, TimeUnit.SECONDS); } /** From 1fea2637973f2baed542629ce9a0e1c388a255d2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 5 Nov 2021 22:12:47 +0200 Subject: [PATCH 046/823] [ML] Avoid passing OpAddEntry across a thread boundary in asyncAddEntry (#12606) * [ML] Avoid passing OpAddEntry across a thread boundary * Retain buffer in current thread (cherry picked from commit 6af747f515677796bba343997b2269ffd27cb601) --- .../mledger/impl/ManagedLedgerImpl.java | 20 ++++++++++++------- .../bookkeeper/mledger/impl/OpAddEntry.java | 18 ++++++++--------- .../mledger/impl/ManagedLedgerTest.java | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 0bc88c561170c..032e53e8f7b59 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -699,10 +699,14 @@ public void asyncAddEntry(ByteBuf buffer, AddEntryCallback callback, Object ctx) log.debug("[{}] asyncAddEntry size={} state={}", name, buffer.readableBytes(), state); } - OpAddEntry addOperation = OpAddEntry.create(this, buffer, callback, ctx); + // retain buffer in this thread + buffer.retain(); // Jump to specific thread to avoid contention from writers writing from different threads - executor.executeOrdered(name, safeRun(() -> internalAsyncAddEntry(addOperation))); + executor.executeOrdered(name, safeRun(() -> { + OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, callback, ctx); + internalAsyncAddEntry(addOperation); + })); } @Override @@ -711,10 +715,14 @@ public void asyncAddEntry(ByteBuf buffer, int numberOfMessages, AddEntryCallback log.debug("[{}] asyncAddEntry size={} state={}", name, buffer.readableBytes(), state); } - OpAddEntry addOperation = OpAddEntry.create(this, buffer, numberOfMessages, callback, ctx); + // retain buffer in this thread + buffer.retain(); // Jump to specific thread to avoid contention from writers writing from different threads - executor.executeOrdered(name, safeRun(() -> internalAsyncAddEntry(addOperation))); + executor.executeOrdered(name, safeRun(() -> { + OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx); + internalAsyncAddEntry(addOperation); + })); } private synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { @@ -1509,9 +1517,7 @@ public synchronized void updateLedgersIdsComplete(Stat stat) { // If op is used by another ledger handle, we need to close it and create a new one if (existsOp.ledger != null) { existsOp.close(); - existsOp = OpAddEntry.create(existsOp.ml, existsOp.data, existsOp.getNumberOfMessages(), existsOp.callback, existsOp.ctx); - // release the extra retain - ReferenceCountUtil.release(existsOp.data); + existsOp = OpAddEntry.createNoRetainBuffer(existsOp.ml, existsOp.data, existsOp.getNumberOfMessages(), existsOp.callback, existsOp.ctx); } existsOp.setLedger(currentLedger); pendingAddEntries.add(existsOp); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index 9106b4f431a3f..e08b204af38c2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -75,16 +75,16 @@ enum State { CLOSED } - public static OpAddEntry create(ManagedLedgerImpl ml, ByteBuf data, AddEntryCallback callback, Object ctx) { - OpAddEntry op = createOpAddEntry(ml, data, callback, ctx); + public static OpAddEntry createNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data, AddEntryCallback callback, Object ctx) { + OpAddEntry op = createOpAddEntryNoRetainBuffer(ml, data, callback, ctx); if (log.isDebugEnabled()) { log.debug("Created new OpAddEntry {}", op); } return op; } - public static OpAddEntry create(ManagedLedgerImpl ml, ByteBuf data, int numberOfMessages, AddEntryCallback callback, Object ctx) { - OpAddEntry op = createOpAddEntry(ml, data, callback, ctx); + public static OpAddEntry createNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data, int numberOfMessages, AddEntryCallback callback, Object ctx) { + OpAddEntry op = createOpAddEntryNoRetainBuffer(ml, data, callback, ctx); op.numberOfMessages = numberOfMessages; if (log.isDebugEnabled()) { log.debug("Created new OpAddEntry {}", op); @@ -92,11 +92,11 @@ public static OpAddEntry create(ManagedLedgerImpl ml, ByteBuf data, int numberOf return op; } - private static OpAddEntry createOpAddEntry(ManagedLedgerImpl ml, ByteBuf data, AddEntryCallback callback, Object ctx) { + private static OpAddEntry createOpAddEntryNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data, AddEntryCallback callback, Object ctx) { OpAddEntry op = RECYCLER.get(); op.ml = ml; op.ledger = null; - op.data = data.retain(); + op.data = data; op.dataLength = data.readableBytes(); op.callback = callback; op.ctx = ctx; @@ -155,7 +155,7 @@ public void addComplete(int rc, final LedgerHandle lh, long entryId, Object ctx) } checkArgument(ledger.getId() == lh.getId(), "ledgerId %s doesn't match with acked ledgerId %s", ledger.getId(), lh.getId()); - + if (!checkAndCompleteOp(ctx)) { // means callback might have been completed by different thread (timeout task thread).. so do nothing return; @@ -255,7 +255,7 @@ private void updateLatency() { /** * Checks if add-operation is completed - * + * * @return true if task is not already completed else returns false. */ private boolean checkAndCompleteOp(Object ctx) { @@ -276,7 +276,7 @@ void handleAddTimeoutFailure(final LedgerHandle ledger, Object ctx) { /** * It handles add failure on the given ledger. it can be triggered when add-entry fails or times out. - * + * * @param lh */ void handleAddFailure(final LedgerHandle lh) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 4358c2f73620b..8ce88951000a1 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -2829,7 +2829,7 @@ public void avoidUseSameOpAddEntryBetweenDifferentLedger() throws Exception { List oldOps = new ArrayList<>(); for (int i = 0; i < 10; i++) { - OpAddEntry op = OpAddEntry.create(ledger, ByteBufAllocator.DEFAULT.buffer(128), null, null); + OpAddEntry op = OpAddEntry.createNoRetainBuffer(ledger, ByteBufAllocator.DEFAULT.buffer(128).retain(), null, null); if (i > 4) { op.setLedger(mock(LedgerHandle.class)); } From 2d092374e189093cce7c792dd31bf9290a3e7d82 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 30 Nov 2021 15:10:39 +0200 Subject: [PATCH 047/823] [Perf] Evaluate the current protocol version once (#13045) - there's no need to dynamically evaluate it on every call (cherry picked from commit 724523f3051def9577d6bd27697866c99f4a7b0e) --- .../java/org/apache/pulsar/common/protocol/Commands.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 3c363e03bd85a..091c9b8b7c6a2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -126,6 +126,10 @@ protected BaseCommand initialValue() throws Exception { } }; + // Return the last ProtocolVersion enum value + private static final int CURRENT_PROTOCOL_VERSION = + ProtocolVersion.values()[ProtocolVersion.values().length - 1].getValue(); + private static BaseCommand localCmd(BaseCommand.Type type) { return LOCAL_BASE_COMMAND.get() .clear() @@ -1736,8 +1740,7 @@ public static byte[] peekStickyKey(ByteBuf metadataAndPayload, String topic, Str } public static int getCurrentProtocolVersion() { - // Return the last ProtocolVersion enum value - return ProtocolVersion.values()[ProtocolVersion.values().length - 1].getValue(); + return CURRENT_PROTOCOL_VERSION; } /** From 8a7acd088f41fb5a074f728c3a95f91b9616d89c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 22 Nov 2021 19:38:22 +0200 Subject: [PATCH 048/823] Fix log level config for pulsar-admin, pulsar-client and pulsar-perf (#12915) - Logger.isDebugEnabled will evaluate to true in the default configuration due to #7789 change (cherry picked from commit 6200ef1c299ed828edbd92582cba93befe9d7a43) --- bin/pulsar-admin-common.sh | 2 ++ bin/pulsar-client | 2 ++ bin/pulsar-perf | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh index 46ab6d8c75818..a8c5d9aadb8aa 100755 --- a/bin/pulsar-admin-common.sh +++ b/bin/pulsar-admin-common.sh @@ -98,6 +98,7 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" # log directory & file PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} @@ -105,4 +106,5 @@ PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" +OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL" OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" diff --git a/bin/pulsar-client b/bin/pulsar-client index dcee2e3b646a9..618f32def771c 100755 --- a/bin/pulsar-client +++ b/bin/pulsar-client @@ -101,12 +101,14 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" # log directory & file PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} #Configure log configuration system properties OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" +OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL" #Change to PULSAR_HOME to support relative paths cd "$PULSAR_HOME" diff --git a/bin/pulsar-perf b/bin/pulsar-perf index aa6e67a9942de..f2a68dfe57aa5 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -137,9 +137,13 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"} PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} +PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} #Configure log configuration system properties OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" +OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" +OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL" OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE" From 137913f17bba5eff09e0701804efedf04d8cad3f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 25 Nov 2021 06:53:18 +0200 Subject: [PATCH 049/823] Revert "Set default root log level to debug" and make PULSAR_LOG_ROOT_LEVEL to default to PULSAR_LOG_LEVEL (#12941) (cherry picked from commit 959430c435538b7338c44e25a1f3c81f3a5a534b) --- bin/pulsar | 2 +- bin/pulsar-admin-common.sh | 2 +- bin/pulsar-client | 2 +- bin/pulsar-perf | 2 +- conf/log4j2.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/pulsar b/bin/pulsar index 69fcd6e48e1de..3ec65f3233609 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -284,8 +284,8 @@ fi # log directory & file PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} -PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"} PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} #Configure log configuration system properties diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh index a8c5d9aadb8aa..a0945bfc2e939 100755 --- a/bin/pulsar-admin-common.sh +++ b/bin/pulsar-admin-common.sh @@ -98,8 +98,8 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" # log directory & file PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} -PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"} PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} #Configure log configuration system properties diff --git a/bin/pulsar-client b/bin/pulsar-client index 618f32def771c..73f82df37b06b 100755 --- a/bin/pulsar-client +++ b/bin/pulsar-client @@ -101,8 +101,8 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" # log directory & file PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"} -PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"} #Configure log configuration system properties OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" diff --git a/bin/pulsar-perf b/bin/pulsar-perf index f2a68dfe57aa5..a2f90b0d7e7cc 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -137,8 +137,8 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"} PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"} -PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"} PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} +PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"} #Configure log configuration system properties OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index a76d58e804f96..71698d7206787 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -33,7 +33,7 @@ Configuration: - name: "pulsar.log.appender" value: "RoutingAppender" - name: "pulsar.log.root.level" - value: "debug" + value: "info" - name: "pulsar.log.level" value: "info" - name: "pulsar.routing.appender.default" From 96065010477063ef185417b65185c7ec45bc1695 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Tue, 2 Nov 2021 02:13:47 +0900 Subject: [PATCH 050/823] Cancel scheduled tasks when deleting ManagedLedgerImpl (#12565) (cherry picked from commit a95a3824e0b1c207ea844fbce724734764f0be62) --- .../mledger/impl/ManagedLedgerImpl.java | 20 ++++++----- .../mledger/impl/ManagedLedgerTest.java | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 032e53e8f7b59..fa9a5cfcf2725 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1332,14 +1332,7 @@ public synchronized void asyncClose(final CloseCallback callback, final Object c factory.close(this); STATE_UPDATER.set(this, State.Closed); - - if (this.timeoutTask != null) { - this.timeoutTask.cancel(false); - } - - if (this.checkLedgerRollTask != null) { - this.checkLedgerRollTask.cancel(false); - } + cancelScheduledTasks(); LedgerHandle lh = currentLedger; @@ -2612,6 +2605,7 @@ public void asyncDelete(final DeleteLedgerCallback callback, final Object ctx) { // Delete the managed ledger without closing, since we are not interested in gracefully closing cursors and // ledgers STATE_UPDATER.set(this, State.Fenced); + cancelScheduledTasks(); List cursors = Lists.newArrayList(this.cursors); if (cursors.isEmpty()) { @@ -4009,4 +4003,14 @@ private void updateLastLedgerCreatedTimeAndScheduleRolloverTask() { } } + private void cancelScheduledTasks() { + if (this.timeoutTask != null) { + this.timeoutTask.cancel(false); + } + + if (this.checkLedgerRollTask != null) { + this.checkLedgerRollTask.cancel(false); + } + } + } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 8ce88951000a1..d837651f1fac5 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -61,6 +61,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -3360,4 +3361,36 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { managedLedgerB.close(); } + + @Test + public void testCancellationOfScheduledTasks() throws Exception { + Field timeoutTaskField = ManagedLedgerImpl.class.getDeclaredField("timeoutTask"); + timeoutTaskField.setAccessible(true); + Field checkLedgerRollTaskField = ManagedLedgerImpl.class.getDeclaredField("checkLedgerRollTask"); + checkLedgerRollTaskField.setAccessible(true); + + ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory.open("my_test_ledger_1"); + ledger1.addEntry("dummy-entry-1".getBytes(Encoding)); + ScheduledFuture timeoutTask1 = (ScheduledFuture) timeoutTaskField.get(ledger1); + assertNotNull(timeoutTask1); + assertFalse(timeoutTask1.isDone()); + ScheduledFuture checkLedgerRollTask1 = (ScheduledFuture) checkLedgerRollTaskField.get(ledger1); + assertNotNull(checkLedgerRollTask1); + assertFalse(checkLedgerRollTask1.isDone()); + ledger1.close(); + assertTrue(timeoutTask1.isCancelled()); + assertTrue(checkLedgerRollTask1.isCancelled()); + + ManagedLedgerImpl ledger2 = (ManagedLedgerImpl) factory.open("my_test_ledger_2"); + ledger2.addEntry("dummy-entry-2".getBytes(Encoding)); + ScheduledFuture timeoutTask2 = (ScheduledFuture) timeoutTaskField.get(ledger2); + assertNotNull(timeoutTask2); + assertFalse(timeoutTask2.isDone()); + ScheduledFuture checkLedgerRollTask2 = (ScheduledFuture) checkLedgerRollTaskField.get(ledger2); + assertNotNull(checkLedgerRollTask2); + assertFalse(checkLedgerRollTask2.isDone()); + ledger2.delete(); + assertTrue(timeoutTask2.isCancelled()); + assertTrue(checkLedgerRollTask2.isCancelled()); + } } From 704277f96c33429d9778e7d5cd63156b282a52f0 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 15 Nov 2021 16:17:03 +0800 Subject: [PATCH 051/823] Add remote cluster info to replicator producer name. (#12734) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the producer is the replicator client, the `remoteCluster` is not right ： ``` Producer{topic=PersistentTopic{topic=persistent://public/default/tp1-partition-0}, client=/127.0.0.1:54628, producerName=pulsar.repl.cluster3, producerId=0, remoteCluster=cluster3, isRemote=true} ``` - Add remote cluster-info to replicator producer builder. (cherry picked from commit 10818e8e39b9f039793b8534da0252f2f4c91a61) --- .../broker/service/AbstractReplicator.java | 3 ++- .../pulsar/broker/service/Producer.java | 11 +++++++++- .../pulsar/broker/service/ReplicatorTest.java | 21 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 59ec74fee29a9..a404f10d3dacb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -83,7 +83,8 @@ public AbstractReplicator(String topicName, String replicatorPrefix, String loca .enableBatching(false) .sendTimeout(0, TimeUnit.SECONDS) // .maxPendingMessages(producerQueueSize) // - .producerName(getReplicatorName(replicatorPrefix, localCluster)); + .producerName(String.format("%s%s%s", getReplicatorName(replicatorPrefix, localCluster), + REPL_PRODUCER_NAME_DELIMITER, remoteCluster)); STATE_UPDATER.set(this, State.Stopped); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index 64e0fd64fd4e1..bef4c5415b17a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -128,7 +128,7 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN this.isRemote = producerName .startsWith(cnx.getBrokerService().pulsar().getConfiguration().getReplicatorPrefix()); - this.remoteCluster = isRemote ? producerName.split("\\.")[2].split(REPL_PRODUCER_NAME_DELIMITER)[0] : null; + this.remoteCluster = parseRemoteClusterName(producerName, isRemote); this.isEncrypted = isEncrypted; this.schemaVersion = schemaVersion; @@ -138,6 +138,15 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN this.clientAddress = cnx.clientSourceAddress(); } + private String parseRemoteClusterName(String producerName, boolean isRemote) { + if (isRemote) { + String clusterName = producerName.split("\\.")[2]; + return clusterName.contains(REPL_PRODUCER_NAME_DELIMITER) + ? clusterName.split(REPL_PRODUCER_NAME_DELIMITER)[0] : clusterName; + } + return null; + } + /** * Method to determine if this producer can replace another producer. * @param other - producer to compare to this one diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 11c8e197f8df9..ac8f233fc8bdf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.UUID; @@ -715,6 +716,26 @@ public void testReplicatorProducerClosing() throws Exception { assertNull(producer); } + @Test(priority = 5, timeOut = 30000) + public void testReplicatorProducerName() throws Exception { + log.info("--- Starting ReplicatorTest::testReplicatorProducerName ---"); + final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/testReplicatorProducerName"); + final TopicName dest = TopicName.get(topicName); + + @Cleanup + MessageProducer producer1 = new MessageProducer(url1, dest); + + Awaitility.await().untilAsserted(() -> { + assertTrue(pulsar2.getBrokerService().getTopicReference(topicName).isPresent()); + }); + Optional topic = pulsar2.getBrokerService().getTopicReference(topicName); + assertTrue(topic.isPresent()); + Set remoteClusters = topic.get().getProducers().values().stream() + .map(org.apache.pulsar.broker.service.Producer::getRemoteCluster) + .collect(Collectors.toSet()); + assertTrue(remoteClusters.contains("r1")); + } + /** * Issue #199 * From de2ad45df1ab7734ec4f20bc9f3290c0aee6429a Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 17 Nov 2021 16:45:31 +0800 Subject: [PATCH 052/823] [Issue 12796][broker] Fix replicator error with user defined ReplicatorPrefix (#12797) (cherry picked from commit bab2a81c17bf2205c0d9498c0d7a73c978777e4b) --- .../pulsar/broker/service/Producer.java | 16 ++++++++---- .../pulsar/broker/service/ReplicatorTest.java | 26 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index bef4c5415b17a..d72f904019a4e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -126,9 +126,10 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN stats.metadata = this.metadata; stats.accessMode = Commands.convertProducerAccessMode(accessMode); - this.isRemote = producerName - .startsWith(cnx.getBrokerService().pulsar().getConfiguration().getReplicatorPrefix()); - this.remoteCluster = parseRemoteClusterName(producerName, isRemote); + + String replicatorPrefix = cnx.getBrokerService().pulsar().getConfiguration().getReplicatorPrefix() + "."; + this.isRemote = producerName.startsWith(replicatorPrefix); + this.remoteCluster = parseRemoteClusterName(producerName, isRemote, replicatorPrefix); this.isEncrypted = isEncrypted; this.schemaVersion = schemaVersion; @@ -138,9 +139,14 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN this.clientAddress = cnx.clientSourceAddress(); } - private String parseRemoteClusterName(String producerName, boolean isRemote) { + /** + * Producer name for replicator is in format. + * "replicatorPrefix.localCluster" (old) + * "replicatorPrefix.localCluster-->remoteCluster" (new) + */ + private String parseRemoteClusterName(String producerName, boolean isRemote, String replicatorPrefix) { if (isRemote) { - String clusterName = producerName.split("\\.")[2]; + String clusterName = producerName.substring(replicatorPrefix.length()); return clusterName.contains(REPL_PRODUCER_NAME_DELIMITER) ? clusterName.split(REPL_PRODUCER_NAME_DELIMITER)[0] : clusterName; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index ac8f233fc8bdf..04f96ec8d898f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -736,6 +736,32 @@ public void testReplicatorProducerName() throws Exception { assertTrue(remoteClusters.contains("r1")); } + @Test(priority = 5, timeOut = 30000) + public void testReplicatorProducerNameWithUserDefinedReplicatorPrefix() throws Exception { + log.info("--- Starting ReplicatorTest::testReplicatorProducerNameWithUserDefinedReplicatorPrefix ---"); + final String topicName = BrokerTestUtil.newUniqueName( + "persistent://pulsar/ns/testReplicatorProducerNameWithUserDefinedReplicatorPrefix"); + final TopicName dest = TopicName.get(topicName); + + pulsar1.getConfiguration().setReplicatorPrefix("user-defined-prefix"); + pulsar2.getConfiguration().setReplicatorPrefix("user-defined-prefix"); + pulsar3.getConfiguration().setReplicatorPrefix("user-defined-prefix"); + + @Cleanup + MessageProducer producer1 = new MessageProducer(url1, dest); + + Awaitility.await().untilAsserted(()->{ + assertTrue(pulsar2.getBrokerService().getTopicReference(topicName).isPresent()); + }); + Optional topic = pulsar2.getBrokerService().getTopicReference(topicName); + assertTrue(topic.isPresent()); + Set remoteClusters = topic.get().getProducers().values().stream() + .map(org.apache.pulsar.broker.service.Producer::getRemoteCluster) + .collect(Collectors.toSet()); + assertTrue(remoteClusters.contains("r1")); + } + + /** * Issue #199 * From a749d564ff361a4cb7cf7d49fb014e5bfcca4dd0 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 9 Dec 2021 18:47:50 +0100 Subject: [PATCH 053/823] Fix namespace policy override ignored when creating subscription (#12699) PR #6471 introduced the possibility to set auto topic creation settings at the namespace level PR #6685 is a fix to take into account the auto topic creation setting when creating a subscription but it uses the broker configuration and doesn't see the namespace level setting that was introduced in #6471 This fix uses the method that fetches config overrides in ZK instead of the one from the broker configuration. This way the method of the pulsar-admin uses the value set for the namespace for auto-topic-creation (cherry picked from commit 965a80f1337697c8714e6b41bf562c196a126eae) --- .../broker/admin/impl/PersistentTopicsBase.java | 4 ++-- .../BrokerServiceAutoTopicCreationTest.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index f223869fe7f93..d97042f15edb6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2024,7 +2024,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su internalCreateSubscriptionForNonPartitionedTopic(asyncResponse, subscriptionName, targetMessageId, authoritative, replicated); } else { - boolean allowAutoTopicCreation = pulsar().getConfiguration().isAllowAutoTopicCreation(); + boolean allowAutoTopicCreation = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName); getPartitionedTopicMetadataAsync(topicName, authoritative, allowAutoTopicCreation).thenAccept(partitionMetadata -> { final int numPartitions = partitionMetadata.partitions; @@ -2109,7 +2109,7 @@ private void internalCreateSubscriptionForNonPartitionedTopic( AsyncResponse asyncResponse, String subscriptionName, MessageIdImpl targetMessageId, boolean authoritative, boolean replicated) { - boolean isAllowAutoTopicCreation = pulsar().getConfiguration().isAllowAutoTopicCreation(); + boolean isAllowAutoTopicCreation = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName); validateTopicOwnershipAsync(topicName, authoritative) .thenCompose(__ -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java index 7b749339e1181..33ed35636db8c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java @@ -350,6 +350,21 @@ public void testNotAllowSubscriptionTopicCreation() throws Exception{ } + @Test + public void testAutoCreationNamespaceOverridesSubscriptionTopicCreation() throws Exception { + pulsar.getConfiguration().setAllowAutoTopicCreation(false); + String topicString = "persistent://prop/ns-abc/non-partitioned-topic" + System.currentTimeMillis(); + String subscriptionName = "non-partitioned-topic-sub"; + final TopicName topicName = TopicName.get(topicString); + pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), + AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType(TopicType.NON_PARTITIONED.toString()) + .build()); + + admin.topics().createSubscription(topicString, subscriptionName, MessageId.earliest); + } + @Test public void testMaxNumPartitionsPerPartitionedTopicTopicCreation() { pulsar.getConfiguration().setAllowAutoTopicCreation(true); From 168bbab4ca5d13afda5eaa173dbcf9f923f22427 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 9 Dec 2021 13:17:13 -0600 Subject: [PATCH 054/823] [Java Client] Send CloseProducer on timeout (#13161) * [Java Client] Send CloseProducer on timeout * Fix failed test from ClientErrorsTest (cherry picked from commit 27d542994703ef40a09363f5de4e92cc3c9d3116) --- .../pulsar/client/api/ClientErrorsTest.java | 58 ++++++++++++++++++- .../pulsar/client/impl/ProducerImpl.java | 12 +++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java index f66b19808f564..d7507d2b47cb9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java @@ -39,6 +39,7 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -170,6 +171,7 @@ private void producerCreateFailAfterRetryTimeout(String topic) throws Exception PulsarClient client = PulsarClient.builder().serviceUrl(mockBrokerService.getBrokerAddress()) .operationTimeout(1, TimeUnit.SECONDS).build(); final AtomicInteger counter = new AtomicInteger(0); + final AtomicInteger closeProducerCounter = new AtomicInteger(0); mockBrokerService.setHandleProducer((ctx, producer) -> { if (counter.incrementAndGet() == 2) { @@ -182,6 +184,10 @@ private void producerCreateFailAfterRetryTimeout(String topic) throws Exception ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.ServiceNotReady, "msg")); }); + mockBrokerService.setHandleCloseProducer((ctx, closeProducer) -> { + closeProducerCounter.incrementAndGet(); + }); + try { client.newProducer().topic(topic).create(); fail("Should have failed"); @@ -189,8 +195,58 @@ private void producerCreateFailAfterRetryTimeout(String topic) throws Exception // we fail even on the retriable error assertTrue(e instanceof PulsarClientException); } + // There is a small race condition here because the producer's timeout both fails the client creation + // and triggers sending CloseProducer. + Awaitility.await().until(() -> closeProducerCounter.get() == 1); + mockBrokerService.resetHandleProducer(); + mockBrokerService.resetHandleCloseProducer(); + } + + @Test + public void testCreatedProducerSendsCloseProducerAfterTimeout() throws Exception { + producerCreatedThenFailsRetryTimeout("persistent://prop/use/ns/t1"); + } + + @Test + public void testCreatedPartitionedProducerSendsCloseProducerAfterTimeout() throws Exception { + producerCreatedThenFailsRetryTimeout("persistent://prop/use/ns/part-t1"); + } + + private void producerCreatedThenFailsRetryTimeout(String topic) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(mockBrokerService.getBrokerAddress()) + .operationTimeout(1, TimeUnit.SECONDS).build(); + final AtomicInteger producerCounter = new AtomicInteger(0); + final AtomicInteger closeProducerCounter = new AtomicInteger(0); + mockBrokerService.setHandleProducer((ctx, producer) -> { + int producerCount = producerCounter.incrementAndGet(); + if (producerCount == 1) { + ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "producer1", + SchemaVersion.Empty)); + // Trigger reconnect + ctx.writeAndFlush(Commands.newCloseProducer(producer.getProducerId(), -1)); + } else if (producerCount != 2) { + // Respond to subsequent requests to prevent timeouts + ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "producer1", + SchemaVersion.Empty)); + } + // Don't respond to the second Producer command to ensure timeout + }); + + mockBrokerService.setHandleCloseProducer((ctx, closeProducer) -> { + closeProducerCounter.incrementAndGet(); + ctx.writeAndFlush(Commands.newSuccess(closeProducer.getRequestId())); + }); + + // Create producer should succeed then upon closure, it should reattempt creation. The first request will + // timeout, which triggers CloseProducer. The client might send send the third Producer command before the + // below assertion, so we pass with 2 or 3. + client.newProducer().topic(topic).create(); + Awaitility.await().until(() -> closeProducerCounter.get() == 1); + Awaitility.await().until(() -> producerCounter.get() == 2 || producerCounter.get() == 3); mockBrokerService.resetHandleProducer(); + mockBrokerService.resetHandleCloseProducer(); } @Test @@ -491,7 +547,6 @@ public void testPartitionedProducerFailOnInitialization() throws Throwable { mockBrokerService.resetHandleProducer(); mockBrokerService.resetHandleCloseProducer(); - client.close(); } // failed to connect to partition at sending step if a producer which connects to broker as lazy-loading mode @@ -552,7 +607,6 @@ public void testPartitionedProducerFailOnSending() throws Throwable { mockBrokerService.resetHandleProducer(); mockBrokerService.resetHandleCloseProducer(); - client.close(); } // if a producer which doesn't connect as lazy-loading mode fails to connect while creating partitioned producer, diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index e33fb5f699463..686fa7de4b8f7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1483,8 +1483,18 @@ public void connectionOpened(final ClientCnx cnx) { cnx.channel().close(); return null; } + + if (cause instanceof TimeoutException) { + // Creating the producer has timed out. We need to ensure the broker closes the producer + // in case it was indeed created, otherwise it might prevent new create producer operation, + // since we are not necessarily closing the connection. + long closeRequestId = client.newRequestId(); + ByteBuf cmd = Commands.newCloseProducer(producerId, closeRequestId); + cnx.sendRequestWithId(cmd, closeRequestId); + } + log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, cause.getMessage()); - // Close the producer since topic does not exists. + // Close the producer since topic does not exist. if (cause instanceof PulsarClientException.TopicDoesNotExistException) { closeAsync().whenComplete((v, ex) -> { if (ex != null) { From 6cdebb7f853e89c83c78eeb835a4beb6c3d3fafc Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Fri, 10 Dec 2021 14:01:03 +0800 Subject: [PATCH 055/823] Bump log4j to 2.15.0 (#13226) (cherry picked from commit 0015daba660e6ee720f58287b4d1eff6e7960ab0) --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 4398d50d861ee..06443fac2adc2 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -39,7 +39,7 @@ 1.8 1.8 3.0.0-M3 - 2.14.0 + 2.15.0 1.7.25 7.3.0 3.11 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 6da3b3a47d91e..cb6441dee7a54 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -383,11 +383,11 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.14.0.jar - - org.apache.logging.log4j-log4j-core-2.14.0.jar - - org.apache.logging.log4j-log4j-slf4j-impl-2.14.0.jar - - org.apache.logging.log4j-log4j-web-2.14.0.jar - - org.apache.logging.log4j-log4j-1.2-api-2.14.0.jar + - org.apache.logging.log4j-log4j-api-2.15.0.jar + - org.apache.logging.log4j-log4j-core-2.15.0.jar + - org.apache.logging.log4j-log4j-slf4j-impl-2.15.0.jar + - org.apache.logging.log4j-log4j-web-2.15.0.jar + - org.apache.logging.log4j-log4j-1.2-api-2.15.0.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - org.apache.bookkeeper-bookkeeper-common-4.14.2.jar diff --git a/pom.xml b/pom.xml index 09c5c6e60b102..64338333bbe75 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ flexible messaging model and an intuitive client API. 6.10.2 1.7.25 3.2.2 - 2.14.0 + 2.15.0 1.69 1.0.2 2.12.3 From fd6286036920d8180da42fdfeaf10089f987995b Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 13 Dec 2021 16:59:50 +0800 Subject: [PATCH 056/823] Fix consume message order issue when use listener. (#13023) (cherry picked from commit e134e372b3cc007bb507f04076011407cc28b7c0) --- .../pulsar/client/impl/ConsumerBase.java | 42 ++++++++++--------- .../pulsar/client/impl/ConsumerImpl.java | 3 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 45505e48a205e..7d6474b10b50f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -916,30 +916,32 @@ private void doPendingBatchReceiveTask(Timeout timeout) { protected void triggerListener() { // Trigger the notification on the message listener in a separate thread to avoid blocking the networking // thread while the message processing happens - try { - // Control executor to call MessageListener one by one. - if (executorQueueSize.get() < 1) { - final Message msg = internalReceive(0, TimeUnit.MILLISECONDS); - if (msg != null) { - executorQueueSize.incrementAndGet(); - if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) { - executorProvider.getExecutor(peekMessageKey(msg)).execute(() -> - callMessageListener(msg)); - } else { - getExternalExecutor(msg).execute(() -> { - callMessageListener(msg); - }); + internalPinnedExecutor.execute(() -> { + try { + // Control executor to call MessageListener one by one. + if (executorQueueSize.get() < 1) { + final Message msg = internalReceive(0, TimeUnit.MILLISECONDS); + if (msg != null) { + executorQueueSize.incrementAndGet(); + if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) { + executorProvider.getExecutor(peekMessageKey(msg)).execute(() -> + callMessageListener(msg)); + } else { + getExternalExecutor(msg).execute(() -> { + callMessageListener(msg); + }); + } } } + } catch (PulsarClientException e) { + log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e); + return; } - } catch (PulsarClientException e) { - log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e); - return; - } - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription); - } + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription); + } + }); } protected void callMessageListener(Message msg) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 11b1def52e0df..72167b226d70f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1229,8 +1229,7 @@ void messageReceived(MessageIdData messageId, int redeliveryCount, List ac uncompressedPayload.release(); } - internalPinnedExecutor.execute(() - -> tryTriggerListener()); + tryTriggerListener(); } From 66b58dcb8853113a2f7503970299020c302d1fc0 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 2 Nov 2021 12:30:30 +0800 Subject: [PATCH 057/823] Fix the reader skips compacted data which original ledger been removed (#12522) * Fix the reader skips compacted data which original ledger been removed The compactor update the compaction cursor(mark delete) first and then update the `compactionHorizon` of the compacted topic. During the compaction cursor move forward, the original ledger will be removed if no other durable cursors. At the same time, if the reader is reading data from the original ledger, the reader will skip the data while the original ledger been removed, details to see https://github.com/apache/pulsar/pull/6787. So the reader might skip the compacted data since the `compactionHorizon` have not updated yet. The approach is: 1. Update the `compactionHorizon` before the compaction cursor move forward, so that the reader will not skip the original data before `compactionHorizon` updated. If the broker crashes before the new compacted Ledger ID been persistent, after the topic been loaded, the compaction can be trigger again and will not loss any data, but we will have an orphan ledger cannot be delete in the BookKeeper cluster. 2. Remove the previous compacted Ledger after the compaction cursor move forward, make sure the new compacted Ledger ID been persistent, Otherwise, we might lost compacted ledger if broker crashes. * Fix checkstyle * Fix tests. * Fix test (cherry picked from commit 74dd9b973f019d6975497f4cbe1bd9925e6137d1) --- .../persistent/CompactorSubscription.java | 28 +++++++++++-------- .../pulsar/compaction/CompactedTopic.java | 3 +- .../pulsar/compaction/CompactedTopicImpl.java | 21 +++++++++----- .../broker/service/PersistentTopicTest.java | 7 +++-- .../pulsar/compaction/CompactedTopicTest.java | 14 +++++++++- 5 files changed, 50 insertions(+), 23 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java index f76dd752834fa..76e54b4ad6a52 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java @@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; @@ -34,7 +33,7 @@ import org.slf4j.LoggerFactory; public class CompactorSubscription extends PersistentSubscription { - private CompactedTopic compactedTopic; + private final CompactedTopic compactedTopic; public CompactorSubscription(PersistentTopic topic, CompactedTopic compactedTopic, String subscriptionName, ManagedCursor cursor) { @@ -65,15 +64,25 @@ public void acknowledgeMessage(List positions, AckType ackType, Map future = new CompletableFuture<>(); - cursor.asyncMarkDelete(position, properties, new MarkDeleteCallback() { + + // The newCompactedLedger must be called at the first step because we need to ensure the reader can read + // complete data from compacted Ledger, otherwise, if the original ledger been deleted the reader cursor + // might move to a subsequent original ledger if `compactionHorizon` have not updated, this will lead to + // the reader skips compacted data at that time, after the `compactionHorizon` updated, the reader able + // to read the complete compacted data again. + // And we can only delete the previous ledger after the mark delete succeed, otherwise we will loss the + // compacted data if mark delete failed. + compactedTopic.newCompactedLedger(position, compactedLedgerId).thenAccept(previousContext -> { + cursor.asyncMarkDelete(position, properties, new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { if (log.isDebugEnabled()) { log.debug("[{}][{}] Mark deleted messages until position on compactor subscription {}", - topicName, subName, position); + topicName, subName, position); + } + if (previousContext != null) { + compactedTopic.deleteCompactedLedger(previousContext.getLedger().getId()); } - future.complete(null); } @Override @@ -81,19 +90,16 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { // TODO: cut consumer connection on markDeleteFailed if (log.isDebugEnabled()) { log.debug("[{}][{}] Failed to mark delete for position on compactor subscription {}", - topicName, subName, ctx, exception); + topicName, subName, ctx, exception); } } }, null); + }); if (topic.getManagedLedger().isTerminated() && cursor.getNumberOfEntriesInBacklog(false) == 0) { // Notify all consumer that the end of topic was reached dispatcher.getConsumers().forEach(Consumer::reachedEndOfTopic); } - - // Once properties have been persisted, we can notify the compacted topic to use - // the new ledger - future.thenAccept((v) -> compactedTopic.newCompactedLedger(position, compactedLedgerId)); } private static final Logger log = LoggerFactory.getLogger(CompactorSubscription.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java index 7c969373dbeb4..31955a5c5c4c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java @@ -26,7 +26,8 @@ import org.apache.pulsar.broker.service.Consumer; public interface CompactedTopic { - CompletableFuture newCompactedLedger(Position p, long compactedLedgerId); + CompletableFuture newCompactedLedger(Position p, long compactedLedgerId); + CompletableFuture deleteCompactedLedger(long compactedLedgerId); void asyncReadEntriesOrWait(ManagedCursor cursor, int numberOfEntriesToRead, boolean isFirstRead, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index 131341318e0f9..bca5682d3efbd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -20,6 +20,7 @@ import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ComparisonChain; import io.netty.buffer.ByteBuf; import java.util.ArrayList; @@ -64,7 +65,7 @@ public CompactedTopicImpl(BookKeeper bk) { } @Override - public CompletableFuture newCompactedLedger(Position p, long compactedLedgerId) { + public CompletableFuture newCompactedLedger(Position p, long compactedLedgerId) { synchronized (this) { compactionHorizon = (PositionImpl) p; @@ -72,15 +73,16 @@ public CompletableFuture newCompactedLedger(Position p, long compactedLedgerI compactedTopicContext = openCompactedLedger(bk, compactedLedgerId); // delete the ledger from the old context once the new one is open - if (previousContext != null) { - return compactedTopicContext.thenCompose((res) -> previousContext) - .thenCompose((res) -> tryDeleteCompactedLedger(bk, res.ledger.getId())); - } else { - return compactedTopicContext; - } + return compactedTopicContext.thenCompose(__ -> + previousContext != null ? previousContext : CompletableFuture.completedFuture(null)); } } + @Override + public CompletableFuture deleteCompactedLedger(long compactedLedgerId) { + return tryDeleteCompactedLedger(bk, compactedLedgerId); + } + @Override public void asyncReadEntriesOrWait(ManagedCursor cursor, int numberOfEntriesToRead, @@ -298,6 +300,11 @@ private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) .compare(p.getLedgerId(), m.getLedgerId()) .compare(p.getEntryId(), m.getEntryId()).result(); } + + @VisibleForTesting + PositionImpl getCompactionHorizon() { + return this.compactionHorizon; + } private static final Logger log = LoggerFactory.getLogger(CompactedTopicImpl.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index cb10d72fe1764..6e68a39539fea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; @@ -124,12 +125,10 @@ import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.compaction.CompactedTopic; +import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.impl.ZKMetadataStore; -import org.apache.pulsar.zookeeper.ZooKeeperCache; -import org.apache.pulsar.zookeeper.ZooKeeperDataCache; -import org.apache.pulsar.broker.admin.AdminResource; import org.apache.zookeeper.ZooKeeper; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -1850,6 +1849,8 @@ public void testClosingReplicationProducerTwice() throws Exception { public void testCompactorSubscription() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); CompactedTopic compactedTopic = mock(CompactedTopic.class); + when(compactedTopic.newCompactedLedger(any(Position.class), anyLong())) + .thenReturn(CompletableFuture.completedFuture(mock(CompactedTopicContext.class))); PersistentSubscription sub = new CompactorSubscription(topic, compactedTopic, Compactor.COMPACTION_SUBSCRIPTION, cursorMock); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 3d410884e3042..0083fc779b701 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -241,7 +241,19 @@ public void testCleanupOldCompactedTopicLedger() throws Exception { Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close(); // update the compacted topic ledger - compactedTopic.newCompactedLedger(new PositionImpl(1,2), newCompactedLedger.getId()).get(); + PositionImpl newHorizon = new PositionImpl(1,3); + compactedTopic.newCompactedLedger(newHorizon, newCompactedLedger.getId()).get(); + + // Make sure the old compacted ledger still exist after the new compacted ledger created. + bk.openLedger(oldCompactedLedger.getId(), + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close(); + + Assert.assertTrue(compactedTopic.getCompactedTopicContext().isPresent()); + Assert.assertEquals(compactedTopic.getCompactedTopicContext().get().getLedger().getId(), + newCompactedLedger.getId()); + Assert.assertEquals(compactedTopic.getCompactionHorizon(), newHorizon); + compactedTopic.deleteCompactedLedger(oldCompactedLedger.getId()).join(); // old ledger should be deleted, new still there try { From 2055ecade9a1e250c9cfdfc625a5fa973d59ed59 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 13 Dec 2021 17:03:20 +0800 Subject: [PATCH 058/823] Close old compacted ledger when open new. (#13210) (cherry picked from commit 07ef9231db8b844586b9217ee2d59237eb9c54b7) --- .../broker/service/persistent/CompactorSubscription.java | 8 ++++++-- .../apache/pulsar/broker/service/PersistentTopicTest.java | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java index 76e54b4ad6a52..f7279968c51bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java @@ -47,8 +47,12 @@ public CompactorSubscription(PersistentTopic topic, CompactedTopic compactedTopi Map properties = cursor.getProperties(); if (properties.containsKey(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY)) { long compactedLedgerId = properties.get(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY); - compactedTopic.newCompactedLedger(cursor.getMarkDeletedPosition(), - compactedLedgerId); + compactedTopic.newCompactedLedger(cursor.getMarkDeletedPosition(), compactedLedgerId) + .thenAccept(previousContext -> { + if (previousContext != null) { + compactedTopic.deleteCompactedLedger(previousContext.getLedger().getId()); + } + }); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 6e68a39539fea..f1a11ad4f9c11 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -1873,6 +1873,8 @@ public void testCompactorSubscriptionUpdatedOnInit() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); CompactedTopic compactedTopic = mock(CompactedTopic.class); + when(compactedTopic.newCompactedLedger(any(Position.class), anyLong())) + .thenReturn(CompletableFuture.completedFuture(null)); new CompactorSubscription(topic, compactedTopic, Compactor.COMPACTION_SUBSCRIPTION, cursorMock); verify(compactedTopic, Mockito.times(1)).newCompactedLedger(position, ledgerId); } From 547526a601fc496a4939d375ea6a608d1046f77c Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Tue, 14 Dec 2021 12:25:48 +0800 Subject: [PATCH 059/823] [Broker] Remove tenant admin verification when listing partitioned topics (#13138) * [Broker] Remove tenant permission verification when list partitioned-topic * Improve test * Update annotation (cherry picked from commit 498810886bc6fd5e1c1aa0e9ae1c3564df0fbdbc) --- .../admin/impl/PersistentTopicsBase.java | 1 - .../broker/admin/v1/PersistentTopics.java | 4 ++-- .../broker/admin/v2/PersistentTopics.java | 4 ++-- .../AuthorizationProducerConsumerTest.java | 20 ++++++++++++++++++- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index d97042f15edb6..62c01f691dec6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -171,7 +171,6 @@ protected List internalGetList() { } protected List internalGetPartitionedTopicList() { - validateAdminAccessForTenant(namespaceName.getTenant()); validateNamespaceOperation(namespaceName, NamespaceOperation.GET_TOPICS); // Validate that namespace exists, throws 404 if it doesn't exist try { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 2917482857363..9ef87f4d7d7b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -69,7 +69,7 @@ public class PersistentTopics extends PersistentTopicsBase { @Path("/{property}/{cluster}/{namespace}") @ApiOperation(hidden = true, value = "Get the list of topics under a namespace.", response = String.class, responseContainer = "List") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin or consume permission on namespace"), @ApiResponse(code = 404, message = "Namespace doesn't exist")}) public void getList(@Suspended final AsyncResponse asyncResponse, @PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace) { @@ -87,7 +87,7 @@ public void getList(@Suspended final AsyncResponse asyncResponse, @PathParam("pr @Path("/{property}/{cluster}/{namespace}/partitioned") @ApiOperation(hidden = true, value = "Get the list of partitioned topics under a namespace.", response = String.class, responseContainer = "List") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin or consume permission on namespace"), @ApiResponse(code = 404, message = "Namespace doesn't exist")}) public List getPartitionedTopicList(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 1aeaccf8114fc..977d54a99d21b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -85,7 +85,7 @@ public class PersistentTopics extends PersistentTopicsBase { response = String.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), - @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 403, message = "Don't have admin or consume permission on namespace"), @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"), @ApiResponse(code = 412, message = "Namespace name is not valid"), @ApiResponse(code = 500, message = "Internal server error")}) @@ -111,7 +111,7 @@ public void getList( response = String.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), - @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 403, message = "Don't have admin or consume permission on namespace"), @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"), @ApiResponse(code = 412, message = "Namespace name is not valid"), @ApiResponse(code = 500, message = "Internal server error")}) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index d852a9bafd92d..65f57ad7e2c6a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -24,6 +24,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.io.IOException; @@ -330,6 +331,7 @@ public void testClearBacklogPermission() throws Exception { conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); + final String tenantRole = "tenant-role"; final String subscriptionRole = "sub-role"; final String subscriptionName = "sub1"; final String namespace = "my-property/my-ns-sub-auth"; @@ -342,6 +344,11 @@ public void testClearBacklogPermission() throws Exception { PulsarAdmin superAdmin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) .authentication(adminAuthentication).build()); + Authentication tenantAdminAuthentication = new ClientAuthentication(tenantRole); + @Cleanup + PulsarAdmin tenantAdmin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) + .authentication(tenantAdminAuthentication).build()); + Authentication subAdminAuthentication = new ClientAuthentication(subscriptionRole); @Cleanup PulsarAdmin sub1Admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) @@ -350,9 +357,11 @@ public void testClearBacklogPermission() throws Exception { superAdmin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); superAdmin.tenants().createTenant("my-property", - new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet("test"))); + new TenantInfoImpl(Sets.newHashSet(tenantRole), Sets.newHashSet("test"))); superAdmin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); superAdmin.topics().createPartitionedTopic(topicName, 1); + assertEquals(tenantAdmin.topics().getPartitionedTopicList(namespace), + Lists.newArrayList(topicName)); // grant topic consume&produce authorization to the subscriptionRole superAdmin.topics().grantPermission(topicName, subscriptionRole, @@ -381,6 +390,13 @@ public void testClearBacklogPermission() throws Exception { .get(subscriptionName).getMsgBacklog(), 10); // subscriptionRole doesn't have namespace-level authorization, so it will fail to clear backlog + try { + sub1Admin.topics().getPartitionedTopicList(namespace); + fail("should have failed with authorization exception"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith( + "Unauthorized to validateNamespaceOperation for operation [GET_TOPICS]")); + } try { sub1Admin.namespaces().clearNamespaceBundleBacklog(namespace, "0x00000000_0xffffffff"); fail("should have failed with authorization exception"); @@ -392,6 +408,8 @@ public void testClearBacklogPermission() throws Exception { superAdmin.namespaces().grantPermissionOnNamespace(namespace, subscriptionRole, Sets.newHashSet(AuthAction.consume)); // now, subscriptionRole have consume authorization on namespace, so it will successfully clear backlog + assertEquals(sub1Admin.topics().getPartitionedTopicList(namespace), + Lists.newArrayList(topicName)); sub1Admin.namespaces().clearNamespaceBundleBacklog(namespace, "0x00000000_0xffffffff"); assertEquals(sub1Admin.topics().getStats(topicName + "-partition-0").getSubscriptions() .get(subscriptionName).getMsgBacklog(), 0); From 7dead7b9523671b95e5b0a7e18e317f4060294e5 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sun, 12 Dec 2021 22:42:39 +0800 Subject: [PATCH 060/823] Add maven.restlet.org repository (#13248) * Add maven.restlet.org repository * Unblock restlet-http * Add maven.restlet.org repository (cherry picked from commit 717e75d82e651156ab464cc898a10bf867ef47bf) --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 64338333bbe75..7d4fbdddc7a6d 100644 --- a/pom.xml +++ b/pom.xml @@ -2294,6 +2294,14 @@ flexible messaging model and an intuitive client API. false + + maven.restlet.org + maven.restlet.org + https://maven.restlet.talend.com + + false + + From ff99f5fab2a976bc954a0f8bdf4da5d1a3c76f56 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 14 Dec 2021 00:53:52 -0600 Subject: [PATCH 061/823] Bump log4j to 2.16.0 (#13277) Apache Log4j 2.16.0 was just released. It is a more complete fix for Log4Shell. See the Apache Log4j mailing list for details: https://lists.apache.org/thread/d6v4r6nosxysyq9rvnr779336yf0woz4 (cherry picked from commit 621ca60cd28d4ba770f55b6a8ad6cccc52321b28) --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 06443fac2adc2..f9b8998ca9adc 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -39,7 +39,7 @@ 1.8 1.8 3.0.0-M3 - 2.15.0 + 2.16.0 1.7.25 7.3.0 3.11 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index cb6441dee7a54..3ce4e4270500d 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -383,11 +383,11 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.15.0.jar - - org.apache.logging.log4j-log4j-core-2.15.0.jar - - org.apache.logging.log4j-log4j-slf4j-impl-2.15.0.jar - - org.apache.logging.log4j-log4j-web-2.15.0.jar - - org.apache.logging.log4j-log4j-1.2-api-2.15.0.jar + - org.apache.logging.log4j-log4j-api-2.16.0.jar + - org.apache.logging.log4j-log4j-core-2.16.0.jar + - org.apache.logging.log4j-log4j-slf4j-impl-2.16.0.jar + - org.apache.logging.log4j-log4j-web-2.16.0.jar + - org.apache.logging.log4j-log4j-1.2-api-2.16.0.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - org.apache.bookkeeper-bookkeeper-common-4.14.2.jar diff --git a/pom.xml b/pom.xml index 7d4fbdddc7a6d..04e9dfb760c04 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ flexible messaging model and an intuitive client API. 6.10.2 1.7.25 3.2.2 - 2.15.0 + 2.16.0 1.69 1.0.2 2.12.3 From c4d6124237f20197218db1870ee4fb0b80a5fb63 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 14 Dec 2021 11:24:08 +0800 Subject: [PATCH 062/823] [CI] Fix Tune Runner VM memory.swappiness no such file or directory Failed CI: https://github.com/apache/pulsar/runs/4501006108?check_suite_focus=true#step:3:1 ### Motivation The Tune Runner VM is flaky. Sometimes, the parent directory of memory.swappiness can't find, we should check it first. ``` Run if [[ "$OSTYPE" == "linux-gnu"* ]]; then 10.1.0.235 fv-az198-145.gjkxsqtjg41etoza45cw35jkyb.bx.internal.cloudapp.net fv-az198-145 1 tee: /sys/fs/cgroup/memory/system.slice/snap-snapd-13640.mount/memory.swappiness: No such file or directory Error: Process completed with exit code 1. ``` ### Modifications Add a check of the parent directory of memory.swappiness (cherry picked from commit 26ac049997bf10d82c344e7db77f503740280d82) --- .github/actions/tune-runner-vm/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index 30cf183e72a45..e8914dbe74f6c 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -33,8 +33,10 @@ runs: # Set vm.swappiness=1 to avoid swapping and allow high RAM usage echo 1 | sudo tee /proc/sys/vm/swappiness # Set swappiness to 1 for all cgroups and sub-groups - for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do - echo 1 | sudo tee $swappiness_file > /dev/null + for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do + if [ -d "swappiness_dir" ]; then + echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null + fi done # use "madvise" Linux Transparent HugePages (THP) setting From f79840ebf4324ebaeaeeaf4b8d7b1d9f67611b15 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 14 Dec 2021 12:10:45 +0100 Subject: [PATCH 063/823] Release 2.9.1 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 126 files changed, 126 insertions(+), 126 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 3ef689c66bc30..257f396902094 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0 + 2.9.1 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 7eb2052850631..a58e46afed585 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0 + 2.9.1 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 71c4882316d2b..726420995c7de 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0 + 2.9.1 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 85b8083e73983..57cfabb0284d4 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index f9b8998ca9adc..e885533bd9f01 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.0 + 2.9.1 jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index a405b8aaf9e65..84e8ee7ff5970 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0 + 2.9.1 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index bb2575ea30e7d..6bfc10dab10b1 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0 + 2.9.1 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index c5bb62fc9667e..ff3b0c63e70f3 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 3206948baf9eb..35e9ea3be0562 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0 + 2.9.1 .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index 77d4d3478c415..f771f22396f9f 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0 + 2.9.1 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 38f75778618bd..2a56ba7e8bd08 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 20d5120e14d7e..4a120af5bfea8 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0 + 2.9.1 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 45dae9e5027ab..86dc7600605c2 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0 + 2.9.1 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 773bc6e473221..989bcfe051f31 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index c4d1da243b32e..05504478b863d 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.0 + 2.9.1 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 5e0f22093cd3e..27ef66377f4a1 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pom.xml b/pom.xml index 04e9dfb760c04..c038be32a2b8a 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index bb7c92c9468b6..cc5e217d41204 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index a5defa992a603..af45d2bb3d371 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 3eacc740afeb4..16866564f0cad 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 941fa473dd584..7e074afc22e68 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index bb7bee7e04722..86bc5621b04ff 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index e155a9d1a979f..45c4a8c858014 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index b79e18a3b020f..4f9b9dcda4a6f 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index eb77cdfef1d4b..f291d1c58a5a3 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 1bced59650786..ce5114459c28b 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 8425a803d91e7..c37d74b8fcb72 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 7e0a3fd46ff24..e4337335f9cec 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index c58ab6cb56807..c5296c3627d89 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 5c9e615965bae..4cdd80b71533d 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 59544f405bcb5..b6256f53b931b 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index d1321b5a0e2d6..bb7d4d61ca2f0 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 90b33e2d1e31a..58e1d3d733eb9 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 181b85570942c..6f3487ddf5d00 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 919654bcd77fe..922b5ebec7a5c 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index c0724a3da4a4e..60e0f7a0f8309 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 8e7d6a8aaff35..0b5564ce3a0c7 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 3294452d83111..ba2502e99893e 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 981b065f18bf0..2319a17180b27 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 972f98cca1a02..9c786d99690ad 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 67318a8f23053..2c16c1fdb5d44 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 56156a4dc964a..e75a9fcf33df3 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 14bc0108d0987..092a71474e167 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 7bd014023b009..f5bf0ea04428e 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 20d5f67c09276..fcc43a8868e34 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 3751fb67f6b2b..846e519f26626 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 116dfe0dbbf01..c1a058f84a596 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 20d7aab8c11ea..316c3009bac9a 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 581a249835d63..4c1f434cbcbaa 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index e9dfe723445a3..2f29075a692aa 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 4731a32492ed2..992415c24bd5b 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0 + 2.9.1 .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index d47e54f913f34..0a9961b7940e0 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 36da3ce64e448..fcd079c48cdf2 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 246cc67f8297e..742ed49cfa1da 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 2dfbc5cdba253..056e13ec4e348 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index ab5aedf555f64..7e32670dbd3e6 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index ebfa960312953..1ead477c20039 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 57441f8f121d8..c643d2040f3bc 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 4704f7795bded..bba433e3990bd 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 50b5e3bbdc8c2..7906e4453cef7 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 0714e741d664d..29aea0e8e92c5 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0 + 2.9.1 pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 219ab2114c1f6..3e651d4fcb2c8 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0 + 2.9.1 pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 5dee7c221c94c..2410c8e1e2470 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0 + 2.9.1 pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 07d779dcda0b4..99ecc53aa3291 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0 + 2.9.1 pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 2af6d304a02ef..9248e2e882e9a 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0 + 2.9.1 pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 07fa2cf79f3bb..245b1684e7583 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index fcf9dadb98261..a3d38fdf3552e 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0 + 2.9.1 pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index cb9606035fdd0..07c9654f3dbb4 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 415f77c942e8f..3562cdc321707 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index f9e264e160b01..b01ac3f0ce46a 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 05d0a8c35730c..3f610661905d5 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 75d4bcdeadd3d..8600a85e2b8fe 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 2e3a418553d23..17257fa3073ca 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0 + 2.9.1 pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index e76a341e1f73e..f3ffa7163bb02 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index dcbd20b2ec623..e80e3662dc612 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index a17eff5430036..a929ab3c914db 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0 + 2.9.1 pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index c7f875116225b..2f93507780ce0 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 90fe56e07aa82..26df287659d9f 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 584e923dcf970..e357409c90cb0 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index c91c778a801c6..774dc3044d691 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 38932936183ae..06ad0cc5b50b4 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index a286f06434f7d..8a61e14ab9b09 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 6c093bb9f54b2..27eb40211bbdd 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 8fc6dc2ff3bd0..f0af99ca394d7 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 78d4c0899d1e9..9be6b78cad54c 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 9a755f145ec14..b1fe4c9ee0438 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 4c0b2d416feb1..011609b6ba7c6 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 4a5916e578db3..8a55c34f03857 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index c8141dbc6a347..a69f436986f8a 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 07450b03d96f6..77b329c796403 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 6d34e9dcf0621..3dc46c0b6a422 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 19a1755f32290..b68dc8da0047e 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0 + 2.9.1 pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index d9bf650361581..04b3e88ab2f87 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0 + 2.9.1 diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index bced6aa1066b7..f6e45a4a8a992 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0 + 2.9.1 pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index c98d9514797a7..4aaba09bb4c81 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index d817d8a599fa7..598c5971a2945 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 5a3396bb9b41d..ba6b957cacdde 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.0 + 2.9.1 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index b7cbed188e1a1..dfab92a8e5d02 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.0 + 2.9.1 .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index a244b58990297..ffb9cd714d752 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 0f242adf5c355..aa59a6a8f8beb 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index d023e705ffdaa..211d4575c0b57 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0 + 2.9.1 pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 54a2a772ab2c7..cc91cb58c5b18 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0 + 2.9.1 pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 7aa74984f2a56..0afa345945868 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0 + 2.9.1 pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index de56bf3d22d2f..2d21778990a54 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index aa115fad92887..704191882006d 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.0 + 2.9.1 pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index be3c23a45a41a..cbe82dbc29e57 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.0 + 2.9.1 pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index a15ee049a8f72..93982b614f0df 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 0c222075594b4..373ce329762f0 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index edd0848a63188..fbb37083eaaa8 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index b576934402030..859a9f686c3d8 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 20a40bc24543b..1babdf1f68349 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.0 + 2.9.1 testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index f4dde997a0762..c6a80c04bf5c1 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 84022b7c7498f..4364c18e8b599 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 1608f82ee35fa..40b4ea322e89f 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index b82298d9b4bfc..f02d10e3c720e 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0 + 2.9.1 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 30d705ba7a8fa..626eb5fa11100 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0 + 2.9.1 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 27567573a65ef..5bee911934e0d 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0 + 2.9.1 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index b7fb9cc8fd1c0..2ac23ba096be1 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 5b8d860748e4f..bfaa069eb8903 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 integration diff --git a/tests/pom.xml b/tests/pom.xml index 1122625d6a667..5eef28b6970c5 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 869b458c24bf3..2a4d3c71f8c38 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 597f882ee4d4d..7fc147fa7ff1b 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 1b7f3333eb2ce..9ed80c851388d 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0 + 2.9.1 pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 55042c233be11..b0c394c2f94ca 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.0 + 2.9.1 .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 2bc10e8370281..f221b7d7cf487 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.0 + 2.9.1 .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 3777d5b492962..e4e935b2ba3b7 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0 + 2.9.1 .. From 4e4c7877d1ef0add6727cd1b58063222d152ed71 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Wed, 15 Dec 2021 19:16:15 +0800 Subject: [PATCH 064/823] Fix incompatibility of BacklogQuota (#13291) (cherry picked from commit 4e89c2bd93bab682d288cf8d069fd77a2c9cb1e4) --- .../common/policies/data/BacklogQuota.java | 9 +++ .../policies/data/impl/BacklogQuotaImpl.java | 77 +++++++++++++++++-- .../policies/data/BacklogQuotaMixIn.java | 26 ------- .../common/util/ObjectMapperFactory.java | 2 - .../common/util/ObjectMapperFactoryTest.java | 27 ------- .../BacklogQuotaCompatibilityTest.java | 60 ++++++++++++++- 6 files changed, 135 insertions(+), 66 deletions(-) delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuotaMixIn.java diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuota.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuota.java index d4b5c4bba1c5b..4604710c3a68c 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuota.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuota.java @@ -28,6 +28,15 @@ */ public interface BacklogQuota { + /** + * Gets quota limit in size. + * Remains for compatible + * + * @return quota limit in bytes + */ + @Deprecated + long getLimit(); + /** * Gets quota limit in size. * diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java index 591e8b8c95a8a..3d97fa06426f9 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java @@ -18,23 +18,86 @@ */ package org.apache.pulsar.common.policies.data.impl; -import lombok.AllArgsConstructor; -import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.ToString; import org.apache.pulsar.common.policies.data.BacklogQuota; -@Data -@AllArgsConstructor +@ToString +@EqualsAndHashCode @NoArgsConstructor public class BacklogQuotaImpl implements BacklogQuota { public static final long BYTES_IN_GIGABYTE = 1024 * 1024 * 1024; - // backlog quota by size in byte - private long limitSize; - // backlog quota by time in second + /** + * backlog quota by size in byte, remains for compatible. + * for the details: https://github.com/apache/pulsar/pull/13291 + * @since 2.9.1 + */ + @Deprecated + private Long limit; + + /** + * backlog quota by size in byte. + */ + private Long limitSize; + + /** + * backlog quota by time in second. + */ private int limitTime; private RetentionPolicy policy; + public BacklogQuotaImpl(long limitSize, int limitTime, RetentionPolicy policy) { + this.limitSize = limitSize; + this.limitTime = limitTime; + this.policy = policy; + } + + @Deprecated + public long getLimit() { + if (limitSize == null) { + // the limitSize and limit can't be both null + return limit; + } + return limitSize; + } + + @Deprecated + public void setLimit(long limit) { + this.limit = limit; + this.limitSize = limit; + } + + public long getLimitSize() { + if (limitSize == null) { + // the limitSize and limit can't be both null + return limit; + } + return limitSize; + } + + public void setLimitSize(long limitSize) { + this.limitSize = limitSize; + this.limit = limitSize; + } + + public int getLimitTime() { + return limitTime; + } + + public void setLimitTime(int limitTime) { + this.limitTime = limitTime; + } + + public RetentionPolicy getPolicy() { + return policy; + } + + public void setPolicy(RetentionPolicy policy) { + this.policy = policy; + } + public static BacklogQuotaImplBuilder builder() { return new BacklogQuotaImplBuilder(); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuotaMixIn.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuotaMixIn.java deleted file mode 100644 index a1562400f0818..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/BacklogQuotaMixIn.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.policies.data; - -import com.fasterxml.jackson.annotation.JsonAlias; - -public abstract class BacklogQuotaMixIn { - @JsonAlias("limit") - private long limitSize; -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ObjectMapperFactory.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ObjectMapperFactory.java index 94e1b7af4a278..ef2e4894a721f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ObjectMapperFactory.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ObjectMapperFactory.java @@ -37,7 +37,6 @@ import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.BacklogQuotaMixIn; import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.BookieInfo; import org.apache.pulsar.common.policies.data.BookiesClusterInfo; @@ -192,7 +191,6 @@ private static void setAnnotationsModule(ObjectMapper mapper) { resolver.addMapping(AutoSubscriptionCreationOverride.class, AutoSubscriptionCreationOverrideImpl.class); // we use MixIn class to add jackson annotations - mapper.addMixIn(BacklogQuotaImpl.class, BacklogQuotaMixIn.class); mapper.addMixIn(ResourceQuota.class, ResourceQuotaMixIn.class); mapper.addMixIn(FunctionConfig.class, JsonIgnorePropertiesMixIn.class); mapper.addMixIn(FunctionState.class, JsonIgnorePropertiesMixIn.class); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/ObjectMapperFactoryTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/ObjectMapperFactoryTest.java index dc8aa8b43fcf3..f9e372941ef0f 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/ObjectMapperFactoryTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/ObjectMapperFactoryTest.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.ToString; -import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ResourceQuota; import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; import org.apache.pulsar.common.stats.Metrics; @@ -28,32 +27,6 @@ import org.testng.annotations.Test; public class ObjectMapperFactoryTest { - @Test - public void testBacklogQuotaMixIn() { - ObjectMapper objectMapper = ObjectMapperFactory.getThreadLocal(); - String json = "{\"limit\":10,\"limitTime\":0,\"policy\":\"producer_request_hold\"}"; - try { - BacklogQuota backlogQuota = objectMapper.readValue(json, BacklogQuota.class); - Assert.assertEquals(backlogQuota.getLimitSize(), 10); - Assert.assertEquals(backlogQuota.getLimitTime(), 0); - Assert.assertEquals(backlogQuota.getPolicy(), BacklogQuota.RetentionPolicy.producer_request_hold); - } catch (Exception ex) { - Assert.fail("shouldn't have thrown exception", ex); - } - - try { - String expectJson = "{\"limitSize\":10,\"limitTime\":0,\"policy\":\"producer_request_hold\"}"; - BacklogQuota backlogQuota = BacklogQuota.builder() - .limitSize(10) - .limitTime(0) - .retentionPolicy(BacklogQuota.RetentionPolicy.producer_request_hold) - .build(); - String writeJson = objectMapper.writeValueAsString(backlogQuota); - Assert.assertEquals(expectJson, writeJson); - } catch (Exception ex) { - Assert.fail("shouldn't have thrown exception", ex); - } - } @Test public void testResourceQuotaMixIn() { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java index 06765e307ef29..659825c74858a 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java @@ -19,15 +19,70 @@ package org.apache.pulsar.metadata; import static org.testng.Assert.assertEquals; + +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.type.TypeFactory; import java.io.IOException; +import java.util.HashMap; + import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; import org.apache.pulsar.metadata.cache.impl.JSONMetadataSerdeSimpleType; +import org.testng.Assert; import org.testng.annotations.Test; public class BacklogQuotaCompatibilityTest { + private final JavaType typeRef = TypeFactory.defaultInstance().constructSimpleType(Policies.class, null); + + private final JSONMetadataSerdeSimpleType simpleType = new JSONMetadataSerdeSimpleType<>(typeRef); + + private final BacklogQuota.RetentionPolicy testPolicy = BacklogQuota.RetentionPolicy.consumer_backlog_eviction; + + @Test + public void testV27ClientSetV28BrokerRead() throws Exception { + Policies writePolicy = new Policies(); + BacklogQuotaImpl writeBacklogQuota = new BacklogQuotaImpl(); + writeBacklogQuota.setLimit(1024); + writeBacklogQuota.setLimitTime(60); + writeBacklogQuota.setPolicy(testPolicy); + HashMap quotaHashMap = new HashMap<>(); + quotaHashMap.put(BacklogQuota.BacklogQuotaType.destination_storage, writeBacklogQuota); + writePolicy.backlog_quota_map = quotaHashMap; + byte[] serialize = simpleType.serialize("/path", writePolicy); + Policies policies = simpleType.deserialize("/path", serialize, null); + BacklogQuota readBacklogQuota = policies.backlog_quota_map.get(BacklogQuota.BacklogQuotaType.destination_storage); + Assert.assertEquals(readBacklogQuota.getLimitSize(), 1024); + Assert.assertEquals(readBacklogQuota.getLimitTime(), 60); + Assert.assertEquals(readBacklogQuota.getPolicy(), testPolicy); + } + + @Test + public void testV28ClientSetV28BrokerRead() throws Exception { + Policies writePolicy = new Policies(); + BacklogQuotaImpl writeBacklogQuota = new BacklogQuotaImpl(); + writeBacklogQuota.setLimitSize(1024); + writeBacklogQuota.setLimitTime(60); + writeBacklogQuota.setPolicy(testPolicy); + HashMap quotaHashMap = new HashMap<>(); + quotaHashMap.put(BacklogQuota.BacklogQuotaType.destination_storage, writeBacklogQuota); + writePolicy.backlog_quota_map = quotaHashMap; + byte[] serialize = simpleType.serialize("/path", writePolicy); + Policies policies = simpleType.deserialize("/path", serialize, null); + BacklogQuota readBacklogQuota = policies.backlog_quota_map.get(BacklogQuota.BacklogQuotaType.destination_storage); + Assert.assertEquals(readBacklogQuota.getLimit(), 1024); + Assert.assertEquals(readBacklogQuota.getLimitTime(), 60); + Assert.assertEquals(readBacklogQuota.getPolicy(), testPolicy); + } + + @Test + public void testV28ClientSetV27BrokerRead() { + BacklogQuotaImpl writeBacklogQuota = new BacklogQuotaImpl(); + writeBacklogQuota.setLimitSize(1024); + Assert.assertEquals(1024, writeBacklogQuota.getLimit()); + } + @Test public void testBackwardCompatibility() throws IOException { String oldPolicyStr = "{\"auth_policies\":{\"namespace_auth\":{},\"destination_auth\":{}," @@ -41,10 +96,7 @@ public void testBackwardCompatibility() throws IOException { + "\"schema_auto_update_compatibility_strategy\":\"Full\",\"schema_compatibility_strategy\":" + "\"UNDEFINED\",\"is_allow_auto_update_schema\":true,\"schema_validation_enforced\":false," + "\"subscription_types_enabled\":[]}\n"; - - JSONMetadataSerdeSimpleType jsonMetadataSerdeSimpleType = new JSONMetadataSerdeSimpleType( - TypeFactory.defaultInstance().constructSimpleType(Policies.class, null)); - Policies policies = (Policies) jsonMetadataSerdeSimpleType.deserialize(null, oldPolicyStr.getBytes(), null); + Policies policies = simpleType.deserialize(null, oldPolicyStr.getBytes(), null); assertEquals(policies.backlog_quota_map.get(BacklogQuota.BacklogQuotaType.destination_storage).getLimitSize(), 1001); assertEquals(policies.backlog_quota_map.get(BacklogQuota.BacklogQuotaType.destination_storage).getLimitTime(), From bce5d56e5f7250d3337838c69cb9a0d0051a65e2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Nov 2021 06:00:33 +0200 Subject: [PATCH 065/823] [ML] Add OpAddEntry to pendingAddEntries after the state check (#12570) - when the state was Fenced, Terminated or Closed, the OpAddEntry instance would remain in pendingAddEntries although the operation is failed immediately. (cherry picked from commit 409239ce27d7d5f9800c5d09e0455ad4c5c0871c) --- .../org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index fa9a5cfcf2725..957cfbb140850 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -729,7 +729,6 @@ private synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { if (!beforeAddEntry(addOperation)) { return; } - pendingAddEntries.add(addOperation); final State state = STATE_UPDATER.get(this); if (state == State.Fenced) { addOperation.failed(new ManagedLedgerFencedException()); @@ -741,10 +740,10 @@ private synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { addOperation.failed(new ManagedLedgerAlreadyClosedException("Managed ledger was already closed")); return; } else if (state == State.WriteFailed) { - pendingAddEntries.remove(addOperation); addOperation.failed(new ManagedLedgerAlreadyClosedException("Waiting to recover from failure")); return; } + pendingAddEntries.add(addOperation); if (state == State.ClosingLedger || state == State.CreatingLedger) { // We don't have a ready ledger to write into From 7d1bc574fe785f7237c313feb5b31b5b84aa5127 Mon Sep 17 00:00:00 2001 From: Massimiliano Mirelli Date: Fri, 3 Dec 2021 16:48:00 +0200 Subject: [PATCH 066/823] Fixing permissions of scripts used to deploy docker images (#11951) (cherry picked from commit caa068fb0b461a806bda8aa9b88dced47a00d158) --- src/assembly-source-package.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/assembly-source-package.xml b/src/assembly-source-package.xml index 2677299357f3f..f31aadc146972 100644 --- a/src/assembly-source-package.xml +++ b/src/assembly-source-package.xml @@ -106,6 +106,7 @@ /docker/pulsar/scripts *.sh + *.py 0755 From 47d91a18662e0902cd0216948f054b2780505472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 9 Dec 2021 08:43:22 +0100 Subject: [PATCH 067/823] [Security] Upgrade OkHttp3 to address CVE-2021-0341 (#13065) * Upgrade OkHttp3 from 3.14.9 to 4.9.3 * Upgrade Okio to the same version of OkHttp3 4.9.3 * Override Okio transitive dependency - Kotlin stdlib - to 1.4.32 in order to address CVE-2020-29582 (cherry picked from commit d24faac38bc8babd1167d9ae684a6273b65930d8) --- .../server/src/assemble/LICENSE.bin.txt | 14 ++++++--- pom.xml | 29 +++++++++++++++++-- pulsar-sql/pom.xml | 29 +++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 3ce4e4270500d..51c7242273c5c 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -446,12 +446,18 @@ The Apache Software License, Version 2.0 * SnakeYaml -- org.yaml-snakeyaml-1.27.jar * RocksDB - org.rocksdb-rocksdbjni-6.10.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar - * Apache Thrifth - org.apache.thrift-libthrift-0.14.2.jar + * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar * OkHttp3 - - com.squareup.okhttp3-logging-interceptor-3.14.9.jar - - com.squareup.okhttp3-okhttp-3.14.9.jar - * Okio - com.squareup.okio-okio-1.17.2.jar + - com.squareup.okhttp3-logging-interceptor-4.9.3.jar + - com.squareup.okhttp3-okhttp-4.9.3.jar + * Okio - com.squareup.okio-okio-2.8.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar + * Kotlin Standard Lib + - org.jetbrains.kotlin-kotlin-stdlib-1.4.32.jar + - org.jetbrains.kotlin-kotlin-stdlib-common-1.4.32.jar + - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.4.32.jar + - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.4.32.jar + - org.jetbrains-annotations-13.0.jar * gRPC - io.grpc-grpc-all-1.33.0.jar - io.grpc-grpc-auth-1.33.0.jar diff --git a/pom.xml b/pom.xml index c038be32a2b8a..9824d9dd58adc 100644 --- a/pom.xml +++ b/pom.xml @@ -192,9 +192,11 @@ flexible messaging model and an intuitive client API. 2.0.2 4.2.0 12.0.1 - 3.14.9 + 4.9.3 - 1.17.2 + 2.8.0 + + 1.4.32 1.0 9.1.3 5.3.1 @@ -1186,12 +1188,35 @@ flexible messaging model and an intuitive client API. okhttp-urlconnection ${okhttp3.version} + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + com.squareup.okio okio ${okio.version} + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin-stdlib.version} + + + org.jetbrains.kotlin + kotlin-stdlib-common + ${kotlin-stdlib.version} + + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin-stdlib.version} + + + diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index aa59a6a8f8beb..0336b16cfabae 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -37,6 +37,13 @@ presto-distribution + + + 3.14.9 + + 1.17.2 + + @@ -104,6 +111,28 @@ jackson-datatype-jsr310 ${jackson.version} + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.okhttp3 + okhttp-urlconnection + ${okhttp3.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + com.squareup.okio + okio + ${okio.version} + From be2610d9445f196a4e0cb840b2608a828f521c73 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sat, 27 Nov 2021 09:21:01 +0800 Subject: [PATCH 068/823] Do not reuse the Failed OpAddEntry object. (#12993) ### Motivation There are 2 ways to complete the OpAddEntry with exception, one is the bk client callback and another one is `ManagedLedgerImple.clearPendingAddEntries`. But, if the OpAddEntry be completed more than once, we will get NPE: ``` java.lang.NullPointerException: null at org.apache.bookkeeper.mledger.impl.OpAddEntry.lambda$handleAddFailure$0(OpAddEntry.java:291) ~[org.apache.pulsar-managed-ledger-2.8.1.jar:2.8.1] at org.apache.bookkeeper.mledger.util.SafeRun$1.safeRun(SafeRun.java:32) ~[org.apache.pulsar-managed-ledger-2.8.1.jar:2.8.1] at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36) [org.apache.bookkeeper-bookkeeper-common-4.14.2-2.jar:4.14.2-2] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] ``` Another one: ``` java.lang.NullPointerException: null at org.apache.bookkeeper.mledger.impl.OpAddEntry.addComplete(OpAddEntry.java:153) ~[org.apache.pulsar-managed-ledger-2.8.1.jar:2.8.1] at org.apache.bookkeeper.client.AsyncCallback$AddCallback.addCompleteWithLatency(AsyncCallback.java:92) ~[org.apache.bookkeeper-bookkeeper-server-4.14.2-2.jar:4.14.2-2] at org.apache.bookkeeper.client.PendingAddOp.submitCallback(PendingAddOp.java:431) ~[org.apache.bookkeeper-bookkeeper-server-4.14.2-2.jar:4.14.2-2] at org.apache.bookkeeper.client.LedgerHandle.errorOutPendingAdds(LedgerHandle.java:1799) ~[org.apache.bookkeeper-bookkeeper-server-4.14.2-2.jar:4.14.2-2] at org.apache.bookkeeper.client.LedgerHandle$5.safeRun(LedgerHandle.java:574) ~[org.apache.bookkeeper-bookkeeper-server-4.14.2-2.jar:4.14.2-2] at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36) [org.apache.bookkeeper-bookkeeper-common-4.14.2-2.jar:4.14.2-2] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] ``` #12364 tries to fix the NPE by change the state of the OpAddEntry to CLOSED, but the OpAddEntry still will be recyled and be reused. And when we get the add entry complete callback from the bk client, we will reach here: https://github.com/apache/pulsar/blob/5dbb7d25849f3a037aa522b5d0767801aa0a5096/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java#L147 But it might to recycle the OpAddEntry already be reused by other entry add operation. It might lead to lost data for this case. ### Modification So the fix is do not recycle the OpAddEntry when call OpAddEntry.failed() which is introduced by #11737. We should contain this fix in 2.8.2, we have encounter serious problem when unloading the bundles, the topic close will be blocked and never complete because of LedgerHandle.errorOutPendingAdds() https://github.com/apache/bookkeeper/blob/87579b0a9f18833ee41fcae37582bb68606d68e7/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java#L574 get NPE but can't throw out and the ledger close callback will never complete. (cherry picked from commit 3e3622c1d4b8b9169b4ba69081a16a539606b72d) --- .../org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 1 - .../main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java | 1 - 2 files changed, 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 957cfbb140850..ce897f3b92048 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1678,7 +1678,6 @@ public ManagedLedgerInterceptor getManagedLedgerInterceptor() { void clearPendingAddEntries(ManagedLedgerException e) { while (!pendingAddEntries.isEmpty()) { OpAddEntry op = pendingAddEntries.poll(); - op.close(); op.failed(e); } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index e08b204af38c2..4f36d56ea2d21 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -137,7 +137,6 @@ public void failed(ManagedLedgerException e) { ReferenceCountUtil.release(data); cb.addFailed(e, ctx); ml.mbean.recordAddEntryError(); - this.recycle(); } } From e3e947d8a6a960da9868f2e3fb5c22c2cc243539 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 30 Nov 2021 20:34:45 +0800 Subject: [PATCH 069/823] Fix Issue #12885, Unordered consuming case in Key_Shared subscription (#12890) (cherry picked from commit 73ef1621ab0bbecfcb2325453a4d93a406fcba3c) --- ...PersistentDispatcherMultipleConsumers.java | 3 ++ ...tStickyKeyDispatcherMultipleConsumers.java | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 06a5c4cad518d..80e3d60536c8d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -83,6 +83,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul protected volatile boolean havePendingRead = false; protected volatile boolean havePendingReplayRead = false; + protected volatile PositionImpl minReplayedPosition = null; protected boolean shouldRewindBeforeReadingOrReplaying = false; protected final String name; @@ -244,6 +245,7 @@ public synchronized void readMoreEntries() { } havePendingReplayRead = true; + minReplayedPosition = messagesToReplayNow.stream().min(PositionImpl::compareTo).orElse(null); Set deletedMessages = topic.isDelayedDeliveryEnabled() ? asyncReplayEntriesInOrder(messagesToReplayNow) : asyncReplayEntries(messagesToReplayNow); // clear already acked positions from replay bucket @@ -267,6 +269,7 @@ public synchronized void readMoreEntries() { consumerList.size()); } havePendingRead = true; + minReplayedPosition = getMessagesToReplayNow(1).stream().findFirst().orElse(null); cursor.asyncReadEntriesOrWait(messagesToRead, bytesToRead, this, ReadType.Normal, topic.getMaxReadPosition()); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index f3bcbf2b2e9c9..420795cad2313 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -169,6 +169,37 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { return; } + // A corner case that we have to retry a readMoreEntries in order to preserver order delivery. + // This may happen when consumer closed. See issue #12885 for details. + if (!allowOutOfOrderDelivery) { + Set messagesToReplayNow = this.getMessagesToReplayNow(1); + if (messagesToReplayNow != null && !messagesToReplayNow.isEmpty() && this.minReplayedPosition != null) { + PositionImpl relayPosition = messagesToReplayNow.stream().findFirst().get(); + // If relayPosition is a new entry wither smaller position is inserted for redelivery during this async + // read, it is possible that this relayPosition should dispatch to consumer first. So in order to + // preserver order delivery, we need to discard this read result, and try to trigger a replay read, + // that containing "relayPosition", by calling readMoreEntries. + if (relayPosition.compareTo(minReplayedPosition) < 0) { + if (log.isDebugEnabled()) { + log.debug("[{}] Position {} (<{}) is inserted for relay during current {} read, discard this " + + "read and retry with readMoreEntries.", + name, relayPosition, minReplayedPosition, readType); + } + if (readType == ReadType.Normal) { + entries.forEach(entry -> { + long stickyKeyHash = getStickyKeyHash(entry); + addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); + entry.release(); + }); + } else if (readType == ReadType.Replay) { + entries.forEach(Entry::release); + } + readMoreEntries(); + return; + } + } + } + nextStuckConsumers.clear(); final Map> groupedEntries = localGroupedEntries.get(); From 2db23b8dd3eb859cc16e30686578be275026c347 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 11 Nov 2021 17:46:55 +0800 Subject: [PATCH 070/823] [Managed Ledger] Fix the incorrect total size when BrokerEntryMetadata is enabled (#12714) ### Motivation When the BrokerEntryMetadata is enabled, the total size in `ManagedLedgerImpl` is inaccurate. Because when the total size is updated in `OpAddEntry#safeRun`, the `dataLength` is the initial size of `data` when `OpAddEntry` is constructed, but `data` could be changed via `setData` method. The inaccurate total size could affect the retention size validation. Because in `ManagedLedgerImpl#internalTrimLedgers`, the total size reduces by the size of `LedgerInfo`, which is assigned from the `LedgerHandle#getLength()`. Therefore, the total size will become 0 or less before all ledgers are removed. ### Modifications - Update `dataLength` field in `setData` method. - Add a `testManagedLedgerTotalSize` test to `BrokerEntryMetadataE2ETest`. It produces 10 messages and trigger the rollover manually so that the first `LedgerInfo` of the managed ledger contains the correct total bytes. Then compare the `totalSize` field with it to verify this fix works. (cherry picked from commit 5dbb7d25849f3a037aa522b5d0767801aa0a5096) --- .../bookkeeper/mledger/impl/OpAddEntry.java | 1 + .../service/BrokerEntryMetadataE2ETest.java | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index 4f36d56ea2d21..0b96b5922e2d9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -313,6 +313,7 @@ public void setNumberOfMessages(int numberOfMessages) { } public void setData(ByteBuf data) { + this.dataLength = data.readableBytes(); this.data = data; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerEntryMetadataE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerEntryMetadataE2ETest.java index d2ea2f1fc261f..49b4742b71dad 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerEntryMetadataE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerEntryMetadataE2ETest.java @@ -18,18 +18,26 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; + +import java.time.Duration; import java.util.List; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; import org.assertj.core.util.Sets; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -354,4 +362,41 @@ public void testConsumerGetBrokerEntryMetadataForBatchMessage() throws Exception producer.close(); consumer.close(); } + + @Test + public void testManagedLedgerTotalSize() throws Exception { + final String topic = newTopicName(); + final int messages = 10; + + admin.topics().createNonPartitionedTopic(topic); + admin.lookups().lookupTopic(topic); + final ManagedLedgerImpl managedLedger = pulsar.getBrokerService().getTopicIfExists(topic).get() + .map(topicObject -> (ManagedLedgerImpl) ((PersistentTopic) topicObject).getManagedLedger()) + .orElse(null); + Assert.assertNotNull(managedLedger); + final ManagedCursor cursor = managedLedger.openCursor("cursor"); // prevent ledgers being removed + + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + for (int i = 0; i < messages; i++) { + producer.send("msg-" + i); + } + + Assert.assertTrue(managedLedger.getTotalSize() > 0); + + managedLedger.getConfig().setMinimumRolloverTime(0, TimeUnit.MILLISECONDS); + managedLedger.getConfig().setMaxEntriesPerLedger(1); + managedLedger.rollCurrentLedgerIfFull(); + + Awaitility.await().atMost(Duration.ofSeconds(3)) + .until(() -> managedLedger.getLedgersInfo().size() > 1); + + final List ledgerInfoList = managedLedger.getLedgersInfoAsList(); + Assert.assertEquals(ledgerInfoList.size(), 2); + Assert.assertEquals(ledgerInfoList.get(0).getSize(), managedLedger.getTotalSize()); + + cursor.close(); + } } From f52ac045f41acbb6c31da21a3463df3cfbe8f1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 15 Dec 2021 22:18:58 +0100 Subject: [PATCH 071/823] [security] Upgrade Netty to 4.1.72 - CVE-2021-43797 (#13328) * [security] Upgrade Netty to 4.1.72 * fix licenses files --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 38 ++++++++++--------- pom.xml | 4 +- pulsar-sql/presto-distribution/LICENSE | 34 +++++++++-------- 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index e885533bd9f01..19b2b4e575d36 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -105,7 +105,7 @@ io.netty netty-common - 4.1.68.Final + 4.1.72.Final test diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 51c7242273c5c..a9570ba1b48a3 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -352,24 +352,26 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.11.jar * Netty - - io.netty-netty-buffer-4.1.68.Final.jar - - io.netty-netty-codec-4.1.68.Final.jar - - io.netty-netty-codec-dns-4.1.68.Final.jar - - io.netty-netty-codec-http-4.1.68.Final.jar - - io.netty-netty-codec-http2-4.1.68.Final.jar - - io.netty-netty-codec-socks-4.1.68.Final.jar - - io.netty-netty-codec-haproxy-4.1.68.Final.jar - - io.netty-netty-common-4.1.68.Final.jar - - io.netty-netty-handler-4.1.68.Final.jar - - io.netty-netty-handler-proxy-4.1.68.Final.jar - - io.netty-netty-resolver-4.1.68.Final.jar - - io.netty-netty-resolver-dns-4.1.68.Final.jar - - io.netty-netty-transport-4.1.68.Final.jar - - io.netty-netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.68.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.68.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.68.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.42.Final.jar + - io.netty-netty-buffer-4.1.72.Final.jar + - io.netty-netty-codec-4.1.72.Final.jar + - io.netty-netty-codec-dns-4.1.72.Final.jar + - io.netty-netty-codec-http-4.1.72.Final.jar + - io.netty-netty-codec-http2-4.1.72.Final.jar + - io.netty-netty-codec-socks-4.1.72.Final.jar + - io.netty-netty-codec-haproxy-4.1.72.Final.jar + - io.netty-netty-common-4.1.72.Final.jar + - io.netty-netty-handler-4.1.72.Final.jar + - io.netty-netty-handler-proxy-4.1.72.Final.jar + - io.netty-netty-resolver-4.1.72.Final.jar + - io.netty-netty-resolver-dns-4.1.72.Final.jar + - io.netty-netty-transport-4.1.72.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.72.Final.jar + - io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.72.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.72.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.72.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar + - io.netty-netty-tcnative-classes-2.0.46.Final.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar diff --git a/pom.xml b/pom.xml index 9824d9dd58adc..f01660d77a6bc 100644 --- a/pom.xml +++ b/pom.xml @@ -108,8 +108,8 @@ flexible messaging model and an intuitive client API. 1.1.7 3.2.5 5.1.0 - 4.1.68.Final - 2.0.42.Final + 4.1.72.Final + 2.0.46.Final 9.4.43.v20210629 2.5.2 2.34 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d35661fa22a4b..94419176d58d1 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -233,23 +233,25 @@ The Apache Software License, Version 2.0 - commons-lang3-3.11.jar * Netty - netty-3.10.6.Final.jar - - netty-buffer-4.1.68.Final.jar - - netty-codec-4.1.68.Final.jar - - netty-codec-dns-4.1.68.Final.jar - - netty-codec-http-4.1.68.Final.jar - - netty-codec-haproxy-4.1.68.Final.jar - - netty-codec-socks-4.1.68.Final.jar - - netty-handler-proxy-4.1.68.Final.jar - - netty-common-4.1.68.Final.jar - - netty-handler-4.1.68.Final.jar + - netty-buffer-4.1.72.Final.jar + - netty-codec-4.1.72.Final.jar + - netty-codec-dns-4.1.72.Final.jar + - netty-codec-http-4.1.72.Final.jar + - netty-codec-haproxy-4.1.72.Final.jar + - netty-codec-socks-4.1.72.Final.jar + - netty-handler-proxy-4.1.72.Final.jar + - netty-common-4.1.72.Final.jar + - netty-handler-4.1.72.Final.jar - netty-reactive-streams-2.0.4.jar - - netty-resolver-4.1.68.Final.jar - - netty-resolver-dns-4.1.68.Final.jar - - netty-tcnative-boringssl-static-2.0.42.Final.jar - - netty-transport-4.1.68.Final.jar - - netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.68.Final.jar - - netty-transport-native-unix-common-4.1.68.Final-linux-x86_64.jar + - netty-resolver-4.1.72.Final.jar + - netty-resolver-dns-4.1.72.Final.jar + - netty-tcnative-boringssl-static-2.0.46.Final.jar + - netty-tcnative-classes-2.0.46.Final.jar + - netty-transport-4.1.72.Final.jar + - netty-transport-classes-epoll-4.1.72.Final.jar + - netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.72.Final.jar + - netty-transport-native-unix-common-4.1.72.Final-linux-x86_64.jar * Joda Time - joda-time-2.10.5.jar * Jetty From 7ea0473fc0b9a3d67d5431669d2996f01a385791 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 18 Dec 2021 20:23:22 +0200 Subject: [PATCH 072/823] [Security] Upgrade to Log4J 2.17.0 to mitigate CVE-2021-45105 (#13392) - more details at https://logging.apache.org/log4j/2.x/security.html (cherry picked from commit 0fa626d6b74734f4c77ec49dcf33d7ab4f0c80c1) --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 19b2b4e575d36..4b6f4e4b0a54f 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -39,7 +39,7 @@ 1.8 1.8 3.0.0-M3 - 2.16.0 + 2.17.0 1.7.25 7.3.0 3.11 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a9570ba1b48a3..27d146688088d 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -385,11 +385,11 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.16.0.jar - - org.apache.logging.log4j-log4j-core-2.16.0.jar - - org.apache.logging.log4j-log4j-slf4j-impl-2.16.0.jar - - org.apache.logging.log4j-log4j-web-2.16.0.jar - - org.apache.logging.log4j-log4j-1.2-api-2.16.0.jar + - org.apache.logging.log4j-log4j-api-2.17.0.jar + - org.apache.logging.log4j-log4j-core-2.17.0.jar + - org.apache.logging.log4j-log4j-slf4j-impl-2.17.0.jar + - org.apache.logging.log4j-log4j-web-2.17.0.jar + - org.apache.logging.log4j-log4j-1.2-api-2.17.0.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - org.apache.bookkeeper-bookkeeper-common-4.14.2.jar diff --git a/pom.xml b/pom.xml index f01660d77a6bc..4dca3ba0f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ flexible messaging model and an intuitive client API. 6.10.2 1.7.25 3.2.2 - 2.16.0 + 2.17.0 1.69 1.0.2 2.12.3 From f1d8d7197765277bacf67c8515827c241053223a Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Wed, 27 Oct 2021 11:41:44 +0800 Subject: [PATCH 073/823] fix delete authentication policies when delete topic. (#12215) (cherry picked from commit 3e578280539aa55c084a35b601902f0e16c5fc2f) --- .../admin/impl/PersistentTopicsBase.java | 99 ++++++++++++------- .../pulsar/broker/service/BrokerService.java | 76 ++++++++++++-- .../service/persistent/PersistentTopic.java | 10 +- .../pulsar/broker/admin/AdminApiTest.java | 4 +- .../AuthenticatedProducerConsumerTest.java | 80 +++++++++++++-- 5 files changed, 213 insertions(+), 56 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 62c01f691dec6..f317e03136977 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -132,6 +132,7 @@ import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -269,7 +270,7 @@ private void grantPermissions(String topicUri, String role, Set acti }); log.info("[{}] Successfully granted access for role {}: {} - topic {}", clientAppId(), role, actions, topicUri); - } catch (org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException e) { + } catch (MetadataStoreException.NotFoundException e) { log.warn("[{}] Failed to grant permissions on topic {}: Namespace does not exist", clientAppId(), topicUri); throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); } catch (Exception e) { @@ -588,42 +589,72 @@ protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, boole } }); } - for (int i = 0; i < numPartitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - try { - pulsar().getAdminClient().topics() - .deleteAsync(topicNamePartition.toString(), force) - .whenComplete((r, ex) -> { - if (ex != null) { - if (ex instanceof NotFoundException) { - // if the sub-topic is not found, the client might not have called create - // producer or it might have been deleted earlier, - //so we ignore the 404 error. - // For all other exception, - //we fail the delete partition method even if a single - // partition is failed to be deleted - if (log.isDebugEnabled()) { - log.debug("[{}] Partition not found: {}", clientAppId(), - topicNamePartition); + // delete authentication policies of the partitioned topic + CompletableFuture deleteAuthFuture = new CompletableFuture<>(); + pulsar().getPulsarResources().getNamespaceResources() + .setPoliciesAsync(topicName.getNamespaceObject(), p -> { + for (int i = 0; i < numPartitions; i++) { + p.auth_policies.getTopicAuthentication().remove(topicName.getPartition(i).toString()); + } + p.auth_policies.getTopicAuthentication().remove(topicName.toString()); + return p; + }).thenAccept(v -> { + log.info("Successfully delete authentication policies for partitioned topic {}", topicName); + deleteAuthFuture.complete(null); + }).exceptionally(ex -> { + if (ex.getCause() instanceof MetadataStoreException.NotFoundException) { + log.warn("Namespace policies of {} not found", topicName.getNamespaceObject()); + deleteAuthFuture.complete(null); + } else { + log.error("Failed to delete authentication policies for partitioned topic {}", + topicName, ex); + deleteAuthFuture.completeExceptionally(ex); + } + return null; + }); + + deleteAuthFuture.whenComplete((r, ex) -> { + if (ex != null) { + future.completeExceptionally(ex); + return; + } + for (int i = 0; i < numPartitions; i++) { + TopicName topicNamePartition = topicName.getPartition(i); + try { + pulsar().getAdminClient().topics() + .deleteAsync(topicNamePartition.toString(), force) + .whenComplete((r1, ex1) -> { + if (ex1 != null) { + if (ex1 instanceof NotFoundException) { + // if the sub-topic is not found, the client might not have called + // create producer or it might have been deleted earlier, + //so we ignore the 404 error. + // For all other exception, + //we fail the delete partition method even if a single + // partition is failed to be deleted + if (log.isDebugEnabled()) { + log.debug("[{}] Partition not found: {}", clientAppId(), + topicNamePartition); + } + } else { + log.error("[{}] Failed to delete partition {}", clientAppId(), + topicNamePartition, ex1); + future.completeExceptionally(ex1); + return; } } else { - log.error("[{}] Failed to delete partition {}", clientAppId(), - topicNamePartition, ex); - future.completeExceptionally(ex); - return; + log.info("[{}] Deleted partition {}", clientAppId(), topicNamePartition); } - } else { - log.info("[{}] Deleted partition {}", clientAppId(), topicNamePartition); - } - if (count.decrementAndGet() == 0) { - future.complete(null); - } - }); - } catch (Exception e) { - log.error("[{}] Failed to delete partition {}", clientAppId(), topicNamePartition, e); - future.completeExceptionally(e); + if (count.decrementAndGet() == 0) { + future.complete(null); + } + }); + } catch (Exception e) { + log.error("[{}] Failed to delete partition {}", clientAppId(), topicNamePartition, e); + future.completeExceptionally(e); + } } - } + }); } else { future.complete(null); } @@ -2654,7 +2685,7 @@ protected PersistentOfflineTopicStats internalGetBacklog(boolean authoritative) // note that we do not want to load the topic and hence skip authorization check try { namespaceResources().getPolicies(namespaceName); - } catch (org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException e) { + } catch (MetadataStoreException.NotFoundException e) { log.warn("[{}] Failed to get topic backlog {}: Namespace does not exist", clientAppId(), namespaceName); throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); } catch (Exception e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 0835eb0939497..76c295bbd4b0e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -971,6 +971,9 @@ public CompletableFuture deleteTopic(String topic, boolean forceDelete, bo } } + if (log.isDebugEnabled()) { + log.debug("Topic {} is not loaded, try to delete from metadata", topic); + } // Topic is not loaded, though we still might be able to delete from metadata TopicName tn = TopicName.get(topic); if (!tn.isPersistent()) { @@ -979,21 +982,76 @@ public CompletableFuture deleteTopic(String topic, boolean forceDelete, bo } CompletableFuture future = new CompletableFuture<>(); - managedLedgerFactory.asyncDelete(tn.getPersistenceNamingEncoding(), new DeleteLedgerCallback() { - @Override - public void deleteLedgerComplete(Object ctx) { - future.complete(null); - } - @Override - public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { - future.completeExceptionally(exception); + CompletableFuture deleteTopicAuthenticationFuture = new CompletableFuture<>(); + deleteTopicAuthenticationWithRetry(topic, deleteTopicAuthenticationFuture, 5); + deleteTopicAuthenticationFuture.whenComplete((v, ex) -> { + if (ex != null) { + future.completeExceptionally(ex); + return; } - }, null); + managedLedgerFactory.asyncDelete(tn.getPersistenceNamingEncoding(), new DeleteLedgerCallback() { + @Override + public void deleteLedgerComplete(Object ctx) { + future.complete(null); + } + + @Override + public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + }); + return future; } + public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture future, int count) { + if (count == 0) { + log.error("The number of retries has exhausted for topic {}", topic); + future.completeExceptionally(new MetadataStoreException("The number of retries has exhausted")); + return; + } + NamespaceName namespaceName = TopicName.get(topic).getNamespaceObject(); + // Check whether there are auth policies for the topic + pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(namespaceName).thenAccept(optPolicies -> { + if (!optPolicies.isPresent() || !optPolicies.get().auth_policies.getTopicAuthentication() + .containsKey(topic)) { + // if there is no auth policy for the topic, just complete and return + if (log.isDebugEnabled()) { + log.debug("Authentication policies not found for topic {}", topic); + } + future.complete(null); + return; + } + pulsar.getPulsarResources().getNamespaceResources() + .setPoliciesAsync(TopicName.get(topic).getNamespaceObject(), p -> { + p.auth_policies.getTopicAuthentication().remove(topic); + return p; + }).thenAccept(v -> { + log.info("Successfully delete authentication policies for topic {}", topic); + future.complete(null); + }).exceptionally(ex1 -> { + if (ex1.getCause() instanceof MetadataStoreException.BadVersionException) { + log.warn( + "Failed to delete authentication policies because of bad version. " + + "Retry to delete authentication policies for topic {}", + topic); + deleteTopicAuthenticationWithRetry(topic, future, count - 1); + } else { + log.error("Failed to delete authentication policies for topic {}", topic, ex1); + future.completeExceptionally(ex1); + } + return null; + }); + }).exceptionally(ex -> { + log.error("Failed to get policies for topic {}", topic, ex); + future.completeExceptionally(ex); + return null; + }); + } + private CompletableFuture> createNonPersistentTopic(String topic) { CompletableFuture> topicFuture = new CompletableFuture<>(); if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index f74060c93f4c8..962edb7e3444b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -145,7 +145,6 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; -import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -1122,10 +1121,12 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, // 2. We want to kick out everyone and forcefully delete the topic. // In this case, we shouldn't care if the usageCount is 0 or not, just proceed if (currentUsageCount() == 0 || (closeIfClientsConnected && !failIfHasSubscriptions)) { - CompletableFuture deleteSchemaFuture = - deleteSchema ? deleteSchema() : CompletableFuture.completedFuture(null); + CompletableFuture deleteTopicAuthenticationFuture = new CompletableFuture<>(); + brokerService.deleteTopicAuthenticationWithRetry(topic, deleteTopicAuthenticationFuture, 5); - deleteSchemaFuture.thenAccept(__ -> deleteTopicPolicies()) + deleteTopicAuthenticationFuture.thenCompose( + __ -> deleteSchema ? deleteSchema() : CompletableFuture.completedFuture(null)) + .thenAccept(__ -> deleteTopicPolicies()) .thenCompose(__ -> transactionBuffer.clearSnapshot()).whenComplete((v, ex) -> { if (ex != null) { log.error("[{}] Error deleting topic", topic, ex); @@ -1152,6 +1153,7 @@ public void deleteLedgerComplete(Object ctx) { brokerService.pulsar().getTopicPoliciesService() .clean(TopicName.get(topic)); + log.info("[{}] Topic deleted", topic); deleteFuture.complete(null); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 7e1c576170636..2c3f704ab58e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -782,9 +782,9 @@ public void namespaces() throws Exception { } // Force topic creation and namespace being loaded - producer = pulsarClient.newProducer(Schema.BYTES).topic("persistent://prop-xyz/use/ns2/my-topic").create(); + producer = pulsarClient.newProducer(Schema.BYTES).topic("persistent://prop-xyz/ns2/my-topic").create(); producer.close(); - admin.topics().delete("persistent://prop-xyz/use/ns2/my-topic"); + admin.topics().delete("persistent://prop-xyz/ns2/my-topic"); // both unload and delete should succeed for ns2 on other broker with a redirect // otheradmin.namespaces().unload("prop-xyz/use/ns2"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java index 6d76ce8cbfe8e..046b26846e2d3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java @@ -19,7 +19,10 @@ package org.apache.pulsar.client.api; import static org.mockito.Mockito.spy; - +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.Sets; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -27,20 +30,20 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; - import javax.ws.rs.InternalServerErrorException; - import org.apache.pulsar.broker.authentication.AuthenticationProviderBasic; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.auth.AuthenticationBasic; import org.apache.pulsar.client.impl.auth.AuthenticationTls; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.zookeeper.KeeperException.Code; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -49,8 +52,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.google.common.collect.Sets; - @Test(groups = "broker-api") public class AuthenticatedProducerConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(AuthenticatedProducerConsumerTest.class); @@ -87,9 +88,10 @@ protected void setup() throws Exception { superUserRoles.add("admin"); conf.setSuperUserRoles(superUserRoles); + conf.setBrokerClientTlsEnabled(true); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SERVER_KEY_FILE_PATH); + "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); @@ -337,4 +339,68 @@ public void testInternalServerExceptionOnLookup() throws Exception { mockZooKeeperGlobal.unsetAlwaysFail(); } + @Test + public void testDeleteAuthenticationPoliciesOfTopic() throws Exception { + Map authParams = new HashMap<>(); + authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); + authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + Authentication authTls = new AuthenticationTls(); + authTls.configure(authParams); + internalSetup(authTls); + + admin.clusters().createCluster("test", ClusterData.builder().build()); + admin.tenants().createTenant("p1", + new TenantInfoImpl(Collections.emptySet(), new HashSet<>(admin.clusters().getClusters()))); + admin.namespaces().createNamespace("p1/ns1"); + + // test for non-partitioned topic + String topic = "persistent://p1/ns1/topic"; + admin.topics().createNonPartitionedTopic(topic); + admin.topics().grantPermission(topic, "test-user", EnumSet.of(AuthAction.consume)); + + Awaitility.await().untilAsserted(() -> { + assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication().containsKey(topic)); + }); + + admin.topics().delete(topic); + + Awaitility.await().untilAsserted(() -> { + assertFalse(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication().containsKey(topic)); + }); + + // test for partitioned topic + String partitionedTopic = "persistent://p1/ns1/partitioned-topic"; + int numPartitions = 5; + + admin.topics().createPartitionedTopic(partitionedTopic, numPartitions); + admin.topics() + .grantPermission(partitionedTopic, "test-user", EnumSet.of(AuthAction.consume)); + + Awaitility.await().untilAsserted(() -> { + assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication().containsKey(partitionedTopic)); + for (int i = 0; i < numPartitions; i++) { + assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication() + .containsKey(TopicName.get(partitionedTopic).getPartition(i).toString())); + } + }); + + admin.topics().deletePartitionedTopic("persistent://p1/ns1/partitioned-topic"); + Awaitility.await().untilAsserted(() -> { + assertFalse(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication().containsKey(partitionedTopic)); + for (int i = 0; i < numPartitions; i++) { + assertFalse(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication() + .containsKey(TopicName.get(partitionedTopic).getPartition(i).toString())); + } + }); + + admin.namespaces().deleteNamespace("p1/ns1"); + admin.tenants().deleteTenant("p1"); + admin.clusters().deleteCluster("test"); + } } From b892196ef7b60d489f1fb27fb3c700fae51eb674 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 26 Oct 2021 08:24:33 +0800 Subject: [PATCH 074/823] [Pulsar SQL] Pulsar SQL support query big entry data (#12448) (cherry picked from commit a8f0788915393e438e5c38aa9497080fea19d733) --- conf/presto/catalog/pulsar.properties | 3 +- .../sql/presto/PulsarConnectorCache.java | 4 +- site2/docs/sql-deployment-configurations.md | 3 + .../integration/containers/BKContainer.java | 1 + .../integration/presto/TestBasicPresto.java | 66 ++++++++++++------- .../presto/TestPrestoQueryTieredStorage.java | 9 +-- .../integration/presto/TestPulsarSQLBase.java | 18 +++-- .../suites/PulsarSQLTestSuite.java | 52 +++++++++++++++ .../integration/topologies/PulsarCluster.java | 6 +- .../topologies/PulsarClusterSpec.java | 4 ++ 10 files changed, 122 insertions(+), 44 deletions(-) diff --git a/conf/presto/catalog/pulsar.properties b/conf/presto/catalog/pulsar.properties index 1f5a89a8b7cc7..e273b98dccc95 100644 --- a/conf/presto/catalog/pulsar.properties +++ b/conf/presto/catalog/pulsar.properties @@ -42,7 +42,8 @@ pulsar.max-split-queue-cache-size=-1 # to prevent erroneous rewriting pulsar.namespace-delimiter-rewrite-enable=false pulsar.rewrite-namespace-delimiter=/ - +# max size of one batch message (default value is 5MB) +# pulsar.max-message-size=5242880 ####### TIERED STORAGE OFFLOADER CONFIGS ####### diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java index bf823de7229f9..9a64c055d07ac 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java @@ -42,6 +42,7 @@ import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -109,7 +110,8 @@ private ManagedLedgerFactory initManagedLedgerFactory(PulsarConnectorConfig puls .setReadEntryTimeout(60) .setThrottleValue(pulsarConnectorConfig.getBookkeeperThrottleValue()) .setNumIOThreads(pulsarConnectorConfig.getBookkeeperNumIOThreads()) - .setNumWorkerThreads(pulsarConnectorConfig.getBookkeeperNumWorkerThreads()); + .setNumWorkerThreads(pulsarConnectorConfig.getBookkeeperNumWorkerThreads()) + .setNettyMaxFrameSizeBytes(pulsarConnectorConfig.getMaxMessageSize() + Commands.MESSAGE_SIZE_FRAME_PADDING); ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig(); managedLedgerFactoryConfig.setMaxCacheSize(pulsarConnectorConfig.getManagedLedgerCacheSizeMB()); diff --git a/site2/docs/sql-deployment-configurations.md b/site2/docs/sql-deployment-configurations.md index 1fe0353f07531..e5c402ebdc14d 100644 --- a/site2/docs/sql-deployment-configurations.md +++ b/site2/docs/sql-deployment-configurations.md @@ -24,6 +24,9 @@ pulsar.entry-read-batch-size=100 # default number of splits to use per query pulsar.target-num-splits=4 + +# max size of one batch message (default value is 5MB) +pulsar.max-message-size=5242880 ``` You can connect Presto to a Pulsar cluster with multiple hosts. To configure multiple hosts for brokers, add multiple URLs to `pulsar.web-service-url`. To configure multiple hosts for ZooKeeper, add multiple URIs to `pulsar.zookeeper-uri`. The following is an example. diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BKContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BKContainer.java index 36f17cd35cd0b..b294cac4e701e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BKContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BKContainer.java @@ -28,5 +28,6 @@ public class BKContainer extends PulsarContainer { public BKContainer(String clusterName, String hostName) { super( clusterName, hostName, hostName, "bin/run-bookie.sh", BOOKIE_PORT, INVALID_PORT); + tailContainerLog(); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java index 91043904076be..62f59c3f36f58 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java @@ -27,7 +27,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -35,6 +34,7 @@ import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; @@ -56,6 +56,7 @@ public class TestBasicPresto extends TestPulsarSQLBase { private void setupPresto() throws Exception { log.info("[TestBasicPresto] setupPresto..."); pulsarCluster.startPrestoWorker(); + initJdbcConnection(); } private void teardownPresto() { @@ -161,31 +162,26 @@ protected int prepareData(TopicName topicName, boolean useNsOffloadPolices, Schema schema, CompressionType compressionType) throws Exception { - @Cleanup - PulsarClient pulsarClient = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); if (schema.getSchemaInfo().getName().equals(Schema.BYTES.getSchemaInfo().getName())) { - prepareDataForBytesSchema(pulsarClient, topicName, isBatch, compressionType); + prepareDataForBytesSchema(topicName, isBatch, compressionType); } else if (schema.getSchemaInfo().getName().equals(Schema.BYTEBUFFER.getSchemaInfo().getName())) { - prepareDataForByteBufferSchema(pulsarClient, topicName, isBatch, compressionType); + prepareDataForByteBufferSchema(topicName, isBatch, compressionType); } else if (schema.getSchemaInfo().getType().equals(SchemaType.STRING)) { - prepareDataForStringSchema(pulsarClient, topicName, isBatch, compressionType); + prepareDataForStringSchema(topicName, isBatch, compressionType); } else if (schema.getSchemaInfo().getType().equals(SchemaType.JSON) || schema.getSchemaInfo().getType().equals(SchemaType.AVRO)) { - prepareDataForStructSchema(pulsarClient, topicName, isBatch, schema, compressionType); + prepareDataForStructSchema(topicName, isBatch, schema, compressionType); } else if (schema.getSchemaInfo().getType().equals(SchemaType.PROTOBUF_NATIVE)) { - prepareDataForProtobufNativeSchema(pulsarClient, topicName, isBatch, schema, compressionType); + prepareDataForProtobufNativeSchema(topicName, isBatch, schema, compressionType); } else if (schema.getSchemaInfo().getType().equals(SchemaType.KEY_VALUE)) { - prepareDataForKeyValueSchema(pulsarClient, topicName, schema, compressionType); + prepareDataForKeyValueSchema(topicName, schema, compressionType); } return NUM_OF_STOCKS; } - private void prepareDataForBytesSchema(PulsarClient pulsarClient, - TopicName topicName, + private void prepareDataForBytesSchema(TopicName topicName, boolean isBatch, CompressionType compressionType) throws PulsarClientException { @Cleanup @@ -201,8 +197,7 @@ private void prepareDataForBytesSchema(PulsarClient pulsarClient, producer.flush(); } - private void prepareDataForByteBufferSchema(PulsarClient pulsarClient, - TopicName topicName, + private void prepareDataForByteBufferSchema(TopicName topicName, boolean isBatch, CompressionType compressionType) throws PulsarClientException { @Cleanup @@ -218,8 +213,7 @@ private void prepareDataForByteBufferSchema(PulsarClient pulsarClient, producer.flush(); } - private void prepareDataForStringSchema(PulsarClient pulsarClient, - TopicName topicName, + private void prepareDataForStringSchema(TopicName topicName, boolean isBatch, CompressionType compressionType) throws PulsarClientException { @Cleanup @@ -235,8 +229,7 @@ private void prepareDataForStringSchema(PulsarClient pulsarClient, producer.flush(); } - private void prepareDataForStructSchema(PulsarClient pulsarClient, - TopicName topicName, + private void prepareDataForStructSchema(TopicName topicName, boolean isBatch, Schema schema, CompressionType compressionType) throws Exception { @@ -254,8 +247,7 @@ private void prepareDataForStructSchema(PulsarClient pulsarClient, producer.flush(); } - private void prepareDataForProtobufNativeSchema(PulsarClient pulsarClient, - TopicName topicName, + private void prepareDataForProtobufNativeSchema(TopicName topicName, boolean isBatch, Schema schema, CompressionType compressionType) throws Exception { @@ -274,8 +266,7 @@ private void prepareDataForProtobufNativeSchema(PulsarClient pulsarClient, producer.flush(); } - private void prepareDataForKeyValueSchema(PulsarClient pulsarClient, - TopicName topicName, + private void prepareDataForKeyValueSchema(TopicName topicName, Schema> schema, CompressionType compressionType) throws Exception { @Cleanup @@ -342,4 +333,33 @@ private void validateContentForKeyValueSchema(int messageNum, String[] contentAr } } + @Test(timeOut = 1000 * 30) + public void testQueueBigEntry() throws Exception { + String tableName = "big_data_" + randomName(5); + String topic = "persistent://public/default/" + tableName; + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .create(); + + // Make sure that the data length bigger than the default maxMessageSize + int dataLength = Commands.DEFAULT_MAX_MESSAGE_SIZE + 2 * 1024 * 1024; + Assert.assertTrue(dataLength < pulsarCluster.getSpec().maxMessageSize()); + byte[] data = new byte[dataLength]; + for (int i = 0; i < dataLength; i++) { + data[i] = 'a'; + } + + int messageCnt = 5; + log.info("start produce big entry data, data length: {}", dataLength); + for (int i = 0 ; i < messageCnt; ++i) { + producer.newMessage().value(data).send(); + } + + int count = selectCount("public/default", tableName); + Assert.assertEquals(count, messageCnt); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java index 881dbe4bd0d41..4c129fadce002 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java @@ -29,7 +29,6 @@ import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -40,8 +39,6 @@ import org.apache.pulsar.tests.integration.containers.S3Container; import org.testcontainers.shaded.org.apache.commons.lang.StringUtils; import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -89,6 +86,7 @@ private void setupExtraContainers() throws Exception { String offloadProperties = getOffloadProperties(BUCKET, null, ENDPOINT); pulsarCluster.startPrestoWorker(OFFLOAD_DRIVER, offloadProperties); pulsarCluster.startPrestoFollowWorkers(1, OFFLOAD_DRIVER, offloadProperties); + initJdbcConnection(); } private String getOffloadProperties(String bucket, String region, String endpoint) { @@ -136,11 +134,6 @@ protected int prepareData(TopicName topicName, Schema schema, CompressionType compressionType) throws Exception { @Cleanup - PulsarClient pulsarClient = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(Stock.class)) .topic(topicName.toString()) .subscriptionName("test") diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java index 026a32d1c8ec6..0626e3522e8c5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java @@ -21,8 +21,6 @@ import static org.assertj.core.api.Assertions.assertThat; import com.google.common.base.Stopwatch; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -196,9 +194,6 @@ private void validateData(TopicName topicName, int messageNum, Schema schema) th ); // test predicate pushdown - String url = String.format("jdbc:presto://%s", pulsarCluster.getPrestoWorkerContainer().getUrl()); - Connection connection = DriverManager.getConnection(url, "test", null); - String query = String.format("select * from pulsar" + ".\"%s\".\"%s\" order by __publish_time__", namespace, topic); log.info("Executing query: {}", query); @@ -267,11 +262,7 @@ private void validateData(TopicName topicName, int messageNum, Schema schema) th log.info("Executing query: result for topic {} returnedTimestamps size: {}", topic, returnedTimestamps.size()); assertThat(returnedTimestamps.size()).isEqualTo(0); - query = String.format("select count(*) from pulsar.\"%s\".\"%s\"", namespace, topic); - log.info("Executing query: {}", query); - res = connection.createStatement().executeQuery(query); - res.next(); - int count = res.getInt("_col0"); + int count = selectCount(namespace, topic); assertThat(count).isGreaterThan(messageNum - 2); } @@ -304,5 +295,12 @@ private static void printCurrent(ResultSet rs) throws SQLException { } + protected int selectCount(String namespace, String tableName) throws SQLException { + String query = String.format("select count(*) from pulsar.\"%s\".\"%s\"", namespace, tableName); + log.info("Executing count query: {}", query); + ResultSet res = connection.createStatement().executeQuery(query); + res.next(); + return res.getInt("_col0"); + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java index 9ed733570452f..762fff751b069 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java @@ -18,13 +18,22 @@ */ package org.apache.pulsar.tests.integration.suites; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.tests.integration.containers.BrokerContainer; import org.apache.pulsar.tests.integration.containers.S3Container; import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +/** + * Pulsar SQL test suite. + */ @Slf4j public abstract class PulsarSQLTestSuite extends PulsarTestSuite { @@ -33,11 +42,15 @@ public abstract class PulsarSQLTestSuite extends PulsarTestSuite { public static final String BUCKET = "pulsar-integtest"; public static final String ENDPOINT = "http://" + S3Container.NAME + ":9090"; + protected Connection connection = null; + protected PulsarClient pulsarClient = null; + @Override protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster(String clusterName, PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) { specBuilder.queryLastMessage(true); specBuilder.clusterName("pulsar-sql-test"); specBuilder.numBrokers(1); + specBuilder.maxMessageSize(2 * Commands.DEFAULT_MAX_MESSAGE_SIZE); return super.beforeSetupCluster(clusterName, specBuilder); } @@ -55,4 +68,43 @@ protected void beforeStartCluster() throws Exception { } } + @Override + public void setupCluster() throws Exception { + super.setupCluster(); + pulsarClient = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + } + + protected void initJdbcConnection() throws SQLException { + if (pulsarCluster.getPrestoWorkerContainer() == null) { + log.error("The presto work container isn't exist."); + return; + } + String url = String.format("jdbc:presto://%s", pulsarCluster.getPrestoWorkerContainer().getUrl()); + connection = DriverManager.getConnection(url, "test", null); + } + + @Override + public void tearDownCluster() throws Exception { + close(); + super.tearDownCluster(); + } + + protected void close() { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + log.error("Failed to close sql connection.", e); + } + } + if (pulsarClient != null) { + try { + pulsarClient.close(); + } catch (PulsarClientException e) { + log.error("Failed to close pulsar client.", e); + } + } + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index 2191799b68622..7117cbb1f9e36 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -80,6 +80,7 @@ public static PulsarCluster forSpec(PulsarClusterSpec spec, CSContainer csContai return new PulsarCluster(spec, csContainer, true); } + @Getter private final PulsarClusterSpec spec; @Getter @@ -157,6 +158,7 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s .withEnv("journalMaxGroupWaitMSec", "0") .withEnv("clusterName", clusterName) .withEnv("diskUsageThreshold", "0.99") + .withEnv("nettyMaxFrameSizeBytes", "" + spec.maxMessageSize) ) ); @@ -173,7 +175,8 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s .withEnv("brokerServiceCompactionMonitorIntervalInSeconds", "1") // used in s3 tests .withEnv("AWS_ACCESS_KEY_ID", "accesskey") - .withEnv("AWS_SECRET_KEY", "secretkey"); + .withEnv("AWS_SECRET_KEY", "secretkey") + .withEnv("maxMessageSize", "" + spec.maxMessageSize); if (spec.queryLastMessage) { brokerContainer.withEnv("bookkeeperExplicitLacIntervalInMills", "10"); brokerContainer.withEnv("bookkeeperUseV2WireProtocol", "false"); @@ -431,6 +434,7 @@ private PrestoWorkerContainer buildPrestoWorkerContainer(String hostName, boolea .withEnv("zookeeperServers", ZKContainer.NAME + ":" + ZKContainer.ZK_PORT) .withEnv("pulsar.zookeeper-uri", ZKContainer.NAME + ":" + ZKContainer.ZK_PORT) .withEnv("pulsar.web-service-url", "http://pulsar-broker-0:8080") + .withEnv("SQL_PREFIX_pulsar.max-message-size", "" + spec.maxMessageSize) .withClasspathResourceMapping( resourcePath, "/pulsar/conf/presto/config.properties", BindMode.READ_WRITE); if (spec.queryLastMessage) { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java index 9dcfcfb725e79..eed604205bdce 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java @@ -29,6 +29,7 @@ import lombok.Singular; import lombok.experimental.Accessors; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.tests.integration.containers.PulsarContainer; import org.testcontainers.containers.GenericContainer; @@ -152,4 +153,7 @@ public class PulsarClusterSpec { * Specify mount files. */ Map brokerMountFiles; + + @Default + int maxMessageSize = Commands.DEFAULT_MAX_MESSAGE_SIZE; } From d07491b9aec7cc751b2c82c22fdb7785442c2927 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 25 Oct 2021 21:15:57 -0700 Subject: [PATCH 075/823] [C++] Fixed connection read error logging (#12492) (cherry picked from commit 6edcaa79c560b3574ae65049333b68b112f83192) --- pulsar-client-cpp/lib/ClientConnection.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 6f947312f01e6..3f82e35031ab2 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -577,9 +577,9 @@ void ClientConnection::handleRead(const boost::system::error_code& err, size_t b if (err || bytesTransferred == 0) { if (err) { if (err == boost::asio::error::operation_aborted) { - LOG_DEBUG(cnxString_ << "Read failed: " << err.message()); + LOG_DEBUG(cnxString_ << "Read operation was canceled: " << err.message()); } else { - LOG_ERROR(cnxString_ << "Read operation was cancelled"); + LOG_ERROR(cnxString_ << "Read operation failed: " << err.message()); } } // else: bytesTransferred == 0, which means server has closed the connection close(); From 1c3d77da2f4c2594975b01f5a65819c3f6aa40c9 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 25 Oct 2021 08:41:05 +0800 Subject: [PATCH 076/823] fix the race of delete subscription and delete topic (#12240) (cherry picked from commit 9137e6b1e561cd1bfc4540e651aa33e06934acf8) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 962edb7e3444b..ffe4bc4597704 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1021,6 +1021,11 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { log.debug("[{}][{}] Error deleting cursor for subscription", topic, subscriptionName, exception); } + if (exception instanceof ManagedLedgerException.ManagedLedgerNotFoundException) { + unsubscribeFuture.complete(null); + lastActive = System.nanoTime(); + return; + } unsubscribeFuture.completeExceptionally(new PersistenceException(exception)); } }, null); From 509a59d5a76df01e18d0382712ea7029fe01f5ae Mon Sep 17 00:00:00 2001 From: YANGLiiN Date: Mon, 25 Oct 2021 12:17:57 +0800 Subject: [PATCH 077/823] Fix the retry topic's `REAL_TOPIC` & `ORIGIN_MESSAGE_ID` property should not be modified once it has been written. (#12451) ### Motivation when reconsumer the message with the configuration maxRedeliveryCount > 1 ,eg deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(3).build()): then when consumer the same message by the third time (RECONSUMETIMES=2) the REAL_TOPIC and the ORIGIN_MESSAGE_ID changed as the second message. but, this should not changed once the REAL_TOPIC ORIGIN_MESSAGE_ID property was written . ``` 2021-10-21T15:03:12,771+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 4:0:-1:0 {} 2021-10-21T15:03:12,909+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 3:1:-1 {REAL_TOPIC=persistent://my-property/my-ns/retry-topic, ORIGIN_MESSAGE_IDY_TIME=4:1:-1:0, DELAY_TIME=1000, RECONSUMETIMES=1} 2021-10-21T15:03:12,965+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 3:10:-1 {REAL_TOPIC=persistent://my-property/my-ns/retry-topic-my-subscription-RETRY, ORIGIN_MESSAGE_IDY_TIME=3:0:-1, DELAY_TIME=1000, RECONSUMETIMES=2} 2021-10-21T15:03:13,026+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 3:20:-1 {REAL_TOPIC=persistent://my-property/my-ns/retry-topic-my-subscription-RETRY, ORIGIN_MESSAGE_IDY_TIME=3:10:-1, DELAY_TIME=1000, RECONSUMETIMES=3} ``` Expected Results (REAL_TOPIC ORIGIN_MESSAGE_ID property equals the frist origin message) ``` 2021-10-21T15:27:01,390+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 4:0:-1:0 {} 2021-10-21T15:27:01,479+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 3:0:-1 {REAL_TOPIC=persistent://my-property/my-ns/retry-topic, ORIGIN_MESSAGE_IDY_TIME=4:0:-1:0, DELAY_TIME=1000, RECONSUMETIMES=1} 2021-10-21T15:27:01,547+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 3:10:-1 {REAL_TOPIC=persistent://my-property/my-ns/retry-topic, ORIGIN_MESSAGE_IDY_TIME=4:0:-1:0, DELAY_TIME=1000, RECONSUMETIMES=2} 2021-10-21T15:27:01,603+0800 [main] ERROR org.apache.pulsar.client.api.RetryTopicTest - consumer received message : 3:20:-1 {REAL_TOPIC=persistent://my-property/my-ns/retry-topic, ORIGIN_MESSAGE_IDY_TIME=4:0:-1:0, DELAY_TIME=1000, RECONSUMETIMES=3} ``` ### Modifications https://github.com/apache/pulsar/blob/master/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java#L652-L653 propertiesMap.put() --> propertiesMap.putIfAbsent() add one testcase to verify the REAL_TOPIC ORIGIN_MESSAGE_ID property (cherry picked from commit 85178830c9bf7462729f6f5d8d9285dee5d2d93d) --- .../pulsar/client/api/RetryTopicTest.java | 105 +++++++++++++++++- .../pulsar/client/impl/ConsumerImpl.java | 4 +- 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java index fc84a62dac1cd..48d20c677c181 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java @@ -18,18 +18,19 @@ */ package org.apache.pulsar.client.api; -import lombok.Cleanup; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import com.google.common.collect.Sets; +import java.util.Set; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.client.util.RetryMessageUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.concurrent.TimeUnit; - -import static org.testng.Assert.assertNull; - @Test(groups = "broker-api") public class RetryTopicTest extends ProducerConsumerBase { @@ -119,6 +120,100 @@ public void testRetryTopic() throws Exception { checkConsumer.close(); } + @Test + public void testRetryTopicProperties() throws Exception { + final String topic = "persistent://my-property/my-ns/retry-topic"; + + final int maxRedeliveryCount = 3; + + final int sendMessages = 10; + + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .enableRetry(true) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0); + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) + .topic("persistent://my-property/my-ns/retry-topic-my-subscription-DLQ") + .subscriptionName("my-subscription") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .create(); + + Set originMessageIds = Sets.newHashSet(); + for (int i = 0; i < sendMessages; i++) { + MessageId msgId = producer.send(String.format("Hello Pulsar [%d]", i).getBytes()); + originMessageIds.add(msgId.toString()); + } + + producer.close(); + + int totalReceived = 0; + Set retryMessageIds = Sets.newHashSet(); + do { + Message message = consumer.receive(); + log.info("consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + // retry message + if (message.hasProperty(RetryMessageUtil.SYSTEM_PROPERTY_RECONSUMETIMES)) { + // check the REAL_TOPIC property + assertEquals(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC), topic); + retryMessageIds.add(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID)); + } + consumer.reconsumeLater(message, 1, TimeUnit.SECONDS); + totalReceived++; + } while (totalReceived < sendMessages * (maxRedeliveryCount + 1)); + + // check the REAL_TOPIC property + assertEquals(retryMessageIds, originMessageIds); + + int totalInDeadLetter = 0; + Set deadLetterMessageIds = Sets.newHashSet(); + do { + Message message = deadLetterConsumer.receive(); + log.info("dead letter consumer received message : {} {}", message.getMessageId(), + new String(message.getData())); + // dead letter message + if (message.hasProperty(RetryMessageUtil.SYSTEM_PROPERTY_RECONSUMETIMES)) { + // check the REAL_TOPIC property + assertEquals(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC), topic); + deadLetterMessageIds.add(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID)); + } + deadLetterConsumer.acknowledge(message); + totalInDeadLetter++; + } while (totalInDeadLetter < sendMessages); + + assertEquals(deadLetterMessageIds, originMessageIds); + + deadLetterConsumer.close(); + consumer.close(); + + Consumer checkConsumer = this.pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Message checkMessage = checkConsumer.receive(3, TimeUnit.SECONDS); + if (checkMessage != null) { + log.info("check consumer received message : {} {}", checkMessage.getMessageId(), + new String(checkMessage.getData())); + } + assertNull(checkMessage); + + checkConsumer.close(); + } + //Issue 9327: do compatibility check in case of the default retry and dead letter topic name changed @Test public void testRetryTopicNameForCompatibility () throws Exception { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 72167b226d70f..d4a6e881d3d2b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -649,8 +649,8 @@ private SortedMap getPropertiesMap(Message message, String or if (message.getProperties() != null) { propertiesMap.putAll(message.getProperties()); } - propertiesMap.put(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC, originTopicNameStr); - propertiesMap.put(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID, originMessageIdStr); + propertiesMap.putIfAbsent(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC, originTopicNameStr); + propertiesMap.putIfAbsent(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID, originMessageIdStr); return propertiesMap; } From 990be1ca9b9147dd8cc4a4061e22eb8f2c363ad9 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Mon, 25 Oct 2021 20:45:20 +0800 Subject: [PATCH 078/823] Remove unused ConsumerImpl.isTxnMessage (#12472) (cherry picked from commit 464a9cdc21609189771f1d95ac211b69eca1454c) --- .../main/java/org/apache/pulsar/client/impl/ConsumerImpl.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index d4a6e881d3d2b..1b8d19cd7f66e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1239,10 +1239,6 @@ private void tryTriggerListener() { } } - private boolean isTxnMessage(MessageMetadata messageMetadata) { - return messageMetadata.hasTxnidMostBits() && messageMetadata.hasTxnidLeastBits(); - } - private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, MessageIdData messageId, ClientCnx cnx) { From 668d52ea01cecf14387e2962f42e84944be70940 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 25 Oct 2021 12:21:13 +0800 Subject: [PATCH 079/823] Remove redundant null check for getInternalListener (#12474) ### Motivation `ServiceConfigurationUtils#getInternalListener` never returns null. The null check is redundant. Otherwise, the null check should also be performed in `CompactorTool#main`. ### Modifications Remove the null check for `getInternalListener` and note the returned value is non-null in method description. (cherry picked from commit dda8cc7b2be8e130495edb0c84376e7611aea3f6) --- .../org/apache/pulsar/broker/ServiceConfigurationUtils.java | 2 +- .../src/main/java/org/apache/pulsar/broker/PulsarService.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java index 8401723ed7a9f..c47b8ca79812d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java @@ -83,7 +83,7 @@ public static String getAppliedAdvertisedAddress(ServiceConfiguration configurat /** * Gets the internal advertised listener for broker-to-broker communication. - * @return an advertised listener + * @return a non-null advertised listener */ public static AdvertisedListener getInternalListener(ServiceConfiguration config) { Map result = MultipleListenerValidator diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index c897f1585a2d3..6b59862ac8cd9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1411,7 +1411,7 @@ public TransactionBufferClient getTransactionBufferClient() { */ protected String brokerUrl(ServiceConfiguration config) { AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config); - return internalListener != null && internalListener.getBrokerServiceUrl() != null + return internalListener.getBrokerServiceUrl() != null ? internalListener.getBrokerServiceUrl().toString() : null; } @@ -1424,7 +1424,7 @@ public static String brokerUrl(String host, int port) { */ public String brokerUrlTls(ServiceConfiguration config) { AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config); - return internalListener != null && internalListener.getBrokerServiceUrlTls() != null + return internalListener.getBrokerServiceUrlTls() != null ? internalListener.getBrokerServiceUrlTls().toString() : null; } From ad4e665e4fb64fda43d67424d9936e27b5bf1736 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Sun, 24 Oct 2021 12:24:06 +0800 Subject: [PATCH 080/823] optimize SecurityUtility code flow (#12431) (cherry picked from commit bc339bb6d72c3fc0dea200ecb1cc035943873076) --- .../apache/pulsar/common/util/SecurityUtility.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index ec55f5349110c..faefc8adcd78e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -422,12 +422,12 @@ public static X509Certificate[] loadCertificatesFromPemStream(InputStream inStre } public static PrivateKey loadPrivateKeyFromPemFile(String keyFilePath) throws KeyManagementException { - PrivateKey privateKey = null; - if (keyFilePath == null || keyFilePath.isEmpty()) { - return privateKey; + return null; } + PrivateKey privateKey; + try (FileInputStream input = new FileInputStream(keyFilePath)) { privateKey = loadPrivateKeyFromPemStream(input); } catch (IOException e) { @@ -438,12 +438,12 @@ public static PrivateKey loadPrivateKeyFromPemFile(String keyFilePath) throws Ke } public static PrivateKey loadPrivateKeyFromPemStream(InputStream inStream) throws KeyManagementException { - PrivateKey privateKey = null; - if (inStream == null) { - return privateKey; + return null; } + PrivateKey privateKey; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, StandardCharsets.UTF_8))) { if (inStream.markSupported()) { inStream.reset(); From e8a629c06d4f23c33612dbc4b994fd4f7c6a25f4 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Sun, 24 Oct 2021 12:41:55 +0800 Subject: [PATCH 081/823] broker resource group test optimize fail msg (#12438) (cherry picked from commit 86f40c1dc127aa9af928c544ff4062666c621ef4) --- ....java => RGUsageMTAggrWaitForAllMsgsTest.java} | 12 +++++------- .../ResourceGroupUsageAggregationTest.java | 15 +++++++-------- .../ResourceUsageTransportManagerTest.java | 1 - 3 files changed, 12 insertions(+), 16 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/{RGUsageMTAggrWaitForAllMesgsTest.java => RGUsageMTAggrWaitForAllMsgsTest.java} (99%) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMesgsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java similarity index 99% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMesgsTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java index 91260098a3c31..fda8693dd8478 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMesgsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java @@ -20,7 +20,6 @@ import com.google.common.collect.Sets; import io.prometheus.client.Summary; -import java.util.Collections; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupUsageStatsType; @@ -59,7 +58,7 @@ // The tenants and namespaces in those topics are associated with a set of resource-groups (RGs). // After sending/receiving all the messages, traffic usage statistics, and Prometheus-metrics // are verified on the RGs. -public class RGUsageMTAggrWaitForAllMesgsTest extends ProducerConsumerBase { +public class RGUsageMTAggrWaitForAllMsgsTest extends ProducerConsumerBase { @BeforeClass @Override protected void setup() throws Exception { @@ -350,13 +349,12 @@ private boolean tenantRGEqualsNamespaceRG(String[] topicStrings) throws PulsarCl } } if ((numEqualRGs + numUnEqualRGs != numTopics) || (numEqualRGs > 0 && numUnEqualRGs > 0)) { - String errMesg = String.format("Found {} topics with equal RGs and {} with unequal, on {} topics", + String errMesg = String.format("Found %s topics with equal RGs and %s with unequal, on %s topics", numEqualRGs, numUnEqualRGs, numTopics); throw new PulsarClientException(errMesg); - } else if (numEqualRGs == numTopics) { - return true; + } else { + return numEqualRGs == numTopics; } - return false; } private void registerTenantsAndNamespaces(String[] topicStrings) throws Exception { @@ -788,7 +786,7 @@ private void verifyRGMetrics(String[] topicStrings, Assert.assertNotEquals(ninetethPercentileValue, 0); } - private static final Logger log = LoggerFactory.getLogger(RGUsageMTAggrWaitForAllMesgsTest.class); + private static final Logger log = LoggerFactory.getLogger(RGUsageMTAggrWaitForAllMsgsTest.class); // Empirically, there appears to be a 45-byte overhead for metadata, imposed by Pulsar runtime. private static final int PER_MESSAGE_METADATA_OHEAD = 45; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java index a89d759e7ab29..08c1f6163f253 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java @@ -130,8 +130,8 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { .subscriptionType(SubscriptionType.Shared) .subscribe(); } catch (PulsarClientException p) { - final String errMesg = String.format("Got exception while building consumer: ex={}", p.getMessage()); - Assert.assertTrue(false, errMesg); + final String errMsg = String.format("Got exception while building consumer: ex=%s", p.getMessage()); + Assert.fail(errMsg); } final TopicName myTopic = TopicName.get(topicString); @@ -146,16 +146,15 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { int recvdNumBytes = 0; int recvdNumMsgs = 0; for (int ix = 0; ix < NumMessagesToSend; ix++) { - MessageId prodMesgId = null; byte[] mesg; try { - mesg = String.format("Hi, ix={}", ix).getBytes(); + mesg = String.format("Hi, ix=%s", ix).getBytes(); producer.send(mesg); sentNumBytes += mesg.length; sentNumMsgs++; } catch (PulsarClientException p) { - final String errMesg = String.format("Got exception while sending {}-th time: ex={}", ix, p.getMessage()); - Assert.assertTrue(false, errMesg); + final String errMsg = String.format("Got exception while sending %s-th time: ex=%s", ix, p.getMessage()); + Assert.fail(errMsg); } } producer.close(); @@ -169,9 +168,9 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { message = consumer.receive(); recvdNumBytes += message.getValue().length; } catch (PulsarClientException p) { - final String errMesg = String.format("Got exception in while receiving {}-th mesg at consumer: ex={}", + final String errMesg = String.format("Got exception in while receiving %s-th mesg at consumer: ex=%s", recvdNumMsgs, p.getMessage()); - Assert.assertTrue(false, errMesg); + Assert.fail(errMesg); } // log.info("consumer received message : {} {}", message.getMessageId(), new String(message.getData())); recvdNumMsgs++; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java index 7332307a612d2..e8182d77a4900 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java @@ -26,7 +26,6 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; From 4269f768c73dc172e6a664b019a45ea9bf3afd2c Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 21 Oct 2021 09:52:47 +0800 Subject: [PATCH 082/823] Fix compactor skips data from last compacted Ledger (#12429) ## Motivation The PR is fixing the compacted data lost during the data compaction. We see a few events deletion but the compacted events obviously dropped a lot. ![image](https://user-images.githubusercontent.com/12592133/138008777-00eb7c0b-358e-4291-bfd4-f4b27cbedbf4.png) After investigating more details about the issue, only the first read operation reads the data from the compacted ledger, since the second read operation, the broker start read data from the original topic. ``` 2021-10-19T23:09:30,021+0800 [broker-topic-workers-OrderedScheduler-7-0] INFO org.apache.pulsar.compaction.CompactedTopicImpl - =====[public/default/persistent/c499d42c-75d7-48d1-9225-2e724c0e1d83] Read from compacted Ledger = cursor position: -1:-1, Horizon: 16:-1, isFirstRead: true 2021-10-19T23:09:30,049+0800 [broker-topic-workers-OrderedScheduler-7-0] INFO org.apache.pulsar.compaction.CompactedTopicImpl - =====[public/default/persistent/c499d42c-75d7-48d1-9225-2e724c0e1d83] Read from original Ledger = cursor position: 16:0, Horizon: 16:-1, isFirstRead: false ``` ## Modifications The compaction task depends on the last snapshot and the incremental entries to build the new snapshot. So for the compaction cursor, we need to force seek the read position to ensure the compactor can read the complete last snapshot because the compactor will read the data before the compaction cursor mark delete position. ## Verifying this change New test added for checking the compacted data will not lost. (cherry picked from commit 1830f90d08acc079c6ee5a5ec05751ab4cbee490) --- .../bookkeeper/mledger/ManagedCursor.java | 6 ++- .../mledger/impl/ManagedCursorImpl.java | 6 +-- .../impl/ManagedCursorContainerTest.java | 2 +- .../pulsar/compaction/CompactedTopicImpl.java | 9 +++- .../pulsar/compaction/CompactedTopicTest.java | 45 +++++++++++++++++++ 5 files changed, 61 insertions(+), 7 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 4af64550fe53b..72ee1a1f9c3ea 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -425,7 +425,11 @@ void markDelete(Position position, Map properties) * @param newReadPosition * the position where to move the cursor */ - void seek(Position newReadPosition); + default void seek(Position newReadPosition) { + seek(newReadPosition, false); + } + + void seek(Position newReadPosition, boolean force); /** * Clear the cursor backlog. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 235736da760dc..dd715ff7588d3 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -2163,18 +2163,16 @@ public void rewind() { } @Override - public void seek(Position newReadPositionInt) { + public void seek(Position newReadPositionInt, boolean force) { checkArgument(newReadPositionInt instanceof PositionImpl); PositionImpl newReadPosition = (PositionImpl) newReadPositionInt; lock.writeLock().lock(); try { - if (newReadPosition.compareTo(markDeletePosition) <= 0) { + if (!force && newReadPosition.compareTo(markDeletePosition) <= 0) { // Make sure the newReadPosition comes after the mark delete position newReadPosition = ledger.getNextValidPosition(markDeletePosition); } - - PositionImpl oldReadPosition = readPosition; readPosition = newReadPosition; } finally { lock.writeLock().unlock(); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index 6b9c0094a127e..57e1964a2c332 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -175,7 +175,7 @@ public void rewind() { } @Override - public void seek(Position newReadPosition) { + public void seek(Position newReadPosition, boolean force) { } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index bca5682d3efbd..6894bcdc67a5e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.compaction; +import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; @@ -124,7 +125,13 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, return readEntries(context.ledger, startPoint, endPoint) .thenAccept((entries) -> { Entry lastEntry = entries.get(entries.size() - 1); - cursor.seek(lastEntry.getPosition().getNext()); + // The compaction task depends on the last snapshot and the incremental + // entries to build the new snapshot. So for the compaction cursor, we + // need to force seek the read position to ensure the compactor can read + // the complete last snapshot because of the compactor will read the data + // before the compaction cursor mark delete position + cursor.seek(lastEntry.getPosition().getNext(), + cursor.getName().equals(COMPACTION_SUBSCRIPTION)); callback.readEntriesComplete(entries, consumer); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 0083fc779b701..e64d805860238 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.compaction; +import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.google.common.collect.Sets; @@ -449,4 +450,48 @@ public void testLastMessageIdForCompactedLedger() throws Exception { reader.readNext(); Assert.assertFalse(reader.hasMessageAvailable()); } + + @Test + public void testDoNotLossTheLastCompactedLedgerData() throws Exception { + String topic = "persistent://my-property/use/my-ns/testDoNotLossTheLastCompactedLedgerData-" + + UUID.randomUUID(); + final int numMessages = 2000; + final int keys = 200; + final String msg = "Test"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .blockIfQueueFull(true) + .maxPendingMessages(numMessages) + .enableBatching(false) + .create(); + CompletableFuture lastMessage = null; + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i % keys + "").value(msg).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, keys); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + }); + admin.topics().unload(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertEquals(stats.ledgers.size(), 1); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + }); + admin.topics().unload(topic); + // Send one more key to and then to trigger the compaction + producer.newMessage().key(keys + "").value(msg).send(); + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertEquals(stats.compactedLedger.entries, keys + 1); + }); + } } From 4388f16df43d161570129cbc0f30912205442b0d Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sun, 24 Oct 2021 23:11:15 +0800 Subject: [PATCH 083/823] #12429 only fixed the compactor skips data issue, but the normal reader/consumer (#12464) also skips data while enabled read compacted data and read from the earliest position. (cherry picked from commit dd90657c33aa71ca86c9988513d92bf79aa68335) --- .../pulsar/compaction/CompactedTopicImpl.java | 4 +--- .../pulsar/compaction/CompactedTopicTest.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index 6894bcdc67a5e..457754070c8dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.compaction; -import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; @@ -130,8 +129,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, // need to force seek the read position to ensure the compactor can read // the complete last snapshot because of the compactor will read the data // before the compaction cursor mark delete position - cursor.seek(lastEntry.getPosition().getNext(), - cursor.getName().equals(COMPACTION_SUBSCRIPTION)); + cursor.seek(lastEntry.getPosition().getNext(), true); callback.readEntriesComplete(entries, consumer); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index e64d805860238..608d99b8307ef 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -493,5 +493,20 @@ public void testDoNotLossTheLastCompactedLedgerData() throws Exception { PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); Assert.assertEquals(stats.compactedLedger.entries, keys + 1); }); + + // Make sure the reader can get all data from the compacted ledger and original ledger. + Reader reader = pulsarClient.newReader(Schema.STRING) + .topic(topic) + .startMessageId(MessageId.earliest) + .readCompacted(true) + .create(); + int received = 0; + while (reader.hasMessageAvailable()) { + reader.readNext(); + received++; + } + Assert.assertEquals(received, keys + 1); + reader.close(); + producer.close(); } } From 4d6eb22f949b0658ca2e3ddc6e25e97a6f1eb364 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Sun, 24 Oct 2021 12:32:29 +0800 Subject: [PATCH 084/823] fix a typo in UnAckedMessageTracker (#12467) (cherry picked from commit 52336c8d0253ca16d51105abc180dcc81e65a5e3) --- .../pulsar/client/impl/NegativeAcksTracker.java | 4 ++-- .../pulsar/client/impl/UnAckedMessageTracker.java | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java index b5e6fd1325163..a0620093ffb96 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java @@ -29,7 +29,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; -import static org.apache.pulsar.client.impl.UnAckedMessageTracker.addChunkedMessageIdsAndRemoveFromSequnceMap; +import static org.apache.pulsar.client.impl.UnAckedMessageTracker.addChunkedMessageIdsAndRemoveFromSequenceMap; class NegativeAcksTracker implements Closeable { @@ -64,7 +64,7 @@ private synchronized void triggerRedelivery(Timeout t) { long now = System.nanoTime(); nackedMessages.forEach((msgId, timestamp) -> { if (timestamp < now) { - addChunkedMessageIdsAndRemoveFromSequnceMap(msgId, messagesToRedeliver, this.consumer); + addChunkedMessageIdsAndRemoveFromSequenceMap(msgId, messagesToRedeliver, this.consumer); messagesToRedeliver.add(msgId); } }); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java index db616f20450a4..ff07156972f62 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java @@ -30,6 +30,7 @@ import java.io.Closeable; import java.util.ArrayDeque; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.Set; @@ -130,7 +131,7 @@ public void run(Timeout t) throws Exception { if (!headPartition.isEmpty()) { log.warn("[{}] {} messages have timed-out", consumerBase, headPartition.size()); headPartition.forEach(messageId -> { - addChunkedMessageIdsAndRemoveFromSequnceMap(messageId, messageIds, consumerBase); + addChunkedMessageIdsAndRemoveFromSequenceMap(messageId, messageIds, consumerBase); messageIds.add(messageId); messageIdPartitionMap.remove(messageId); }); @@ -150,14 +151,12 @@ public void run(Timeout t) throws Exception { }, this.tickDurationInMs, TimeUnit.MILLISECONDS); } - public static void addChunkedMessageIdsAndRemoveFromSequnceMap(MessageId messageId, Set messageIds, - ConsumerBase consumerBase) { + public static void addChunkedMessageIdsAndRemoveFromSequenceMap(MessageId messageId, Set messageIds, + ConsumerBase consumerBase) { if (messageId instanceof MessageIdImpl) { MessageIdImpl[] chunkedMsgIds = consumerBase.unAckedChunkedMessageIdSequenceMap.get((MessageIdImpl) messageId); if (chunkedMsgIds != null && chunkedMsgIds.length > 0) { - for (MessageIdImpl msgId : chunkedMsgIds) { - messageIds.add(msgId); - } + Collections.addAll(messageIds, chunkedMsgIds); } consumerBase.unAckedChunkedMessageIdSequenceMap.remove((MessageIdImpl) messageId); } From bc50bac64ab53e4c2b06932b459fb9aeaefe5924 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Sun, 24 Oct 2021 12:28:29 +0800 Subject: [PATCH 085/823] Optimize the code: remove extra spaces (#12470) (cherry picked from commit 95e6a0edd74d51b2275ae24083276259d7a92f91) --- .../main/java/org/apache/pulsar/client/impl/ConsumerBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 7d6474b10b50f..9312df5368255 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -717,13 +717,13 @@ protected void onNegativeAcksSend(Set messageIds) { protected void onAckTimeoutSend(Set messageIds) { if (interceptors != null) { - interceptors. onAckTimeoutSend(this, messageIds); + interceptors.onAckTimeoutSend(this, messageIds); } } protected void onPartitionsChange(String topicName, int partitions) { if (interceptors != null) { - interceptors. onPartitionsChange(topicName, partitions); + interceptors.onPartitionsChange(topicName, partitions); } } From 86681ae42cc1e43eef8ede9aa6c33e4fd22fed00 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Fri, 22 Oct 2021 16:23:42 +0800 Subject: [PATCH 086/823] Future completed twice in the method of impl.MLPendingAckStore#closeAsync (#12362) (cherry picked from commit 10371ee870d26a079d7d744154e7c4f67ac09fae) --- .../pendingack/impl/MLPendingAckStore.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index b60f4ba21dc29..fb88878d8c826 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -120,12 +120,23 @@ public CompletableFuture closeAsync() { cursor.asyncClose(new AsyncCallbacks.CloseCallback() { @Override public void closeComplete(Object ctx) { - try { - managedLedger.close(); - } catch (Exception e) { - completableFuture.completeExceptionally(e); - } - completableFuture.complete(null); + managedLedger.asyncClose(new AsyncCallbacks.CloseCallback() { + + @Override + public void closeComplete(Object ctx) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] MLPendingAckStore closed successfully！", managedLedger.getName(), ctx); + } + completableFuture.complete(null); + } + + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}][{}] MLPendingAckStore closed failed,exception={}", managedLedger.getName(), + ctx, exception); + completableFuture.completeExceptionally(exception); + } + }, ctx); } @Override From 5e64b51b31e9d4b50907194b75dacca8de584f07 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Fri, 22 Oct 2021 11:15:55 +0800 Subject: [PATCH 087/823] Fix the read performance issue in the offload readAsync (#12443) --- *Motivation* In the #12123, I add the seek operation at the readAsync method. It makes sure the data stream always seek to the first entry position to read and will not introduce EOF exception. But in the offload index entry, it groups a set of entries into a range, the seek operation will seek the posistion to the first entry in the range. That will introduce a performance issue because every read opeartion will read from the first entry in the range until it find the actual first read entry. But if we remove the seek operation, that will cause a EOF exception from the readAsync method. This PR adds a limitation of the seek opeartion. *Modifications* Add available method in the backedInputStream to get know how many bytes we can read from the stream. (cherry picked from commit b4d05ac1bf5cddb613d93806e505ef8788e1acc0) --- .../impl/BlobStoreBackedInputStreamImpl.java | 5 ++++ .../impl/BlobStoreBackedReadHandleImpl.java | 26 ++++++++++++------- .../BlobStoreBackedInputStreamTest.java | 23 ++++++++++++++++ .../BlobStoreManagedLedgerOffloaderTest.java | 19 ++++++++++++++ 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java index 6a204d56de951..e3fc68ab7e218 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java @@ -141,4 +141,9 @@ public void seekForward(long position) throws IOException { public void close() { buffer.release(); } + + @Override + public int available() throws IOException { + return (int)(objectLen - cursor) + buffer.readableBytes(); + } } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java index 2bf380d8c4126..98fdff442800f 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java @@ -105,6 +105,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr CompletableFuture promise = new CompletableFuture<>(); executor.submit(() -> { List entries = new ArrayList(); + boolean seeked = false; try { if (firstEntry > lastEntry || firstEntry < 0 @@ -115,14 +116,13 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr long entriesToRead = (lastEntry - firstEntry) + 1; long nextExpectedId = firstEntry; - // seek the position to the first entry position, otherwise we will get the unexpected entry ID when doing - // the first read, that would cause read an unexpected entry id which is out of range between firstEntry - // and lastEntry - // for example, when we get 1-10 entries at first, then the next request is get 2-9, the following code - // will read the entry id from the stream and that is not the correct entry id, so it will seek to the - // correct position then read the stream as normal. But the entry id may exceed the last entry id, that - // will cause we are hardly to know the edge of the request range. - inputStream.seek(index.getIndexEntryForEntry(firstEntry).getDataOffset()); + // checking the data stream has enough data to read to avoid throw EOF exception when reading data. + // 12 bytes represent the stream have the length and entryID to read. + if (dataStream.available() < 12) { + log.warn("There hasn't enough data to read, current available data has {} bytes," + + " seek to the first entry {} to avoid EOF exception", inputStream.available(), firstEntry); + inputStream.seek(index.getIndexEntryForEntry(firstEntry).getDataOffset()); + } while (entriesToRead > 0) { if (state == State.Closed) { @@ -149,14 +149,20 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr log.warn("The read entry {} is not the expected entry {} but in the range of {} - {}," + " seeking to the right position", entryId, nextExpectedId, nextExpectedId, lastEntry); inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); - continue; } else if (entryId < nextExpectedId && !index.getIndexEntryForEntry(nextExpectedId).equals(index.getIndexEntryForEntry(entryId))) { log.warn("Read an unexpected entry id {} which is smaller than the next expected entry id {}" + ", seeking to the right position", entries, nextExpectedId); inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); - continue; } else if (entryId > lastEntry) { + // in the normal case, the entry id should increment in order. But if there has random access in + // the read method, we should allow to seek to the right position and the entry id should + // never over to the last entry again. + if (!seeked) { + inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); + seeked = true; + continue; + } log.info("Expected to read {}, but read {}, which is greater than last entry {}", nextExpectedId, entryId, lastEntry); throw new BKException.BKUnexpectedConditionException(); diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java index ffe8fb20e3f21..36541b42c7527 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java @@ -260,4 +260,27 @@ public void testSeekForward() throws Exception { toTest.seekForward(after); assertStreamsMatch(toTest, toCompare); } + + @Test + public void testAvailable() throws IOException { + String objectKey = "testAvailable"; + int objectSize = 2048; + RandomInputStream toWrite = new RandomInputStream(0, objectSize); + Payload payload = Payloads.newInputStreamPayload(toWrite); + payload.getContentMetadata().setContentLength((long)objectSize); + Blob blob = blobStore.blobBuilder(objectKey) + .payload(payload) + .contentLength(objectSize) + .build(); + String ret = blobStore.putBlob(BUCKET, blob); + BackedInputStream bis = new BlobStoreBackedInputStreamImpl( + blobStore, BUCKET, objectKey, (k, md) -> {}, objectSize, 512); + Assert.assertEquals(bis.available(), objectSize); + bis.seek(500); + Assert.assertEquals(bis.available(), objectSize - 500); + bis.seek(1024); + Assert.assertEquals(bis.available(), 1024); + bis.seek(2048); + Assert.assertEquals(bis.available(), 0); + } } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java index 90d8b1198f489..77dfc55b7770e 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java @@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; import java.io.IOException; import java.util.Collections; @@ -477,4 +478,22 @@ public void testReadUnknownIndexVersion() throws Exception { Assert.assertTrue(e.getCause().getMessage().contains("Invalid object version")); } } + + @Test + public void testReadEOFException() throws Throwable { + ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 1); + LedgerOffloader offloader = getOffloader(); + UUID uuid = UUID.randomUUID(); + offloader.offload(toWrite, uuid, new HashMap<>()).get(); + + ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, Collections.emptyMap()).get(); + Assert.assertEquals(toTest.getLastAddConfirmed(), toWrite.getLastAddConfirmed()); + toTest.readAsync(0, toTest.getLastAddConfirmed()).get(); + + try { + toTest.readAsync(0, 0).get(); + } catch (Exception e) { + fail("Get unexpected exception when reading entries", e); + } + } } From 56aaa70b2be15e37a1bff4fc794bd56e6abd7769 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Fri, 22 Oct 2021 17:03:38 +0800 Subject: [PATCH 088/823] Reduce the readFailureBackoff time (#12444) ### Motivation When reading entries met exception, the dispatcher will backoff 15s to start next read. The 15s is unacceptable at most of case for the consumer. So it's better to reduce the interval. ### Modifications Make the read failure backoff start from 1s. (cherry picked from commit 99c90a548085d0dbefcc613eac729368933811c6) --- conf/broker.conf | 9 ++++++++ conf/standalone.conf | 9 ++++++++ .../pulsar/broker/ServiceConfiguration.java | 21 +++++++++++++++++++ ...sistentDispatcherSingleActiveConsumer.java | 7 +++++-- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 1fc578a287f6f..3dd85905055e2 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -391,6 +391,15 @@ dispatcherMinReadBatchSize=1 # Max number of entries to dispatch for a shared subscription. By default it is 20 entries. dispatcherMaxRoundRobinBatchSize=20 +# The read failure backoff initial time in milliseconds. By default it is 15s. +dispatcherReadFailureBackoffInitialTimeInMs=15000 + +# The read failure backoff max time in milliseconds. By default it is 60s. +dispatcherReadFailureBackoffMaxTimeInMs=60000 + +# The read failure backoff mandatory stop time in milliseconds. By default it is 0s. +dispatcherReadFailureBackoffMandatoryStopTimeInMs=0 + # Precise dispathcer flow control according to history message number of each entry preciseDispatcherFlowControl=false diff --git a/conf/standalone.conf b/conf/standalone.conf index 2e0273a2f3ea0..878a852a586e9 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -243,6 +243,15 @@ dispatchThrottlingRateRelativeToPublishRate=false # backlog. dispatchThrottlingOnNonBacklogConsumerEnabled=true +# The read failure backoff initial time in milliseconds. By default it is 15s. +dispatcherReadFailureBackoffInitialTimeInMs=15000 + +# The read failure backoff max time in milliseconds. By default it is 60s. +dispatcherReadFailureBackoffMaxTimeInMs=60000 + +# The read failure backoff mandatory stop time in milliseconds. By default it is 0s. +dispatcherReadFailureBackoffMandatoryStopTimeInMs=0 + # Precise dispathcer flow control according to history message number of each entry preciseDispatcherFlowControl=false diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 4c7ee850ad195..69a3d35645713 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -779,6 +779,27 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private int dispatcherMinReadBatchSize = 1; + @FieldContext( + dynamic = true, + category = CATEGORY_SERVER, + doc = "The read failure backoff initial time in milliseconds. By default it is 15s." + ) + private int dispatcherReadFailureBackoffInitialTimeInMs = 15000; + + @FieldContext( + dynamic = true, + category = CATEGORY_SERVER, + doc = "The read failure backoff max time in milliseconds. By default it is 60s." + ) + private int dispatcherReadFailureBackoffMaxTimeInMs = 60000; + + @FieldContext( + dynamic = true, + category = CATEGORY_SERVER, + doc = "The read failure backoff mandatory stop time in milliseconds. By default it is 0s." + ) + private int dispatcherReadFailureBackoffMandatoryStopTimeInMs = 0; + @FieldContext( dynamic = true, category = CATEGORY_SERVER, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index fa7ac03a18221..653c1c1e2194f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -65,8 +65,7 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher protected volatile boolean havePendingRead = false; protected volatile int readBatchSize; - protected final Backoff readFailureBackoff = new Backoff(15, TimeUnit.SECONDS, - 1, TimeUnit.MINUTES, 0, TimeUnit.MILLISECONDS); + protected final Backoff readFailureBackoff; private volatile ScheduledFuture readOnActiveConsumerTask = null; private final RedeliveryTracker redeliveryTracker; @@ -80,6 +79,10 @@ public PersistentDispatcherSingleActiveConsumer(ManagedCursor cursor, SubType su : ""/* NonDurableCursor doesn't have name */); this.cursor = cursor; this.readBatchSize = serviceConfig.getDispatcherMaxReadBatchSize(); + this.readFailureBackoff = new Backoff(serviceConfig.getDispatcherReadFailureBackoffInitialTimeInMs(), + TimeUnit.MILLISECONDS, serviceConfig.getDispatcherReadFailureBackoffMaxTimeInMs(), + TimeUnit.MILLISECONDS, serviceConfig.getDispatcherReadFailureBackoffMandatoryStopTimeInMs(), + TimeUnit.MILLISECONDS); this.redeliveryTracker = RedeliveryTrackerDisabled.REDELIVERY_TRACKER_DISABLED; this.initializeDispatchRateLimiterIfNeeded(Optional.empty()); } From dc99ff57c58d73a9b7ebf36eeb64c9e0f724e6a0 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Fri, 22 Oct 2021 11:22:57 +0800 Subject: [PATCH 089/823] Add retry to tolerate the offload index file read failure (#12452) * Add retry to tolerate the offload index file read failure --- *Motivation* We met the ReadLedgerMetadata exception when reading the index file. The index file only read once, so it may not read all the data from the stream and cause the metadata read failed. We need to ensure the all data is read from the stream or the stream is end. When the stream is end, we will receive the EOF exception, so we need to use `readFully` not `read`. Add the retry logic to tolerate the failure cause by the network. Because the stream is from the HTTP, so it's may break on some case. Add a small retry to avoid it to backoff by the dispatcher. *Modifications* - Use `readFully` to replace the `read` method - Add a small retry for handling the index block build * Add comments and enrich log (cherry picked from commit 33bcc17bbe07ffd9683556edecd9ce546b8fd93f) --- .../impl/BlobStoreBackedReadHandleImpl.java | 32 +++++++++++++++---- .../jcloud/impl/OffloadIndexBlockImpl.java | 6 +--- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java index 98fdff442800f..f4dc1b8b4e421 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java @@ -224,12 +224,32 @@ public static ReadHandle open(ScheduledExecutorService executor, VersionCheck versionCheck, long ledgerId, int readBufferSize) throws IOException { - Blob blob = blobStore.getBlob(bucket, indexKey); - versionCheck.check(indexKey, blob); - OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create(); - OffloadIndexBlock index; - try (InputStream payLoadStream = blob.getPayload().openStream()) { - index = (OffloadIndexBlock) indexBuilder.fromStream(payLoadStream); + int retryCount = 3; + OffloadIndexBlock index = null; + IOException lastException = null; + // The following retry is used to avoid to some network issue cause read index file failure. + // If it can not recovery in the retry, we will throw the exception and the dispatcher will schedule to + // next read. + // If we use a backoff to control the retry, it will introduce a concurrent operation. + // We don't want to make it complicated, because in the most of case it shouldn't in the retry loop. + while (retryCount-- > 0) { + Blob blob = blobStore.getBlob(bucket, indexKey); + versionCheck.check(indexKey, blob); + OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create(); + try (InputStream payLoadStream = blob.getPayload().openStream()) { + index = (OffloadIndexBlock) indexBuilder.fromStream(payLoadStream); + } catch (IOException e) { + // retry to avoid the network issue caused read failure + log.warn("Failed to get index block from the offoaded index file {}, still have {} times to retry", + indexKey, retryCount, e); + lastException = e; + continue; + } + lastException = null; + break; + } + if (lastException != null) { + throw lastException; } BackedInputStream inputStream = new BlobStoreBackedInputStreamImpl(blobStore, bucket, key, diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadIndexBlockImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadIndexBlockImpl.java index 2f64089c81cd9..a3fa14e763a06 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadIndexBlockImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadIndexBlockImpl.java @@ -338,11 +338,7 @@ private OffloadIndexBlock fromStream(DataInputStream dis) throws IOException { int segmentMetadataLength = dis.readInt(); byte[] metadataBytes = new byte[segmentMetadataLength]; - - if (segmentMetadataLength != dis.read(metadataBytes)) { - log.error("Read ledgerMetadata from bytes failed"); - throw new IOException("Read ledgerMetadata from bytes failed"); - } + dis.readFully(metadataBytes); this.segmentMetadata = parseLedgerMetadata(metadataBytes); for (int i = 0; i < indexEntryCount; i++) { From 3e40197c1f8ae902b07431f019d4aadff98c5b92 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Thu, 21 Oct 2021 09:19:03 +0800 Subject: [PATCH 090/823] Fix some test not enabled in integration tests. (#12417) (cherry picked from commit 7d9154bc0812311a77afb2f88b52f9c32c8989f4) --- .../src/test/resources/pulsar-io-sources.xml | 2 ++ .../src/test/resources/pulsar-messaging.xml | 3 +- .../src/test/resources/pulsar-python.xml | 28 +++++++++++++++++++ .../src/test/resources/pulsar-schema.xml | 1 + .../src/test/resources/pulsar-semantics.xml | 28 +++++++++++++++++++ .../src/test/resources/pulsar-upgrade.xml | 28 +++++++++++++++++++ .../integration/src/test/resources/pulsar.xml | 3 ++ 7 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 tests/integration/src/test/resources/pulsar-python.xml create mode 100644 tests/integration/src/test/resources/pulsar-semantics.xml create mode 100644 tests/integration/src/test/resources/pulsar-upgrade.xml diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml index a5afc5d8d0322..636b3e479195f 100644 --- a/tests/integration/src/test/resources/pulsar-io-sources.xml +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -23,6 +23,8 @@ + + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index 6421561fa8a75..cfbdb22587034 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -26,8 +26,9 @@ + + - \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-python.xml b/tests/integration/src/test/resources/pulsar-python.xml new file mode 100644 index 0000000000000..a5faa6389e0f1 --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-python.xml @@ -0,0 +1,28 @@ + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml index c24b4fae0ccb1..e07fdf2b2d86f 100644 --- a/tests/integration/src/test/resources/pulsar-schema.xml +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -24,6 +24,7 @@ + diff --git a/tests/integration/src/test/resources/pulsar-semantics.xml b/tests/integration/src/test/resources/pulsar-semantics.xml new file mode 100644 index 0000000000000..5b5402af4623b --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-semantics.xml @@ -0,0 +1,28 @@ + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-upgrade.xml b/tests/integration/src/test/resources/pulsar-upgrade.xml new file mode 100644 index 0000000000000..a52db54753372 --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-upgrade.xml @@ -0,0 +1,28 @@ + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index 7993d0c6cd238..5382d9b5f5bef 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -36,5 +36,8 @@ + + + From 72804d4e60001c903634f78669adde920ec13b4c Mon Sep 17 00:00:00 2001 From: Aloys Date: Thu, 21 Oct 2021 15:11:06 +0800 Subject: [PATCH 091/823] remove redundant code (#12424) ### Motivation Remove reduncant code ### Modifications Remove one line reduncant code (cherry picked from commit 4d4db6baa8e24b2d0bd465f053685af5f58dd4d0) --- .../pulsar/broker/validator/MultipleListenerValidator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java index 1ba46e12739e2..d56398675115f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java @@ -96,7 +96,6 @@ public static Map validateAndAnalysisAdvertisedListe } } String hostPort = String.format("%s:%d", uri.getHost(), uri.getPort()); - reverseMappings.computeIfAbsent(hostPort, k -> Sets.newTreeSet()); Set sets = reverseMappings.computeIfAbsent(hostPort, k -> Sets.newTreeSet()); sets.add(entry.getKey()); if (sets.size() > 1) { From b0f4d9f078cd0ce36fbb072f4a425d0cfa709c6a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 21 Oct 2021 14:29:43 +0800 Subject: [PATCH 092/823] Fix frequent segmentation fault of Python tests by refactoring ExecutorService (#12427) * Refactor ExecutorService to avoid usage of pointers * Remove unnecessary friend class * Fix CentOS 7 build * Fix PeriodicalTest (cherry picked from commit af0ea69a0368a2804cedbc93a8911b0aebf26dbc) --- pulsar-client-cpp/lib/ClientConnection.cc | 10 ++-- pulsar-client-cpp/lib/ExecutorService.cc | 55 ++++++++++++--------- pulsar-client-cpp/lib/ExecutorService.h | 42 +++++++--------- pulsar-client-cpp/tests/PeriodicTaskTest.cc | 12 ++--- 4 files changed, 61 insertions(+), 58 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 3f82e35031ab2..b9f2209e0ae0e 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -162,11 +162,11 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: resolver_(executor_->createTcpResolver()), socket_(executor_->createSocket()), #if BOOST_VERSION >= 107000 - strand_(boost::asio::make_strand(executor_->io_service_->get_executor())), + strand_(boost::asio::make_strand(executor_->getIOService().get_executor())), #elif BOOST_VERSION >= 106600 - strand_(executor_->io_service_->get_executor()), + strand_(executor_->getIOService().get_executor()), #else - strand_(*(executor_->io_service_)), + strand_(executor_->getIOService()), #endif logicalAddress_(logicalAddress), physicalAddress_(physicalAddress), @@ -183,7 +183,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: #if BOOST_VERSION >= 105400 boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12_client); #else - boost::asio::ssl::context ctx(*executor_->io_service_, boost::asio::ssl::context::tlsv1_client); + boost::asio::ssl::context ctx(executor_->getIOService(), boost::asio::ssl::context::tlsv1_client); #endif Url serviceUrl; Url::parse(physicalAddress, serviceUrl); @@ -240,7 +240,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: } } - tlsSocket_ = executor_->createTlsSocket(socket_, ctx); + tlsSocket_ = ExecutorService::createTlsSocket(socket_, ctx); LOG_DEBUG("TLS SNI Host: " << serviceUrl.host()); if (!SSL_set_tlsext_host_name(tlsSocket_->native_handle(), serviceUrl.host().c_str())) { diff --git a/pulsar-client-cpp/lib/ExecutorService.cc b/pulsar-client-cpp/lib/ExecutorService.cc index 4db31124fc0e6..9cfbd82881d3e 100644 --- a/pulsar-client-cpp/lib/ExecutorService.cc +++ b/pulsar-client-cpp/lib/ExecutorService.cc @@ -27,22 +27,40 @@ DECLARE_LOG_OBJECT() namespace pulsar { -ExecutorService::ExecutorService() - : io_service_(new boost::asio::io_service()), - work_(new BackgroundWork(*io_service_)), - worker_(std::bind(&ExecutorService::startWorker, this, io_service_)) {} +ExecutorService::ExecutorService() {} ExecutorService::~ExecutorService() { close(); } -void ExecutorService::startWorker(std::shared_ptr io_service) { io_service_->run(); } +void ExecutorService::start() { + auto self = shared_from_this(); + std::thread t{[self] { + if (self->isClosed()) { + return; + } + boost::system::error_code ec; + self->getIOService().run(ec); + if (ec) { + LOG_ERROR("Failed to run io_service: " << ec.message()); + } + }}; + t.detach(); +} + +ExecutorServicePtr ExecutorService::create() { + // make_shared cannot access the private constructor, so we need to expose the private constructor via a + // derived class. + struct ExecutorServiceImpl : public ExecutorService {}; + + auto executor = std::make_shared(); + executor->start(); + return std::static_pointer_cast(executor); +} /* * factory method of boost::asio::ip::tcp::socket associated with io_service_ instance * @ returns shared_ptr to this socket */ -SocketPtr ExecutorService::createSocket() { - return SocketPtr(new boost::asio::ip::tcp::socket(*io_service_)); -} +SocketPtr ExecutorService::createSocket() { return SocketPtr(new boost::asio::ip::tcp::socket(io_service_)); } TlsSocketPtr ExecutorService::createTlsSocket(SocketPtr &socket, boost::asio::ssl::context &ctx) { return std::shared_ptr >( @@ -54,11 +72,11 @@ TlsSocketPtr ExecutorService::createTlsSocket(SocketPtr &socket, boost::asio::ss * @returns shraed_ptr to resolver object */ TcpResolverPtr ExecutorService::createTcpResolver() { - return TcpResolverPtr(new boost::asio::ip::tcp::resolver(*io_service_)); + return TcpResolverPtr(new boost::asio::ip::tcp::resolver(io_service_)); } DeadlineTimerPtr ExecutorService::createDeadlineTimer() { - return DeadlineTimerPtr(new boost::asio::deadline_timer(*io_service_)); + return DeadlineTimerPtr(new boost::asio::deadline_timer(io_service_)); } void ExecutorService::close() { @@ -67,21 +85,10 @@ void ExecutorService::close() { return; } - io_service_->stop(); - work_.reset(); - // Detach the worker thread instead of join to avoid potential deadlock - if (worker_.joinable()) { - try { - worker_.detach(); - } catch (const std::system_error &e) { - // This condition will happen if we're forking the process, therefore the thread was not ported to - // the child side of the fork and the detach would be failing. - LOG_DEBUG("Failed to detach thread: " << e.what()); - } - } + io_service_.stop(); } -void ExecutorService::postWork(std::function task) { io_service_->post(task); } +void ExecutorService::postWork(std::function task) { io_service_.post(task); } ///////////////////// @@ -93,7 +100,7 @@ ExecutorServicePtr ExecutorServiceProvider::get() { int idx = executorIdx_++ % executors_.size(); if (!executors_[idx]) { - executors_[idx] = std::make_shared(); + executors_[idx] = ExecutorService::create(); } return executors_[idx]; diff --git a/pulsar-client-cpp/lib/ExecutorService.h b/pulsar-client-cpp/lib/ExecutorService.h index 6746936190566..6b0909194b7d9 100644 --- a/pulsar-client-cpp/lib/ExecutorService.h +++ b/pulsar-client-cpp/lib/ExecutorService.h @@ -25,7 +25,6 @@ #include #include #include -#include #include #include @@ -34,51 +33,48 @@ typedef std::shared_ptr SocketPtr; typedef std::shared_ptr > TlsSocketPtr; typedef std::shared_ptr TcpResolverPtr; typedef std::shared_ptr DeadlineTimerPtr; -class PULSAR_PUBLIC ExecutorService : private boost::noncopyable { - friend class ClientConnection; - +class PULSAR_PUBLIC ExecutorService : public std::enable_shared_from_this { public: - ExecutorService(); + using IOService = boost::asio::io_service; + using SharedPtr = std::shared_ptr; + + static SharedPtr create(); ~ExecutorService(); + ExecutorService(const ExecutorService &) = delete; + ExecutorService &operator=(const ExecutorService &) = delete; + SocketPtr createSocket(); - TlsSocketPtr createTlsSocket(SocketPtr &socket, boost::asio::ssl::context &ctx); + static TlsSocketPtr createTlsSocket(SocketPtr &socket, boost::asio::ssl::context &ctx); TcpResolverPtr createTcpResolver(); DeadlineTimerPtr createDeadlineTimer(); void postWork(std::function task); + void close(); - boost::asio::io_service &getIOService() { return *io_service_; } + IOService &getIOService() { return io_service_; } + bool isClosed() const noexcept { return closed_; } private: - /* - * only called once and within lock so no need to worry about thread-safety - */ - void startWorker(std::shared_ptr io_service); - /* * io_service is our interface to os, io object schedule async ops on this object */ - std::shared_ptr io_service_; + IOService io_service_; /* * work will not let io_service.run() return even after it has finished work * it will keep it running in the background so we don't have to take care of it */ - typedef boost::asio::io_service::work BackgroundWork; - std::unique_ptr work_; - - /* - * worker thread which runs until work object is destroyed, it's running io_service::run in - * background invoking async handlers as they are finished and result is available from - * io_service - */ - std::thread worker_; + IOService::work work_{io_service_}; std::atomic_bool closed_{false}; + + ExecutorService(); + + void start(); }; -typedef std::shared_ptr ExecutorServicePtr; +using ExecutorServicePtr = ExecutorService::SharedPtr; class PULSAR_PUBLIC ExecutorServiceProvider { public: diff --git a/pulsar-client-cpp/tests/PeriodicTaskTest.cc b/pulsar-client-cpp/tests/PeriodicTaskTest.cc index 11c1c62ec3f2b..2c1da70e80e3c 100644 --- a/pulsar-client-cpp/tests/PeriodicTaskTest.cc +++ b/pulsar-client-cpp/tests/PeriodicTaskTest.cc @@ -29,11 +29,11 @@ DECLARE_LOG_OBJECT() using namespace pulsar; TEST(PeriodicTaskTest, testCountdownTask) { - ExecutorService executor; + auto executor = ExecutorService::create(); std::atomic_int count{5}; - auto task = std::make_shared(executor.getIOService(), 200); + auto task = std::make_shared(executor->getIOService(), 200); task->setCallback([task, &count](const PeriodicTask::ErrorCode& ec) { if (--count <= 0) { task->stop(); @@ -56,13 +56,13 @@ TEST(PeriodicTaskTest, testCountdownTask) { ASSERT_EQ(count.load(), 0); task->stop(); - executor.close(); + executor->close(); } TEST(PeriodicTaskTest, testNegativePeriod) { - ExecutorService executor; + auto executor = ExecutorService::create(); - auto task = std::make_shared(executor.getIOService(), -1); + auto task = std::make_shared(executor->getIOService(), -1); std::atomic_bool callbackTriggered{false}; task->setCallback([&callbackTriggered](const PeriodicTask::ErrorCode& ec) { callbackTriggered = true; }); @@ -71,5 +71,5 @@ TEST(PeriodicTaskTest, testNegativePeriod) { ASSERT_EQ(callbackTriggered.load(), false); task->stop(); - executor.close(); + executor->close(); } From 7c7dfedb0cb5b48c03b3e687ad342786ff4f5cf0 Mon Sep 17 00:00:00 2001 From: Jason918 Date: Thu, 21 Oct 2021 07:35:36 +0800 Subject: [PATCH 093/823] Fix wrong property name in NamespaceIsolationDataImpl#secondary (#12433) (cherry picked from commit be0afead7fa15896123215e5d442a130073e677d) --- .../pulsar/common/policies/data/NamespaceIsolationDataImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java index 5ca9d07c56558..4c2d99cc5d7bb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java @@ -59,7 +59,7 @@ public class NamespaceIsolationDataImpl implements NamespaceIsolationData { private List primary; @ApiModelProperty( - name = "primary", + name = "secondary", value = "The list of secondary brokers for serving the list of namespaces in this isolation policy" ) private List secondary; From eaf629851ec13051a176c6ab8a814c4fce76f0e0 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 21 Oct 2021 14:30:34 +0800 Subject: [PATCH 094/823] Change the producer fence error log to debug level (#12447) (cherry picked from commit 64c441945ece5b2ddd7f2a3e82323a5130774438) --- .../org/apache/pulsar/client/impl/ProducerImpl.java | 10 ++++++++++ .../apache/pulsar/functions/worker/WorkerUtils.java | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 686fa7de4b8f7..f8e59a2006592 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1495,6 +1495,16 @@ public void connectionOpened(final ClientCnx cnx) { log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, cause.getMessage()); // Close the producer since topic does not exist. + if (cause instanceof PulsarClientException.ProducerFencedException) { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Failed to create producer: {}", + topic, producerName, cause.getMessage()); + } + } else { + log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, cause.getMessage()); + } + + // Close the producer since topic does not exists. if (cause instanceof PulsarClientException.TopicDoesNotExistException) { closeAsync().whenComplete((v, ex) -> { if (ex != null) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java index d7c6a71f8c36f..4f0287f19ffc7 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java @@ -352,7 +352,9 @@ public static Producer createExclusiveProducerWithRetry(PulsarClient cli } tries++; if (tries % 6 == 0) { - log.warn("Failed to acquire exclusive producer to topic {} after {} attempts. Will retry if we are still the leader.", topic, tries); + if (log.isDebugEnabled()) { + log.debug("Failed to acquire exclusive producer to topic {} after {} attempts. Will retry if we are still the leader.", topic, tries); + } } Thread.sleep(sleepInBetweenMs); } while (isLeader.get()); From 5b0fe1bf1f55590b7b24ebde2b716af8b852ff44 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 19 Oct 2021 20:36:26 +0800 Subject: [PATCH 095/823] [Transaction] Fix bugs, Exception thrower by TB::appendBufferToTxn must be ManagedLedgerException. (#12376) Exception thrower by TB::appendBufferToTxn must be ManagedLedgerException. So we handle this in (cherry picked from commit 1eb08f3136dbe96e473b63038da9337043be7a0d) --- .../service/persistent/PersistentTopic.java | 4 + .../buffer/impl/TopicTransactionBuffer.java | 2 +- .../broker/transaction/TransactionTest.java | 78 +++++++++++++++++-- 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index ffe4bc4597704..f917aab44cd34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2998,6 +2998,10 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon decrementPendingWriteOpsAndCheck(); }) .exceptionally(throwable -> { + throwable = throwable.getCause(); + if (!(throwable instanceof ManagedLedgerException)) { + throwable = new ManagedLedgerException(throwable); + } addFailed((ManagedLedgerException) throwable, publishContext); return null; }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 735927cc99b3f..78f92959abc34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -219,7 +219,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("Failed to append buffer to txn {}", txnId, exception); - completableFuture.completeExceptionally(new PersistenceException(exception)); + completableFuture.completeExceptionally(exception); } }, null); return completableFuture; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 34ea362130501..12a6b285ccd5c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -18,21 +18,16 @@ */ package org.apache.pulsar.broker.transaction; -import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore.PENDING_ACK_STORE_SUFFIX; import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import static org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore.PENDING_ACK_STORE_SUFFIX; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import io.netty.buffer.Unpooled; import java.lang.reflect.Field; import java.util.List; import java.util.Optional; @@ -43,6 +38,7 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; @@ -369,4 +365,72 @@ public void testTakeSnapshotBeforeBuildTxnProducer() throws Exception { Assert.assertEquals(snapshot1.getMaxReadPositionEntryId(), 1); }); } + + @Test + public void testAppendBufferWithNotManageLedgerExceptionCanCastToMLE() + throws Exception { + String topic = "persistent://pulsar/system/testReCreateTopic"; + admin.topics().createNonPartitionedTopic(topic); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(topic, false) + .get().get(); + CountDownLatch countDownLatch = new CountDownLatch(1); + Topic.PublishContext publishContext = new Topic.PublishContext() { + + @Override + public String getProducerName() { + return "test"; + } + + public long getSequenceId() { + return 30; + } + /** + * Return the producer name for the original producer. + * + * For messages published locally, this will return the same local producer name, though in case of + * replicated messages, the original producer name will differ + */ + public String getOriginalProducerName() { + return "test"; + } + + public long getOriginalSequenceId() { + return 30; + } + + public long getHighestSequenceId() { + return 30; + } + + public long getOriginalHighestSequenceId() { + return 30; + } + + public long getNumberOfMessages() { + return 30; + } + + @Override + public void completed(Exception e, long ledgerId, long entryId) { + Assert.assertTrue(e.getCause() instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException); + countDownLatch.countDown(); + } + }; + + //Close topic manageLedger. + persistentTopic.getManagedLedger().close(); + + //Publish to a closed managerLedger to test ManagerLedgerException. + persistentTopic.publishTxnMessage(new TxnID(123L, 321L), + Unpooled.copiedBuffer("message", UTF_8), publishContext); + + //If it times out, it means that the assertTrue in publishContext.completed is failed. + Awaitility.await().until(() -> { + countDownLatch.await(); + return true; + }); + } } From 71a8b2e9931e2f7d2b89a3e59ac6817e043eb265 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Tue, 19 Oct 2021 10:33:26 +0800 Subject: [PATCH 096/823] fix windows test path probleam (#12398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Make unit test path compatible to windows， make these tests can run on windows ### Modifications Same to #12329 , change `"file://" + tokenFile` to `"file:///" + tokenFile.toString().replace('\\', '/')` Run in windows can't recognize format like `file://C\xxx\yyy\zzz` It should be `file:///C` and the `\` should be `/` (cherry picked from commit 4da43109066ebea7a0b43a5a5c598b4a923e9d9b) --- .../AuthenticationProviderTokenTest.java | 3 +-- .../rest/api/v2/FunctionApiV2ResourceTest.java | 16 ++++++++-------- .../rest/api/v3/FunctionApiV3ResourceTest.java | 17 ++++++++--------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java index 6582a6ab69d89..b05ad4ca4839d 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -240,7 +239,7 @@ public void testAuthSecretKeyFromFile() throws Exception { AuthenticationProviderToken provider = new AuthenticationProviderToken(); Properties properties = new Properties(); - properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file://" + secretKeyFile.toString()); + properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file:///" + secretKeyFile.toString().replace('\\', '/')); ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index f37d5df1fe9a0..89fb32adb5941 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -1031,8 +1031,8 @@ public void testUpdateFunctionWithUrl() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; FunctionConfig functionConfig = new FunctionConfig(); functionConfig.setOutput(outputTopic); @@ -1427,10 +1427,10 @@ public void testDownloadFunctionHttpUrl() throws Exception { public void testDownloadFunctionFile() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); + String fileLocation = file.getAbsolutePath().replace('\\', '/'); String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file://" + fileLocation, null).getEntity(); + StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation, null).getEntity(); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1446,8 +1446,8 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); @@ -1479,8 +1479,8 @@ public void testRegisterFunctionWithConflictingFields() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index d2e7aca1cfa87..837e3211368c9 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -29,7 +29,6 @@ import static org.powermock.api.mockito.PowerMockito.doThrow; import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import com.google.common.collect.Lists; @@ -1525,7 +1524,7 @@ public void testDownloadFunctionHttpUrl() throws Exception { public void testDownloadFunctionFile() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); + String fileLocation = file.getAbsolutePath().replace('\\', '/'); String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); PulsarWorkerService worker = mock(PulsarWorkerService.class); doReturn(true).when(worker).isInitialized(); @@ -1533,7 +1532,7 @@ public void testDownloadFunctionFile() throws Exception { when(config.isAuthorizationEnabled()).thenReturn(false); when(worker.getWorkerConfig()).thenReturn(config); FunctionsImpl function = new FunctionsImpl(() -> worker); - StreamingOutput streamOutput = function.downloadFunction("file://" + fileLocation, null, null); + StreamingOutput streamOutput = function.downloadFunction("file:///" + fileLocation, null, null); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1549,8 +1548,8 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); @@ -1577,8 +1576,8 @@ public void testRegisterFunctionWithConflictingFields() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); @@ -1601,8 +1600,8 @@ public void testCreateFunctionWithoutSettingRuntime() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); From 2c364d90e1293eeb994b6c96cab808fb420fa799 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 18 Oct 2021 19:47:57 -0700 Subject: [PATCH 097/823] [C++] Use weak ref to ClientConnection for timeout task (#12409) ### Motivation Fixes #12408. Using a weak reference in the timeout task for `ClientConnection` to break a circular reference dependency between the connection instance and the task. (cherry picked from commit 4e43a1dd85809f0242e354aef7a27973820e0dda) --- pulsar-client-cpp/lib/ClientConnection.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index b9f2209e0ae0e..212868920266b 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -532,18 +532,25 @@ void ClientConnection::handleResolve(const boost::system::error_code& err, return; } - auto self = shared_from_this(); - connectTimeoutTask_->setCallback([this, self](const PeriodicTask::ErrorCode& ec) { - if (state_ != Ready) { - LOG_ERROR(cnxString_ << "Connection was not established in " << connectTimeoutTask_->getPeriodMs() - << " ms, close the socket"); + auto self = ClientConnectionWeakPtr(shared_from_this()); + + connectTimeoutTask_->setCallback([self](const PeriodicTask::ErrorCode& ec) { + ClientConnectionPtr ptr = self.lock(); + if (!ptr) { + // Connection was already destroyed + return; + } + + if (ptr->state_ != Ready) { + LOG_ERROR(ptr->cnxString_ << "Connection was not established in " + << ptr->connectTimeoutTask_->getPeriodMs() << " ms, close the socket"); PeriodicTask::ErrorCode err; - socket_->close(err); + ptr->socket_->close(err); if (err) { - LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); + LOG_WARN(ptr->cnxString_ << "Failed to close socket: " << err.message()); } } - connectTimeoutTask_->stop(); + ptr->connectTimeoutTask_->stop(); }); LOG_DEBUG(cnxString_ << "Connecting to " << endpointIterator->endpoint() << "..."); From c90dff205cca23718cf4eb550994fc971c416868 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Mon, 18 Oct 2021 14:02:30 +0800 Subject: [PATCH 098/823] [Admin] Get schema validation enforce add applied. (#12349) now, namespace get schema validation enforce don't return broker config, if namespace policy the schema validation enforce is false and we use --applied, we should return this config in broker level. (cherry picked from commit 69fb80236d61e7594fdf8e84724f435d7820cc3f) --- .../broker/admin/impl/NamespacesBase.java | 9 ++++-- .../pulsar/broker/admin/v2/Namespaces.java | 5 ++-- .../AdminApiSchemaValidationEnforced.java | 9 ++++++ .../pulsar/client/admin/Namespaces.java | 28 +++++++++++++++++-- .../client/admin/internal/NamespacesImpl.java | 19 ++++++++++--- .../pulsar/admin/cli/PulsarAdminToolTest.java | 3 ++ .../pulsar/admin/cli/CmdNamespaces.java | 5 +++- 7 files changed, 67 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 07c676af4854b..f2327e28b93c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2387,10 +2387,15 @@ protected void internalSetSchemaCompatibilityStrategy(SchemaCompatibilityStrateg "schemaCompatibilityStrategy"); } - protected boolean internalGetSchemaValidationEnforced() { + protected boolean internalGetSchemaValidationEnforced(boolean applied) { validateNamespacePolicyOperation(namespaceName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); - return getNamespacePolicies(namespaceName).schema_validation_enforced; + boolean schemaValidationEnforced = getNamespacePolicies(namespaceName).schema_validation_enforced; + if (!schemaValidationEnforced && applied) { + return pulsar().getConfiguration().isSchemaValidationEnforced(); + } else { + return schemaValidationEnforced; + } } protected void internalSetSchemaValidationEnforced(boolean schemaValidationEnforced) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 53eab242d3d55..cf84374f34bc9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1626,9 +1626,10 @@ public void setSubscriptionTypesEnabled( @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") }) public boolean getSchemaValidtionEnforced(@PathParam("tenant") String tenant, - @PathParam("namespace") String namespace) { + @PathParam("namespace") String namespace, + @QueryParam("applied") @DefaultValue("false") boolean applied) { validateNamespaceName(tenant, namespace); - return internalGetSchemaValidationEnforced(); + return internalGetSchemaValidationEnforced(applied); } @POST diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java index b7747de67595e..75d77dc3da875 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java @@ -66,6 +66,15 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testGetSchemaValidationEnforcedApplied() throws Exception { + String namespace = "schema-validation-enforced/testApplied"; + admin.namespaces().createNamespace(namespace); + this.conf.setSchemaValidationEnforced(true); + assertTrue(admin.namespaces().getSchemaValidationEnforced(namespace, true)); + assertFalse(admin.namespaces().getSchemaValidationEnforced(namespace, false)); + } + @Test public void testDisableSchemaValidationEnforcedNoSchema() throws Exception { admin.namespaces().createNamespace("schema-validation-enforced/default-no-schema"); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index f6f8654f2d4a7..6ea4bab025cd3 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -3539,6 +3539,7 @@ void setSchemaAutoUpdateCompatibilityStrategy(String namespace, /** * Get schema validation enforced for namespace. + * @param namespace namespace for this command. * @return the schema validation enforced flag * @throws NotAuthorizedException * Don't have admin permission @@ -3547,16 +3548,39 @@ void setSchemaAutoUpdateCompatibilityStrategy(String namespace, * @throws PulsarAdminException * Unexpected error */ - boolean getSchemaValidationEnforced(String namespace) - throws PulsarAdminException; + boolean getSchemaValidationEnforced(String namespace) throws PulsarAdminException; /** * Get schema validation enforced for namespace asynchronously. + * @param namespace namespace for this command. * * @return the schema validation enforced flag */ CompletableFuture getSchemaValidationEnforcedAsync(String namespace); + /** + * Get schema validation enforced for namespace. + * @param namespace namespace for this command. + * @param applied applied for this command. + * @return the schema validation enforced flag + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Tenant or Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + boolean getSchemaValidationEnforced(String namespace, boolean applied) throws PulsarAdminException; + + /** + * Get schema validation enforced for namespace asynchronously. + * @param namespace namespace for this command. + * @param applied applied for this command. + * + * @return the schema validation enforced flag + */ + CompletableFuture getSchemaValidationEnforcedAsync(String namespace, boolean applied); + /** * Set schema validation enforced for namespace. * if a producer without a schema attempts to produce to a topic with schema in this the namespace, the diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 8307a5f172583..610423b18c101 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -3060,11 +3060,21 @@ public void setSchemaAutoUpdateCompatibilityStrategy(String namespace, } @Override - public boolean getSchemaValidationEnforced(String namespace) + public boolean getSchemaValidationEnforced(String namespace) throws PulsarAdminException { + return getSchemaValidationEnforced(namespace, false); + } + + @Override + public CompletableFuture getSchemaValidationEnforcedAsync(String namespace) { + return getSchemaValidationEnforcedAsync(namespace, false); + } + + @Override + public boolean getSchemaValidationEnforced(String namespace, boolean applied) throws PulsarAdminException { try { - return getSchemaValidationEnforcedAsync(namespace). - get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + return getSchemaValidationEnforcedAsync(namespace, applied) + .get(this.readTimeoutMs, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { throw (PulsarAdminException) e.getCause(); } catch (InterruptedException e) { @@ -3076,9 +3086,10 @@ public boolean getSchemaValidationEnforced(String namespace) } @Override - public CompletableFuture getSchemaValidationEnforcedAsync(String namespace) { + public CompletableFuture getSchemaValidationEnforcedAsync(String namespace, boolean applied) { NamespaceName ns = NamespaceName.get(namespace); WebTarget path = namespacePath(ns, "schemaValidationEnforced"); + path = path.queryParam("applied", applied); final CompletableFuture future = new CompletableFuture<>(); asyncGetRequest(path, new InvocationCallback() { diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 32befc8a95386..60fb6acf1d9a6 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -368,6 +368,9 @@ public void namespaces() throws Exception { namespaces.run(split("get-subscription-types-enabled myprop/clust/ns1")); verify(mockNamespaces).getSubscriptionTypesEnabled("myprop/clust/ns1"); + namespaces.run(split("get-schema-validation-enforce myprop/clust/ns1 -ap")); + verify(mockNamespaces).getSchemaValidationEnforced("myprop/clust/ns1", true); + namespaces .run(split("set-bookie-affinity-group myprop/clust/ns1 --primary-group test1 --secondary-group test2")); verify(mockNamespaces).setBookieAffinityGroup("myprop/clust/ns1", diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 435b5d8b45ddb..c642288383aeb 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1877,11 +1877,14 @@ private class GetSchemaValidationEnforced extends CliCommand { @Parameter(description = "tenant/namespace", required = true) private java.util.List params; + @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the namespace") + private boolean applied = false; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - System.out.println(getAdmin().namespaces().getSchemaValidationEnforced(namespace)); + System.out.println(getAdmin().namespaces().getSchemaValidationEnforced(namespace, applied)); } } From 11c01cdd3be36b8a5d40e09aef352f84e85e6abd Mon Sep 17 00:00:00 2001 From: litao Date: Mon, 18 Oct 2021 09:23:05 +0800 Subject: [PATCH 099/823] Fix java doc for MultipleListenerValidator (#12389) ### Motivation *Fix the java doc mentioned in [#12353](https://github.com/apache/pulsar/pull/12353/files#r729887373)* ### Modifications *Fix java doc for MultipleListenerValidator#validateAndAnalysisAdvertisedListener.* (cherry picked from commit 9d10b8b3aa19647f6ff81e39866bd5d409c4133a) --- .../broker/validator/MultipleListenerValidator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java index d56398675115f..ce02da974d52e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java @@ -39,10 +39,11 @@ public final class MultipleListenerValidator { /** * Validate the configuration of `advertisedListeners`, `internalListenerName`. - * 2. the listener name in `advertisedListeners` must not duplicate. - * 3. user can not assign same 'host:port' to different listener. - * 4. if `internalListenerName` is absent, the first `listener` in the `advertisedListeners` will be the `internalListenerName`. - * 5. if pulsar do not specify `brokerServicePortTls`, should only contain one entry of `pulsar://` per listener name. + * 1. `advertisedListeners` consists of a comma-separated list of endpoints. + * 2. Each endpoint consists of a listener name and an associated address (`listener:scheme://host:port`). + * 3. A listener name may be repeated to define both a non-TLS and a TLS endpoint. + * 4. Duplicate definitions are disallowed. + * 5. If `internalListenerName` is absent, set it to the first listener defined in `advertisedListeners`. * @param config the pulsar broker configure. * @return */ From 2014457ae2b56eb9b231bfe2c6e8bee3e656cf53 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 29 Oct 2021 17:54:27 -0700 Subject: [PATCH 100/823] Websocket should pass the encryption context to the consumers (#12539) (cherry picked from commit d20efe5b0becd73ef33816c4bf33eafa2e28efa4) --- .../proxy/ProxyPublishConsumeTest.java | 62 +++++++++++++++++++ .../websocket/proxy/SimpleConsumerSocket.java | 3 + .../pulsar/websocket/ConsumerHandler.java | 1 + .../websocket/data/ConsumerMessage.java | 3 + 4 files changed, 69 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java index 3e3cf9a9e823f..6386c515f1cf1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java @@ -29,12 +29,14 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -54,9 +56,11 @@ import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerAccessMode; import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.TopicType; @@ -935,6 +939,64 @@ public void ackBatchMessageTest() throws Exception { } } + @Test(timeOut = 20000) + public void consumeEncryptedMessages() throws Exception { + final String subscription = "my-sub"; + final String topic = "my-property/my-ns/encrypted" + UUID.randomUUID(); + final String consumerUri = "ws://localhost:" + proxyServer.getListenPortHTTP().get() + + "/ws/v2/consumer/persistent/" + topic + "/" + subscription + "?cryptoFailureAction=CONSUME"; + final int messages = 10; + + WebSocketClient consumerClient = new WebSocketClient(); + SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket(); + + + final String rsaPublicKeyData = "data:application/x-pem-file;base64,LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF0S1d3Z3FkblRZck9DditqMU1rVApXZlNIMHdDc0haWmNhOXdBVzNxUDR1dWhsQnZuYjEwSmNGZjVaanpQOUJTWEsrdEhtSTh1b04zNjh2RXY2eWhVClJITTR5dVhxekN4enVBd2tRU28zOXJ6WDhQR0M3cWRqQ043TERKM01ucWlCSXJVc1NhRVAxd3JOc0Ixa0krbzkKRVIxZTVPL3VFUEFvdFA5MzNoSFEwSjJoTUVla0hxTDdzQmxKOThoNk5tc2ljRWFVa2FyZGswVE9YcmxrakMrYwpNZDhaYkdTY1BxSTlNMzhibW4zT0x4RlRuMXZ0aHB2blhMdkNtRzRNKzZ4dFl0RCtucGNWUFp3MWkxUjkwZk1zCjdwcFpuUmJ2OEhjL0RGZE9LVlFJZ2FtNkNEZG5OS2dXN2M3SUJNclAwQUVtMzdIVHUwTFNPalAyT0hYbHZ2bFEKR1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg=="; + final String rsaPrivateKeyData = "data:application/x-pem-file;base64,LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdEtXd2dxZG5UWXJPQ3YrajFNa1RXZlNIMHdDc0haWmNhOXdBVzNxUDR1dWhsQnZuCmIxMEpjRmY1Wmp6UDlCU1hLK3RIbUk4dW9OMzY4dkV2NnloVVJITTR5dVhxekN4enVBd2tRU28zOXJ6WDhQR0MKN3FkakNON0xESjNNbnFpQklyVXNTYUVQMXdyTnNCMWtJK285RVIxZTVPL3VFUEFvdFA5MzNoSFEwSjJoTUVlawpIcUw3c0JsSjk4aDZObXNpY0VhVWthcmRrMFRPWHJsa2pDK2NNZDhaYkdTY1BxSTlNMzhibW4zT0x4RlRuMXZ0Cmhwdm5YTHZDbUc0TSs2eHRZdEQrbnBjVlBadzFpMVI5MGZNczdwcFpuUmJ2OEhjL0RGZE9LVlFJZ2FtNkNEZG4KTktnVzdjN0lCTXJQMEFFbTM3SFR1MExTT2pQMk9IWGx2dmxRR1FJREFRQUJBb0lCQUFhSkZBaTJDN3UzY05yZgpBc3RZOXZWRExvTEl2SEZabGtCa3RqS1pEWW1WSXNSYitoU0NWaXdWVXJXTEw2N1I2K0l2NGVnNERlVE9BeDAwCjhwbmNYS2daVHcyd0liMS9RalIvWS9SamxhQzhsa2RtUldsaTd1ZE1RQ1pWc3lodVNqVzZQajd2cjhZRTR3b2oKRmhOaWp4RUdjZjl3V3JtTUpyemRuVFdRaVhCeW8rZVR2VVE5QlBnUEdyUmpzTVptVGtMeUFWSmZmMkRmeE81YgpJV0ZEWURKY3lZQU1DSU1RdTd2eXMvSTUwb3U2aWxiMUNPNlFNNlo3S3BQZU9vVkZQd3R6Ymg4Y2Y5eE04VU5TCmo2Si9KbWRXaGdJMzRHUzNOQTY4eFRRNlBWN3pqbmhDYytpY2NtM0pLeXpHWHdhQXBBWitFb2NlLzlqNFdLbXUKNUI0emlSMENnWUVBM2wvOU9IYmwxem15VityUnhXT0lqL2kyclR2SHp3Qm5iblBKeXVlbUw1Vk1GZHBHb2RRMwp2d0h2eVFtY0VDUlZSeG1Yb2pRNFF1UFBIczNxcDZ3RUVGUENXeENoTFNUeGxVYzg1U09GSFdVMk85OWpWN3pJCjcrSk9wREsvTXN0c3g5bkhnWGR1SkYrZ2xURnRBM0xIOE9xeWx6dTJhRlBzcHJ3S3VaZjk0UThDZ1lFQXovWngKYWtFRytQRU10UDVZUzI4Y1g1WGZqc0lYL1YyNkZzNi9zSDE2UWpVSUVkZEU1VDRmQ3Vva3hDalNpd1VjV2htbApwSEVKNVM1eHAzVllSZklTVzNqUlczcXN0SUgxdHBaaXBCNitTMHpUdUptTEpiQTNJaVdFZzJydE10N1gxdUp2CkEvYllPcWUwaE9QVHVYdVpkdFZaMG5NVEtrN0dHOE82VmtCSTdGY0NnWUVBa0RmQ21zY0pnczdKYWhsQldIbVgKekg5cHdlbStTUEtqSWMvNE5CNk4rZGdpa3gyUHAwNWhwUC9WaWhVd1lJdWZ2cy9MTm9nVllOUXJ0SGVwVW5yTgoyK1RtYkhiWmdOU3YxTGR4dDgyVWZCN3kwRnV0S3U2bGhtWEh5TmVjaG8zRmk4c2loMFYwYWlTV21ZdUhmckFICkdhaXNrRVpLbzFpaVp2UVhKSXg5TzJNQ2dZQVRCZjByOWhUWU10eXh0YzZIMy9zZGQwMUM5dGhROGdEeTB5alAKMFRxYzBkTVNKcm9EcW1JV2tvS1lldzkvYmhGQTRMVzVUQ25Xa0NBUGJIbU50RzRmZGZiWXdta0gvaGRuQTJ5MApqS2RscGZwOEdYZVVGQUdIR3gxN0ZBM3NxRnZnS1VoMGVXRWdSSFVMN3ZkUU1WRkJnSlM5M283elFNOTRmTGdQCjZjT0I4d0tCZ0ZjR1Y0R2pJMld3OWNpbGxhQzU1NE12b1NqZjhCLyswNGtYekRPaDhpWUlJek85RVVpbDFqaksKSnZ4cDRobkx6VEtXYnV4M01FV3F1ckxrWWFzNkdwS0JqdytpTk9DYXI2WWRxV0dWcU0zUlV4N1BUVWFad2tLeApVZFA2M0lmWTdpWkNJVC9RYnlIUXZJVWUyTWFpVm5IK3VseGRrSzZZNWU3Z3hjYmNrSUg0Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg=="; + + Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(false) + .defaultCryptoKeyReader(rsaPublicKeyData) + .addEncryptionKey("ws-consumer-a") + .create(); + + try { + consumerClient.start(); + ClientUpgradeRequest consumerRequest = new ClientUpgradeRequest(); + Future consumerFuture = consumerClient.connect(consumeSocket, URI.create(consumerUri), consumerRequest); + + assertTrue(consumerFuture.get().isOpen()); + assertEquals(consumeSocket.getReceivedMessagesCount(), 0); + + for (int i = 0; i < messages; i++) { + producer.sendAsync(String.valueOf(i).getBytes(StandardCharsets.UTF_8)); + } + + producer.flush(); + consumeSocket.sendPermits(messages); + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(consumeSocket.getReceivedMessagesCount(), messages)); + + for (JsonObject msg : consumeSocket.messages) { + assertTrue(msg.has("encryptionContext")); + JsonObject encryptionCtx = msg.getAsJsonObject("encryptionContext"); + JsonObject keys = encryptionCtx.getAsJsonObject("keys"); + assertTrue(keys.has("ws-consumer-a")); + + assertTrue(keys.getAsJsonObject("ws-consumer-a").has("keyValue")); + } + + // The message should not be acked since we only acked 1 message of the batch message + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(admin.topics().getStats(topic).getSubscriptions() + .get(subscription).getMsgBacklog(), 0)); + + } finally { + stopWebSocketClient(consumerClient); + } + } + private void verifyTopicStat(Client client, String baseUrl, String topic) { String statUrl = baseUrl + topic + "/stats"; WebTarget webTarget = client.target(statUrl); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/SimpleConsumerSocket.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/SimpleConsumerSocket.java index 749bfdcd2ba25..b1a9908d7234e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/SimpleConsumerSocket.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/SimpleConsumerSocket.java @@ -44,6 +44,7 @@ public class SimpleConsumerSocket { private final CountDownLatch closeLatch; private Session session; private final ArrayList consumerBuffer; + final ArrayList messages; private final AtomicInteger receivedMessages = new AtomicInteger(); // Custom message handler to override standard message processing, if it's needed private SimpleConsumerMessageHandler customMessageHandler; @@ -51,6 +52,7 @@ public class SimpleConsumerSocket { public SimpleConsumerSocket() { this.closeLatch = new CountDownLatch(1); consumerBuffer = new ArrayList<>(); + this.messages = new ArrayList<>(); } public boolean awaitClose(int duration, TimeUnit unit) throws InterruptedException { @@ -79,6 +81,7 @@ public void onConnect(Session session) throws InterruptedException { public synchronized void onMessage(String msg) throws JsonParseException, IOException { receivedMessages.incrementAndGet(); JsonObject message = new Gson().fromJson(msg, JsonObject.class); + this.messages.add(message); if (message.get(X_PULSAR_MESSAGE_ID) != null) { String messageId = message.get(X_PULSAR_MESSAGE_ID).getAsString(); consumerBuffer.add(messageId); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index 54a5a62a6bbf3..0192188b12549 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -160,6 +160,7 @@ private void receiveMessage() { dm.properties = msg.getProperties(); dm.publishTime = DateFormatter.format(msg.getPublishTime()); dm.redeliveryCount = msg.getRedeliveryCount(); + dm.encryptionContext = msg.getEncryptionCtx().orElse(null); if (msg.getEventTime() != 0) { dm.eventTime = DateFormatter.format(msg.getEventTime()); } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java index 9660c95be65ca..9091a7eb64ee9 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; +import org.apache.pulsar.common.api.EncryptionContext; @JsonInclude(Include.NON_NULL) public class ConsumerMessage { @@ -32,5 +33,7 @@ public class ConsumerMessage { public int redeliveryCount; public String eventTime; + public EncryptionContext encryptionContext; + public String key; } From c3459ef6977505c6041faee5edbfe93cf55f1bb9 Mon Sep 17 00:00:00 2001 From: ran Date: Sat, 30 Oct 2021 08:17:45 +0800 Subject: [PATCH 101/823] [Python Client] Python client support using custom Avro schema definition (#12516) ### Motivation Currently, the Python client didn't support using schema definition to generate `AvroSchema`, so users couldn't use the schema definition file in the Python client. ### Modifications Add a new init-param `schema_definition` for `AvroSchema` to support initializing the `AvroSchema` by an Avro schema definition. ``` class AvroSchema(Schema): def __init__(self, record_cls, schema_definition=None): if record_cls is None and schema_definition is None: raise AssertionError("The param record_cls and schema_definition shouldn't be both None.") if record_cls is not None: self._schema = record_cls.schema() else: self._schema = schema_definition super(AvroSchema, self).__init__(record_cls, _pulsar.SchemaType.AVRO, self._schema, 'AVRO') ``` ### How to use Assume that there is a company Avro schema definition file `company.avsc` like this. ``` { "doc": "this is doc", "namespace": "example.avro", "type": "record", "name": "Company", "fields": [ {"name": "name", "type": ["null", "string"]}, {"name": "address", "type": ["null", "string"]}, {"name": "employees", "type": ["null", {"type": "array", "items": { "type": "record", "name": "Employee", "fields": [ {"name": "name", "type": ["null", "string"]}, {"name": "age", "type": ["null", "int"]} ] }}]}, {"name": "labels", "type": ["null", {"type": "map", "values": "string"}]} ] } ``` Users could load schema from file by `avro.schema` or `fastavro.schema` > refer to [load_schema](https://fastavro.readthedocs.io/en/latest/schema.html#fastavro._schema_py.load_schema) or [Avro Schema](http://avro.apache.org/docs/current/gettingstartedpython.html) ``` schema_definition = load_schema("examples/company.avsc") # schema_definition = avro.schema.parse(open("examples/company.avsc", "rb").read()).to_json() avro_schema = AvroSchema(None, schema_definition=schema_definition) producer = client.create_producer( topic=topic, schema=avro_schema) consumer = client.subscribe(topic, 'test', schema=avro_schema) company = { "name": "company-name" + str(i), "address": 'xxx road xxx street ' + str(i), "employees": [ {"name": "user" + str(i), "age": 20 + i}, {"name": "user" + str(i), "age": 30 + i}, {"name": "user" + str(i), "age": 35 + i}, ], "labels": { "industry": "software" + str(i), "scale": ">100", "funds": "1000000.0" } } producer.send(company) msg = consumer.receive() # Users could get a dict object by `value()` method. msg.value() ``` (cherry picked from commit 85575f4f5d8eb54516c1c02e1cfcca0d936f2e49) --- .../python/examples/company.avsc | 19 +++ .../python/pulsar/schema/schema_avro.py | 29 ++-- pulsar-client-cpp/python/schema_test.py | 128 +++++++++++++++++- 3 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 pulsar-client-cpp/python/examples/company.avsc diff --git a/pulsar-client-cpp/python/examples/company.avsc b/pulsar-client-cpp/python/examples/company.avsc new file mode 100644 index 0000000000000..cdb595f41d257 --- /dev/null +++ b/pulsar-client-cpp/python/examples/company.avsc @@ -0,0 +1,19 @@ +{ + "doc": "this is doc", + "namespace": "example.avro", + "type": "record", + "name": "Company", + "fields": [ + {"name": "name", "type": ["null", "string"]}, + {"name": "address", "type": ["null", "string"]}, + {"name": "employees", "type": ["null", {"type": "array", "items": { + "type": "record", + "name": "Employee", + "fields": [ + {"name": "name", "type": ["null", "string"]}, + {"name": "age", "type": ["null", "int"]} + ] + }}]}, + {"name": "labels", "type": ["null", {"type": "map", "values": "string"}]} + ] +} \ No newline at end of file diff --git a/pulsar-client-cpp/python/pulsar/schema/schema_avro.py b/pulsar-client-cpp/python/pulsar/schema/schema_avro.py index e76fc51affbe8..58615053ca16f 100644 --- a/pulsar-client-cpp/python/pulsar/schema/schema_avro.py +++ b/pulsar-client-cpp/python/pulsar/schema/schema_avro.py @@ -32,10 +32,15 @@ if HAS_AVRO: class AvroSchema(Schema): - def __init__(self, record_cls): - super(AvroSchema, self).__init__(record_cls, _pulsar.SchemaType.AVRO, - record_cls.schema(), 'AVRO') - self._schema = record_cls.schema() + def __init__(self, record_cls, schema_definition=None): + if record_cls is None and schema_definition is None: + raise AssertionError("The param record_cls and schema_definition shouldn't be both None.") + + if record_cls is not None: + self._schema = record_cls.schema() + else: + self._schema = schema_definition + super(AvroSchema, self).__init__(record_cls, _pulsar.SchemaType.AVRO, self._schema, 'AVRO') def _get_serialized_value(self, x): if isinstance(x, enum.Enum): @@ -53,9 +58,14 @@ def _get_serialized_value(self, x): return x def encode(self, obj): - self._validate_object_type(obj) buffer = io.BytesIO() - m = self.encode_dict(obj.__dict__) + m = obj + if self._record_cls is not None: + self._validate_object_type(obj) + m = self.encode_dict(obj.__dict__) + elif not isinstance(obj, dict): + raise ValueError('If using the custom schema, the record data should be dict type.') + fastavro.schemaless_writer(buffer, self._schema, m) return buffer.getvalue() @@ -68,11 +78,14 @@ def encode_dict(self, d): def decode(self, data): buffer = io.BytesIO(data) d = fastavro.schemaless_reader(buffer, self._schema) - return self._record_cls(**d) + if self._record_cls is not None: + return self._record_cls(**d) + else: + return d else: class AvroSchema(Schema): - def __init__(self, _record_cls): + def __init__(self, _record_cls, _schema_definition): raise Exception("Avro library support was not found. Make sure to install Pulsar client " + "with Avro support: pip3 install 'pulsar-client[avro]'") diff --git a/pulsar-client-cpp/python/schema_test.py b/pulsar-client-cpp/python/schema_test.py index 7adbcbe50e852..d2554da5a7365 100755 --- a/pulsar-client-cpp/python/schema_test.py +++ b/pulsar-client-cpp/python/schema_test.py @@ -25,6 +25,7 @@ from pulsar.schema import * from enum import Enum import json +from fastavro.schema import load_schema class SchemaTest(TestCase): @@ -1145,12 +1146,127 @@ def produce_consume_test(schema_type): client.close() - def test(self): - class NamespaceDemo(Record): - _namespace = 'xxx.xxx.xxx' - x = String() - y = Integer() - print('schema: ', NamespaceDemo.schema()) + def custom_schema_test(self): + + def encode_and_decode(schema_definition): + avro_schema = AvroSchema(None, schema_definition=schema_definition) + + company = { + "name": "company-name", + "address": 'xxx road xxx street', + "employees": [ + {"name": "user1", "age": 25}, + {"name": "user2", "age": 30}, + {"name": "user3", "age": 35}, + ], + "labels": { + "industry": "software", + "scale": ">100", + "funds": "1000000.0" + } + } + data = avro_schema.encode(company) + company_decode = avro_schema.decode(data) + self.assertEqual(company, company_decode) + + schema_definition = { + 'doc': 'this is doc', + 'namespace': 'example.avro', + 'type': 'record', + 'name': 'Company', + 'fields': [ + {'name': 'name', 'type': ['null', 'string']}, + {'name': 'address', 'type': ['null', 'string']}, + {'name': 'employees', 'type': ['null', {'type': 'array', 'items': { + 'type': 'record', + 'name': 'Employee', + 'fields': [ + {'name': 'name', 'type': ['null', 'string']}, + {'name': 'age', 'type': ['null', 'int']} + ] + }}]}, + {'name': 'labels', 'type': ['null', {'type': 'map', 'values': 'string'}]} + ] + } + encode_and_decode(schema_definition) + # Users could load schema from file by `fastavro.schema` + # Or use `avro.schema` like this `avro.schema.parse(open("examples/company.avsc", "rb").read()).to_json()` + encode_and_decode(load_schema("examples/company.avsc")) + + def custom_schema_produce_and_consume_test(self): + client = pulsar.Client(self.serviceUrl) + + def produce_and_consume(topic, schema_definition): + print('custom schema produce and consume test topic - ', topic) + example_avro_schema = AvroSchema(None, schema_definition=schema_definition) + + producer = client.create_producer( + topic=topic, + schema=example_avro_schema) + consumer = client.subscribe(topic, 'test', schema=example_avro_schema) + + for i in range(0, 10): + company = { + "name": "company-name" + str(i), + "address": 'xxx road xxx street ' + str(i), + "employees": [ + {"name": "user" + str(i), "age": 20 + i}, + {"name": "user" + str(i), "age": 30 + i}, + {"name": "user" + str(i), "age": 35 + i}, + ], + "labels": { + "industry": "software" + str(i), + "scale": ">100", + "funds": "1000000.0" + } + } + producer.send(company) + + for i in range(0, 10): + msg = consumer.receive() + company = { + "name": "company-name" + str(i), + "address": 'xxx road xxx street ' + str(i), + "employees": [ + {"name": "user" + str(i), "age": 20 + i}, + {"name": "user" + str(i), "age": 30 + i}, + {"name": "user" + str(i), "age": 35 + i}, + ], + "labels": { + "industry": "software" + str(i), + "scale": ">100", + "funds": "1000000.0" + } + } + self.assertEqual(msg.value(), company) + consumer.acknowledge(msg) + + consumer.close() + producer.close() + + schema_definition = { + 'doc': 'this is doc', + 'namespace': 'example.avro', + 'type': 'record', + 'name': 'Company', + 'fields': [ + {'name': 'name', 'type': ['null', 'string']}, + {'name': 'address', 'type': ['null', 'string']}, + {'name': 'employees', 'type': ['null', {'type': 'array', 'items': { + 'type': 'record', + 'name': 'Employee', + 'fields': [ + {'name': 'name', 'type': ['null', 'string']}, + {'name': 'age', 'type': ['null', 'int']} + ] + }}]}, + {'name': 'labels', 'type': ['null', {'type': 'map', 'values': 'string'}]} + ] + } + produce_and_consume('custom-schema-test-1', schema_definition=schema_definition) + produce_and_consume('custom-schema-test-2', schema_definition=load_schema("examples/company.avsc")) + + client.close() if __name__ == '__main__': main() From eabd17d3ee4bfa239c3e14e873c571ddd4de15b5 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 27 Oct 2021 00:50:47 +0800 Subject: [PATCH 102/823] Update Producer stats on producer close() (#12500) (cherry picked from commit 848690621299353284932e9281e4689813835855) --- .../impl/ProducerStatsRecorderImpl.java | 90 ++++++++++--------- .../impl/PartitionedProducerImplTest.java | 33 +++++++ .../impl/ProducerStatsRecorderImplTest.java | 20 +++++ 3 files changed, 100 insertions(+), 43 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java index faf73cb3e2788..6b435d683032b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java @@ -111,49 +111,7 @@ private void init(ProducerConfigurationData conf) { } try { - long now = System.nanoTime(); - double elapsed = (now - oldTime) / 1e9; - oldTime = now; - - long currentNumMsgsSent = numMsgsSent.sumThenReset(); - long currentNumBytesSent = numBytesSent.sumThenReset(); - long currentNumSendFailedMsgs = numSendFailed.sumThenReset(); - long currentNumAcksReceived = numAcksReceived.sumThenReset(); - - totalMsgsSent.add(currentNumMsgsSent); - totalBytesSent.add(currentNumBytesSent); - totalSendFailed.add(currentNumSendFailedMsgs); - totalAcksReceived.add(currentNumAcksReceived); - - synchronized (ds) { - latencyPctValues = ds.getQuantiles(PERCENTILES); - ds.reset(); - } - - sendMsgsRate = currentNumMsgsSent / elapsed; - sendBytesRate = currentNumBytesSent / elapsed; - - if ((currentNumMsgsSent | currentNumSendFailedMsgs | currentNumAcksReceived - | currentNumMsgsSent) != 0) { - - for (int i = 0; i < latencyPctValues.length; i++) { - if (Double.isNaN(latencyPctValues[i])) { - latencyPctValues[i] = 0; - } - } - - log.info("[{}] [{}] Pending messages: {} --- Publish throughput: {} msg/s --- {} Mbit/s --- " - + "Latency: med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - max: {} ms --- " - + "Ack received rate: {} ack/s --- Failed messages: {}", producer.getTopic(), - producer.getProducerName(), producer.getPendingQueueSize(), - THROUGHPUT_FORMAT.format(sendMsgsRate), - THROUGHPUT_FORMAT.format(sendBytesRate / 1024 / 1024 * 8), - DEC.format(latencyPctValues[0]), DEC.format(latencyPctValues[2]), - DEC.format(latencyPctValues[3]), DEC.format(latencyPctValues[4]), - DEC.format(latencyPctValues[5]), - THROUGHPUT_FORMAT.format(currentNumAcksReceived / elapsed), currentNumSendFailedMsgs); - } - + updateStats(); } catch (Exception e) { log.error("[{}] [{}]: {}", producer.getTopic(), producer.getProducerName(), e.getMessage()); } finally { @@ -171,6 +129,51 @@ Timeout getStatTimeout() { return statTimeout; } + protected void updateStats() { + long now = System.nanoTime(); + double elapsed = (now - oldTime) / 1e9; + oldTime = now; + + long currentNumMsgsSent = numMsgsSent.sumThenReset(); + long currentNumBytesSent = numBytesSent.sumThenReset(); + long currentNumSendFailedMsgs = numSendFailed.sumThenReset(); + long currentNumAcksReceived = numAcksReceived.sumThenReset(); + + totalMsgsSent.add(currentNumMsgsSent); + totalBytesSent.add(currentNumBytesSent); + totalSendFailed.add(currentNumSendFailedMsgs); + totalAcksReceived.add(currentNumAcksReceived); + + synchronized (ds) { + latencyPctValues = ds.getQuantiles(PERCENTILES); + ds.reset(); + } + + sendMsgsRate = currentNumMsgsSent / elapsed; + sendBytesRate = currentNumBytesSent / elapsed; + + if ((currentNumMsgsSent | currentNumSendFailedMsgs | currentNumAcksReceived + | currentNumMsgsSent) != 0) { + + for (int i = 0; i < latencyPctValues.length; i++) { + if (Double.isNaN(latencyPctValues[i])) { + latencyPctValues[i] = 0; + } + } + + log.info("[{}] [{}] Pending messages: {} --- Publish throughput: {} msg/s --- {} Mbit/s --- " + + "Latency: med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - max: {} ms --- " + + "Ack received rate: {} ack/s --- Failed messages: {}", producer.getTopic(), + producer.getProducerName(), producer.getPendingQueueSize(), + THROUGHPUT_FORMAT.format(sendMsgsRate), + THROUGHPUT_FORMAT.format(sendBytesRate / 1024 / 1024 * 8), + DEC.format(latencyPctValues[0]), DEC.format(latencyPctValues[2]), + DEC.format(latencyPctValues[3]), DEC.format(latencyPctValues[4]), + DEC.format(latencyPctValues[5]), + THROUGHPUT_FORMAT.format(currentNumAcksReceived / elapsed), currentNumSendFailedMsgs); + } + } + @Override public void updateNumMsgsSent(long numMsgs, long totalMsgsSize) { numMsgsSent.add(numMsgs); @@ -297,6 +300,7 @@ public double getSendLatencyMillisMax() { } public void cancelStatsTimeout() { + this.updateStats(); if (statTimeout != null) { statTimeout.cancel(); statTimeout = null; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java index 1f9496bbd505a..ad2c992b34c86 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java @@ -202,4 +202,37 @@ public void testGetStats() throws Exception { impl.getStats(); } + @Test + public void testGetStatsWithoutArriveUpdateInterval() throws Exception { + String topicName = "test-stats-without-arrive-interval"; + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("pulsar://localhost:6650"); + conf.setStatsIntervalSeconds(100); + + ThreadFactory threadFactory = + new DefaultThreadFactory("client-test-stats", Thread.currentThread().isDaemon()); + EventLoopGroup eventLoopGroup = EventLoopUtil + .newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); + + PulsarClientImpl clientImpl = new PulsarClientImpl(conf, eventLoopGroup); + + ProducerConfigurationData producerConfData = new ProducerConfigurationData(); + producerConfData.setMessageRoutingMode(MessageRoutingMode.CustomPartition); + producerConfData.setCustomMessageRouter(new CustomMessageRouter()); + + assertEquals(Long.parseLong("100"), clientImpl.getConfiguration().getStatsIntervalSeconds()); + + PartitionedProducerImpl impl = new PartitionedProducerImpl<>( + clientImpl, topicName, producerConfData, + 1, null, null, null); + + impl.getProducers().get(0).getStats().incrementSendFailed(); + ProducerStatsRecorderImpl stats = impl.getStats(); + assertEquals(stats.getTotalSendFailed(), 0); + // When close producer, the ProducerStatsRecorder will update stats immediately + impl.close(); + stats = impl.getStats(); + assertEquals(stats.getTotalSendFailed(), 1); + } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java index d654158e9efa6..f6e7f284ce639 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java @@ -54,4 +54,24 @@ public void testIncrementNumAcksReceived() throws Exception { Thread.sleep(1200); assertEquals(1000.0, recorder.getSendLatencyMillisMax(), 0.5); } + + @Test + public void testGetStatsAndCancelStatsTimeoutWithoutArriveUpdateInterval() { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setStatsIntervalSeconds(60); + PulsarClientImpl client = mock(PulsarClientImpl.class); + when(client.getConfiguration()).thenReturn(conf); + Timer timer = new HashedWheelTimer(); + when(client.timer()).thenReturn(timer); + ProducerImpl producer = mock(ProducerImpl.class); + when(producer.getTopic()).thenReturn("topic-test"); + when(producer.getProducerName()).thenReturn("producer-test"); + when(producer.getPendingQueueSize()).thenReturn(1); + ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); + ProducerStatsRecorderImpl recorder = new ProducerStatsRecorderImpl(client, producerConfigurationData, producer); + long latencyNs = TimeUnit.SECONDS.toNanos(1); + recorder.incrementNumAcksReceived(latencyNs); + recorder.cancelStatsTimeout(); + assertEquals(1000.0, recorder.getSendLatencyMillisMax(), 0.5); + } } From 4d29ea63f2f3d4461e1b1d15222deda1aa993dc7 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Mon, 1 Nov 2021 16:03:58 +0800 Subject: [PATCH 103/823] [pulsar-admin] Modify exception of set-properties for namespace (#12436) when I execute ./pulsar-admin namespaces set-properties --properties a=a=b test/app1 we hope to display a readable prompt of set-properties for namespace when exception occurs. Modifications Change IllegalArgumentException to ParameterException (cherry picked from commit cd9a65ee3c648443ee13f3248add1fd4e10cae89) --- .../java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index c642288383aeb..f6fa1234c4c34 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -2198,15 +2198,18 @@ void run() throws Exception { String namespace = validateNamespace(params); Map map = new HashMap<>(); if (properties.size() == 0) { - throw new IllegalArgumentException("Required at least one property for the namespace."); + throw new ParameterException(String.format("Required at least one property for the namespace, " + + "but found %d.", properties.size())); } for (String property : properties) { if (!property.contains("=")) { - throw new IllegalArgumentException("Invalid key value pair format."); + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); } else { String[] keyValue = property.split("="); if (keyValue.length != 2) { - throw new IllegalArgumentException("Invalid key value pair format."); + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); } map.put(keyValue[0], keyValue[1]); } From f7692269f9155b3da9e9a2e0ff028dbaa7e4b4fc Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Mon, 1 Nov 2021 16:01:32 +0800 Subject: [PATCH 104/823] =?UTF-8?q?The=20count=20of=20topics=20on=20the=20?= =?UTF-8?q?bundle=20is=20less=20than=202=EF=BC=8Cskip=20split=20(#12527)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The count of topics on the bundle is less than 2，skip split (cherry picked from commit 384c5dce8b9948ef8d01a20d18f3fd310e63f091) --- .../pulsar/broker/loadbalance/impl/BundleSplitterTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java index e81fb506f290e..fa48618bd205c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java @@ -72,8 +72,8 @@ public Set findBundlesToSplit(final LoadData loadData, final PulsarServi for (final Map.Entry entry : localData.getLastStats().entrySet()) { final String bundle = entry.getKey(); final NamespaceBundleStats stats = entry.getValue(); - if (stats.topics == 1) { - log.info("namespace bundle {} only have 1 topic", bundle); + if (stats.topics < 2) { + log.info("The count of topics on the bundle {} is less than 2，skip split!", bundle); continue; } double totalMessageRate = 0; From 849cfe4ac6bf366bce2a15a379e0fae0a14a7600 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 1 Nov 2021 23:36:38 +0800 Subject: [PATCH 105/823] Optimize Tests (#12560) (cherry picked from commit cb703cab9b58bc7fe2da5694e2404538c44dab7a) --- .../TopicTransactionBufferRecoverTest.java | 36 +--------- .../TransactionClientConnectTest.java | 31 +-------- .../transaction/TransactionProduceTest.java | 42 ++--------- .../broker/transaction/TransactionTest.java | 26 +------ .../transaction/TransactionTestBase.java | 69 ++++++++++++++----- .../buffer/TransactionLowWaterMarkTest.java | 39 +---------- .../buffer/TransactionStablePositionTest.java | 26 +------ .../TransactionMetaStoreAssignmentTest.java | 14 +--- .../PendingAckInMemoryDeleteTest.java | 43 +----------- .../pendingack/PendingAckPersistentTest.java | 35 ++-------- .../client/impl/TransactionEndToEndTest.java | 38 +--------- 11 files changed, 72 insertions(+), 327 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 3607b45a64a28..335cecc44138d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.broker.transaction; -import com.google.common.collect.Sets; - import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -55,10 +53,7 @@ import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.EventsTopicNames; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; @@ -75,8 +70,6 @@ @Slf4j public class TopicTransactionBufferRecoverTest extends TransactionTestBase { - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final String RECOVER_COMMIT = NAMESPACE1 + "/recover-commit"; private static final String RECOVER_ABORT = NAMESPACE1 + "/recover-abort"; private static final String SUBSCRIPTION_NAME = "test-recover"; @@ -85,36 +78,9 @@ public class TopicTransactionBufferRecoverTest extends TransactionTestBase { private static final int NUM_PARTITIONS = 16; @BeforeMethod protected void setup() throws Exception { - setBrokerCount(1); - internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - admin.topics().createNonPartitionedTopic(RECOVER_COMMIT); + setUpBase(1, NUM_PARTITIONS, RECOVER_COMMIT, 0); admin.topics().createNonPartitionedTopic(RECOVER_ABORT); admin.topics().createNonPartitionedTopic(TAKE_SNAPSHOT); - - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); - - if (pulsarClient != null) { - pulsarClient.shutdown(); - } - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - - - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java index 42eadfe98d87c..a51eae82f006e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java @@ -18,20 +18,14 @@ */ package org.apache.pulsar.broker.transaction; -import com.google.common.collect.Sets; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.TransactionMetadataStoreService; import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.TransactionMetaStoreHandler; import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; @@ -51,33 +45,12 @@ public class TransactionClientConnectTest extends TransactionTestBase { - private static final String RECONNECT_TOPIC = "persistent://public/txn/txn-client-reconnect-test"; + private static final String RECONNECT_TOPIC = NAMESPACE1 + "/txn-client-reconnect-test"; private static final int NUM_PARTITIONS = 1; @BeforeMethod(alwaysRun = true) public void setup() throws Exception { - setBrokerCount(1); - super.internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace("public/txn", 10); - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createNonPartitionedTopic(RECONNECT_TOPIC); + setUpBase(1, NUM_PARTITIONS, RECONNECT_TOPIC, 0); admin.topics().createSubscription(RECONNECT_TOPIC, "test", MessageId.latest); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); - - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index 0e63f533b58c0..cbae03b1a8b94 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.transaction; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; @@ -50,10 +49,7 @@ import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.api.proto.MarkerType; import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; import org.testng.Assert; @@ -69,9 +65,6 @@ public class TransactionProduceTest extends TransactionTestBase { private static final int TOPIC_PARTITION = 3; - - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final String PRODUCE_COMMIT_TOPIC = NAMESPACE1 + "/produce-commit"; private static final String PRODUCE_ABORT_TOPIC = NAMESPACE1 + "/produce-abort"; private static final String ACK_COMMIT_TOPIC = NAMESPACE1 + "/ack-commit"; @@ -79,37 +72,10 @@ public class TransactionProduceTest extends TransactionTestBase { private static final int NUM_PARTITIONS = 16; @BeforeMethod protected void setup() throws Exception { - setBrokerCount(1); - internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - admin.topics().createPartitionedTopic(PRODUCE_COMMIT_TOPIC, 3); - admin.topics().createPartitionedTopic(PRODUCE_ABORT_TOPIC, 3); - admin.topics().createPartitionedTopic(ACK_COMMIT_TOPIC, 3); - admin.topics().createPartitionedTopic(ACK_ABORT_TOPIC, 3); - - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); - - if (pulsarClient != null) { - pulsarClient.shutdown(); - } - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - - - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); + setUpBase(1, NUM_PARTITIONS, PRODUCE_COMMIT_TOPIC, TOPIC_PARTITION); + admin.topics().createPartitionedTopic(PRODUCE_ABORT_TOPIC, TOPIC_PARTITION); + admin.topics().createPartitionedTopic(ACK_COMMIT_TOPIC, TOPIC_PARTITION); + admin.topics().createPartitionedTopic(ACK_ABORT_TOPIC, TOPIC_PARTITION); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 12a6b285ccd5c..31d8113795d68 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -86,36 +86,12 @@ @Test(groups = "broker") public class TransactionTest extends TransactionTestBase { - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final int NUM_BROKERS = 1; private static final int NUM_PARTITIONS = 1; @BeforeMethod protected void setup() throws Exception { - this.setBrokerCount(NUM_BROKERS); - this.internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length - 1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder() - .serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); - pulsarClient.close(); - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); + setUpBase(NUM_BROKERS, NUM_PARTITIONS, NAMESPACE1 + "/test", 0); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 1dba73a0378a5..e13365d15f971 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; +import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import io.netty.channel.EventLoopGroup; import java.util.ArrayList; @@ -45,10 +46,10 @@ import org.apache.pulsar.broker.auth.SameThreadOrderedSafeExecutor; import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor; import org.apache.pulsar.broker.namespace.NamespaceService; -import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; -import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; -import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -61,6 +62,7 @@ import org.apache.zookeeper.MockZooKeeperSession; import org.apache.zookeeper.ZooKeeper; import org.awaitility.Awaitility; +import org.testng.Assert; @Slf4j public abstract class TransactionTestBase extends TestRetrySupport { @@ -83,6 +85,9 @@ public abstract class TransactionTestBase extends TestRetrySupport { private OrderedExecutor bkExecutor; private NonClosableMockBookKeeper mockBookKeeper; + public static final String TENANT = "tnx"; + protected static final String NAMESPACE1 = TENANT + "/ns1"; + public void internalSetup() throws Exception { incrementSetupNumber(); init(); @@ -108,6 +113,40 @@ private void init() throws Exception { mockBookKeeper = createMockBookKeeper(bkExecutor); startBroker(); } + protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int numPartitions) throws Exception{ + setBrokerCount(numBroker); + internalSetup(); + + String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); + String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; + admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + + webServicePort).build()); + + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), numPartitionsOfTC); + if (topic != null) { + admin.tenants().createTenant(TENANT, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NAMESPACE1); + if (numPartitions == 0) { + admin.topics().createNonPartitionedTopic(topic); + } else { + admin.topics().createPartitionedTopic(topic, numPartitions); + } + } + if (pulsarClient != null) { + pulsarClient.shutdown(); + } + pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(true) + .build(); + // wait tc init success to ready state + waitForCoordinatorToBeAvailable(numPartitionsOfTC); + } protected void startBroker() throws Exception { for (int i = 0; i < brokerCount; i++) { @@ -295,20 +334,12 @@ protected final void internalCleanup() { } public void waitForCoordinatorToBeAvailable(int numOfTCPerBroker){ // wait tc init success to ready state - Awaitility.await().until(() -> { - Map stores = - getPulsarServiceList().get(brokerCount-1).getTransactionMetadataStoreService().getStores(); - if (stores.size() == numOfTCPerBroker) { - for (TransactionCoordinatorID transactionCoordinatorID : stores.keySet()) { - if (((MLTransactionMetadataStore) stores.get(transactionCoordinatorID)).getState() - != TransactionMetadataStoreState.State.Ready) { - return false; - } - } - return true; - } else { - return false; - } - }); + Awaitility.await() + .untilAsserted(() -> { + int transactionMetaStoreCount = pulsarServiceList.stream() + .mapToInt(pulsarService -> pulsarService.getTransactionMetadataStoreService().getStores().size()) + .sum(); + Assert.assertEquals(transactionMetaStoreCount, numOfTCPerBroker); + }); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java index db9d4073b024f..873509ff6bf97 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java @@ -23,17 +23,12 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - -import com.google.common.collect.Sets; - import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - -import javax.validation.constraints.AssertTrue; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -47,7 +42,6 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; @@ -55,17 +49,13 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.transaction.TransactionImpl; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; + import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; -import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; import org.testng.Assert; @@ -80,35 +70,12 @@ @Test(groups = "broker") public class TransactionLowWaterMarkTest extends TransactionTestBase { - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final String TOPIC = NAMESPACE1 + "/test-topic"; @BeforeMethod(alwaysRun = true) protected void setup() throws Exception { - setBrokerCount(1); - internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - admin.topics().createNonPartitionedTopic(TOPIC); - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); - - if (pulsarClient != null) { - pulsarClient.shutdown(); - } - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); + setUpBase(1, 16, TOPIC, 0); + Map stores = getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores(); Awaitility.await().until(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java index e43f2625c3306..ef1c76182467d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java @@ -54,35 +54,11 @@ @Test(groups = "broker") public class TransactionStablePositionTest extends TransactionTestBase { - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final String TOPIC = NAMESPACE1 + "/test-topic"; @BeforeMethod protected void setup() throws Exception { - internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - admin.topics().createNonPartitionedTopic(TOPIC); - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); - - if (pulsarClient != null) { - pulsarClient.shutdown(); - } - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - + setUpBase(1, 16, TOPIC, 0); Awaitility.await().until(() -> ((PulsarClientImpl) pulsarClient) .getTcClient().getState() == TransactionCoordinatorClient.State.READY); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java index 172530566a98b..01027868e272b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java @@ -20,15 +20,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.collect.Sets; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.awaitility.Awaitility; import org.testng.Assert; @@ -41,15 +37,7 @@ public class TransactionMetaStoreAssignmentTest extends TransactionTestBase { @Override @BeforeMethod(alwaysRun = true) protected void setup() throws Exception { - setBrokerCount(3); - super.internalSetup(); - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); + setUpBase(3, 16, null, 0); pulsarClient.close(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index fc952c42baedb..bc22473e285f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.transaction.pendingack; -import com.google.common.collect.Sets; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -35,21 +34,11 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; -import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; -import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -58,7 +47,6 @@ import java.lang.reflect.Field; import java.util.HashMap; -import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; @@ -71,39 +59,10 @@ @Test(groups = "broker") public class PendingAckInMemoryDeleteTest extends TransactionTestBase { - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final int NUM_PARTITIONS = 16; @BeforeMethod protected void setup() throws Exception { - setBrokerCount(1); - internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); - - if (pulsarClient != null) { - pulsarClient.shutdown(); - } - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - - Map stores = - getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores(); - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); + setUpBase(1, NUM_PARTITIONS, NAMESPACE1 +"/test", 0); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index 3820ebc6cee79..97f8f51d3e89b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -62,36 +62,13 @@ @Slf4j public class PendingAckPersistentTest extends TransactionTestBase { - private static final String PENDING_ACK_REPLAY_TOPIC = "persistent://public/txn/pending-ack-replay"; - - private static final String NAMESPACE = "public/txn"; + private static final String PENDING_ACK_REPLAY_TOPIC = NAMESPACE1 + "/pending-ack-replay"; private static final int NUM_PARTITIONS = 16; @BeforeMethod public void setup() throws Exception { - setBrokerCount(1); - super.internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterDataImpl.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE, 10); - admin.topics().createNonPartitionedTopic(PENDING_ACK_REPLAY_TOPIC); - - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); + setUpBase(1, NUM_PARTITIONS, PENDING_ACK_REPLAY_TOPIC, 0); } @AfterMethod(alwaysRun = true) @@ -312,7 +289,7 @@ private void testDeleteSubThenDeletePendingAckManagedLedger() throws Exception { String subName = "test-delete"; String topic = TopicName.get(TopicDomain.persistent.toString(), - NamespaceName.get(NAMESPACE), "test-delete").toString(); + NamespaceName.get(NAMESPACE1), "test-delete").toString(); @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(topic) @@ -325,7 +302,7 @@ private void testDeleteSubThenDeletePendingAckManagedLedger() throws Exception { admin.topics().deleteSubscription(topic, subName); - List topics = admin.namespaces().getTopics(NAMESPACE); + List topics = admin.namespaces().getTopics(NAMESPACE1); TopicStats topicStats = admin.topics().getStats(topic, false); @@ -341,7 +318,7 @@ private void testDeleteTopicThenDeletePendingAckManagedLedger() throws Exception String subName2 = "test-delete"; String topic = TopicName.get(TopicDomain.persistent.toString(), - NamespaceName.get(NAMESPACE), "test-delete").toString(); + NamespaceName.get(NAMESPACE1), "test-delete").toString(); @Cleanup Consumer consumer1 = pulsarClient.newConsumer() .topic(topic) @@ -364,7 +341,7 @@ private void testDeleteTopicThenDeletePendingAckManagedLedger() throws Exception admin.topics().delete(topic); - List topics = admin.namespaces().getTopics(NAMESPACE); + List topics = admin.namespaces().getTopics(NAMESPACE1); assertFalse(topics.contains(MLPendingAckStore.getTransactionPendingAckStoreSuffix(topic, subName1))); assertFalse(topics.contains(MLPendingAckStore.getTransactionPendingAckStoreSuffix(topic, subName2))); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 9c30f4a651251..463044969ac10 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -23,9 +23,6 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - -import com.google.common.collect.Sets; - import java.lang.reflect.Field; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -63,11 +60,8 @@ import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.api.proto.CommandAck; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; @@ -87,42 +81,14 @@ public class TransactionEndToEndTest extends TransactionTestBase { private static final int TOPIC_PARTITION = 3; - - private static final String TENANT = "tnx"; - private static final String NAMESPACE1 = TENANT + "/ns1"; private static final String TOPIC_OUTPUT = NAMESPACE1 + "/output"; private static final String TOPIC_MESSAGE_ACK_TEST = NAMESPACE1 + "/message-ack-test"; private static final int NUM_PARTITIONS = 16; @BeforeMethod protected void setup() throws Exception { - setBrokerCount(1); - internalSetup(); - - String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); - String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; - admin.clusters().createCluster(CLUSTER_NAME, ClusterData.builder().serviceUrl("http://localhost:" + webServicePort).build()); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); - admin.topics().createPartitionedTopic(TOPIC_OUTPUT, TOPIC_PARTITION); + setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); admin.topics().createPartitionedTopic(TOPIC_MESSAGE_ACK_TEST, 1); - - admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); - admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); - - if (pulsarClient != null) { - pulsarClient.close(); - } - pulsarClient = PulsarClient.builder() - .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) - .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); - - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(NUM_PARTITIONS); } + } @AfterMethod(alwaysRun = true) protected void cleanup() { From 33b5d5fa5b965fc7da52542cb6a7f004e852a408 Mon Sep 17 00:00:00 2001 From: Yuto Furuta Date: Tue, 2 Nov 2021 21:05:31 +0900 Subject: [PATCH 106/823] [CAPI] Support setting priority for consumers (#12526) ### Motivation We would like to make it possible for C API based client library to set the priority level for consumers. ### Modifications Add methods to set/get priority level for consumers. Co-authored-by: k2la (cherry picked from commit a75a9746a12a6461c87518434f0f550563718e9d) --- .../include/pulsar/c/consumer_configuration.h | 6 ++++++ pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h b/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h index efe353ad167c4..a11e11e480f52 100644 --- a/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h +++ b/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h @@ -275,6 +275,12 @@ PULSAR_PUBLIC void pulsar_consumer_set_subscription_initial_position( PULSAR_PUBLIC void pulsar_consumer_configuration_set_property(pulsar_consumer_configuration_t *conf, const char *name, const char *value); +PULSAR_PUBLIC void pulsar_consumer_configuration_set_priority_level( + pulsar_consumer_configuration_t *consumer_configuration, int priority_level); + +PULSAR_PUBLIC int pulsar_consumer_configuration_get_priority_level( + pulsar_consumer_configuration_t *consumer_configuration); + // const CryptoKeyReaderPtr getCryptoKeyReader() // // const; diff --git a/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc b/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc index 90c60dfc991ad..aaec12cff6f8d 100644 --- a/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc +++ b/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc @@ -185,3 +185,13 @@ int pulsar_consumer_get_subscription_initial_position( pulsar_consumer_configuration_t *consumer_configuration) { return consumer_configuration->consumerConfiguration.getSubscriptionInitialPosition(); } + +void pulsar_consumer_configuration_set_priority_level(pulsar_consumer_configuration_t *consumer_configuration, + int priority_level) { + consumer_configuration->consumerConfiguration.setPriorityLevel(priority_level); +} + +int pulsar_consumer_configuration_get_priority_level( + pulsar_consumer_configuration_t *consumer_configuration) { + return consumer_configuration->consumerConfiguration.getPriorityLevel(); +} From ed519055161f637c81349282176cb151a6dafc7a Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 3 Nov 2021 12:57:06 +0800 Subject: [PATCH 107/823] [Transaction]Fix maxReadPosition with normal publish (#12386) now Transaction buffer syncMaxReadPositionForNormalPublish don't wait recover success. Transaction buffer recover is asynchronous, so we need to wait buffer recover success then syncMaxReadPositionForNormalPublish when producer normal message. We should not change maxReadPosition before TransactionBuffer recovers completely. It will change when the state of TB is NoSnapshot or the state of TB is Ready but the ongoingTxn is empty. (cherry picked from commit 32c8f865f731f739065ef900126973be258d3801) --- .../buffer/impl/TopicTransactionBuffer.java | 7 +- .../transaction/TransactionConsumeTest.java | 17 +++- .../broker/transaction/TransactionTest.java | 81 ++++++++++++++++++- 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 78f92959abc34..84b209c6cf3a4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -455,9 +455,14 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { // when ongoing transaction is empty, proved that lastAddConfirm is can read max position, because callback // thread is the same tread, in this time the lastAddConfirm don't content transaction message. synchronized (TopicTransactionBuffer.this) { - if (ongoingTxns.isEmpty()) { + if (checkIfNoSnapshot()) { maxReadPosition = position; changeMaxReadPositionAndAddAbortTimes.incrementAndGet(); + } else if (checkIfReady()) { + if (ongoingTxns.isEmpty()) { + maxReadPosition = position; + changeMaxReadPositionAndAddAbortTimes.incrementAndGet(); + } } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index 82fd1d1f9eb5e..d381487a82bb3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -36,12 +36,16 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -75,6 +79,11 @@ public void setup() throws Exception { new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME))); admin.namespaces().createNamespace("public/txn", 10); admin.topics().createNonPartitionedTopic(CONSUME_TOPIC); + + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 1); } @AfterMethod(alwaysRun = true) @@ -229,7 +238,12 @@ private void sendNormalMessages(Producer producer, int startMsgCnt, private List appendTransactionMessages( TxnID txnID, PersistentTopic topic, int transactionMsgCnt, List sendMessageList) - throws ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException, PulsarClientException { + //Change the state of TB to Ready. + @Cleanup + Producer producer = PulsarClient.builder().serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .enableTransaction(true).build() + .newProducer(Schema.STRING).topic(CONSUME_TOPIC).sendTimeout(0, TimeUnit.SECONDS).create(); List positionList = new ArrayList<>(); for (int i = 0; i < transactionMsgCnt; i++) { final int j = i; @@ -239,7 +253,6 @@ private List appendTransactionMessages( .setTxnidMostBits(txnID.getMostSigBits()) .setTxnidLeastBits(txnID.getLeastSigBits()) .setPublishTime(System.currentTimeMillis()); - String msg = TXN_MSG_CONTENT + i; sendMessageList.add(msg); ByteBuf headerAndPayload = Commands.serializeMetadataAndPayload( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 31d8113795d68..a237314331662 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -26,7 +26,6 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Sets; import io.netty.buffer.Unpooled; import java.lang.reflect.Field; import java.util.List; @@ -42,10 +41,13 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.buffer.matadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; @@ -68,9 +70,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.RetentionPolicies; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; @@ -409,4 +409,79 @@ public void completed(Exception e, long ledgerId, long entryId) { return true; }); } + + @Test + public void testMaxReadPositionForNormalPublish() throws Exception{ + String topic = "persistent://" + NAMESPACE1 + "/NormalPublish"; + admin.topics().createNonPartitionedTopic(topic); + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(topic, false).get().get(); + + TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); + PulsarClient noTxnClient = PulsarClient.builder().enableTransaction(false) + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()).build(); + + //test the state of TransactionBuffer is NoSnapshot + //before build Producer by pulsarClient that enables transaction. + Producer normalProducer = noTxnClient.newProducer(Schema.STRING) + .producerName("testNormalPublish") + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(topicTransactionBuffer.checkIfNoSnapshot())); + + //test publishing normal messages will change maxReadPosition in the state of NoSnapshot. + MessageIdImpl messageId = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); + PositionImpl position = topicTransactionBuffer.getMaxReadPosition(); + Assert.assertEquals(position.getLedgerId(), messageId.getLedgerId()); + Assert.assertEquals(position.getEntryId(), messageId.getEntryId()); + + //test the state of TransactionBuffer is Ready after build Producer by pulsarClient that enables transaction. + Producer txnProducer = pulsarClient.newProducer(Schema.STRING) + .producerName("testTransactionPublish") + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + Awaitility.await().untilAsserted(() ->Assert.assertTrue(topicTransactionBuffer.checkIfReady())); + //test publishing txn messages will not change maxReadPosition if don`t commit or abort. + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); + MessageIdImpl messageId1 = (MessageIdImpl) txnProducer.newMessage(transaction).value("txn message").send(); + PositionImpl position1 = topicTransactionBuffer.getMaxReadPosition(); + Assert.assertEquals(position1.getLedgerId(), messageId.getLedgerId()); + Assert.assertEquals(position1.getEntryId(), messageId.getEntryId()); + + MessageIdImpl messageId2 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); + PositionImpl position2 = topicTransactionBuffer.getMaxReadPosition(); + Assert.assertEquals(position2.getLedgerId(), messageId.getLedgerId()); + Assert.assertEquals(position2.getEntryId(), messageId.getEntryId()); + transaction.commit().get(); + PositionImpl position3 = topicTransactionBuffer.getMaxReadPosition(); + + Assert.assertEquals(position3.getLedgerId(), messageId2.getLedgerId()); + Assert.assertEquals(position3.getEntryId(), messageId2.getEntryId() + 1); + + //test publishing normal messages will change maxReadPosition if the state of TB + //is Ready and ongoingTxns is empty. + MessageIdImpl messageId4 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); + PositionImpl position4 = topicTransactionBuffer.getMaxReadPosition(); + Assert.assertEquals(position4.getLedgerId(), messageId4.getLedgerId()); + Assert.assertEquals(position4.getEntryId(), messageId4.getEntryId()); + + //test publishing normal messages will not change maxReadPosition if the state o TB is Initializing. + Class transactionBufferStateClass = + (Class) topicTransactionBuffer.getClass().getSuperclass(); + Field field = transactionBufferStateClass.getDeclaredField("state"); + field.setAccessible(true); + Class topicTransactionBufferClass = TopicTransactionBuffer.class; + Field maxReadPositionField = topicTransactionBufferClass.getDeclaredField("maxReadPosition"); + maxReadPositionField.setAccessible(true); + field.set(topicTransactionBuffer, TopicTransactionBufferState.State.Initializing); + MessageIdImpl messageId5 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); + PositionImpl position5 = (PositionImpl) maxReadPositionField.get(topicTransactionBuffer); + Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); + Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); + + } } From 038bb43e7c1e7749348807864cca63586a5a321c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 3 Nov 2021 23:10:15 +0800 Subject: [PATCH 108/823] [C++] Fix request timeout for GetLastMessageId doesn't work (#12586) * Fix request timeout for GetLastMessageId doesn't work * Fix CentOS 7 build error * Revert refactors * Remove redundant clear for listeners * Use swap instead of move (cherry picked from commit a54c6c003c626cb16d90200ad81dd3ec37be2133) --- pulsar-client-cpp/lib/ClientConnection.cc | 7 +- pulsar-client-cpp/lib/Future.h | 32 +++++---- pulsar-client-cpp/tests/PromiseTest.cc | 84 +++++++++++++++++++++++ 3 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 pulsar-client-cpp/tests/PromiseTest.cc diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 212868920266b..3ad6f4062f4b9 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -1620,7 +1620,12 @@ Future ClientConnection::newGetLastMessageId(uint64_t consume pendingGetLastMessageIdRequests_.insert(std::make_pair(requestId, promise)); lock.unlock(); - sendRequestWithId(Commands::newGetLastMessageId(consumerId, requestId), requestId); + sendRequestWithId(Commands::newGetLastMessageId(consumerId, requestId), requestId) + .addListener([promise](Result result, const ResponseData& data) { + if (result != ResultOk) { + promise.setFailed(result); + } + }); return promise.getFuture(); } diff --git a/pulsar-client-cpp/lib/Future.h b/pulsar-client-cpp/lib/Future.h index cafb63f11f74c..b695e5e8c2a95 100644 --- a/pulsar-client-cpp/lib/Future.h +++ b/pulsar-client-cpp/lib/Future.h @@ -90,7 +90,8 @@ class Promise { public: Promise() : state_(std::make_shared >()) {} - bool setValue(const Type& value) { + bool setValue(const Type& value) const { + static Result DEFAULT_RESULT; InternalState* state = state_.get(); Lock lock(state->mutex); @@ -99,21 +100,24 @@ class Promise { } state->value = value; - state->result = Result(); + state->result = DEFAULT_RESULT; state->complete = true; - typename std::list::iterator it; - for (it = state->listeners.begin(); it != state->listeners.end(); ++it) { - ListenerCallback& callback = *it; - callback(state->result, state->value); + decltype(state->listeners) listeners; + listeners.swap(state->listeners); + + lock.unlock(); + + for (auto& callback : listeners) { + callback(DEFAULT_RESULT, value); } - state->listeners.clear(); state->condition.notify_all(); return true; } - bool setFailed(Result result) { + bool setFailed(Result result) const { + static Type DEFAULT_VALUE; InternalState* state = state_.get(); Lock lock(state->mutex); @@ -124,13 +128,15 @@ class Promise { state->result = result; state->complete = true; - typename std::list::iterator it; - for (it = state->listeners.begin(); it != state->listeners.end(); ++it) { - ListenerCallback& callback = *it; - callback(state->result, state->value); + decltype(state->listeners) listeners; + listeners.swap(state->listeners); + + lock.unlock(); + + for (auto& callback : listeners) { + callback(result, DEFAULT_VALUE); } - state->listeners.clear(); state->condition.notify_all(); return true; } diff --git a/pulsar-client-cpp/tests/PromiseTest.cc b/pulsar-client-cpp/tests/PromiseTest.cc new file mode 100644 index 0000000000000..73c6f8c230846 --- /dev/null +++ b/pulsar-client-cpp/tests/PromiseTest.cc @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include +#include +#include + +using namespace pulsar; + +TEST(PromiseTest, testSetValue) { + Promise promise; + std::thread t{[promise] { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + promise.setValue("hello"); + }}; + t.detach(); + + std::string value; + ASSERT_EQ(promise.getFuture().get(value), 0); + ASSERT_EQ(value, "hello"); +} + +TEST(PromiseTest, testSetFailed) { + Promise promise; + std::thread t{[promise] { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + promise.setFailed(-1); + }}; + t.detach(); + + std::string value; + ASSERT_EQ(promise.getFuture().get(value), -1); + ASSERT_EQ(value, ""); +} + +TEST(PromiseTest, testListeners) { + Promise promise; + auto future = promise.getFuture(); + + bool resultSetFailed = true; + bool resultSetValue = true; + std::vector results; + std::vector values; + + future + .addListener([promise, &resultSetFailed, &results, &values](int result, const std::string& value) { + resultSetFailed = promise.setFailed(-1L); + results.emplace_back(result); + values.emplace_back(value); + }) + .addListener([promise, &resultSetValue, &results, &values](int result, const std::string& value) { + resultSetValue = promise.setValue("WRONG"); + results.emplace_back(result); + values.emplace_back(value); + }); + + promise.setValue("hello"); + std::string value; + ASSERT_EQ(future.get(value), 0); + ASSERT_EQ(value, "hello"); + + ASSERT_FALSE(resultSetFailed); + ASSERT_FALSE(resultSetValue); + ASSERT_EQ(results, (std::vector(2, 0))); + ASSERT_EQ(values, (std::vector(2, "hello"))); +} From 10bbe0a0bb09f3c626cfe94eca765a3806466474 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 4 Nov 2021 23:03:13 +0800 Subject: [PATCH 109/823] [Transaction] Merge transactionBuffer exception into a class (#12358) (cherry picked from commit 15a23030a7162ee5d5613d1b3bd4f483c1e8100f) --- .../TransactionMetadataStoreService.java | 6 +- ...PersistentDispatcherMultipleConsumers.java | 4 +- ...sistentDispatcherSingleActiveConsumer.java | 4 +- .../StreamingEntryReader.java | 4 +- .../buffer/TransactionBufferReader.java | 5 +- .../transaction/buffer/TransactionMeta.java | 6 +- .../NoTxnsCommittedAtLedgerException.java | 31 ------- .../TransactionNotSealedException.java | 31 ------- .../TransactionSealedException.java | 33 ------- .../TransactionStatusException.java | 37 -------- .../UnsupportedTxnActionException.java | 34 ------- .../buffer/impl/InMemTransactionBuffer.java | 48 +++++----- .../impl/InMemTransactionBufferReader.java | 4 +- .../exception/TransactionException.java | 83 +++++++++++++++++ .../buffer/TransactionBufferException.java | 91 +++++++++++++++++++ .../buffer}/package-info.java | 2 +- .../TransactionCoordinatorException.java | 56 ++++++++++++ .../coordinator/package-info.java} | 12 +-- .../package-info.java} | 13 +-- .../TransactionPendingAckException.java} | 25 +++-- .../pendingack/package-info.java} | 13 +-- ...ctionPendingAckStoreProviderException.java | 32 ------- .../impl/MLPendingAckStoreProvider.java | 5 +- .../InMemTransactionBufferReaderTest.java | 4 +- .../buffer/TransactionBufferTest.java | 16 ++-- 25 files changed, 311 insertions(+), 288 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/NoTxnsCommittedAtLedgerException.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotSealedException.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionSealedException.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionStatusException.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/UnsupportedTxnActionException.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/TransactionException.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/TransactionBufferException.java rename pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/{buffer/exceptions => exception/buffer}/package-info.java (93%) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/TransactionCoordinatorException.java rename pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/{buffer/exceptions/TransactionBufferProviderException.java => exception/coordinator/package-info.java} (73%) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/{buffer/exceptions/EndOfTransactionException.java => exception/package-info.java} (70%) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/{buffer/exceptions/TransactionBufferException.java => exception/pendingack/TransactionPendingAckException.java} (58%) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/{buffer/exceptions/TransactionNotFoundException.java => exception/pendingack/package-info.java} (68%) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 31a87144465dc..607f05e481673 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -36,7 +36,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.UnsupportedTxnActionException; +import org.apache.pulsar.broker.transaction.exception.coordinator.TransactionCoordinatorException; import org.apache.pulsar.broker.transaction.recover.TransactionRecoverTrackerImpl; import org.apache.pulsar.broker.transaction.timeout.TransactionTimeoutTrackerFactoryImpl; import org.apache.pulsar.client.api.PulsarClientException.BrokerPersistenceException; @@ -331,8 +331,8 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea newStatus = ABORTING; break; default: - UnsupportedTxnActionException exception = - new UnsupportedTxnActionException(txnID, txnAction); + TransactionCoordinatorException.UnsupportedTxnActionException exception = + new TransactionCoordinatorException.UnsupportedTxnActionException(txnID, txnAction); LOG.error(exception.getMessage()); completableFuture.completeExceptionally(exception); return completableFuture; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 80e3d60536c8d..0d060056ae6ee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -55,7 +55,7 @@ import org.apache.pulsar.broker.service.StickyKeyConsumerSelector; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotSealedException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -610,7 +610,7 @@ public synchronized void readEntriesFailed(ManagedLedgerException exception, Obj // Notify the consumer only if all the messages were already acknowledged consumerList.forEach(Consumer::reachedEndOfTopic); } - } else if (exception.getCause() instanceof TransactionNotSealedException) { + } else if (exception.getCause() instanceof TransactionBufferException.TransactionNotSealedException) { waitTimeMillis = 1; if (log.isDebugEnabled()) { log.debug("[{}] Error reading transaction entries : {}, Read Type {} - Retrying to read in {} seconds", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 653c1c1e2194f..848cbb40a5ac4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -45,7 +45,7 @@ import org.apache.pulsar.broker.service.SendMessageInfo; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotSealedException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.policies.data.DispatchRate; @@ -469,7 +469,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep // Notify the consumer only if all the messages were already acknowledged consumers.forEach(Consumer::reachedEndOfTopic); } - } else if (exception.getCause() instanceof TransactionNotSealedException) { + } else if (exception.getCause() instanceof TransactionBufferException.TransactionNotSealedException) { waitTimeMillis = 1; if (log.isDebugEnabled()) { log.debug("[{}] Error reading transaction entries : {}, - Retrying to read in {} seconds", name, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java index 12f5600c51a14..85083da93c072 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java @@ -37,7 +37,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.util.SafeRun; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotSealedException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.impl.Backoff; /** @@ -197,7 +197,7 @@ private void internalReadEntryFailed(ManagedLedgerException exception, Object ct PositionImpl readPosition = pendingReadEntryRequest.position; pendingReadEntryRequest.retry++; long waitTimeMillis = readFailureBackoff.next(); - if (exception.getCause() instanceof TransactionNotSealedException) { + if (exception.getCause() instanceof TransactionBufferException.TransactionNotSealedException) { waitTimeMillis = 1; if (log.isDebugEnabled()) { log.debug("[{}] Error reading transaction entries : {}, - Retrying to read in {} seconds", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferReader.java index 4547de00dc8df..54dafd55410a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferReader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferReader.java @@ -21,7 +21,8 @@ import com.google.common.annotations.Beta; import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.transaction.buffer.exceptions.EndOfTransactionException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; + /** * A reader to read entries of a given transaction from transaction buffer. @@ -38,7 +39,7 @@ public interface TransactionBufferReader extends AutoCloseable { * * @param numEntries the number of entries to read from transaction buffer. * @return a future represents the result of the read operations. - * @throws EndOfTransactionException if reaching end of the transaction and no + * @throws TransactionBufferException.EndOfTransactionException if reaching end of the transaction and no * more entries to return. */ CompletableFuture> readNext(int numEntries); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionMeta.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionMeta.java index 9afc05e039640..347ab74564780 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionMeta.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionMeta.java @@ -22,7 +22,7 @@ import java.util.SortedMap; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionStatusException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; @@ -57,9 +57,9 @@ public interface TransactionMeta { * Return messages number in one transaction. * * @return the number of transaction messages - * @throws TransactionStatusException + * @throws TransactionBufferException.TransactionStatusException */ - int numMessageInTxn() throws TransactionStatusException; + int numMessageInTxn() throws TransactionBufferException.TransactionStatusException; /** * Return the committed ledger id at data ledger. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/NoTxnsCommittedAtLedgerException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/NoTxnsCommittedAtLedgerException.java deleted file mode 100644 index b20b29a575e13..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/NoTxnsCommittedAtLedgerException.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - -/** - * Exception is thrown when no transactions found committed at a given ledger. - */ -public class NoTxnsCommittedAtLedgerException extends TransactionBufferException { - - private static final long serialVersionUID = 0L; - - public NoTxnsCommittedAtLedgerException(String message) { - super(message); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotSealedException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotSealedException.java deleted file mode 100644 index c732c8856a3be..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotSealedException.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - -/** - * Exception is thrown when opening a reader on a transaction that is not sealed yet. - */ -public class TransactionNotSealedException extends TransactionBufferException { - - private static final long serialVersionUID = 0L; - - public TransactionNotSealedException(String message) { - super(message); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionSealedException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionSealedException.java deleted file mode 100644 index 13fdec5cd7769..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionSealedException.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - -/** - * Exception thrown if a transaction is already sealed. - * - *

    If a transaction is sealed, no more entries should be appended to this transaction. - */ -public class TransactionSealedException extends TransactionBufferException { - - private static final long serialVersionUID = 5366602873819540477L; - - public TransactionSealedException(String message) { - super(message); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionStatusException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionStatusException.java deleted file mode 100644 index 008f27f58a411..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionStatusException.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - -import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; - -/** - * Exceptions are thrown when operations are applied to a transaction which is not in expected txn status. - */ -public class TransactionStatusException extends TransactionBufferException { - - private static final long serialVersionUID = 0L; - - public TransactionStatusException(TxnID txnId, - TxnStatus expectedStatus, - TxnStatus actualStatus) { - super("Transaction `" + txnId + "` is not in an expected status `" + expectedStatus - + "`, but is in status `" + actualStatus + "`"); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/UnsupportedTxnActionException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/UnsupportedTxnActionException.java deleted file mode 100644 index e4a83410c0159..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/UnsupportedTxnActionException.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - -import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.TxnAction; - -/** - * Exceptions are thrown when txnAction is unsupported. - */ -public class UnsupportedTxnActionException extends TransactionBufferException { - - private static final long serialVersionUID = 0L; - - public UnsupportedTxnActionException(TxnID txnId, int txnAction) { - super("Transaction `" + txnId + "` receive unsupported txnAction " + TxnAction.valueOf(txnAction)); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java index 2c32d0b68b1e8..f66b263812da3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java @@ -36,10 +36,7 @@ import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotFoundException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotSealedException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionSealedException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionStatusException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionInBufferStats; @@ -86,7 +83,7 @@ public int numEntries() { } @Override - public int numMessageInTxn() throws TransactionStatusException { + public int numMessageInTxn() throws TransactionBufferException.TransactionStatusException { return -1; } @@ -125,7 +122,7 @@ public CompletableFuture committingTxn() { public CompletableFuture commitTxn(long committedAtLedgerId, long committedAtEntryId) { try { return CompletableFuture.completedFuture(commitAt(committedAtLedgerId, committedAtEntryId)); - } catch (TransactionStatusException e) { + } catch (TransactionBufferException.TransactionStatusException e) { return FutureUtil.failedFuture(e); } } @@ -134,23 +131,23 @@ public CompletableFuture commitTxn(long committedAtLedgerId, lo public CompletableFuture abortTxn() { try { return CompletableFuture.completedFuture(abort()); - } catch (TransactionStatusException e) { + } catch (TransactionBufferException.TransactionStatusException e) { return FutureUtil.failedFuture(e); } } - synchronized TxnBuffer abort() throws TransactionStatusException { + synchronized TxnBuffer abort() throws TransactionBufferException.TransactionStatusException { if (TxnStatus.OPEN != status) { - throw new TransactionStatusException(txnid, TxnStatus.OPEN, status); + throw new TransactionBufferException.TransactionStatusException(txnid, TxnStatus.OPEN, status); } this.status = TxnStatus.ABORTED; return this; } synchronized TxnBuffer commitAt(long committedAtLedgerId, long committedAtEntryId) - throws TransactionStatusException { + throws TransactionBufferException.TransactionStatusException { if (TxnStatus.OPEN != status) { - throw new TransactionStatusException(txnid, TxnStatus.OPEN, status); + throw new TransactionBufferException.TransactionStatusException(txnid, TxnStatus.OPEN, status); } this.committedAtLedgerId = committedAtLedgerId; @@ -170,11 +167,13 @@ public void close() { } } - public void appendEntry(long sequenceId, ByteBuf entry) throws TransactionSealedException { + public void appendEntry(long sequenceId, ByteBuf entry) throws + TransactionBufferException.TransactionSealedException { synchronized (this) { if (TxnStatus.OPEN != status) { // the transaction is not open anymore, reject the append operations - throw new TransactionSealedException("Transaction `" + txnid + "` is already sealed"); + throw new TransactionBufferException + .TransactionSealedException("Transaction `" + txnid + "` is already sealed"); } } @@ -183,11 +182,13 @@ public void appendEntry(long sequenceId, ByteBuf entry) throws TransactionSealed } } - public TransactionBufferReader newReader(long sequenceId) throws TransactionNotSealedException { + public TransactionBufferReader newReader(long sequenceId) throws + TransactionBufferException.TransactionNotSealedException { synchronized (this) { if (TxnStatus.COMMITTED != status) { // the transaction is not committed yet, hence the buffer is not sealed - throw new TransactionNotSealedException("Transaction `" + txnid + "` is not sealed yet"); + throw new TransactionBufferException + .TransactionNotSealedException("Transaction `" + txnid + "` is not sealed yet"); } } @@ -220,17 +221,17 @@ public CompletableFuture getTransactionMeta(TxnID txnID) { CompletableFuture getFuture = new CompletableFuture<>(); try { getFuture.complete(getTxnBufferOrThrowNotFoundException(txnID)); - } catch (TransactionNotFoundException e) { + } catch (TransactionBufferException.TransactionNotFoundException e) { getFuture.completeExceptionally(e); } return getFuture; } private TxnBuffer getTxnBufferOrThrowNotFoundException(TxnID txnID) - throws TransactionNotFoundException { + throws TransactionBufferException.TransactionNotFoundException { TxnBuffer buffer = buffers.get(txnID); if (null == buffer) { - throw new TransactionNotFoundException( + throw new TransactionBufferException.TransactionNotFoundException( "Transaction `" + txnID + "` doesn't exist in the transaction buffer"); } return buffer; @@ -262,7 +263,7 @@ public CompletableFuture appendBufferToTxn(TxnID txnId, try { txnBuffer.appendEntry(sequenceId, buffer); appendFuture.complete(null); - } catch (TransactionSealedException e) { + } catch (TransactionBufferException.TransactionSealedException e) { appendFuture.completeExceptionally(e); } return appendFuture; @@ -276,7 +277,8 @@ public CompletableFuture openTransactionBufferReader( TxnBuffer txnBuffer = getTxnBufferOrThrowNotFoundException(txnID); TransactionBufferReader reader = txnBuffer.newReader(startSequenceId); openFuture.complete(reader); - } catch (TransactionNotFoundException | TransactionNotSealedException e) { + } catch (TransactionBufferException.TransactionNotFoundException + | TransactionBufferException.TransactionNotSealedException e) { openFuture.completeExceptionally(e); } return openFuture; @@ -295,7 +297,8 @@ public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { addTxnToTxnIdex(txnID, committedAtLedgerId); } commitFuture.complete(null); - } catch (TransactionNotFoundException | TransactionStatusException e) { + } catch (TransactionBufferException.TransactionNotFoundException + | TransactionBufferException.TransactionStatusException e) { commitFuture.completeExceptionally(e); } return commitFuture; @@ -318,7 +321,8 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { txnBuffer.abort(); buffers.remove(txnID, txnBuffer); abortFuture.complete(null); - } catch (TransactionNotFoundException | TransactionStatusException e) { + } catch (TransactionBufferException.TransactionNotFoundException + | TransactionBufferException.TransactionStatusException e) { abortFuture.completeExceptionally(e); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBufferReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBufferReader.java index 91b095ede5b94..d81ce02aebe7b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBufferReader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBufferReader.java @@ -27,7 +27,7 @@ import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionEntry; -import org.apache.pulsar.broker.transaction.buffer.exceptions.EndOfTransactionException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.api.transaction.TxnID; /** @@ -78,7 +78,7 @@ public synchronized CompletableFuture> readNext(int numEn } if (txnEntries.isEmpty()) { - readFuture.completeExceptionally(new EndOfTransactionException( + readFuture.completeExceptionally(new TransactionBufferException.EndOfTransactionException( "No more entries found in transaction `" + txnId + "`" )); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/TransactionException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/TransactionException.java new file mode 100644 index 0000000000000..f81fc8a724d5a --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/TransactionException.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.exception; + +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; + +/** + * The base exception class for the errors thrown from Transaction. + */ +public abstract class TransactionException extends Exception { + + private static final long serialVersionUID = 0L; + + public TransactionException(String message) { + super(message); + } + + public TransactionException(String message, Throwable cause) { + super(message, cause); + } + + public TransactionException(Throwable cause) { + super(cause); + } + + /** + * Exception is thrown when opening a reader on a transaction that is not sealed yet. + */ + public static class TransactionNotSealedException extends TransactionException { + + private static final long serialVersionUID = 0L; + + public TransactionNotSealedException(String message) { + super(message); + } + } + + /** + * Exception thrown if a transaction is already sealed. + * + *

    If a transaction is sealed, no more entries should be appended to this transaction. + */ + public static class TransactionSealedException extends TransactionException { + + private static final long serialVersionUID = 5366602873819540477L; + + public TransactionSealedException(String message) { + super(message); + } + } + + /** + * Exceptions are thrown when operations are applied to a transaction which is not in expected txn status. + */ + public static class TransactionStatusException extends TransactionException { + + private static final long serialVersionUID = 0L; + + public TransactionStatusException(TxnID txnId, + TxnStatus expectedStatus, + TxnStatus actualStatus) { + super("Transaction `q" + txnId + "` is not in an expected status `" + expectedStatus + + "`, but is in status `" + actualStatus + "`"); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/TransactionBufferException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/TransactionBufferException.java new file mode 100644 index 0000000000000..ab301c9017dda --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/TransactionBufferException.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.exception.buffer; + +import org.apache.pulsar.broker.transaction.exception.TransactionException; + +/** + * The base exception class for the errors thrown from Transaction Buffer. + */ +public abstract class TransactionBufferException extends TransactionException { + + private static final long serialVersionUID = 0L; + + public TransactionBufferException(String message) { + super(message); + } + + public TransactionBufferException(String message, Throwable cause) { + super(message, cause); + } + + public TransactionBufferException(Throwable cause) { + super(cause); + } + + + /** + * Exception thrown when reaching end of a transaction. + */ + public static class EndOfTransactionException extends TransactionBufferException { + + private static final long serialVersionUID = 0L; + + public EndOfTransactionException(String message) { + super(message); + } + } + + /** + * Exception is thrown when no transactions found committed at a given ledger. + */ + public class NoTxnsCommittedAtLedgerException extends TransactionBufferException { + + private static final long serialVersionUID = 0L; + + public NoTxnsCommittedAtLedgerException(String message) { + super(message); + } + } + + /** + * Transaction buffer provider exception. + */ + public class TransactionBufferProviderException extends TransactionBufferException { + + public TransactionBufferProviderException(String message) { + super(message); + } + + } + + /** + * Exception is thrown when the transaction is not found in the transaction buffer. + */ + public static class TransactionNotFoundException extends TransactionBufferException { + + private static final long serialVersionUID = 0L; + + public TransactionNotFoundException(String message) { + super(message); + } + } + + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/package-info.java similarity index 93% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/package-info.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/package-info.java index 2aee740f6d9b8..60bb5ee2a0245 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/package-info.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/buffer/package-info.java @@ -19,4 +19,4 @@ /** * Exceptions thrown when encountering errors in transaction buffer. */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; +package org.apache.pulsar.broker.transaction.exception.buffer; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/TransactionCoordinatorException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/TransactionCoordinatorException.java new file mode 100644 index 0000000000000..8da04ac174eb2 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/TransactionCoordinatorException.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.exception.coordinator; + +import org.apache.pulsar.broker.transaction.exception.TransactionException; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.TxnAction; + +/** + * The base exception class for the errors thrown from Transaction Coordinator. + */ +public abstract class TransactionCoordinatorException extends TransactionException { + + private static final long serialVersionUID = 0L; + + public TransactionCoordinatorException(String message) { + super(message); + } + + public TransactionCoordinatorException(String message, Throwable cause) { + super(message, cause); + } + + public TransactionCoordinatorException(Throwable cause) { + super(cause); + } + + + /** + * Exceptions are thrown when txnAction is unsupported. + */ + public static class UnsupportedTxnActionException extends TransactionCoordinatorException { + + private static final long serialVersionUID = 0L; + + public UnsupportedTxnActionException(TxnID txnId, int txnAction) { + super("Transaction `" + txnId + "` receive unsupported txnAction " + TxnAction.valueOf(txnAction)); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionBufferProviderException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/package-info.java similarity index 73% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionBufferProviderException.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/package-info.java index e7a7a62cc43af..ceaff6ac80311 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionBufferProviderException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/coordinator/package-info.java @@ -16,15 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - /** - * Transaction buffer provider exception. + * Exceptions thrown when encountering errors in transaction buffer. */ -public class TransactionBufferProviderException extends TransactionBufferException { - - public TransactionBufferProviderException(String message) { - super(message); - } - -} +package org.apache.pulsar.broker.transaction.exception.coordinator; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/EndOfTransactionException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/package-info.java similarity index 70% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/EndOfTransactionException.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/package-info.java index 57fce9a0a6e37..222d871178629 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/EndOfTransactionException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/package-info.java @@ -16,16 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - /** - * Exception thrown when reaching end of a transaction. + * Exceptions thrown when encountering errors in transaction buffer. */ -public class EndOfTransactionException extends TransactionBufferException { - - private static final long serialVersionUID = 0L; - - public EndOfTransactionException(String message) { - super(message); - } -} +package org.apache.pulsar.broker.transaction.exception; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionBufferException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/pendingack/TransactionPendingAckException.java similarity index 58% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionBufferException.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/pendingack/TransactionPendingAckException.java index 3ecbf2b555d3b..74999bf1cb7ab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionBufferException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/pendingack/TransactionPendingAckException.java @@ -16,24 +16,37 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; +package org.apache.pulsar.broker.transaction.exception.pendingack; + +import org.apache.pulsar.broker.transaction.exception.TransactionException; /** - * The base exception class for the errors thrown from Transaction Buffer. + * The base exception class for the errors thrown from Transaction Pending ACk. */ -public abstract class TransactionBufferException extends Exception { +public abstract class TransactionPendingAckException extends TransactionException { private static final long serialVersionUID = 0L; - public TransactionBufferException(String message) { + public TransactionPendingAckException(String message) { super(message); } - public TransactionBufferException(String message, Throwable cause) { + public TransactionPendingAckException(String message, Throwable cause) { super(message, cause); } - public TransactionBufferException(Throwable cause) { + public TransactionPendingAckException(Throwable cause) { super(cause); } + + /** + * Transaction pending ack store provider exception. + */ + public static class TransactionPendingAckStoreProviderException extends TransactionPendingAckException { + + public TransactionPendingAckStoreProviderException(String message) { + super(message); + } + + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotFoundException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/pendingack/package-info.java similarity index 68% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotFoundException.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/pendingack/package-info.java index 0f1dc4696e4c1..0d0df63bd7673 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/exceptions/TransactionNotFoundException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/exception/pendingack/package-info.java @@ -16,16 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.transaction.buffer.exceptions; - /** - * Exception is thrown when the transaction is not found in the transaction buffer. + * Exceptions thrown when encountering errors in transaction buffer. */ -public class TransactionNotFoundException extends TransactionBufferException { - - private static final long serialVersionUID = 0L; - - public TransactionNotFoundException(String message) { - super(message); - } -} +package org.apache.pulsar.broker.transaction.exception.pendingack; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java deleted file mode 100644 index 0f41e2cd3c455..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.pendingack.exceptions; - -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionBufferException; - -/** - * Transaction pending ack store provider exception. - */ -public class TransactionPendingAckStoreProviderException extends TransactionBufferException { - - public TransactionPendingAckStoreProviderException(String message) { - super(message); - } - -} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index 548dd8a434a90..5417caec3d7cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -26,9 +26,9 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.exception.pendingack.TransactionPendingAckException; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; -import org.apache.pulsar.broker.transaction.pendingack.exceptions.TransactionPendingAckStoreProviderException; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.naming.TopicName; @@ -45,7 +45,8 @@ public CompletableFuture newPendingAckStore(PersistentSubscript if (subscription == null) { pendingAckStoreFuture.completeExceptionally( - new TransactionPendingAckStoreProviderException("The subscription is null.")); + new TransactionPendingAckException + .TransactionPendingAckStoreProviderException("The subscription is null.")); return pendingAckStoreFuture; } PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/InMemTransactionBufferReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/InMemTransactionBufferReaderTest.java index 513a78da429dc..d4234fbd16610 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/InMemTransactionBufferReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/InMemTransactionBufferReaderTest.java @@ -33,7 +33,7 @@ import java.util.TreeMap; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.EndOfTransactionException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferReader; import org.apache.pulsar.client.api.transaction.TxnID; import org.testng.annotations.Test; @@ -116,7 +116,7 @@ public void testEndOfTransactionException() throws Exception { reader.readNext(1).get(); fail("should fail to read entries if there is no more in the transaction buffer"); } catch (ExecutionException ee) { - assertTrue(ee.getCause() instanceof EndOfTransactionException); + assertTrue(ee.getCause() instanceof TransactionBufferException.EndOfTransactionException); } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferTest.java index 223849048d57f..c10444afe3c19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferTest.java @@ -33,11 +33,9 @@ import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferProvider; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotFoundException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotSealedException; -import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionStatusException; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -86,7 +84,7 @@ public void testOpenReaderOnNonExistentTxn() throws Exception { buffer.openTransactionBufferReader(txnId, 0L).get(); fail("Should fail to open reader if a transaction doesn't exist"); } catch (ExecutionException ee) { - assertTrue(ee.getCause() instanceof TransactionNotFoundException); + assertTrue(ee.getCause() instanceof TransactionBufferException.TransactionNotFoundException); } } @@ -102,7 +100,7 @@ public void testOpenReaderOnAnOpenTxn() throws Exception { buffer.openTransactionBufferReader(txnId, 0L).get(); fail("Should fail to open a reader on an OPEN transaction"); } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionNotSealedException); + assertTrue(e.getCause() instanceof TransactionBufferException.TransactionNotSealedException); } } @@ -136,7 +134,7 @@ public void testCommitNonExistentTxn() throws Exception { buffer.commitTxn(txnId, Long.MIN_VALUE).get(); fail("Should fail to commit a transaction if it doesn't exist"); } catch (ExecutionException ee) { - assertTrue(ee.getCause() instanceof TransactionNotFoundException); + assertTrue(ee.getCause() instanceof TransactionBufferException.TransactionNotFoundException); } } @@ -160,7 +158,7 @@ public void testAbortNonExistentTxn() throws Exception { buffer.abortTxn(txnId, Long.MIN_VALUE).get(); fail("Should fail to abort a transaction if it doesn't exist"); } catch (ExecutionException ee) { - assertTrue(ee.getCause() instanceof TransactionNotFoundException); + assertTrue(ee.getCause() instanceof TransactionBufferException.TransactionNotFoundException); } } @@ -181,7 +179,7 @@ public void testAbortCommittedTxn() throws Exception { buffer.abortTxn(txnId, Long.MIN_VALUE).get(); fail("Should fail to abort a committed transaction"); } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionStatusException); + assertTrue(e.getCause() instanceof TransactionBufferException.TransactionStatusException); } txnMeta = buffer.getTransactionMeta(txnId).get(); assertEquals(txnId, txnMeta.id()); @@ -277,7 +275,7 @@ private void verifyTxnNotExist(TxnID txnID) throws Exception { buffer.getTransactionMeta(txnID).get(); fail("Should fail to get transaction metadata if it doesn't exist"); } catch (ExecutionException ee) { - assertTrue(ee.getCause() instanceof TransactionNotFoundException); + assertTrue(ee.getCause() instanceof TransactionBufferException.TransactionNotFoundException); } } } From 0b5430af68b1b7082569009b291f323ea4bae35e Mon Sep 17 00:00:00 2001 From: litao Date: Thu, 4 Nov 2021 22:50:38 +0800 Subject: [PATCH 110/823] [docs] Fix doc for pulsar-admin bookies cmd (#12542) (cherry picked from commit 938dd54ce6f4633e102df9e2ae1f35daa0fdb2bd) --- .../src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java index e6551b9eb5230..c15e8dbc60d6a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java @@ -41,7 +41,7 @@ void run() throws Exception { @Parameters(commandDescription = "Gets the rack placement information for a specific bookie in the cluster") private class GetBookie extends CliCommand { - @Parameter(names = { "-b", "--bookie" }, description = "bookie address", required = true) + @Parameter(names = { "-b", "--bookie" }, description = "Bookie address (format: `address:port`)", required = true) private String bookieAddress; @Override @@ -62,7 +62,7 @@ void run() throws Exception { @Parameters(commandDescription = "Remove rack placement information for a specific bookie in the cluster") private class RemoveBookie extends CliCommand { - @Parameter(names = { "-b", "--bookie" }, description = "bookie address", required = true) + @Parameter(names = { "-b", "--bookie" }, description = "Bookie address (format: `address:port`)", required = true) private String bookieAddress; @Override From efb36feec70e57cf3856f10d384b7ed8b7de370a Mon Sep 17 00:00:00 2001 From: Jason918 Date: Thu, 4 Nov 2021 22:44:04 +0800 Subject: [PATCH 111/823] fix log typo in NamespaceService#checkHeartbeatNamespace and NamespaceService#checkHeartbeatNamespaceV2 (#12582) (cherry picked from commit 6afbee6d0e7a6d21eb0fca400a19e38cbbad102e) --- .../org/apache/pulsar/broker/namespace/NamespaceService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index dc8f38390a4fe..f6cba9c7129a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1323,7 +1323,7 @@ public static NamespaceName getSLAMonitorNamespace(String host, ServiceConfigura public static String checkHeartbeatNamespace(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { - LOG.debug("SLAMonitoring namespace matched the lookup namespace {}", ns.getNamespaceObject().toString()); + LOG.debug("Heartbeat namespace matched the lookup namespace {}", ns.getNamespaceObject().toString()); return String.format("http://%s", m.group(1)); } else { return null; @@ -1333,7 +1333,7 @@ public static String checkHeartbeatNamespace(ServiceUnitId ns) { public static String checkHeartbeatNamespaceV2(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { - LOG.debug("SLAMonitoring namespace matched the lookup namespace {}", ns.getNamespaceObject().toString()); + LOG.debug("Heartbeat namespace v2 matched the lookup namespace {}", ns.getNamespaceObject().toString()); return String.format("http://%s", m.group(1)); } else { return null; From 2cc110e1df8505a4bfbc9ea072ee912c9fe29463 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 4 Nov 2021 21:01:24 +0800 Subject: [PATCH 112/823] Fix additional servlets nar might extract to null directory (#12585) ### Motivation The additional servlets use NAR package to implantation plugin mechanism, it need extract to specific directory. However, the `narExtractionDirectory` is from `Properties`, but the properties has only the configuration in the configuration file. The default value of `narExtractionDirectory ` in `ServiceConfiguration` can't be use. ### Modifications When `narExtractionDirectory ` configuration is not set, use `NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR` as default directory. (cherry picked from commit 9ecd613c0fdaf7f5306fc7f633a4d12218a7a4d2) --- .../plugin/servlet/AdditionalServlets.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java index 2451cf5af04f3..080e1c7509643 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java @@ -21,13 +21,12 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.util.Arrays; -import java.util.List; import java.util.Map; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.configuration.PulsarConfiguration; +import org.apache.pulsar.common.nar.NarClassLoader; /** * A collection of loaded additional servlets. @@ -71,18 +70,23 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti if (additionalServlets == null) { additionalServlets = conf.getProperties().getProperty(PROXY_ADDITIONAL_SERVLETS); } + + String narExtractionDirectory = conf.getProperties().getProperty(NAR_EXTRACTION_DIRECTORY); + if(narExtractionDirectory == null) { + narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; + } + if (additionalServletDirectory == null || additionalServlets == null) { return null; } AdditionalServletDefinitions definitions = AdditionalServletUtils.searchForServlets(additionalServletDirectory - , null); + , narExtractionDirectory); ImmutableMap.Builder builder = ImmutableMap.builder(); - List additionalServletsList = Arrays.asList(additionalServlets.split(",")); - additionalServletsList.forEach(servletName -> { - + String[] additionalServletsList = additionalServlets.split(","); + for (String servletName : additionalServletsList) { AdditionalServletMetadata definition = definitions.servlets().get(servletName); if (null == definition) { throw new RuntimeException("No additional servlet is found for name `" + servletName @@ -91,8 +95,7 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti AdditionalServletWithClassLoader servletWithClassLoader; try { - servletWithClassLoader = AdditionalServletUtils.load(definition, - conf.getProperties().getProperty(NAR_EXTRACTION_DIRECTORY)); + servletWithClassLoader = AdditionalServletUtils.load(definition, narExtractionDirectory); if (servletWithClassLoader != null) { builder.put(servletName, servletWithClassLoader); } @@ -101,7 +104,7 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti log.error("Failed to load the additional servlet for name `" + servletName + "`", e); throw new RuntimeException("Failed to load the additional servlet for name `" + servletName + "`"); } - }); + } Map servlets = builder.build(); if (servlets != null && !servlets.isEmpty()) { From 7e97e6cc8ddac16b26a8ab6161b91221ceac470b Mon Sep 17 00:00:00 2001 From: Zhanpeng Wu Date: Thu, 4 Nov 2021 23:15:34 +0800 Subject: [PATCH 113/823] Fix invalid firstSentAt value (#12588) (cherry picked from commit 2ca96e4e0e63f99af0b079f3d590bbc8aa179676) --- .../main/java/org/apache/pulsar/client/impl/ProducerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index f8e59a2006592..e28721c50973d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1227,7 +1227,7 @@ void sendComplete(final Exception e) { "%s : createdAt %s ns ago, firstSentAt %s ns ago, lastSentAt %s ns ago, retryCount %s", te.getMessage(), ns - this.createdAt, - ns - this.firstSentAt, + this.firstSentAt <= 0 ? ns - this.lastSentAt : ns - this.firstSentAt, ns - this.lastSentAt, retryCount ); From 91a2b1230223a3e134785dd09d17705fb0ebeb24 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 4 Nov 2021 09:51:35 +0800 Subject: [PATCH 114/823] [Test] Cleanup ProxyPublishConsumeTest (#12607) (cherry picked from commit 556ba0ea57a518af931edc935a8a6b4799ead10e) --- .../proxy/ProxyPublishConsumeTest.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java index 6386c515f1cf1..b12f670b8b22e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java @@ -29,14 +29,12 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -56,11 +54,9 @@ import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; -import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerAccessMode; import org.apache.pulsar.client.api.ProducerConsumerBase; -import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.TopicType; @@ -82,7 +78,6 @@ import org.glassfish.jersey.logging.LoggingFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -295,14 +290,14 @@ public void unsubscribeTest() throws Exception { Future consumerFuture = consumeClient.connect(consumeSocket, consumeUri, consumeRequest); consumerFuture.get(); List subs = admin.topics().getSubscriptions(topic); - Assert.assertEquals(subs.size(), 1); - Assert.assertEquals(subs.get(0), subscription); + assertEquals(subs.size(), 1); + assertEquals(subs.get(0), subscription); // do unsubscribe consumeSocket.unsubscribe(); //wait for delete Thread.sleep(1000); subs = admin.topics().getSubscriptions(topic); - Assert.assertEquals(subs.size(), 0); + assertEquals(subs.size(), 0); } finally { stopWebSocketClient(consumeClient); } @@ -927,11 +922,11 @@ public void ackBatchMessageTest() throws Exception { producer.flush(); consumeSocket.sendPermits(messages); Awaitility.await().untilAsserted(() -> - Assert.assertEquals(consumeSocket.getReceivedMessagesCount(), messages)); + assertEquals(consumeSocket.getReceivedMessagesCount(), messages)); // The message should not be acked since we only acked 1 message of the batch message Awaitility.await().untilAsserted(() -> - Assert.assertEquals(admin.topics().getStats(topic).getSubscriptions() + assertEquals(admin.topics().getStats(topic).getSubscriptions() .get(subscription).getMsgBacklog(), 0)); } finally { @@ -976,7 +971,7 @@ public void consumeEncryptedMessages() throws Exception { producer.flush(); consumeSocket.sendPermits(messages); Awaitility.await().untilAsserted(() -> - Assert.assertEquals(consumeSocket.getReceivedMessagesCount(), messages)); + assertEquals(consumeSocket.getReceivedMessagesCount(), messages)); for (JsonObject msg : consumeSocket.messages) { assertTrue(msg.has("encryptionContext")); @@ -989,7 +984,7 @@ public void consumeEncryptedMessages() throws Exception { // The message should not be acked since we only acked 1 message of the batch message Awaitility.await().untilAsserted(() -> - Assert.assertEquals(admin.topics().getStats(topic).getSubscriptions() + assertEquals(admin.topics().getStats(topic).getSubscriptions() .get(subscription).getMsgBacklog(), 0)); } finally { @@ -1051,13 +1046,13 @@ private void verifyProxyStats(Client client, String baseUrl, String topic) { // number of consumers are connected = 2 (one is reader) assertEquals(stats.consumerStats.size(), 2); ConsumerStats consumerStats = stats.consumerStats.iterator().next(); - // Assert.assertTrue(consumerStats.numberOfMsgDelivered > 0); + assertTrue(consumerStats.numberOfMsgDelivered > 0); assertNotNull(consumerStats.remoteConnection); // number of producers are connected = 1 assertEquals(stats.producerStats.size(), 1); ProducerStats producerStats = stats.producerStats.iterator().next(); - // Assert.assertTrue(producerStats.numberOfMsgPublished > 0); + assertTrue(producerStats.numberOfMsgPublished > 0); assertNotNull(producerStats.remoteConnection); } From 6e884ccdb9a0e478dae2757c7cd8ff30e80f38b3 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 5 Nov 2021 22:11:16 +0800 Subject: [PATCH 115/823] Clean up the metadata of the non-persistent partitioned topics. (#12550) (cherry picked from commit 6ecef2ec9c02e3a79fce88ac1a3cf44dd4f82509) --- .../nonpersistent/NonPersistentTopic.java | 19 +++++++++++++++++++ .../service/NonPersistentTopicE2ETest.java | 14 ++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 916aa53db8e8e..4b1c57221969c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -39,6 +39,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.AbstractReplicator; +import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -881,6 +882,7 @@ public void checkGC() { } stopReplProducers().thenCompose(v -> delete(true, false, true)) + .thenAccept(__ -> tryToDeletePartitionedMetadata()) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { Throwable throwable = e.getCause(); @@ -902,6 +904,23 @@ public void checkGC() { } } + private CompletableFuture tryToDeletePartitionedMetadata() { + if (TopicName.get(topic).isPartitioned() && !deletePartitionedTopicMetadataWhileInactive()) { + return CompletableFuture.completedFuture(null); + } + TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName()); + try { + NamespaceResources.PartitionedTopicResources partitionedTopicResources = brokerService.pulsar() + .getPulsarResources().getNamespaceResources().getPartitionedTopicResources(); + if (!partitionedTopicResources.partitionedTopicExists(topicName)) { + return CompletableFuture.completedFuture(null); + } + return partitionedTopicResources.deletePartitionedTopicAsync(topicName); + } catch (Exception e) { + return FutureUtil.failedFuture(e); + } + } + @Override public void checkInactiveSubscriptions() { TopicName name = TopicName.get(topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NonPersistentTopicE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NonPersistentTopicE2ETest.java index 1543d3e384469..9525944adc74e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NonPersistentTopicE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NonPersistentTopicE2ETest.java @@ -52,6 +52,8 @@ public class NonPersistentTopicE2ETest extends BrokerTestBase { @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + conf.setBrokerDeleteInactivePartitionedTopicMetadataEnabled(true); + conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1); super.baseSetup(); } @@ -228,5 +230,17 @@ public void testGC() throws Exception { producer2.close(); assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent()); + + // 6. Test for partitioned topic to delete the partitioned metadata + String topicGc = "non-persistent://prop/ns-abc/topic-gc"; + int partitions = 5; + admin.topics().createPartitionedTopic(topicGc, partitions); + Producer producer3 = pulsarClient.newProducer().topic(topicGc).create(); + producer3.close(); + assertTrue(pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( + TopicName.get(topicGc)).join().partitions == partitions); + runGC(); + assertTrue(pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( + TopicName.get(topicGc)).join().partitions == 0); } } From 588dbaa3ac48f757f1e7fc6f1a4b3aef162b2309 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Fri, 5 Nov 2021 00:55:59 +0800 Subject: [PATCH 116/823] [Compaction] Do not move the non-durable cursor position when trimming ledgers while topic with compaction (#12602) * [Compaction] Do not move the non-durable cursor position when trimming ledgers while topic with compaction. For the non-durable cursor, the ledgers trimming task will cause skip the removed ledgers to avoid readers introduced backlogs and make sure the data can be removed if over the retention, more details to see #6787. But for a topic which enabled compaction, this will lead to the reader skips the compacted data. The new added test can illustrate this problem well. For reading compacted data, reading a message ID that earlier that the first message ID of the original data is a normal behavior, so we should not move forward the cursor which will read the compacted data. * Fix checkstyle. * Fix tests. * Fix tests. (cherry picked from commit a6b1b34a5c028b74bd44c5b8f32b42752b6cec14) --- .../mledger/impl/ManagedLedgerImpl.java | 3 +- .../mledger/impl/NonDurableCursorImpl.java | 10 +++ ...bstractDispatcherSingleActiveConsumer.java | 10 ++- ...sistentDispatcherSingleActiveConsumer.java | 2 +- ...sistentDispatcherSingleActiveConsumer.java | 4 +- .../pulsar/compaction/CompactedTopicTest.java | 73 +++++++++++++++++++ 6 files changed, 95 insertions(+), 7 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index ce897f3b92048..a59ce222da543 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2548,7 +2548,8 @@ private void advanceCursorsIfNecessary(List ledgersToDelete) { // move the mark delete position to the highestPositionToDelete only if it is smaller than the add confirmed // to prevent the edge case where the cursor is caught up to the latest and highestPositionToDelete may be larger than the last add confirmed if (highestPositionToDelete.compareTo((PositionImpl) cursor.getMarkDeletedPosition()) > 0 - && highestPositionToDelete.compareTo((PositionImpl) cursor.getManagedLedger().getLastConfirmedEntry()) <= 0 ) { + && highestPositionToDelete.compareTo((PositionImpl) cursor.getManagedLedger().getLastConfirmedEntry()) <= 0 + && !(!cursor.isDurable() && cursor instanceof NonDurableCursorImpl && ((NonDurableCursorImpl) cursor).isReadCompacted())) { cursor.asyncMarkDelete(highestPositionToDelete, new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java index 15b1f047dab3f..0f7ffe41f8a0b 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java @@ -33,6 +33,8 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { + private volatile boolean readCompacted; + NonDurableCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, String cursorName, PositionImpl startCursorPosition, CommandSubscribe.InitialPosition initialPosition) { super(bookkeeper, config, ledger, cursorName); @@ -116,6 +118,14 @@ public void asyncDeleteCursor(final String consumerName, final DeleteCursorCallb callback.deleteCursorComplete(ctx); } + public void setReadCompacted(boolean readCompacted) { + this.readCompacted = readCompacted; + } + + public boolean isReadCompacted() { + return readCompacted; + } + @Override public synchronized String toString() { return MoreObjects.toStringHelper(this).add("ledger", ledger.getName()).add("ackPos", markDeletePosition) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java index 690a5984b4828..4c7ea45f1a4bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java @@ -26,6 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.impl.NonDurableCursorImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; @@ -45,7 +47,7 @@ public abstract class AbstractDispatcherSingleActiveConsumer extends AbstractBas protected boolean isKeyHashRangeFiltered = false; protected CompletableFuture closeFuture = null; protected final int partitionIndex; - + protected final ManagedCursor cursor; // This dispatcher supports both the Exclusive and Failover subscription types protected final SubType subscriptionType; @@ -59,12 +61,13 @@ public abstract class AbstractDispatcherSingleActiveConsumer extends AbstractBas public AbstractDispatcherSingleActiveConsumer(SubType subscriptionType, int partitionIndex, String topicName, Subscription subscription, - ServiceConfiguration serviceConfig) { + ServiceConfiguration serviceConfig, ManagedCursor cursor) { super(subscription, serviceConfig); this.topicName = topicName; this.consumers = new CopyOnWriteArrayList<>(); this.partitionIndex = partitionIndex; this.subscriptionType = subscriptionType; + this.cursor = cursor; ACTIVE_CONSUMER_UPDATER.set(this, null); } @@ -178,6 +181,9 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce consumer.notifyActiveConsumerChange(currentActiveConsumer); } } + if (cursor != null && !cursor.isDurable() && cursor instanceof NonDurableCursorImpl) { + ((NonDurableCursorImpl) cursor).setReadCompacted(ACTIVE_CONSUMER_UPDATER.get(this).readCompacted()); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java index 6094ab71df2cc..5cdbff17b8171 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java @@ -44,7 +44,7 @@ public final class NonPersistentDispatcherSingleActiveConsumer extends AbstractD public NonPersistentDispatcherSingleActiveConsumer(SubType subscriptionType, int partitionIndex, NonPersistentTopic topic, Subscription subscription) { super(subscriptionType, partitionIndex, topic.getName(), subscription, - topic.getBrokerService().pulsar().getConfiguration()); + topic.getBrokerService().pulsar().getConfiguration(), null); this.topic = topic; this.subscription = subscription; this.msgDrop = new Rate(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 848cbb40a5ac4..6161847e42dc0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -58,7 +58,6 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher implements Dispatcher, ReadEntriesCallback { protected final PersistentTopic topic; - protected final ManagedCursor cursor; protected final String name; private Optional dispatchRateLimiter = Optional.empty(); @@ -73,11 +72,10 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher public PersistentDispatcherSingleActiveConsumer(ManagedCursor cursor, SubType subscriptionType, int partitionIndex, PersistentTopic topic, Subscription subscription) { super(subscriptionType, partitionIndex, topic.getName(), subscription, - topic.getBrokerService().pulsar().getConfiguration()); + topic.getBrokerService().pulsar().getConfiguration(), cursor); this.topic = topic; this.name = topic.getName() + " / " + (cursor.getName() != null ? Codec.decode(cursor.getName()) : ""/* NonDurableCursor doesn't have name */); - this.cursor = cursor; this.readBatchSize = serviceConfig.getDispatcherMaxReadBatchSize(); this.readFailureBackoff = new Backoff(serviceConfig.getDispatcherReadFailureBackoffInitialTimeInMs(), TimeUnit.MILLISECONDS, serviceConfig.getDispatcherReadFailureBackoffMaxTimeInMs(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 608d99b8307ef..cbe73721483fb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -509,4 +509,77 @@ public void testDoNotLossTheLastCompactedLedgerData() throws Exception { reader.close(); producer.close(); } + + @Test + public void testReadCompactedDataWhenLedgerRolloverKickIn() throws Exception { + String topic = "persistent://my-property/use/my-ns/testReadCompactedDataWhenLedgerRolloverKickIn-" + + UUID.randomUUID(); + final int numMessages = 2000; + final int keys = 200; + final String msg = "Test"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .blockIfQueueFull(true) + .maxPendingMessages(numMessages) + .enableBatching(false) + .create(); + CompletableFuture lastMessage = null; + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i % keys + "").value(msg).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, keys); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + }); + // Send more 200 keys + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key((i % keys + keys) + "").value(msg).sendAsync(); + } + producer.flush(); + lastMessage.join(); + + // Make sure we have more than 1 original ledgers + admin.topics().unload(topic); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().getInternalStats(topic).ledgers.size(), 2); + }); + + // Start a new reader to reading messages + Reader reader = pulsarClient.newReader(Schema.STRING) + .topic(topic) + .startMessageId(MessageId.earliest) + .readCompacted(true) + .receiverQueueSize(10) + .create(); + + // Send more 200 keys + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key((i % keys + keys * 2) + "").value(msg).sendAsync(); + } + producer.flush(); + lastMessage.join(); + + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, keys * 3); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + }); + + // The reader should read all 600 keys + int received = 0; + while (reader.hasMessageAvailable()) { + System.out.println(reader.readNext().getKey()); + received++; + } + Assert.assertEquals(received, keys * 3); + } } From 47b8e44660c2f15934381c8d5d13edd0d9d9090a Mon Sep 17 00:00:00 2001 From: baomingyu Date: Fri, 5 Nov 2021 22:05:04 +0800 Subject: [PATCH 117/823] modify check waitingForPingResponse with volatile (#12615) (cherry picked from commit 62e2547bea445c4f67935a57f59886757facbd2d) --- .../java/org/apache/pulsar/common/protocol/PulsarHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index 557dbfa9a92c0..cdf372dca02c8 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -37,7 +37,7 @@ public abstract class PulsarHandler extends PulsarDecoder { protected SocketAddress remoteAddress; private int remoteEndpointProtocolVersion = ProtocolVersion.v0.getValue(); private final long keepAliveIntervalSeconds; - private boolean waitingForPingResponse = false; + private volatile boolean waitingForPingResponse = false; private ScheduledFuture keepAliveTask; public int getRemoteEndpointProtocolVersion() { From 4421d1704fa1a8472110f6680c4614f1815a2c07 Mon Sep 17 00:00:00 2001 From: liuchangqing Date: Fri, 5 Nov 2021 14:31:48 +0800 Subject: [PATCH 118/823] Fix String should use equals but not ==. (#12619) Fix String should use equals but not == (cherry picked from commit c2165e898f4e5cda7e89f5a7106263e7a1ec48ac) --- .../org/apache/bookkeeper/test/BookKeeperClusterTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java index 006c287610c93..105405fe41c2c 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java @@ -228,7 +228,7 @@ protected ServerConfiguration newServerConfiguration(int port, String zkServers, conf.setBookiePort(port); if (ledgerRootPath != "") { conf.setMetadataServiceUri("zk://" + zkUtil.getZooKeeperConnectString() + ledgerRootPath); - }else { + } else { conf.setZkServers(zkServers); } conf.setJournalDirName(journalDir.getPath()); From 5e7f9cd60f57ca4591944de3a089bf6cc746ac8c Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Fri, 5 Nov 2021 15:39:37 +0800 Subject: [PATCH 119/823] Remove wrong visible testing annotation in function workers (#12621) * Remove wrong visible testing annotation * Remove unused method since 2017 (cherry picked from commit 8496afc58bdd27c47cde8a9ba3c76b80ab796320) --- .../worker/FunctionMetaDataManager.java | 19 +++++++------------ .../worker/FunctionRuntimeManager.java | 9 --------- .../functions/worker/PulsarWorkerService.java | 4 ---- .../functions/worker/SchedulerManager.java | 2 -- .../functions/worker/WorkerService.java | 1 - .../functions/worker/dlog/DLInputStream.java | 11 ----------- .../functions/worker/rest/WorkerServer.java | 13 ------------- .../worker/rest/api/ComponentImpl.java | 5 +---- .../worker/rest/api/FunctionsImpl.java | 2 -- .../functions/worker/rest/api/SinksImpl.java | 2 -- .../worker/rest/api/SourcesImpl.java | 2 -- .../functions/worker/rest/api/WorkerImpl.java | 6 +----- .../service/WorkerServiceWithClassLoader.java | 2 -- .../worker/FunctionMetaDataManagerTest.java | 12 ++++++------ 14 files changed, 15 insertions(+), 75 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java index 1ff1b2bdf17ef..5388bd703312f 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.functions.worker; -import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.Collection; import java.util.LinkedList; @@ -62,7 +61,6 @@ public class FunctionMetaDataManager implements AutoCloseable { // Represents the global state // tenant -> namespace -> (function name, FunctionRuntimeInfo) - @VisibleForTesting final Map>> functionMetaDataMap = new ConcurrentHashMap<>(); private final SchedulerManager schedulerManager; @@ -240,7 +238,7 @@ public synchronized void updateFunctionOnLeader(FunctionMetaData functionMetaDat } lastMessageSeen = builder.send(); if (delete) { - needsScheduling = proccessDeregister(functionMetaData); + needsScheduling = processDeregister(functionMetaData); } else { needsScheduling = processUpdate(functionMetaData); } @@ -359,7 +357,7 @@ private void processUncompactedMetaDataTopicMessage(Message message) thr this.processUpdate(serviceRequest.getFunctionMetaData()); break; case DELETE: - this.proccessDeregister(serviceRequest.getFunctionMetaData()); + this.processDeregister(serviceRequest.getFunctionMetaData()); break; default: log.warn("Received request with unrecognized type: {}", serviceRequest); @@ -373,7 +371,7 @@ private void processCompactedMetaDataTopicMessage(Message message) throw String functionName = FunctionCommon.extractNameFromFullyQualifiedName(message.getKey()); if (message.getData() == null || message.getData().length == 0) { // this is a delete message - this.proccessDeregister(tenant, namespace, functionName, version); + this.processDeregister(tenant, namespace, functionName, version); } else { FunctionMetaData functionMetaData = FunctionMetaData.parseFrom(message.getData()); this.processUpdate(functionMetaData); @@ -404,16 +402,15 @@ private boolean containsFunctionMetaData(String tenant, String namespace, String return false; } - @VisibleForTesting - synchronized boolean proccessDeregister(FunctionMetaData deregisterRequestFs) throws IllegalArgumentException { + synchronized boolean processDeregister(FunctionMetaData deregisterRequestFs) throws IllegalArgumentException { String functionName = deregisterRequestFs.getFunctionDetails().getName(); String tenant = deregisterRequestFs.getFunctionDetails().getTenant(); String namespace = deregisterRequestFs.getFunctionDetails().getNamespace(); - return proccessDeregister(tenant, namespace, functionName, deregisterRequestFs.getVersion()); + return processDeregister(tenant, namespace, functionName, deregisterRequestFs.getVersion()); } - synchronized boolean proccessDeregister(String tenant, String namespace, - String functionName, long version) throws IllegalArgumentException { + synchronized boolean processDeregister(String tenant, String namespace, + String functionName, long version) throws IllegalArgumentException { boolean needsScheduling = false; @@ -437,7 +434,6 @@ synchronized boolean proccessDeregister(String tenant, String namespace, return needsScheduling; } - @VisibleForTesting synchronized boolean processUpdate(FunctionMetaData updateRequestFs) throws IllegalArgumentException { log.debug("Process update request: {}", updateRequestFs); @@ -481,7 +477,6 @@ private boolean isRequestOutdated(String tenant, String namespace, String functi return currentFunctionMetaData.getVersion() >= version; } - @VisibleForTesting void setFunctionMetaData(FunctionMetaData functionMetaData) { Function.FunctionDetails functionDetails = functionMetaData.getFunctionDetails(); if (!this.functionMetaDataMap.containsKey(functionDetails.getTenant())) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index eaad1320d7ce8..d37049fe4029e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -79,12 +79,10 @@ public class FunctionRuntimeManager implements AutoCloseable{ // all assignments // WorkerId -> Function Fully Qualified InstanceId -> List - @VisibleForTesting Map> workerIdToAssignments = new ConcurrentHashMap<>(); // All the runtime info related to functions executed by this worker // Fully Qualified InstanceId - > FunctionRuntimeInfo - @VisibleForTesting class FunctionRuntimeInfos { private Map functionRuntimeInfoMap = new ConcurrentHashMap<>(); @@ -114,10 +112,8 @@ public int size() { } } - @VisibleForTesting final FunctionRuntimeInfos functionRuntimeInfos = new FunctionRuntimeInfos(); - @VisibleForTesting @Getter final WorkerConfig workerConfig; @@ -266,10 +262,6 @@ public MessageId initialize() { } } - /** - * Starts the function runtime manager - */ - /** * Get current assignments * @return a map of current assignments in the following format @@ -827,7 +819,6 @@ public synchronized void deleteAssignment(String fullyQualifiedInstanceId) { } } - @VisibleForTesting void deleteAssignment(Assignment assignment) { String fullyQualifiedInstanceId = FunctionCommon.getFullyQualifiedInstanceId(assignment.getInstance()); Map assignmentMap = this.workerIdToAssignments.get(assignment.getWorkerId()); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index 9c47dfe8b2bb3..3056a0dc64baa 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -45,7 +45,6 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authorization.AuthorizationService; -import org.apache.pulsar.broker.cache.ConfigurationCacheService; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -55,12 +54,10 @@ import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.naming.NamedEntity; import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.policies.path.PolicyPath; import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; import org.apache.pulsar.functions.worker.rest.api.FunctionsImplV2; @@ -188,7 +185,6 @@ public void generateFunctionsStats(SimpleTextOutputStream out) { ); } - @VisibleForTesting public void init(WorkerConfig workerConfig, URI dlogUri, boolean runAsStandalone) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java index 130e839618fba..2fed4991b93b8 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java @@ -371,7 +371,6 @@ private synchronized Set getCurrentAvailableWorkers() { return currentMembership; } - @VisibleForTesting void invokeScheduler() { long startTime = System.nanoTime(); @@ -561,7 +560,6 @@ void clearAssignmentsMovedInLastDrain() { assignmentsMovedInLastDrain = null; } - @VisibleForTesting List invokeDrain(String workerId) { long startTime = System.nanoTime(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java index f76f7536dfb01..f5293ceded295 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java @@ -21,7 +21,6 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authorization.AuthorizationService; -import org.apache.pulsar.broker.cache.ConfigurationCacheService; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.util.SimpleTextOutputStream; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/dlog/DLInputStream.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/dlog/DLInputStream.java index 10acb54350020..27af304741280 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/dlog/DLInputStream.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/dlog/DLInputStream.java @@ -40,10 +40,8 @@ public class DLInputStream extends InputStream { // Cache the input stream for a log record. private static class LogRecordWithInputStream { private final InputStream payloadStream; - private final LogRecordWithDLSN logRecord; LogRecordWithInputStream(LogRecordWithDLSN logRecord) { - this.logRecord = logRecord; this.payloadStream = logRecord.getPayLoadInputStream(); } @@ -51,15 +49,6 @@ InputStream getPayLoadInputStream() { return payloadStream; } - LogRecordWithDLSN getLogRecord() { - return logRecord; - } - - // The last txid of the log record is the position of the next byte in the stream. - // Subtract length to get starting offset. - long getOffset() { - return logRecord.getTransactionId() - logRecord.getPayload().length; - } } /** diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java index c7414c23734d7..e531084442466 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.functions.worker.rest; -import com.google.common.annotations.VisibleForTesting; import io.prometheus.client.jetty.JettyStatisticsCollector; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationService; @@ -46,8 +45,6 @@ import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; -import java.net.BindException; -import java.net.URI; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -69,15 +66,6 @@ public class WorkerServer { private ServerConnector httpConnector; private ServerConnector httpsConnector; - private static String getErrorMessage(Server server, int port, Exception ex) { - if (ex instanceof BindException) { - final URI uri = server.getURI(); - return String.format("%s http://%s:%d", ex.getMessage(), uri.getHost(), port); - } - - return ex.getMessage(); - } - public WorkerServer(WorkerService workerService, AuthenticationService authenticationService) { this.workerConfig = workerService.getWorkerConfig(); this.workerService = workerService; @@ -191,7 +179,6 @@ public static ServletContextHandler newServletContextHandler(String contextPath, return contextHandler; } - @VisibleForTesting public void stop() { if (this.server != null) { try { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index c1bc9960b9ab8..e8d67a5c60930 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -267,10 +267,7 @@ boolean isWorkerServiceAvailable() { if (workerService == null) { return false; } - if (!workerService.isInitialized()) { - return false; - } - return true; + return workerService.isInitialized(); } PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaData functionMetaData, diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 8c3585100cb95..b00846de3f027 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -32,7 +32,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -64,7 +63,6 @@ import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Functions; -import org.apache.pulsar.packages.management.core.common.PackageType; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @Slf4j diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index bbb16806b5539..ab69ab9182cf9 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -30,7 +30,6 @@ import java.io.InputStream; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -64,7 +63,6 @@ import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Sinks; -import org.apache.pulsar.packages.management.core.common.PackageType; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @Slf4j diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index 82a818d773d97..df2dca813e770 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -30,7 +30,6 @@ import java.io.InputStream; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -64,7 +63,6 @@ import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Sources; -import org.apache.pulsar.packages.management.core.common.PackageType; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @Slf4j diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java index f32f6d1c7bb7d..6c2180cee7076 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java @@ -19,7 +19,6 @@ package org.apache.pulsar.functions.worker.rest.api; import lombok.extern.slf4j.Slf4j; -import lombok.val; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -76,10 +75,7 @@ private boolean isWorkerServiceAvailable() { if (workerService == null) { return false; } - if (!workerService.isInitialized()) { - return false; - } - return true; + return workerService.isInitialized(); } @Override diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceWithClassLoader.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceWithClassLoader.java index 9439d72540c8a..6959616ce3d7d 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceWithClassLoader.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceWithClassLoader.java @@ -25,7 +25,6 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authorization.AuthorizationService; -import org.apache.pulsar.broker.cache.ConfigurationCacheService; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.nar.NarClassLoader; @@ -38,7 +37,6 @@ import org.apache.pulsar.functions.worker.service.api.Sinks; import org.apache.pulsar.functions.worker.service.api.Sources; import org.apache.pulsar.functions.worker.service.api.Workers; -import org.apache.pulsar.zookeeper.ZooKeeperCache; /** * A worker service with its classloader. diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java index e5221bdd6611b..f7d133db32117 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java @@ -299,7 +299,7 @@ public void testProcessRequest() throws PulsarClientException, IOException { mockPulsarClient(), ErrorNotifier.getDefaultImpl())); doReturn(true).when(functionMetaDataManager).processUpdate(any(Function.FunctionMetaData.class)); - doReturn(true).when(functionMetaDataManager).proccessDeregister(any(Function.FunctionMetaData.class)); + doReturn(true).when(functionMetaDataManager).processDeregister(any(Function.FunctionMetaData.class)); Request.ServiceRequest serviceRequest = Request.ServiceRequest.newBuilder().setServiceRequestType( @@ -324,9 +324,9 @@ public void testProcessRequest() throws PulsarClientException, IOException { doReturn(serviceRequest.toByteArray()).when(msg).getData(); functionMetaDataManager.processMetaDataTopicMessage(msg); - verify(functionMetaDataManager, times(1)).proccessDeregister( + verify(functionMetaDataManager, times(1)).processDeregister( any(Function.FunctionMetaData.class)); - verify(functionMetaDataManager).proccessDeregister(serviceRequest.getFunctionMetaData()); + verify(functionMetaDataManager).processDeregister(serviceRequest.getFunctionMetaData()); } @Test @@ -393,7 +393,7 @@ public void processDeregister() throws PulsarClientException { .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).build(); - Assert.assertFalse(functionMetaDataManager.proccessDeregister(m1)); + Assert.assertFalse(functionMetaDataManager.processDeregister(m1)); verify(functionMetaDataManager, times(0)) .setFunctionMetaData(any(Function.FunctionMetaData.class)); verify(schedulerManager, times(0)).schedule(); @@ -411,7 +411,7 @@ public void processDeregister() throws PulsarClientException { // outdated delete request try { - functionMetaDataManager.proccessDeregister(m1); + functionMetaDataManager.processDeregister(m1); Assert.assertTrue(false); } catch (IllegalArgumentException e) { Assert.assertEquals(e.getMessage(), "Delete request ignored because it is out of date. Please try again."); @@ -426,7 +426,7 @@ public void processDeregister() throws PulsarClientException { // delete now m1 = m1.toBuilder().setVersion(2).build(); - Assert.assertTrue(functionMetaDataManager.proccessDeregister(m1)); + Assert.assertTrue(functionMetaDataManager.processDeregister(m1)); verify(functionMetaDataManager, times(1)) .setFunctionMetaData(any(Function.FunctionMetaData.class)); verify(schedulerManager, times(0)).schedule(); From 1aa80b7554672cd307e4d730ae946c954d1b695f Mon Sep 17 00:00:00 2001 From: litao Date: Sat, 6 Nov 2021 11:14:31 +0800 Subject: [PATCH 120/823] [pulsar-broker] Add git branch information for PulsarVersion (#12541) (cherry picked from commit bc8e40c7c26791ea5e030eb790318f8443014887) --- .../src/main/java/org/apache/pulsar/broker/PulsarService.java | 1 + .../main/java-templates/org/apache/pulsar/PulsarVersion.java | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 6b59862ac8cd9..28ccbb1013426 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -582,6 +582,7 @@ public void start() throws PulsarServerException { LOG.info("Starting Pulsar Broker service; version: '{}'", (brokerVersion != null ? brokerVersion : "unknown")); LOG.info("Git Revision {}", PulsarVersion.getGitSha()); + LOG.info("Git Branch {}", PulsarVersion.getGitBranch()); LOG.info("Built by {} on {} at {}", PulsarVersion.getBuildUser(), PulsarVersion.getBuildHost(), diff --git a/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java b/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java index 07f97cd611e2b..e1e57e1909d7e 100644 --- a/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java +++ b/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java @@ -82,6 +82,10 @@ public static String getGitSha() { } } + public static String getGitBranch() { + return "${git.branch}"; + } + public static String getBuildUser() { String email = "${git.build.user.email}"; String name = "${git.build.user.name}"; From e46a6f7d1f005912b0c1536de9f3d8f630c1e129 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Sat, 6 Nov 2021 12:07:54 +0900 Subject: [PATCH 121/823] Add @Test annotation to test methods (#12640) (cherry picked from commit 370f47fe17a23b56f34c1e12e7abb28dd794c3b8) --- .../apache/pulsar/broker/admin/PersistentTopicsTest.java | 1 + .../intercept/BrokerInterceptorWithClassLoaderTest.java | 1 + .../broker/protocol/ProtocolHandlerWithClassLoaderTest.java | 1 + .../pulsar/broker/stats/ManagedCursorMetricsTest.java | 1 + .../pulsar/client/api/SimpleProducerConsumerStatTest.java | 1 + .../src/test/java/org/apache/pulsar/schema/SchemaTest.java | 1 + .../java/org/apache/pulsar/admin/cli/TestCmdClusters.java | 3 ++- .../test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java | 6 +++--- .../java/org/apache/pulsar/admin/cli/TestCmdSources.java | 6 +++--- .../runtime/kubernetes/KubernetesRuntimeFactoryTest.java | 2 +- .../functions/runtime/kubernetes/KubernetesRuntimeTest.java | 3 ++- .../io/elasticsearch/ElasticSearchClientSslTests.java | 2 +- .../pulsar/io/elasticsearch/ElasticSearchSinkTests.java | 2 +- .../proxy/extensions/ProxyExtensionWithClassLoaderTest.java | 1 + .../tests/integration/functions/PulsarFunctionsTest.java | 4 ++-- .../pulsar/tests/integration/offload/TestBaseOffload.java | 6 +++--- .../pulsar/tests/integration/topologies/PulsarTestBase.java | 6 +++--- .../offload/jcloud/impl/BufferedOffloadStreamTest.java | 2 +- 18 files changed, 29 insertions(+), 20 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 4a27dbb245122..a471caba4ad3a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -915,6 +915,7 @@ public void testSetReplicatedSubscriptionStatus() { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); } + @Test public void testGetMessageById() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test")); admin.tenants().createTenant("tenant-xyz", tenantInfo); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java index aa4a5bc4562f1..ae669373ded70 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java @@ -33,6 +33,7 @@ @Test(groups = "broker") public class BrokerInterceptorWithClassLoaderTest { + @Test public void testWrapper() throws Exception { BrokerInterceptor h = mock(BrokerInterceptor.class); NarClassLoader loader = mock(NarClassLoader.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoaderTest.java index 42d26d53f27c6..75d8c646b6bc8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoaderTest.java @@ -74,6 +74,7 @@ public void testWrapper() throws Exception { verify(h, times(1)).getProtocolDataToAdvertise(); } + @Test public void testClassLoaderSwitcher() throws Exception { NarClassLoader loader = mock(NarClassLoader.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java index 9b3b354aa28df..5e20c09fed1dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java @@ -49,6 +49,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Test public void testManagedCursorMetrics() throws Exception { final String subName = "my-sub"; final String topicName = "persistent://my-namespace/use/my-ns/my-topic1"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java index cc83ba2c331ca..480d83544b28b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java @@ -335,6 +335,7 @@ public void testSendTimeout(int batchMessageDelayMs) throws Exception { log.info("-- Exiting {} test --", methodName); } + @Test public void testBatchMessagesRateOut() throws PulsarClientException, InterruptedException, PulsarAdminException { log.info("-- Starting {} test --", methodName); String topicName = "persistent://my-property/cluster/my-ns/testBatchMessagesRateOut"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index dab9d08ac143e..db56fbce5a19b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -800,6 +800,7 @@ public void testNullKey() throws Exception { assertEquals("foo", message.getValue()); } + @Test public void testConsumeMultipleSchemaMessages() throws Exception { final String namespace = "test-namespace-" + randomName(16); String ns = PUBLIC_TENANT + "/" + namespace; diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java index 0ab093d56c26b..cd4c22f8b2473 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java @@ -62,7 +62,8 @@ public void testCmdClusterConfigFile() throws Exception { testCmdClusterConfigFile(clusterData, clusterData); } - public void testCmdClusterConfigFile(ClusterData testClusterData, ClusterData expectedClusterData) throws Exception { + private void testCmdClusterConfigFile(ClusterData testClusterData, ClusterData expectedClusterData) + throws Exception { File file = Files.createTempFile("tmp_cluster", ".yaml").toFile(); ObjectMapperFactory.getThreadLocalYaml().writeValue(file, testClusterData); Assert.assertEquals(testClusterData, CmdUtils.loadConfig(file.getAbsolutePath(), ClusterData.class)); diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java index 87f9a3fff7eda..27402bddbf0e8 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java @@ -341,7 +341,7 @@ public void testMissingConfig() throws Exception { ); } - public void testCmdSinkCliMissingArgs( + private void testCmdSinkCliMissingArgs( String tenant, String namespace, String name, @@ -492,7 +492,7 @@ public void testCmdSinkConfigFileInvalidJar() throws Exception { testCmdSinkConfigFile(testSinkConfig, expectedSinkConfig); } - public void testCmdSinkConfigFile(SinkConfig testSinkConfig, SinkConfig expectedSinkConfig) throws Exception { + private void testCmdSinkConfigFile(SinkConfig testSinkConfig, SinkConfig expectedSinkConfig) throws Exception { File file = Files.createTempFile("", "").toFile(); @@ -572,7 +572,7 @@ public void testCliOverwriteConfigFile() throws Exception { ); } - public void testMixCliAndConfigFile( + private void testMixCliAndConfigFile( String tenant, String namespace, String name, diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java index 5b4d906b81b75..ff75d26534153 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java @@ -261,7 +261,7 @@ public void testMissingConfig() throws Exception { ); } - public void testCmdSourceCliMissingArgs( + private void testCmdSourceCliMissingArgs( String tenant, String namespace, String name, @@ -500,7 +500,7 @@ public void testCliOverwriteConfigFile() throws Exception { ); } - public void testMixCliAndConfigFile( + private void testMixCliAndConfigFile( String tenant, String namespace, String name, @@ -674,4 +674,4 @@ public void testParseConfigs() throws Exception { Assert.assertEquals(config.get("float_string"), "1000.0"); Assert.assertEquals(config.get("created_at"), "Mon Jul 02 00:33:15 +0000 2018"); } -} \ No newline at end of file +} diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java index 1b8946b64b41d..dc0119cab4625 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java @@ -333,7 +333,7 @@ public void testValidateResourcesGranularityAndProportion() throws Exception { "Per instance ram requested, 0, for function should be positive and a multiple of the granularity, 1000"); } - public void testAuthProvider(Optional authProvider) throws Exception { + private void testAuthProvider(Optional authProvider) throws Exception { factory = createKubernetesRuntimeFactory(null, null, null, null, false, authProvider, Optional.empty()); } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 928497a25cc14..7f6c36acc6104 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -358,7 +358,8 @@ public void testResources() throws Exception { testResources(1.0 / 1.5, 1000, 1.3, 1.0); } - public void testResources(double userCpuRequest, long userMemoryRequest, double cpuOverCommitRatio, double memoryOverCommitRatio) throws Exception { + private void testResources(double userCpuRequest, long userMemoryRequest, double cpuOverCommitRatio, + double memoryOverCommitRatio) throws Exception { Function.Resources resources = Function.Resources.newBuilder() .setRam(userMemoryRequest).setCpu(userCpuRequest).setDisk(10000L).build(); diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java index 9f878aad1ad17..7cbd24991df8b 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java @@ -157,7 +157,7 @@ public void testSslWithClientAuth() throws IOException { } - public void testIndexExists(ElasticSearchClient client) throws IOException { + private void testIndexExists(ElasticSearchClient client) throws IOException { assertFalse(client.indexExists("mynewindex")); assertTrue(client.createIndexIfNeeded("mynewindex")); assertTrue(client.indexExists("mynewindex")); diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java index 9e3363c84d76c..c7523543beeef 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java @@ -292,7 +292,7 @@ public void testNullValueDelete() throws Exception { testNullValue(ElasticSearchConfig.NullValueAction.DELETE); } - public void testNullValue(ElasticSearchConfig.NullValueAction action) throws Exception { + private void testNullValue(ElasticSearchConfig.NullValueAction action) throws Exception { String index = "testnullvalue" + action.toString().toLowerCase(Locale.ROOT); map.put("indexName", index); map.put("keyIgnore", "false"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionWithClassLoaderTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionWithClassLoaderTest.java index b43eb22ab8952..57882a6a4b33b 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionWithClassLoaderTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionWithClassLoaderTest.java @@ -70,6 +70,7 @@ public void testWrapper() throws Exception { verify(h, times(1)).start(service); } + @Test public void testClassLoaderSwitcher() throws Exception { NarClassLoader loader = mock(NarClassLoader.class); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 87357a04cf6fb..ad97740c80895 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -120,7 +120,7 @@ protected Map produceMessagesToInputTopic(String inputTopicName, return kvs; } - public void testFunctionLocalRun(Runtime runtime) throws Exception { + protected void testFunctionLocalRun(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD) { return; } @@ -243,7 +243,7 @@ public void testFunctionLocalRun(Runtime runtime) throws Exception { } - public void testWindowFunction(String type, String[] expectedResults) throws Exception { + protected void testWindowFunction(String type, String[] expectedResults) throws Exception { int NUM_OF_MESSAGES = 100; int windowLengthCount = 10; int slidingIntervalCount = 5; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java index ad7a8fdf1d048..f5a320e582072 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java @@ -48,7 +48,7 @@ private static byte[] buildEntry(String pattern) { return entry; } - public void testPublishOffloadAndConsumeViaCLI(String serviceUrl, String adminUrl) throws Exception { + protected void testPublishOffloadAndConsumeViaCLI(String serviceUrl, String adminUrl) throws Exception { final String tenant = "offload-test-cli-" + randomName(4); final String namespace = tenant + "/ns1"; final String topic = "persistent://" + namespace + "/topic1"; @@ -120,7 +120,7 @@ public void testPublishOffloadAndConsumeViaCLI(String serviceUrl, String adminUr } } - public void testPublishOffloadAndConsumeViaThreshold(String serviceUrl, String adminUrl) throws Exception { + protected void testPublishOffloadAndConsumeViaThreshold(String serviceUrl, String adminUrl) throws Exception { final String tenant = "offload-test-threshold-" + randomName(4); final String namespace = tenant + "/ns1"; final String topic = "persistent://" + namespace + "/topic1"; @@ -240,7 +240,7 @@ public boolean ledgerExistsInBookKeeper(long ledgerId) throws Exception { } } - public void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String adminUrl) throws Exception { + protected void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String adminUrl) throws Exception { final String tenant = "offload-test-deletion-lag-" + randomName(4); final String namespace = tenant + "/ns1"; final String topic = "persistent://" + namespace + "/topic1"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java index ebdfbe84de401..778c67b27b538 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarTestBase.java @@ -76,7 +76,7 @@ protected static String generateTopicName(String namespace, String topicPrefix, } } - public void testPublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception { + protected void testPublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception { String topicName = generateTopicName("testpubconsume", isPersistent); int numMessages = 10; @@ -107,7 +107,7 @@ public void testPublishAndConsume(String serviceUrl, boolean isPersistent) throw } } - public void testBatchMessagePublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception { + protected void testBatchMessagePublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception { String topicName = generateTopicName("test-batch-publish-consume", isPersistent); final int numMessages = 10000; @@ -142,7 +142,7 @@ public void testBatchMessagePublishAndConsume(String serviceUrl, boolean isPersi } } - public void testBatchIndexAckDisabled(String serviceUrl) throws Exception { + protected void testBatchIndexAckDisabled(String serviceUrl) throws Exception { String topicName = generateTopicName("test-batch-index-ack-disabled", true); final int numMessages = 100; try (PulsarClient client = PulsarClient.builder() diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BufferedOffloadStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BufferedOffloadStreamTest.java index 4ebf9434952eb..a5dc7ba17076f 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BufferedOffloadStreamTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BufferedOffloadStreamTest.java @@ -42,7 +42,7 @@ public class BufferedOffloadStreamTest { final Random random = new Random(); - public void testWithPadding(int paddingLen) throws Exception { + private void testWithPadding(int paddingLen) throws Exception { int blockSize = StreamingDataBlockHeaderImpl.getDataStartOffset(); List entryBuffer = new LinkedList<>(); final UUID uuid = UUID.randomUUID(); From 0ca7151e6a4e7d521bd2eb0986605e00e9ba97a1 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Fri, 5 Nov 2021 19:59:51 -0700 Subject: [PATCH 122/823] Upgrade debezium to 1.7.1 (#12644) (cherry picked from commit ccfe395ed86886f56269a10e43789d9829c0c00c) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4dca3ba0f9004..f142eb93b9394 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ flexible messaging model and an intuitive client API. 332 2.13 2.13.6 - 1.7.0.Final + 1.7.1.Final 0.11.1 0.18.0 2.3.0 From 3f809e65fd5f85b6b10c1169177d053759cfaf3f Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Sat, 6 Nov 2021 22:14:19 +0800 Subject: [PATCH 123/823] [Broker] Optimize exception information for schemas (#12647) (cherry picked from commit 36f151ce442ec928c6fcf5840b825284f7f5ae88) --- .../pulsar/broker/admin/impl/SchemasResourceBase.java | 11 +++++++---- .../apache/pulsar/tests/integration/cli/CLITest.java | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 87fc3aecbd07d..dfd870a053d99 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -272,9 +272,11 @@ private static GetSchemaResponse convertSchemaAndMetadataToGetSchemaResponse(Sch private static void handleGetSchemaResponse(AsyncResponse response, SchemaAndMetadata schema, Throwable error) { if (isNull(error)) { if (isNull(schema)) { - response.resume(Response.status(Response.Status.NOT_FOUND).build()); + response.resume(Response.status( + Response.Status.NOT_FOUND.getStatusCode(), "Schema not found").build()); } else if (schema.schema.isDeleted()) { - response.resume(Response.status(Response.Status.NOT_FOUND).build()); + response.resume(Response.status( + Response.Status.NOT_FOUND.getStatusCode(), "Schema is deleted").build()); } else { response.resume(Response.ok().encoding(MediaType.APPLICATION_JSON) .entity(convertSchemaAndMetadataToGetSchemaResponse(schema)).build()); @@ -290,7 +292,8 @@ private static void handleGetAllSchemasResponse(AsyncResponse response, List Date: Sat, 6 Nov 2021 22:13:02 +0800 Subject: [PATCH 124/823] Close Zk database on unit tests (#12649) (cherry picked from commit 701df3206d274682d752d4e6405fb3ce65ffe197) --- .../src/test/java/org/apache/pulsar/metadata/TestZKServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java index 4f83906ad96f9..9e1f7bcc4f273 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java @@ -100,6 +100,7 @@ public void checkContainers() throws Exception { public void stop() throws Exception { if (zks != null) { zks.shutdown(); + zks.getZKDatabase().close(); zks = null; } From b55bba3c75089b2abb3c5cc8450ceec75af95eab Mon Sep 17 00:00:00 2001 From: penghui Date: Mon, 20 Dec 2021 23:18:36 +0800 Subject: [PATCH 125/823] Fix cherry-pick check style issue --- .../pulsar/broker/service/nonpersistent/NonPersistentTopic.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 4b1c57221969c..43245b9e672e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -38,8 +38,8 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; From e904735203f78e003ac8aeb30edb68ed279bd62d Mon Sep 17 00:00:00 2001 From: feynmanlin Date: Fri, 5 Nov 2021 20:43:43 +0800 Subject: [PATCH 126/823] Support retry when creating reader of Topic Policies (#12622) (cherry picked from commit 2ecafbfa4ca540230accb6f4f18f774fa4757adf) --- .../SystemTopicBasedTopicPoliciesService.java | 23 +++++++++++-- ...temTopicBasedTopicPoliciesServiceTest.java | 34 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index f5dc4b778aa09..7055682638fca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -35,6 +36,8 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.Backoff; +import org.apache.pulsar.client.util.RetryUtil; import org.apache.pulsar.common.events.ActionType; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.PulsarEvent; @@ -194,12 +197,10 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); result.complete(null); } else { - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory - .createTopicPoliciesSystemTopicClient(namespace); ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); policyCacheInitMap.put(namespace, false); CompletableFuture> readerCompletableFuture = - systemTopicClient.newReaderAsync(); + creatSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); readerCompletableFuture.whenComplete((reader, ex) -> { if (ex != null) { @@ -215,6 +216,22 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name return result; } + protected CompletableFuture> creatSystemTopicClientWithRetry( + NamespaceName namespace) { + SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory + .createTopicPoliciesSystemTopicClient(namespace); + CompletableFuture> result = new CompletableFuture<>(); + Backoff backoff = new Backoff(1, TimeUnit.SECONDS, 3, TimeUnit.SECONDS, 10, TimeUnit.SECONDS); + RetryUtil.retryAsynchronously(() -> { + try { + return systemTopicClient.newReader(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + }, backoff, pulsarService.getExecutor(), result); + return result; + } + @Override public CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { NamespaceName namespace = namespaceBundle.getNamespaceObject(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 61365fe4d3e6b..49d86993e330f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -18,11 +18,18 @@ */ package org.apache.pulsar.broker.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNull; import static org.testng.AssertJUnit.assertTrue; import com.google.common.collect.Sets; +import java.lang.reflect.Field; import java.util.List; import java.util.Map; import java.util.UUID; @@ -31,9 +38,15 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.systopic.TopicPoliciesSystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; +import org.apache.pulsar.client.impl.ReaderImpl; +import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; @@ -245,4 +258,25 @@ public void testGetPolicyTimeout() throws Exception { long cost = System.currentTimeMillis() - start; assertTrue("actual:" + cost, cost >= 5000 - 1000); } + + @Test + public void testCreatSystemTopicClientWithRetry() throws Exception { + SystemTopicBasedTopicPoliciesService service = + spy((SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService()); + Field field = SystemTopicBasedTopicPoliciesService.class + .getDeclaredField("namespaceEventsSystemTopicFactory"); + field.setAccessible(true); + NamespaceEventsSystemTopicFactory factory = spy((NamespaceEventsSystemTopicFactory) field.get(service)); + SystemTopicClient client = mock(TopicPoliciesSystemTopicClient.class); + doReturn(client).when(factory).createTopicPoliciesSystemTopicClient(any()); + field.set(service, factory); + + SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class); + // Throw an exception first, create successfully after retrying + doThrow(new PulsarClientException("test")).doReturn(reader).when(client).newReader(); + + SystemTopicClient.Reader reader1 = service.creatSystemTopicClientWithRetry(null).get(); + + assertEquals(reader1, reader); + } } From 736dd6a853d724f652c4aa945076e40384728abd Mon Sep 17 00:00:00 2001 From: ran Date: Sun, 7 Nov 2021 21:24:33 +0800 Subject: [PATCH 127/823] [Python Schema] Python schema support custom Avro configurations for Enum type (#12642) ### Motivation Currently, the Python client didn't support setting configurations `required`, `default`, `required_default` for Enum type in Record. ### Modifications Modify the `_Enum` class to `CustomEnum` class, the `_Enum` wasn't exposed to users, the new class `CustomEnum` will be exposed to users, they could set Avro definition configurations `required`, `default`, `required_default`. ### How to use ``` class Color(Enum): red = 1 green = 2 blue = 3 class NestedObj(Record): a = Integer() color = CustomEnum(Color, required_default=True, default=Color.blue) ``` The schema definition will be like this ``` { 'type': 'record', 'name': 'NestedObj', 'fields': [ {'name': 'a', 'type': ['null', 'int']}, {'name': 'color', 'default': 'blue', 'type': ['null', {'type': 'enum', 'name': 'Color', 'symbols': ['red', 'green', 'blue']}]} ] } ``` The old way of use will also be preserved. This feature could work well with Java client. (cherry picked from commit e7389ed965cb712058c4ed1e8931d7563ca10bc8) --- .../python/examples/company.avsc | 4 +- .../python/pulsar/schema/__init__.py | 2 +- .../python/pulsar/schema/definition.py | 18 ++- pulsar-client-cpp/python/schema_test.py | 118 +++++++++--------- 4 files changed, 74 insertions(+), 68 deletions(-) diff --git a/pulsar-client-cpp/python/examples/company.avsc b/pulsar-client-cpp/python/examples/company.avsc index cdb595f41d257..5fb186092182b 100644 --- a/pulsar-client-cpp/python/examples/company.avsc +++ b/pulsar-client-cpp/python/examples/company.avsc @@ -14,6 +14,8 @@ {"name": "age", "type": ["null", "int"]} ] }}]}, - {"name": "labels", "type": ["null", {"type": "map", "values": "string"}]} + {"name": "labels", "type": ["null", {"type": "map", "values": "string"}]}, + {"name": "companyType", "type": ["null", {"type": "enum", "name": "CompanyType", "symbols": + ["companyType1", "companyType2", "companyType3"]}]} ] } \ No newline at end of file diff --git a/pulsar-client-cpp/python/pulsar/schema/__init__.py b/pulsar-client-cpp/python/pulsar/schema/__init__.py index 150629d0f2f25..efa680666a729 100644 --- a/pulsar-client-cpp/python/pulsar/schema/__init__.py +++ b/pulsar-client-cpp/python/pulsar/schema/__init__.py @@ -18,7 +18,7 @@ # from .definition import Record, Field, Null, Boolean, Integer, Long, \ - Float, Double, Bytes, String, Array, Map + Float, Double, Bytes, String, Array, Map, CustomEnum from .schema import Schema, BytesSchema, StringSchema, JsonSchema from .schema_avro import AvroSchema diff --git a/pulsar-client-cpp/python/pulsar/schema/definition.py b/pulsar-client-cpp/python/pulsar/schema/definition.py index fd778f3293677..9b6c86106cefd 100644 --- a/pulsar-client-cpp/python/pulsar/schema/definition.py +++ b/pulsar-client-cpp/python/pulsar/schema/definition.py @@ -44,8 +44,7 @@ def _get_fields(cls, dct): fields = OrderedDict() for name, value in dct.items(): if issubclass(type(value), EnumMeta): - # Wrap Python enums - value = _Enum(value) + value = CustomEnum(value) elif type(value) == RecordMeta: # We expect an instance of a record rather than the class itself value = value() @@ -125,6 +124,12 @@ def schema_info(cls, defined_names): schema['namespace'] = cls._avro_namespace schema['fields'] = [] + def get_filed_default_value(value): + if isinstance(value, Enum): + return value.name + else: + return value + if cls._sorted_fields: fields = sorted(cls._fields.keys()) else: @@ -135,7 +140,7 @@ def schema_info(cls, defined_names): if field._required else ['null', field.schema_info(defined_names)] schema['fields'].append({ 'name': name, - 'default': field.default(), + 'default': get_filed_default_value(field.default()), 'type': field_type }) if field.required_default() else schema['fields'].append({ 'name': name, @@ -360,15 +365,16 @@ def default(self): # Complex types -class _Enum(Field): - def __init__(self, enum_type): + +class CustomEnum(Field): + def __init__(self, enum_type, default=None, required=False, required_default=False): if not issubclass(enum_type, Enum): raise Exception(enum_type + " is not a valid Enum type") self.enum_type = enum_type self.values = {} for x in enum_type.__members__.values(): self.values[x.value] = x - super(_Enum, self).__init__() + super(CustomEnum, self).__init__(default, required, required_default) def type(self): return 'enum' diff --git a/pulsar-client-cpp/python/schema_test.py b/pulsar-client-cpp/python/schema_test.py index d2554da5a7365..077f2bb076387 100755 --- a/pulsar-client-cpp/python/schema_test.py +++ b/pulsar-client-cpp/python/schema_test.py @@ -49,6 +49,7 @@ class Example(Record): g = Double() h = Bytes() i = Map(String()) + j = CustomEnum(Color) fastavro.parse_schema(Example.schema()) self.assertEqual(Example.schema(), { @@ -74,16 +75,23 @@ class Example(Record): {"name": "i", "type": ["null", { "type": "map", "values": "string"}] - }, + }, + {"name": "j", "type": ["null", "Color"]} ] }) def test_complex(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + class MySubRecord(Record): _sorted_fields = True x = Integer() y = Long() z = String() + color = CustomEnum(Color) class Example(Record): _sorted_fields = True @@ -101,9 +109,12 @@ class Example(Record): "type": ["null", { "name": "MySubRecord", "type": "record", - "fields": [{"name": "x", "type": ["null", "int"]}, - {"name": "y", "type": ["null", "long"]}, - {"name": "z", "type": ["null", "string"]}] + "fields": [ + {'name': 'color', 'type': ['null', {'type': 'enum', 'name': 'Color', 'symbols': + ['red', 'green', 'blue']}]}, + {"name": "x", "type": ["null", "int"]}, + {"name": "y", "type": ["null", "long"]}, + {"name": "z", "type": ["null", "string"]}] }] }, {"name": "sub2", @@ -630,6 +641,8 @@ class MyEnum(Enum): class Example(Record): name = String() v = MyEnum + w = CustomEnum(MyEnum) + x = CustomEnum(MyEnum, required=True, default=MyEnum.A, required_default=True) topic = 'my-json-enum-topic' @@ -641,13 +654,15 @@ class Example(Record): consumer = client.subscribe(topic, 'test', schema=JsonSchema(Example)) - r = Example(name='test', v=MyEnum.C) + r = Example(name='test', v=MyEnum.C, w=MyEnum.B) producer.send(r) msg = consumer.receive() self.assertEqual('test', msg.value().name) self.assertEqual(MyEnum.C, MyEnum(msg.value().v)) + self.assertEqual(MyEnum.B, MyEnum(msg.value().w)) + self.assertEqual(MyEnum.A, MyEnum(msg.value().x)) client.close() def test_avro_enum(self): @@ -659,6 +674,8 @@ class MyEnum(Enum): class Example(Record): name = String() v = MyEnum + w = CustomEnum(MyEnum) + x = CustomEnum(MyEnum, required=True, default=MyEnum.B, required_default=True) topic = 'my-avro-enum-topic' @@ -670,12 +687,14 @@ class Example(Record): consumer = client.subscribe(topic, 'test', schema=AvroSchema(Example)) - r = Example(name='test', v=MyEnum.C) + r = Example(name='test', v=MyEnum.C, w=MyEnum.A) producer.send(r) msg = consumer.receive() msg.value() self.assertEqual(MyEnum.C, msg.value().v) + self.assertEqual(MyEnum.A, MyEnum(msg.value().w)) + self.assertEqual(MyEnum.B, MyEnum(msg.value().x)) client.close() def test_avro_map_array(self): @@ -913,6 +932,11 @@ class MyRecord(Record): client.close() def test_serialize_schema_complex(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + class NestedObj1(Record): _sorted_fields = True na1 = String() @@ -925,6 +949,8 @@ class NestedObj2(Record): nc2 = NestedObj1() class NestedObj3(Record): + _sorted_fields = True + color = CustomEnum(Color) na3 = Integer() class NestedObj4(Record): @@ -933,11 +959,6 @@ class NestedObj4(Record): na4 = String() nb4 = Integer() - class Color(Enum): - red = 1 - green = 2 - blue = 3 - class ComplexRecord(Record): _avro_namespace = 'xxx.xxx' _sorted_fields = True @@ -945,6 +966,7 @@ class ComplexRecord(Record): b = Integer() color = Color color2 = Color + color3 = CustomEnum(Color, required=True, default=Color.red, required_default=True) nested = NestedObj2() nested2 = NestedObj2() mapNested = Map(NestedObj3()) @@ -970,8 +992,10 @@ class ComplexRecord(Record): {'name': 'color', 'type': ['null', {'type': 'enum', 'name': 'Color', 'symbols': [ 'red', 'green', 'blue']}]}, {'name': 'color2', 'type': ['null', 'Color']}, + {'name': 'color3', 'default': 'red', 'type': 'Color'}, {'name': 'mapNested', 'type': ['null', {'type': 'map', 'values': {'name': 'NestedObj3', 'type': 'record', 'fields': [ + {'name': 'color', 'type': ['null', 'Color']}, {'name': 'na3', 'type': ['null', 'int']} ]}} ]}, @@ -998,12 +1022,12 @@ def encode_and_decode(schema_type): r = ComplexRecord(a=1, b=2, color=Color.red, color2=Color.blue, nested=nested_obj2, nested2=nested_obj2, mapNested={ - 'a': NestedObj3(na3=1), + 'a': NestedObj3(na3=1, color=Color.green), 'b': NestedObj3(na3=2), - 'c': NestedObj3(na3=3) + 'c': NestedObj3(na3=3, color=Color.red) }, mapNested2={ - 'd': NestedObj3(na3=4), - 'e': NestedObj3(na3=5), + 'd': NestedObj3(na3=4, color=Color.red), + 'e': NestedObj3(na3=5, color=Color.blue), 'f': NestedObj3(na3=6) }, arrayNested=[ NestedObj4(na4='value na4 1', nb4=100), @@ -1017,32 +1041,9 @@ def encode_and_decode(schema_type): data_decode = data_schema.decode(data_encode) self.assertEqual(data_decode.__class__.__name__, 'ComplexRecord') self.assertEqual(data_decode, r) - self.assertEqual(data_decode.a, 1) - self.assertEqual(data_decode.b, 2) - self.assertEqual(data_decode.color, Color.red) - self.assertEqual(data_decode.color2, Color.blue) - self.assertEqual(data_decode.nested.na2, 22) - self.assertEqual(data_decode.nested.nb2, True) - self.assertEqual(data_decode.nested.nc2.na1, 'na1 value') - self.assertEqual(data_decode.nested.nc2.nb1, 20.5) - self.assertEqual(data_decode.nested2.na2, 22) - self.assertEqual(data_decode.nested2.nb2, True) - self.assertEqual(data_decode.nested2.nc2.na1, 'na1 value') - self.assertEqual(data_decode.nested2.nc2.nb1, 20.5) - self.assertEqual(data_decode.mapNested['a'].na3, 1) - self.assertEqual(data_decode.mapNested['b'].na3, 2) - self.assertEqual(data_decode.mapNested['c'].na3, 3) - self.assertEqual(data_decode.mapNested2['d'].na3, 4) - self.assertEqual(data_decode.mapNested2['e'].na3, 5) - self.assertEqual(data_decode.mapNested2['f'].na3, 6) - self.assertEqual(data_decode.arrayNested[0].na4, 'value na4 1') - self.assertEqual(data_decode.arrayNested[0].nb4, 100) - self.assertEqual(data_decode.arrayNested[1].na4, 'value na4 2') - self.assertEqual(data_decode.arrayNested[1].nb4, 200) - self.assertEqual(data_decode.arrayNested2[0].na4, 'value na4 3') - self.assertEqual(data_decode.arrayNested2[0].nb4, 300) - self.assertEqual(data_decode.arrayNested2[1].na4, 'value na4 4') - self.assertEqual(data_decode.arrayNested2[1].nb4, 400) + self.assertEqual(r.color3, Color.red) + self.assertEqual(r.mapNested['a'].color, Color.green) + self.assertEqual(r.mapNested['b'].color, None) print('Encode and decode complex schema finish. schema_type: ', schema_type) encode_and_decode('avro') @@ -1069,8 +1070,12 @@ class NestedObj2(Record): self.assertEqual(data_decode.na2, 1) self.assertTrue(data_decode.nb2) - def test_produce_and_consume_complex_schema_data(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + class NestedObj1(Record): na1 = String() nb1 = Double() @@ -1082,6 +1087,7 @@ class NestedObj2(Record): class NestedObj3(Record): na3 = Integer() + color = CustomEnum(Color, required=True, required_default=True, default=Color.blue) class NestedObj4(Record): na4 = String() @@ -1090,6 +1096,7 @@ class NestedObj4(Record): class ComplexRecord(Record): a = Integer() b = Integer() + color = CustomEnum(Color) nested = NestedObj2() mapNested = Map(NestedObj3()) arrayNested = Array(NestedObj4()) @@ -1112,8 +1119,8 @@ def produce_consume_test(schema_type): nested_obj1 = NestedObj1(na1='na1 value', nb1=20.5) nested_obj2 = NestedObj2(na2=22, nb2=True, nc2=nested_obj1) r = ComplexRecord(a=1, b=2, nested=nested_obj2, mapNested={ - 'a': NestedObj3(na3=1), - 'b': NestedObj3(na3=2), + 'a': NestedObj3(na3=1, color=Color.red), + 'b': NestedObj3(na3=2, color=Color.green), 'c': NestedObj3(na3=3) }, arrayNested=[ NestedObj4(na4='value na4 1', nb4=100), @@ -1125,19 +1132,6 @@ def produce_consume_test(schema_type): value = msg.value() self.assertEqual(value.__class__.__name__, 'ComplexRecord') self.assertEqual(value, r) - self.assertEqual(value.a, 1) - self.assertEqual(value.b, 2) - self.assertEqual(value.nested.na2, 22) - self.assertEqual(value.nested.nb2, True) - self.assertEqual(value.nested.nc2.na1, 'na1 value') - self.assertEqual(value.nested.nc2.nb1, 20.5) - self.assertEqual(value.mapNested['a'].na3, 1) - self.assertEqual(value.mapNested['b'].na3, 2) - self.assertEqual(value.mapNested['c'].na3, 3) - self.assertEqual(value.arrayNested[0].na4, 'value na4 1') - self.assertEqual(value.arrayNested[0].nb4, 100) - self.assertEqual(value.arrayNested[1].na4, 'value na4 2') - self.assertEqual(value.arrayNested[1].nb4, 200) print('Produce and consume complex schema data finish. schema_type', schema_type) @@ -1163,7 +1157,8 @@ def encode_and_decode(schema_definition): "industry": "software", "scale": ">100", "funds": "1000000.0" - } + }, + "companyType": "companyType1" } data = avro_schema.encode(company) company_decode = avro_schema.decode(data) @@ -1185,7 +1180,9 @@ def encode_and_decode(schema_definition): {'name': 'age', 'type': ['null', 'int']} ] }}]}, - {'name': 'labels', 'type': ['null', {'type': 'map', 'values': 'string'}]} + {'name': 'labels', 'type': ['null', {'type': 'map', 'values': 'string'}]}, + {'name': 'companyType', 'type': ['null', {'type': 'enum', 'name': 'CompanyType', 'symbols': + ['companyType1', 'companyType2', 'companyType3']}]} ] } encode_and_decode(schema_definition) @@ -1218,7 +1215,8 @@ def produce_and_consume(topic, schema_definition): "industry": "software" + str(i), "scale": ">100", "funds": "1000000.0" - } + }, + "companyType": "companyType" + str((i % 3) + 1) } producer.send(company) From 006d42e3d2bf7dcf8e49fb4808c78ad3521244b4 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Sun, 7 Nov 2021 21:27:38 +0800 Subject: [PATCH 128/823] [broker] remove useless method "PersistentTopic#getPersistentTopic" (#12655) (cherry picked from commit 40356eb3f6cf65ac91189295c8245633dd24e9f3) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index f917aab44cd34..2e459729d95c6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3146,10 +3146,6 @@ private void initializeTopicSubscribeRateLimiterIfNeeded(Optional } } - private PersistentTopic getPersistentTopic() { - return this; - } - private void registerTopicPolicyListener() { if (brokerService.pulsar().getConfig().isSystemTopicEnabled() && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { From c4e491b827c60ada575862eb2a64988ac558020a Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Mon, 8 Nov 2021 19:12:37 +0800 Subject: [PATCH 129/823] [pulsar-admin] Perfect judgment conditions of pulsar-admin (#12315) ### Motivation Perfect judgment conditions of `bin/pulsar-admin`, included parameter `CONF_FILE_PATH` and field `serviceUrl`. **Details are as follows:** **First of all**, the configuration file `CONF_FILE_PATH` is a required option rather than an optional option (from the [source code](https://github.com/apache/pulsar/blob/master/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java#L288-L299), the configuration file is regarded as an optional option) , because some very important [default values](https://github.com/apache/pulsar/blob/master/conf/client.conf#L22-L30) (such as `webServiceUrl=http://localhost:8080/`, `brokerServiceUrl=pulsar://localhost:6650/`) cannot be obtained without a configuration file, and will mistake args[0] as the configuration file, as follows: ``` ./bin/pulsar-admin topics list-partitioned-topics test/app1 Exception in thread "main" java.io.FileNotFoundException: topics (No such file or directory) at java.io.FileInputStream.open0(Native Method) at java.io.FileInputStream.open(FileInputStream.java:195) at java.io.FileInputStream.(FileInputStream.java:138) at java.io.FileInputStream.(FileInputStream.java:93) at org.apache.pulsar.admin.cli.PulsarAdminTool.main(PulsarAdminTool.java:296) ``` so we first need to judge whether the configuration file `CONF_FILE_PATH` exists, and give the correct command line format (`Usage: pulsar-admin CONF_FILE_PATH [options] [command] [command options]`) if it is illegal. **Secondly**, `serviceUrl` is a very important field, because pulsarAdmin can connect to the pulsar server through this value. If the configuration file does not have releated `key`=`value`(such as `webServiceUrl`=, `brokerServiceUrl`=) and the user does not specify option `--admin-url`, then NPE will appear, as follows: ``` ./bin/pulsar-admin topics list-partitioned-topics test/app1 java.lang.NullPointerException at org.apache.pulsar.client.admin.internal.PulsarAdminImpl.(PulsarAdminImpl.java:189) at org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl.build(PulsarAdminBuilderImpl.java:47) at org.apache.pulsar.admin.cli.PulsarAdminTool.lambda$main$2(PulsarAdminTool.java:320) at org.apache.pulsar.admin.cli.PulsarAdminTool$PulsarAdminSupplier.get(PulsarAdminTool.java:174) at org.apache.pulsar.admin.cli.PulsarAdminTool$PulsarAdminSupplier.get(PulsarAdminTool.java:161) at org.apache.pulsar.admin.cli.CmdBase.getAdmin(CmdBase.java:111) at org.apache.pulsar.admin.cli.CmdTopics.getTopics(CmdTopics.java:2360) at org.apache.pulsar.admin.cli.CmdTopics.access$11000(CmdTopics.java:73) at org.apache.pulsar.admin.cli.CmdTopics$PartitionedTopicListCmd.run(CmdTopics.java:275) at org.apache.pulsar.admin.cli.CmdBase.run(CmdBase.java:86) at org.apache.pulsar.admin.cli.PulsarAdminTool.run(PulsarAdminTool.java:282) at org.apache.pulsar.admin.cli.PulsarAdminTool.main(PulsarAdminTool.java:330) class java.lang.NullPointerException: null ``` so we need to judge whether field `serviceUrl` exists. (cherry picked from commit f976500029514e5275dc97e1e51748e8ed4fc0b2) --- .../pulsar/admin/cli/PulsarAdminTool.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index 5f0cd58256815..a41f44438fed5 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -37,6 +37,8 @@ import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import static org.apache.commons.lang3.StringUtils.isBlank; + public class PulsarAdminTool { private static boolean allowSystemExit = true; @@ -249,6 +251,11 @@ boolean run(String[] args, Function a return false; } + if (isBlank(serviceUrl)) { + jcommander.usage(); + return false; + } + if (version) { System.out.println("Current version of pulsar admin client is: " + PulsarVersion.getVersion()); return true; @@ -285,11 +292,12 @@ boolean run(String[] args, Function a public static void main(String[] args) throws Exception { lastExitCode = 0; - String configFile = null; - if (args.length > 0) { - configFile = args[0]; - args = Arrays.copyOfRange(args, 1, args.length); + if (args.length == 0) { + System.out.println("Usage: pulsar-admin CONF_FILE_PATH [options] [command] [command options]"); + exit(0); + return; } + String configFile = args[0]; Properties properties = new Properties(); if (configFile != null) { @@ -301,6 +309,7 @@ public static void main(String[] args) throws Exception { PulsarAdminTool tool = new PulsarAdminTool(properties); int cmdPos; + args = Arrays.copyOfRange(args, 1, args.length); for (cmdPos = 0; cmdPos < args.length; cmdPos++) { if (tool.commandMap.containsKey(args[cmdPos])) { break; From bb868772d0c11c1c37822d071a9279543246d9c7 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Mon, 8 Nov 2021 19:13:02 +0800 Subject: [PATCH 130/823] [pulsar-admin] Add corresponding get command for namespace (#12322) ### Motivation CLI `./bin/pulsar-admin` provides many `set` commands for namespace and also provides corresponding `get` commands, but I found that the following `set` commands lack corresponding `get` commands. - [grant-subscription-permission](https://pulsar.apache.org/tools/pulsar-admin/2.9.0-SNAPSHOT/#-em-grant-subscription-permission-em-) - [set-auto-topic-creation](https://pulsar.apache.org/tools/pulsar-admin/2.9.0-SNAPSHOT/#-em-set-auto-topic-creation-em-) - [set-auto-subscription-creation](https://pulsar.apache.org/tools/pulsar-admin/2.9.0-SNAPSHOT/#-em-set-auto-subscription-creation-em-) - [set-encryption-required](https://pulsar.apache.org/tools/pulsar-admin/2.9.0-SNAPSHOT/#-em-set-encryption-required-em-) - [set-subscription-auth-mode](https://pulsar.apache.org/tools/pulsar-admin/2.9.0-SNAPSHOT/#-em-set-subscription-auth-mode-em-) The purpose of this PR is to supplement these `get` commands, as follow: - subscription-permission - get-auto-topic-creation - get-auto-subscription-creation - get-encryption-required - get-subscription-auth-mode (cherry picked from commit d055ebd1186ffae2d2af1382e05b21c60e9f9574) --- .../broker/admin/impl/NamespacesBase.java | 24 +++ .../pulsar/broker/admin/v1/Namespaces.java | 65 +++++++ .../pulsar/broker/admin/v2/Namespaces.java | 59 ++++++ .../pulsar/broker/auth/AuthorizationTest.java | 6 + ...erServiceAutoSubscriptionCreationTest.java | 19 +- .../BrokerServiceAutoTopicCreationTest.java | 37 ++-- .../AuthorizationProducerConsumerTest.java | 7 + .../pulsar/client/admin/Namespaces.java | 81 +++++++++ .../client/admin/internal/NamespacesImpl.java | 170 ++++++++++++++++++ .../pulsar/admin/cli/PulsarAdminToolTest.java | 6 + .../pulsar/admin/cli/CmdNamespaces.java | 65 +++++++ 11 files changed, 518 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index f2327e28b93c8..dd02f67321942 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -830,6 +830,12 @@ protected void internalSetSubscriptionExpirationTime(Integer expirationTime) { }); } + protected AutoTopicCreationOverride internalGetAutoTopicCreation() { + validateNamespacePolicyOperation(namespaceName, PolicyName.AUTO_TOPIC_CREATION, PolicyOperation.READ); + Policies policies = getNamespacePolicies(namespaceName); + return policies.autoTopicCreationOverride; + } + protected void internalSetAutoTopicCreation(AsyncResponse asyncResponse, AutoTopicCreationOverride autoTopicCreationOverride) { final int maxPartitions = pulsar().getConfig().getMaxNumPartitionsPerPartitionedTopic(); @@ -902,6 +908,12 @@ protected void internalSetAutoSubscriptionCreation( }); } + protected AutoSubscriptionCreationOverride internalGetAutoSubscriptionCreation() { + validateNamespacePolicyOperation(namespaceName, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.READ); + Policies policies = getNamespacePolicies(namespaceName); + return policies.autoSubscriptionCreationOverride; + } + protected void internalRemoveAutoSubscriptionCreation(AsyncResponse asyncResponse) { internalSetAutoSubscriptionCreation(asyncResponse, null); } @@ -1727,6 +1739,12 @@ protected void internalSetSubscriptionAuthMode(SubscriptionAuthMode subscription } } + protected SubscriptionAuthMode internalGetSubscriptionAuthMode() { + validateNamespacePolicyOperation(namespaceName, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.READ); + Policies policies = getNamespacePolicies(namespaceName); + return policies.subscription_auth_mode; + } + protected void internalModifyEncryptionRequired(boolean encryptionRequired) { validateNamespacePolicyOperation(namespaceName, PolicyName.ENCRYPTION, PolicyOperation.WRITE); validatePoliciesReadOnlyAccess(); @@ -1745,6 +1763,12 @@ protected void internalModifyEncryptionRequired(boolean encryptionRequired) { } } + protected Boolean internalGetEncryptionRequired() { + validateNamespacePolicyOperation(namespaceName, PolicyName.ENCRYPTION, PolicyOperation.READ); + Policies policies = getNamespacePolicies(namespaceName); + return policies.encryption_required; + } + protected DelayedDeliveryPolicies internalGetDelayedDelivery() { validateNamespacePolicyOperation(namespaceName, PolicyName.DELAYED_DELIVERY, PolicyOperation.READ); return getNamespacePolicies(namespaceName).delayed_delivery_policies; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index dac7e44b9ef1c..51728d63e1b41 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -244,6 +244,22 @@ public Map> getPermissions(@PathParam("property") String return policies.auth_policies.getNamespaceAuthentication(); } + @GET + @Path("/{property}/{cluster}/{namespace}/permissions/subscription") + @ApiOperation(value = "Retrieve the permissions for a subscription.") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist"), + @ApiResponse(code = 409, message = "Namespace is not empty")}) + public Map> getPermissionOnSubscription(@PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace) { + validateNamespaceName(property, cluster, namespace); + validateNamespaceOperation(NamespaceName.get(property, namespace), NamespaceOperation.GET_PERMISSION); + + Policies policies = getNamespacePolicies(namespaceName); + return policies.auth_policies.getSubscriptionAuthentication(); + } + @POST @Path("/{property}/{cluster}/{namespace}/permissions/{role}") @ApiOperation(hidden = true, value = "Grant a new permission to a role on a namespace.") @@ -460,6 +476,18 @@ public void modifyDeduplication(@PathParam("property") String property, @PathPar internalModifyDeduplication(enableDeduplication); } + @GET + @Path("/{property}/{cluster}/{namespace}/autoTopicCreation") + @ApiOperation(value = "Get autoTopicCreation info in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist")}) + public AutoTopicCreationOverride getAutoTopicCreation(@PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace) { + validateNamespaceName(property, cluster, namespace); + return internalGetAutoTopicCreation(); + } + @POST @Path("/{property}/{cluster}/{namespace}/autoTopicCreation") @ApiOperation(value = "Override broker's allowAutoTopicCreation setting for a namespace") @@ -521,6 +549,18 @@ public void setAutoSubscriptionCreation( } } + @GET + @Path("/{property}/{cluster}/{namespace}/autoSubscriptionCreation") + @ApiOperation(value = "Get autoSubscriptionCreation info in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist")}) + public AutoSubscriptionCreationOverride getAutoSubscriptionCreation(@PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace) { + validateNamespaceName(property, cluster, namespace); + return internalGetAutoSubscriptionCreation(); + } + @DELETE @Path("/{property}/{cluster}/{namespace}/autoSubscriptionCreation") @ApiOperation(value = "Remove override of broker's allowAutoSubscriptionCreation in a namespace") @@ -964,6 +1004,18 @@ public void setSubscriptionAuthMode(@PathParam("property") String property, @Pat internalSetSubscriptionAuthMode(subscriptionAuthMode); } + @GET + @Path("/{property}/{cluster}/{namespace}/subscriptionAuthMode") + @ApiOperation(value = "Get subscription auth mode in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist")}) + public SubscriptionAuthMode getSubscriptionAuthMode(@PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace) { + validateNamespaceName(property, cluster, namespace); + return internalGetSubscriptionAuthMode(); + } + @POST @Path("/{property}/{cluster}/{namespace}/encryptionRequired") @ApiOperation(hidden = true, value = "Message encryption is required or not for all topics in a namespace") @@ -976,6 +1028,19 @@ public void modifyEncryptionRequired(@PathParam("property") String property, @Pa internalModifyEncryptionRequired(encryptionRequired); } + @GET + @Path("/{property}/{cluster}/{namespace}/encryptionRequired") + @ApiOperation(value = "Get message encryption required status in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist")}) + public Boolean getEncryptionRequired(@PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace) { + validateAdminAccessForTenant(property); + validateNamespaceName(property, cluster, namespace); + return internalGetEncryptionRequired(); + } + @GET @Path("/{property}/{cluster}/{namespace}/maxProducersPerTopic") @ApiOperation(value = "Get maxProducersPerTopic config on a namespace.") diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index cf84374f34bc9..e1e892c88a235 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -192,6 +192,21 @@ public Map> getPermissions(@PathParam("tenant") String t return policies.auth_policies.getNamespaceAuthentication(); } + @GET + @Path("/{tenant}/{namespace}/permissions/subscription") + @ApiOperation(value = "Retrieve the permissions for a subscription.") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), + @ApiResponse(code = 409, message = "Namespace is not empty")}) + public Map> getPermissionOnSubscription(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + validateNamespaceOperation(NamespaceName.get(tenant, namespace), NamespaceOperation.GET_PERMISSION); + + Policies policies = getNamespacePolicies(namespaceName); + return policies.auth_policies.getSubscriptionAuthentication(); + } + @POST @Path("/{tenant}/{namespace}/permissions/{role}") @ApiOperation(value = "Grant a new permission to a role on a namespace.") @@ -382,6 +397,17 @@ public void removeDeduplication(@PathParam("tenant") String tenant, @PathParam(" internalModifyDeduplication(null); } + @GET + @Path("/{tenant}/{namespace}/autoTopicCreation") + @ApiOperation(value = "Get autoTopicCreation info in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) + public AutoTopicCreationOverride getAutoTopicCreation(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetAutoTopicCreation(); + } + @POST @Path("/{tenant}/{namespace}/autoTopicCreation") @ApiOperation(value = "Override broker's allowAutoTopicCreation setting for a namespace") @@ -443,6 +469,17 @@ public void setAutoSubscriptionCreation( } } + @GET + @Path("/{tenant}/{namespace}/autoSubscriptionCreation") + @ApiOperation(value = "Get autoSubscriptionCreation info in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) + public AutoSubscriptionCreationOverride getAutoSubscriptionCreation(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetAutoSubscriptionCreation(); + } + @DELETE @Path("/{tenant}/{namespace}/autoSubscriptionCreation") @ApiOperation(value = "Remove override of broker's allowAutoSubscriptionCreation in a namespace") @@ -972,6 +1009,17 @@ public void setSubscriptionAuthMode(@PathParam("tenant") String tenant, internalSetSubscriptionAuthMode(subscriptionAuthMode); } + @GET + @Path("/{tenant}/{namespace}/subscriptionAuthMode") + @ApiOperation(value = "Get subscription auth mode in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) + public SubscriptionAuthMode getSubscriptionAuthMode(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetSubscriptionAuthMode(); + } + @POST @Path("/{tenant}/{namespace}/encryptionRequired") @ApiOperation(value = "Message encryption is required or not for all topics in a namespace") @@ -987,6 +1035,17 @@ public void modifyEncryptionRequired( internalModifyEncryptionRequired(encryptionRequired); } + @GET + @Path("/{tenant}/{namespace}/encryptionRequired") + @ApiOperation(value = "Get message encryption required status in a namespace") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) + public Boolean getEncryptionRequired(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetEncryptionRequired(); + } + @GET @Path("/{tenant}/{namespace}/delayedDelivery") @ApiOperation(value = "Get delayed delivery messages config on a namespace.") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index dcdc602985b64..b7a54d137fd70 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -32,6 +32,7 @@ import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -201,7 +202,12 @@ public void simple() throws Exception { // tests for subscription auth mode admin.namespaces().grantPermissionOnNamespace("p1/c1/ns1", "*", EnumSet.of(AuthAction.consume)); + admin.namespaces().setSubscriptionAuthMode("p1/c1/ns1", SubscriptionAuthMode.None); + Assert.assertEquals(admin.namespaces().getSubscriptionAuthMode("p1/c1/ns1"), + SubscriptionAuthMode.None); admin.namespaces().setSubscriptionAuthMode("p1/c1/ns1", SubscriptionAuthMode.Prefix); + Assert.assertEquals(admin.namespaces().getSubscriptionAuthMode("p1/c1/ns1"), + SubscriptionAuthMode.Prefix); waitForChange(); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java index c635968907ab5..d0eb3bfad2db4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java @@ -26,6 +26,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -98,10 +99,13 @@ public void testAutoSubscriptionCreationNamespaceAllowOverridesBroker() throws E admin.topics().createNonPartitionedTopic(topicName.toString()); pulsar.getConfiguration().setAllowAutoSubscriptionCreation(false); + AutoSubscriptionCreationOverride autoSubscriptionCreationOverride = AutoSubscriptionCreationOverride.builder() + .allowAutoSubscriptionCreation(true) + .build(); pulsar.getAdminClient().namespaces().setAutoSubscriptionCreation(topicName.getNamespace(), - AutoSubscriptionCreationOverride.builder() - .allowAutoSubscriptionCreation(true) - .build()); + autoSubscriptionCreationOverride); + Assert.assertEquals(pulsar.getAdminClient().namespaces().getAutoSubscriptionCreation(topicName.getNamespace()), + autoSubscriptionCreationOverride); // Subscribe operation should be successful pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName(subscriptionName).subscribe(); @@ -117,10 +121,13 @@ public void testAutoSubscriptionCreationNamespaceDisallowOverridesBroker() throw admin.topics().createNonPartitionedTopic(topicName.toString()); pulsar.getConfiguration().setAllowAutoSubscriptionCreation(true); + AutoSubscriptionCreationOverride autoSubscriptionCreationOverride = AutoSubscriptionCreationOverride.builder() + .allowAutoSubscriptionCreation(false) + .build(); pulsar.getAdminClient().namespaces().setAutoSubscriptionCreation(topicName.getNamespace(), - AutoSubscriptionCreationOverride.builder() - .allowAutoSubscriptionCreation(false) - .build()); + autoSubscriptionCreationOverride); + Assert.assertEquals(pulsar.getAdminClient().namespaces().getAutoSubscriptionCreation(topicName.getNamespace()), + autoSubscriptionCreationOverride); try { pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName(subscriptionName).subscribe(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java index 33ed35636db8c..1b3ca1616fa61 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java @@ -29,6 +29,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.TopicType; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -168,11 +169,13 @@ public void testAutoCreationNamespaceAllowOverridesBroker() throws Exception { final String subscriptionName = "test-topic-sub-4"; final TopicName topicName = TopicName.get(topicString); pulsar.getConfiguration().setAllowAutoTopicCreation(false); - pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), - AutoTopicCreationOverride.builder() - .allowAutoTopicCreation(true) - .topicType(TopicType.NON_PARTITIONED.toString()) - .build()); + AutoTopicCreationOverride autoTopicCreationOverride = AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType(TopicType.NON_PARTITIONED.toString()) + .build(); + pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), autoTopicCreationOverride); + Assert.assertEquals(pulsar.getAdminClient().namespaces().getAutoTopicCreation(topicName.getNamespace()), + autoTopicCreationOverride); pulsarClient.newConsumer().topic(topicString).subscriptionName(subscriptionName).subscribe(); assertTrue(admin.namespaces().getTopics("prop/ns-abc").contains(topicString)); @@ -185,10 +188,12 @@ public void testAutoCreationNamespaceDisallowOverridesBroker() throws Exception final String subscriptionName = "test-topic-sub-5"; final TopicName topicName = TopicName.get(topicString); pulsar.getConfiguration().setAllowAutoTopicCreation(true); - pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), - AutoTopicCreationOverride.builder() - .allowAutoTopicCreation(false) - .build()); + AutoTopicCreationOverride autoTopicCreationOverride = AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(false) + .build(); + pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), autoTopicCreationOverride); + Assert.assertEquals(pulsar.getAdminClient().namespaces().getAutoTopicCreation(topicName.getNamespace()), + autoTopicCreationOverride); try { pulsarClient.newConsumer().topic(topicString).subscriptionName(subscriptionName).subscribe(); @@ -205,12 +210,14 @@ public void testAutoCreationNamespaceOverrideAllowsPartitionedTopics() throws Ex final TopicName topicName = TopicName.get(topicString); pulsar.getConfiguration().setAllowAutoTopicCreation(false); - pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), - AutoTopicCreationOverride.builder() - .allowAutoTopicCreation(true) - .topicType(TopicType.PARTITIONED.toString()) - .defaultNumPartitions(4) - .build()); + AutoTopicCreationOverride autoTopicCreationOverride = AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType(TopicType.PARTITIONED.toString()) + .defaultNumPartitions(4) + .build(); + pulsar.getAdminClient().namespaces().setAutoTopicCreation(topicName.getNamespace(), autoTopicCreationOverride); + Assert.assertEquals(pulsar.getAdminClient().namespaces().getAutoTopicCreation(topicName.getNamespace()), + autoTopicCreationOverride); final String subscriptionName = "test-topic-sub-6"; pulsarClient.newConsumer().topic(topicString).subscriptionName(subscriptionName).subscribe(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 65f57ad7e2c6a..5eeac534780fa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -280,6 +281,9 @@ public void testSubscriberPermission() throws Exception { String otherPrincipal = "Principal-1-to-access-sub"; tenantAdmin.namespaces().grantPermissionOnSubscription(namespace, subscriptionName, Collections.singleton(otherPrincipal)); + TreeMap> permissionOnSubscription = new TreeMap<>(); + permissionOnSubscription.put(subscriptionName, Collections.singleton(otherPrincipal)); + Assert.assertEquals(tenantAdmin.namespaces().getPermissionOnSubscription(namespace), permissionOnSubscription); // now, subscriptionRole doesn't have subscription level access so, it will fail to access subscription try { @@ -300,6 +304,9 @@ public void testSubscriberPermission() throws Exception { // now, grant subscription-access to subscriptionRole as well superAdmin.namespaces().grantPermissionOnSubscription(namespace, subscriptionName, Sets.newHashSet(otherPrincipal, subscriptionRole)); + TreeMap> permissionOnSubscription1 = new TreeMap<>(); + permissionOnSubscription1.put(subscriptionName, Sets.newHashSet(otherPrincipal, subscriptionRole)); + Assert.assertEquals(tenantAdmin.namespaces().getPermissionOnSubscription(namespace), permissionOnSubscription1); sub1Admin.topics().skipAllMessages(topicName, subscriptionName); sub1Admin.topics().skipMessages(topicName, subscriptionName, 1); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 6ea4bab025cd3..fb1b67e7f1d9b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -687,6 +687,19 @@ public interface Namespaces { */ CompletableFuture revokePermissionsOnNamespaceAsync(String namespace, String role); + /** + * Get permission to role to access subscription's admin-api. + * @param namespace + * @throws PulsarAdminException + */ + Map> getPermissionOnSubscription(String namespace) throws PulsarAdminException; + + /** + * Get permission to role to access subscription's admin-api asynchronously. + * @param namespace + */ + CompletableFuture>> getPermissionOnSubscriptionAsync(String namespace); + /** * Grant permission to role to access subscription's admin-api. * @param namespace @@ -1237,6 +1250,23 @@ void setAutoTopicCreation(String namespace, AutoTopicCreationOverride autoTopicC CompletableFuture setAutoTopicCreationAsync( String namespace, AutoTopicCreationOverride autoTopicCreationOverride); + /** + * Get the autoTopicCreation info within a namespace. + * + * @param namespace + * @return + * @throws PulsarAdminException + */ + AutoTopicCreationOverride getAutoTopicCreation(String namespace) throws PulsarAdminException; + + /** + * Get the autoTopicCreation info within a namespace asynchronously. + * + * @param namespace + * @return + */ + CompletableFuture getAutoTopicCreationAsync(String namespace); + /** * Removes the autoTopicCreation policy for a given namespace. *

    @@ -1322,6 +1352,23 @@ void setAutoSubscriptionCreation( CompletableFuture setAutoSubscriptionCreationAsync( String namespace, AutoSubscriptionCreationOverride autoSubscriptionCreationOverride); + /** + * Get the autoSubscriptionCreation info within a namespace. + * + * @param namespace + * @return + * @throws PulsarAdminException + */ + AutoSubscriptionCreationOverride getAutoSubscriptionCreation(String namespace) throws PulsarAdminException; + + /** + * Get the autoSubscriptionCreation info within a namespace asynchronously. + * + * @param namespace + * @return + */ + CompletableFuture getAutoSubscriptionCreationAsync(String namespace); + /** * Sets the subscriptionTypesEnabled policy for a given namespace, overriding broker settings. * @@ -2438,6 +2485,23 @@ CompletableFuture clearNamespaceBundleBacklogForSubscriptionAsync(String n */ void setEncryptionRequiredStatus(String namespace, boolean encryptionRequired) throws PulsarAdminException; + /** + * Get the encryption required status within a namespace. + * + * @param namespace + * @return + * @throws PulsarAdminException + */ + Boolean getEncryptionRequiredStatus(String namespace) throws PulsarAdminException; + + /** + * Get the encryption required status within a namespace asynchronously. + * + * @param namespace + * @return + */ + CompletableFuture getEncryptionRequiredStatusAsync(String namespace); + /** * Set the encryption required status for all topics within a namespace asynchronously. *

    @@ -2646,6 +2710,23 @@ void setSubscriptionAuthMode(String namespace, SubscriptionAuthMode subscription */ CompletableFuture setSubscriptionAuthModeAsync(String namespace, SubscriptionAuthMode subscriptionAuthMode); + /** + * Get the subscriptionAuthMode within a namespace. + * + * @param namespace + * @return + * @throws PulsarAdminException + */ + SubscriptionAuthMode getSubscriptionAuthMode(String namespace) throws PulsarAdminException; + + /** + * Get the subscriptionAuthMode within a namespace asynchronously. + * + * @param namespace + * @return + */ + CompletableFuture getSubscriptionAuthModeAsync(String namespace); + /** * Get the deduplicationSnapshotInterval for a namespace. * diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 610423b18c101..04da2f0f411f5 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -513,6 +513,39 @@ public CompletableFuture revokePermissionsOnNamespaceAsync(String namespac return asyncDeleteRequest(path); } + @Override + public Map> getPermissionOnSubscription(String namespace) throws PulsarAdminException { + try { + return getPermissionOnSubscriptionAsync(namespace).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw (PulsarAdminException) e.getCause(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } + } + + @Override + public CompletableFuture>> getPermissionOnSubscriptionAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "permissions", "subscription"); + final CompletableFuture>> future = new CompletableFuture<>(); + asyncGetRequest(path, + new InvocationCallback>>() { + @Override + public void completed(Map> permissions) { + future.complete(permissions); + } + + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }); + return future; + } @Override public void grantPermissionOnSubscription(String namespace, String subscription, Set roles) @@ -995,6 +1028,40 @@ public CompletableFuture setAutoTopicCreationAsync( return asyncPostRequest(path, Entity.entity(autoTopicCreationOverride, MediaType.APPLICATION_JSON)); } + @Override + public AutoTopicCreationOverride getAutoTopicCreation(String namespace) throws PulsarAdminException { + try { + return getAutoTopicCreationAsync(namespace).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw (PulsarAdminException) e.getCause(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } + } + + @Override + public CompletableFuture getAutoTopicCreationAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "autoTopicCreation"); + final CompletableFuture future = new CompletableFuture<>(); + asyncGetRequest(path, + new InvocationCallback() { + @Override + public void completed(AutoTopicCreationOverride autoTopicCreationOverride) { + future.complete(autoTopicCreationOverride); + } + + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }); + return future; + } + @Override public void removeAutoTopicCreation(String namespace) throws PulsarAdminException { try { @@ -1040,6 +1107,41 @@ public CompletableFuture setAutoSubscriptionCreationAsync(String namespace return asyncPostRequest(path, Entity.entity(autoSubscriptionCreationOverride, MediaType.APPLICATION_JSON)); } + @Override + public AutoSubscriptionCreationOverride getAutoSubscriptionCreation(String namespace) throws PulsarAdminException { + try { + return getAutoSubscriptionCreationAsync(namespace).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw (PulsarAdminException) e.getCause(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } + } + + @Override + public CompletableFuture getAutoSubscriptionCreationAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "autoSubscriptionCreation"); + final CompletableFuture future = new CompletableFuture<>(); + asyncGetRequest(path, + new InvocationCallback() { + @Override + public void completed(AutoSubscriptionCreationOverride autoSubscriptionCreation) { + future.complete(autoSubscriptionCreation); + } + + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }); + return future; + } + + @Override public void setSubscriptionTypesEnabled( String namespace, Set subscriptionTypesEnabled) throws PulsarAdminException { @@ -2105,6 +2207,40 @@ public CompletableFuture setSubscriptionAuthModeAsync( return asyncPostRequest(path, Entity.entity(subscriptionAuthMode, MediaType.APPLICATION_JSON)); } + @Override + public SubscriptionAuthMode getSubscriptionAuthMode(String namespace) throws PulsarAdminException { + try { + return getSubscriptionAuthModeAsync(namespace).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw (PulsarAdminException) e.getCause(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } + } + + @Override + public CompletableFuture getSubscriptionAuthModeAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "subscriptionAuthMode"); + final CompletableFuture future = new CompletableFuture<>(); + asyncGetRequest(path, + new InvocationCallback() { + @Override + public void completed(SubscriptionAuthMode subscriptionAuthMode) { + future.complete(subscriptionAuthMode); + } + + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }); + return future; + } + @Override public void setEncryptionRequiredStatus(String namespace, boolean encryptionRequired) throws PulsarAdminException { try { @@ -2127,6 +2263,40 @@ public CompletableFuture setEncryptionRequiredStatusAsync(String namespace return asyncPostRequest(path, Entity.entity(encryptionRequired, MediaType.APPLICATION_JSON)); } + @Override + public Boolean getEncryptionRequiredStatus(String namespace) throws PulsarAdminException { + try { + return getEncryptionRequiredStatusAsync(namespace).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw (PulsarAdminException) e.getCause(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } + } + + @Override + public CompletableFuture getEncryptionRequiredStatusAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "encryptionRequired"); + final CompletableFuture future = new CompletableFuture<>(); + asyncGetRequest(path, + new InvocationCallback() { + @Override + public void completed(Boolean enabled) { + future.complete(enabled); + } + + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }); + return future; + } + @Override public DelayedDeliveryPolicies getDelayedDelivery(String namespace) throws PulsarAdminException { try { diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 60fb6acf1d9a6..5cdaba27efa3e 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -508,6 +508,9 @@ public void namespaces() throws Exception { .topicType(TopicType.NON_PARTITIONED.toString()) .build()); + namespaces.run(split("get-auto-topic-creation myprop/clust/ns1")); + verify(mockNamespaces).getAutoTopicCreation("myprop/clust/ns1"); + namespaces.run(split("remove-auto-topic-creation myprop/clust/ns1")); verify(mockNamespaces).removeAutoTopicCreation("myprop/clust/ns1"); @@ -515,6 +518,9 @@ public void namespaces() throws Exception { verify(mockNamespaces).setAutoSubscriptionCreation("myprop/clust/ns1", AutoSubscriptionCreationOverride.builder().allowAutoSubscriptionCreation(true).build()); + namespaces.run(split("get-auto-subscription-creation myprop/clust/ns1")); + verify(mockNamespaces).getAutoSubscriptionCreation("myprop/clust/ns1"); + namespaces.run(split("remove-auto-subscription-creation myprop/clust/ns1")); verify(mockNamespaces).removeAutoSubscriptionCreation("myprop/clust/ns1"); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index f6fa1234c4c34..a326f82b19d1a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -235,6 +235,18 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Get permissions to access subscription admin-api") + private class SubscriptionPermissions extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(getAdmin().namespaces().getPermissionOnSubscription(namespace)); + } + } + @Parameters(commandDescription = "Grant permissions to access subscription admin-api") private class GrantSubscriptionPermissions extends CliCommand { @Parameter(description = "tenant/namespace", required = true) @@ -588,6 +600,18 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Get autoTopicCreation info for a namespace") + private class GetAutoTopicCreation extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(getAdmin().namespaces().getAutoTopicCreation(namespace)); + } + } + @Parameters(commandDescription = "Remove override of autoTopicCreation for a namespace") private class RemoveAutoTopicCreation extends CliCommand { @Parameter(description = "tenant/namespace", required = true) @@ -619,6 +643,18 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Get the autoSubscriptionCreation for a namespace") + private class GetAutoSubscriptionCreation extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(getAdmin().namespaces().getAutoSubscriptionCreation(namespace)); + } + } + @Parameters(commandDescription = "Remove override of autoSubscriptionCreation for a namespace") private class RemoveAutoSubscriptionCreation extends CliCommand { @Parameter(description = "tenant/namespace", required = true) @@ -1274,6 +1310,18 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Get encryption required for a namespace") + private class GetEncryptionRequired extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(getAdmin().namespaces().getEncryptionRequiredStatus(namespace)); + } + } + @Parameters(commandDescription = "Get the delayed delivery policy for a namespace") private class GetDelayedDelivery extends CliCommand { @Parameter(description = "tenant/namespace", required = true) @@ -1406,6 +1454,18 @@ void run() throws Exception { } } + @Parameters(commandDescription = "Get subscriptionAuthMod for a namespace") + private class GetSubscriptionAuthMode extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(getAdmin().namespaces().getSubscriptionAuthMode(namespace)); + } + } + @Parameters(commandDescription = "Get deduplicationSnapshotInterval for a namespace") private class GetDeduplicationSnapshotInterval extends CliCommand { @Parameter(description = "tenant/namespace", required = true) @@ -2331,6 +2391,7 @@ public CmdNamespaces(Supplier admin) { jcommander.addCommand("grant-permission", new GrantPermissions()); jcommander.addCommand("revoke-permission", new RevokePermissions()); + jcommander.addCommand("subscription-permission", new SubscriptionPermissions()); jcommander.addCommand("grant-subscription-permission", new GrantSubscriptionPermissions()); jcommander.addCommand("revoke-subscription-permission", new RevokeSubscriptionPermissions()); @@ -2370,9 +2431,11 @@ public CmdNamespaces(Supplier admin) { jcommander.addCommand("remove-deduplication", new RemoveDeduplication()); jcommander.addCommand("set-auto-topic-creation", new SetAutoTopicCreation()); + jcommander.addCommand("get-auto-topic-creation", new GetAutoTopicCreation()); jcommander.addCommand("remove-auto-topic-creation", new RemoveAutoTopicCreation()); jcommander.addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation()); + jcommander.addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation()); jcommander.addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation()); jcommander.addCommand("get-retention", new GetRetention()); @@ -2412,7 +2475,9 @@ public CmdNamespaces(Supplier admin) { jcommander.addCommand("unsubscribe", new Unsubscribe()); jcommander.addCommand("set-encryption-required", new SetEncryptionRequired()); + jcommander.addCommand("get-encryption-required", new GetEncryptionRequired()); jcommander.addCommand("set-subscription-auth-mode", new SetSubscriptionAuthMode()); + jcommander.addCommand("get-subscription-auth-mode", new GetSubscriptionAuthMode()); jcommander.addCommand("set-delayed-delivery", new SetDelayedDelivery()); jcommander.addCommand("get-delayed-delivery", new GetDelayedDelivery()); From 02b2e3e80292ada52339da8e276fc0fba755e214 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Mon, 8 Nov 2021 20:19:02 +0900 Subject: [PATCH 131/823] Enable CLI to publish non-batched messages (#12641) ### Motivation Currently, messages produced by the `pulsar-client` command are always batched. However, zero queue consumers cannot receive these batched messages. I think it would be useful to be able to easily produce non-batched messages. ### Modifications Added an option to disable batching to the `pulsar-client` command: ```sh $ ./bin/pulsar-client produce -m hello -n 10 --disable-batching persistent://public/default/t1 ``` (cherry picked from commit a1bad71728d0b39e5b38fdd4ea9a7578629ac975) --- .../client/cli/PulsarClientToolTest.java | 46 +++++++++++++++++-- .../apache/pulsar/client/cli/CmdProduce.java | 5 ++ site2/docs/reference-cli-tools.md | 1 + 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java index d56308911ddf4..b1e067035d273 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java @@ -33,6 +33,9 @@ import lombok.Cleanup; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; import org.testng.Assert; @@ -106,7 +109,7 @@ public void testNonDurableSubscribe() throws Exception { properties.setProperty("serviceUrl", brokerUrl.toString()); properties.setProperty("useTls", "false"); - final String topicName = "persistent://prop/ns-abc/test/topic-" + UUID.randomUUID().toString(); + final String topicName = getTopicWithRandomSuffix("non-durable"); int numberOfMessages = 10; @Cleanup("shutdownNow") @@ -155,7 +158,7 @@ public void testDurableSubscribe() throws Exception { properties.setProperty("serviceUrl", brokerUrl.toString()); properties.setProperty("useTls", "false"); - final String topicName = "persistent://prop/ns-abc/test/topic-" + UUID.randomUUID().toString(); + final String topicName = getTopicWithRandomSuffix("durable"); int numberOfMessages = 10; @Cleanup("shutdownNow") @@ -197,7 +200,7 @@ public void testEncryption() throws Exception { properties.setProperty("serviceUrl", brokerUrl.toString()); properties.setProperty("useTls", "false"); - final String topicName = "persistent://prop/ns-abc/test/topic-" + UUID.randomUUID().toString(); + final String topicName = getTopicWithRandomSuffix("encryption"); final String keyUriBase = "file:../pulsar-broker/src/test/resources/certificate/"; final int numberOfMessages = 10; @@ -234,4 +237,41 @@ public void testEncryption() throws Exception { } } + @Test(timeOut = 20000) + public void testDisableBatching() throws Exception { + Properties properties = new Properties(); + properties.setProperty("serviceUrl", brokerUrl.toString()); + properties.setProperty("useTls", "false"); + + final String topicName = getTopicWithRandomSuffix("disable-batching"); + final int numberOfMessages = 5; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe(); + + PulsarClientTool pulsarClientTool1 = new PulsarClientTool(properties); + String[] args1 = {"produce", "-m", "batched", "-n", Integer.toString(numberOfMessages), topicName}; + Assert.assertEquals(pulsarClientTool1.run(args1), 0); + + PulsarClientTool pulsarClientTool2 = new PulsarClientTool(properties); + String[] args2 = {"produce", "-m", "non-batched", "-n", Integer.toString(numberOfMessages), "-db", topicName}; + Assert.assertEquals(pulsarClientTool2.run(args2), 0); + + for (int i = 0; i < numberOfMessages * 2; i++) { + Message msg = consumer.receive(10, TimeUnit.SECONDS); + Assert.assertNotNull(msg); + if (i < numberOfMessages) { + Assert.assertEquals(new String(msg.getData()), "batched"); + Assert.assertTrue(msg.getMessageId() instanceof BatchMessageIdImpl); + } else { + Assert.assertEquals(new String(msg.getData()), "non-batched"); + Assert.assertFalse(msg.getMessageId() instanceof BatchMessageIdImpl); + } + } + } + + private static String getTopicWithRandomSuffix(String localNameBase) { + return String.format("persistent://prop/ns-abc/test/%s-%s", localNameBase, UUID.randomUUID().toString()); + } + } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java index 1ae2d38e106ed..0ec22ba2e7b48 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java @@ -107,6 +107,9 @@ public class CmdProduce { description = "Rate (in msg/sec) at which to produce," + " value 0 means to produce messages as fast as possible.") private double publishRate = 0; + + @Parameter(names = { "-db", "--disable-batching" }, description = "Disable batch sending of messages") + private boolean disableBatching = false; @Parameter(names = { "-c", "--chunking" }, description = "Should split the message and publish in chunks if message size is larger than allowed max size") @@ -247,6 +250,8 @@ private int publish(String topic) { if (this.chunkingAllowed) { producerBuilder.enableChunking(true); producerBuilder.enableBatching(false); + } else if (this.disableBatching) { + producerBuilder.enableBatching(false); } if (isNotBlank(this.encKeyName) && isNotBlank(this.encKeyValue)) { producerBuilder.addEncryptionKey(this.encKeyName); diff --git a/site2/docs/reference-cli-tools.md b/site2/docs/reference-cli-tools.md index 6a128186dc284..1911fe3217463 100644 --- a/site2/docs/reference-cli-tools.md +++ b/site2/docs/reference-cli-tools.md @@ -312,6 +312,7 @@ Options |`-m`, `--messages`|Comma-separated string of messages to send; either -m or -f must be specified|[]| |`-n`, `--num-produce`|The number of times to send the message(s); the count of messages/files * num-produce should be below 1000|1| |`-r`, `--rate`|Rate (in messages per second) at which to produce; a value 0 means to produce messages as fast as possible|0.0| +|`-db`, `--disable-batching`|Disable batch sending of messages|false| |`-c`, `--chunking`|Split the message and publish in chunks if the message size is larger than the allowed max size|false| |`-s`, `--separator`|Character to split messages string with.|","| |`-k`, `--key`|Message key to add|key=value string, like k1=v1,k2=v2.| From 85757732f6c46a9b56fa056efbdafc1c0be97c10 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Wed, 10 Nov 2021 15:58:51 +0800 Subject: [PATCH 132/823] ### Motivation (#12698) Fix lost compaction data due to compaction properties missed during reset-cursor. 1. The compaction reader will seek to the earliest position to read data from the topic, but the compaction properties missed during the cursor reset, this will lead to the inited compaction subscribe without compaction horizon, so the compaction reader will skip the last compacted data. It will only happen when init the compaction subscription, so can introduced by the loadbalance or topic unloading manually. 2. Advance the cursor should also keep the properties, otherwise, the properties will lost during the cursor trimming. ### Changes 1. Keep the properties for resetting the cursor while the cursor is for data compaction. 2. Copy the properties to the new mark delete entry while advance the cursor, this is triggered byt the managed ledger internal, so it's not only for compacted topic, the internal task should not loss the properties when trimming the cursor. ### Tests New tests added to make sure the compaction will not loss data during topic unloading and the reader can read all the compacted data after the compaction task complete (cherry picked from commit 98e2c66a7b2d5fd42641527cd9ad6c3b497d65c7) --- .../mledger/impl/ManagedCursorImpl.java | 10 ++- .../mledger/impl/ManagedLedgerImpl.java | 2 +- .../intercept/CounterBrokerInterceptor.java | 8 +-- .../pulsar/compaction/CompactedTopicTest.java | 70 ++++++++++++++++++- 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index dd715ff7588d3..f13fa932fd2bd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -194,6 +194,7 @@ public class ManagedCursorImpl implements ManagedCursor { private long entriesReadCount; private long entriesReadSize; private int individualDeletedMessagesSerializedSize; + private static final String COMPACTION_CURSOR_NAME = "__compaction"; class MarkDeleteEntry { final PositionImpl newPosition; @@ -1068,7 +1069,8 @@ public void operationComplete() { Range.closedOpen(markDeletePosition, newMarkDeletePosition))); } markDeletePosition = newMarkDeletePosition; - lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, Collections.emptyMap(), + lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor() ? + getProperties() : Collections.emptyMap(), null, null); individualDeletedMessages.clear(); if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { @@ -1118,7 +1120,7 @@ public void operationFailed(ManagedLedgerException exception) { }; - internalAsyncMarkDelete(newPosition, Collections.emptyMap(), new MarkDeleteCallback() { + internalAsyncMarkDelete(newPosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { finalCallback.operationComplete(); @@ -3066,5 +3068,9 @@ public boolean checkAndUpdateReadPositionChanged() { return isReadPositionOnTail || isReadPositionChanged; } + private boolean isCompactionCursor() { + return COMPACTION_CURSOR_NAME.equals(name); + } + private static final Logger log = LoggerFactory.getLogger(ManagedCursorImpl.class); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index a59ce222da543..caff2960e1c37 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2550,7 +2550,7 @@ private void advanceCursorsIfNecessary(List ledgersToDelete) { if (highestPositionToDelete.compareTo((PositionImpl) cursor.getMarkDeletedPosition()) > 0 && highestPositionToDelete.compareTo((PositionImpl) cursor.getManagedLedger().getLastConfirmedEntry()) <= 0 && !(!cursor.isDurable() && cursor instanceof NonDurableCursorImpl && ((NonDurableCursorImpl) cursor).isReadCompacted())) { - cursor.asyncMarkDelete(highestPositionToDelete, new MarkDeleteCallback() { + cursor.asyncMarkDelete(highestPositionToDelete, cursor.getProperties(), new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java index dc51c3dc5154e..1462cfab89505 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java @@ -56,14 +56,14 @@ public void beforeSendMessage(Subscription subscription, Entry entry, long[] ackSet, MessageMetadata msgMetadata) { - log.info("Send message to topic {}, subscription {}", + log.debug("Send message to topic {}, subscription {}", subscription.getTopic(), subscription.getName()); beforeSendCount++; } @Override public void onPulsarCommand(BaseCommand command, ServerCnx cnx) { - log.info("[{}] On [{}] Pulsar command", count, command.getType().name()); + log.debug("[{}] On [{}] Pulsar command", count, command.getType().name()); count ++; } @@ -75,13 +75,13 @@ public void onConnectionClosed(ServerCnx cnx) { @Override public void onWebserviceRequest(ServletRequest request) { count ++; - log.info("[{}] On [{}] Webservice request", count, ((HttpServletRequest)request).getRequestURL().toString()); + log.debug("[{}] On [{}] Webservice request", count, ((HttpServletRequest)request).getRequestURL().toString()); } @Override public void onWebserviceResponse(ServletRequest request, ServletResponse response) { count ++; - log.info("[{}] On [{}] Webservice response {}", count, ((HttpServletRequest)request).getRequestURL().toString(), response); + log.debug("[{}] On [{}] Webservice response {}", count, ((HttpServletRequest)request).getRequestURL().toString(), response); if (response instanceof Response) { Response res = (Response) response; responseList.add(new ResponseEvent(res.getHttpChannel().getRequest().getRequestURI(), res.getStatus())); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index cbe73721483fb..69d66d9d0f1b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -33,6 +33,7 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.IntStream; @@ -577,9 +578,76 @@ public void testReadCompactedDataWhenLedgerRolloverKickIn() throws Exception { // The reader should read all 600 keys int received = 0; while (reader.hasMessageAvailable()) { - System.out.println(reader.readNext().getKey()); + reader.readNext(); received++; } Assert.assertEquals(received, keys * 3); + reader.close(); + producer.close(); + } + + @Test(timeOut = 120000) + public void testCompactionWithTopicUnloading() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactionWithTopicUnloading-" + + UUID.randomUUID(); + final int numMessages = 2000; + final int keys = 500; + final String msg = "Test"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .blockIfQueueFull(true) + .maxPendingMessages(numMessages) + .enableBatching(false) + .create(); + CompletableFuture lastMessage = null; + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i % keys + "").value(msg).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().triggerCompaction(topic); + Awaitility.await().pollInterval(5, TimeUnit.SECONDS).untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, keys); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + }); + + admin.topics().unload(topic); + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key((i % keys + keys) + "").value(msg).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().triggerCompaction(topic); + Thread.sleep(100); + admin.topics().unload(topic); + admin.topics().triggerCompaction(topic); + Awaitility.await().pollInterval(3, TimeUnit.SECONDS).atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, keys * 2); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + }); + + // Start a new reader to reading messages + Reader reader = pulsarClient.newReader(Schema.STRING) + .topic(topic) + .startMessageId(MessageId.earliest) + .readCompacted(true) + .receiverQueueSize(10) + .create(); + + // The reader should read all 600 keys + int received = 0; + while (reader.hasMessageAvailable()) { + reader.readNext(); + received++; + } + Assert.assertEquals(received, keys * 2); + reader.close(); + producer.close(); } } From 905f54a732bd2172e45b486a618aa96a3e58e4f6 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Tue, 9 Nov 2021 23:16:16 -0800 Subject: [PATCH 133/823] [pulsar-client] Fix pending queue-size stats for batch messages (#12704) (cherry picked from commit 9a3e7ecb326d045a10c3438f10bda63002d4603c) --- .../client/api/SimpleProducerConsumerStatTest.java | 7 ++++++- .../org/apache/pulsar/client/impl/ProducerImpl.java | 10 +++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java index 480d83544b28b..8f93494fab3b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java @@ -33,7 +33,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import com.google.gson.Gson; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.client.admin.PulsarAdminException; import org.slf4j.Logger; @@ -46,6 +45,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.RateLimiter; +import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -76,6 +76,11 @@ public Object[][] ackTimeoutSecProvider() { return new Object[][] { { 0, 0 }, { 0, 2 }, { 1000, 0 }, { 1000, 2 } }; } + @DataProvider(name = "batchingEnabled") + public Object[][] batchingEnabled() { + return new Object[][] { { true }, { false } }; + } + @Test(dataProvider = "batch_with_timeout") public void testSyncProducerAndConsumer(int batchMessageDelayMs, int ackTimeoutSec) throws Exception { log.info("-- Starting {} test --", methodName); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index e28721c50973d..cc978a11ee90c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -57,6 +57,7 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Message; @@ -1946,7 +1947,14 @@ public String getConnectedSince() { } public int getPendingQueueSize() { - return pendingMessages.size(); + if (!isBatchMessagingEnabled()) { + return pendingMessages.size(); + } + MutableInt size = new MutableInt(0); + pendingMessages.forEach(op -> { + size.add(Math.max(op.numMessagesInBatch, 1)); + }); + return size.getValue(); } @Override From 0008873b2ea440d7defea47d4843af269348bbfa Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Wed, 10 Nov 2021 16:24:49 +0800 Subject: [PATCH 134/823] [pulsar-admin] Print topic internal info as formatted json (#12709) (cherry picked from commit bcc8243c97a99173c9907500f25031fc905cc802) --- .../src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index fbd037a7f2984..d0ef10c85de27 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -28,6 +28,8 @@ import com.google.common.collect.Lists; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; @@ -594,7 +596,8 @@ private class GetInternalInfo extends CliCommand { @Override void run() throws PulsarAdminException { String topic = validateTopicName(params); - String result = getTopics().getInternalInfo(topic); + String internalInfo = getTopics().getInternalInfo(topic); + JsonObject result = JsonParser.parseString(internalInfo).getAsJsonObject(); Gson gson = new GsonBuilder().setPrettyPrinting().create(); System.out.println(gson.toJson(result)); } From d584b989dfb1b51c84cdcf792137efbdfbb33ad7 Mon Sep 17 00:00:00 2001 From: feynmanlin Date: Wed, 10 Nov 2021 18:46:35 +0800 Subject: [PATCH 135/823] Remove unused code in PersistentTopic.java (#12715) (cherry picked from commit 8551868468327cc04ca6d4e9fe4bcb718ad38cbc) --- .../apache/pulsar/broker/service/persistent/PersistentTopic.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 2e459729d95c6..c898bbb2d8fcd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2829,7 +2829,6 @@ private CompletableFuture getMessageTTL() { if (messageTtl.isPresent()) { return CompletableFuture.completedFuture(messageTtl.get()); } - TopicName name = TopicName.get(topic); return brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) From 37628b5650e27231af6d757a519ed8e53784f6bc Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Thu, 11 Nov 2021 09:10:03 +0800 Subject: [PATCH 136/823] Remove unnecessary powermock annotation (#12713) (cherry picked from commit 8cae63557a318240e95697f382b4f61c22b70d64) --- .../pulsar/broker/admin/PersistentTopicsTest.java | 4 ---- .../org/apache/pulsar/broker/admin/TopicsTest.java | 4 ---- ...istentStickyKeyDispatcherMultipleConsumersTest.java | 2 -- .../service/persistent/PersistentSubscriptionTest.java | 7 ------- .../streamingdispatch/StreamingEntryReaderTests.java | 10 ---------- 5 files changed, 27 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index a471caba4ad3a..6e5ef99efe893 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -80,8 +80,6 @@ import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.zookeeper.KeeperException; import org.mockito.ArgumentCaptor; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.reflect.Whitebox; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -89,8 +87,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -@PrepareForTest(PersistentTopics.class) -@PowerMockIgnore("com.sun.management.*") @Slf4j @Test(groups = "broker") public class PersistentTopicsTest extends MockedPulsarServiceBaseTest { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java index 3f57806c17767..7b77b1a74f20c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java @@ -25,13 +25,11 @@ import lombok.NoArgsConstructor; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.EncoderFactory; import org.apache.avro.io.JsonEncoder; import org.apache.avro.reflect.ReflectDatumWriter; import org.apache.avro.util.Utf8; import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.broker.admin.v2.PersistentTopics; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -70,7 +68,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.reflect.Whitebox; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -97,7 +94,6 @@ import static org.powermock.api.mockito.PowerMockito.mock; import static org.powermock.api.mockito.PowerMockito.spy; -@PrepareForTest(PersistentTopics.class) public class TopicsTest extends MockedPulsarServiceBaseTest { private Topics topics; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java index 990bd8f4b5acf..4a2b0f48e7c50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -71,7 +71,6 @@ public class NonPersistentStickyKeyDispatcherMultipleConsumersTest { private NonPersistentTopic topicMock; private NonPersistentSubscription subscriptionMock; private ServiceConfiguration configMock; - private ChannelPromise channelMock; private NonPersistentStickyKeyDispatcherMultipleConsumers nonpersistentDispatcher; @@ -100,7 +99,6 @@ public void setup() throws Exception { doReturn(brokerMock).when(topicMock).getBrokerService(); doReturn(topicName).when(topicMock).getName(); - channelMock = mock(ChannelPromise.class); subscriptionMock = mock(NonPersistentSubscription.class); PowerMockito.mockStatic(DispatchRateLimiter.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index 76f485ebb42a3..a76c6374e5c68 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -21,7 +21,6 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -73,20 +72,14 @@ import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; -import org.apache.pulsar.zookeeper.ZooKeeperCache; -import org.apache.pulsar.zookeeper.ZooKeeperDataCache; import org.apache.zookeeper.ZooKeeper; import org.awaitility.Awaitility; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -@PrepareForTest({ ZooKeeperDataCache.class, BrokerService.class }) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) @Test(groups = "broker") public class PersistentSubscriptionTest { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java index e58859d5f201e..217ac0f290306 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java @@ -38,8 +38,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.testng.annotations.Test; import java.nio.charset.Charset; @@ -66,15 +64,7 @@ /** * Tests for {@link StreamingEntryReader} */ -@PowerMockIgnore({ - "javax.management.*", - "javax.xml.parsers.*", - "com.sun.org.apache.xerces.internal.jaxp.*", - "ch.qos.logback.*", - "org.slf4j.*", - "org.apache.logging.*"}) @Test(groups = "flaky") -@PrepareForTest({ManagedLedgerImpl.class}) public class StreamingEntryReaderTests extends MockedBookKeeperTestCase { private static final Charset Encoding = Charsets.UTF_8; From d7ae988bea0651095632a5ca1ad0f50c5c2331a7 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 11 Nov 2021 23:37:42 +0800 Subject: [PATCH 137/823] Remove catching `NamingException`. (#12725) (cherry picked from commit 6ab7401056f6c0f6bdf695fecc2764fdff16fcac) --- .../java/org/apache/pulsar/broker/service/BrokerService.java | 2 +- .../pulsar/broker/service/persistent/PersistentTopic.java | 2 +- .../apache/pulsar/broker/service/persistent/SystemTopic.java | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 76c295bbd4b0e..b3fca060cd811 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1362,7 +1362,7 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { return null; }); - } catch (NamingException | PulsarServerException e) { + } catch (PulsarServerException e) { log.warn("Failed to create topic {}-{}", topic, e.getMessage()); pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.completeExceptionally(e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index c898bbb2d8fcd..c2320717f3549 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -250,7 +250,7 @@ public void reset() { } } - public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) throws NamingException { + public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) { super(topic, brokerService); this.ledger = ledger; this.subscriptions = new ConcurrentOpenHashMap<>(16, 1); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java index aaf83a9fd689e..6e3173f0c1708 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java @@ -23,12 +23,10 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.BrokerServiceException; public class SystemTopic extends PersistentTopic { - public SystemTopic(String topic, ManagedLedger ledger, BrokerService brokerService) - throws BrokerServiceException.NamingException, PulsarServerException { + public SystemTopic(String topic, ManagedLedger ledger, BrokerService brokerService) throws PulsarServerException { super(topic, ledger, brokerService); } From a813c52931a8078a959b06cdba750419bc71dad5 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Thu, 11 Nov 2021 07:32:35 -0800 Subject: [PATCH 138/823] [pulsar-broker] Handle lookup redirect for V1-topics with different cluster (#12743) (cherry picked from commit 25cbfadbf0a1aba8c149c978063902500ab08379) --- .../PulsarAuthorizationProvider.java | 23 +++++++++++-------- .../broker/resources/ClusterResources.java | 4 ++++ .../pulsar/broker/service/ReplicatorTest.java | 21 +++++++++++++++++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index b6b1bafd4c377..b50d7de0ab637 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -364,20 +364,23 @@ private CompletableFuture updateSubscriptionPermissionAsync(NamespaceName } private CompletableFuture checkAuthorization(TopicName topicName, String role, AuthAction action) { - return checkPermission(topicName, role, action) - .thenApply(isPermission -> isPermission && checkCluster(topicName)); + return checkPermission(topicName, role, action). + thenApply(isPermission -> isPermission). + thenCompose(permission -> permission ? checkCluster(topicName) : + CompletableFuture.completedFuture(false)); } - private boolean checkCluster(TopicName topicName) { + private CompletableFuture checkCluster(TopicName topicName) { if (topicName.isGlobal() || conf.getClusterName().equals(topicName.getCluster())) { - return true; - } else { - if (log.isDebugEnabled()) { - log.debug("Topic [{}] does not belong to local cluster [{}]", topicName.toString(), - conf.getClusterName()); - } - return false; + return CompletableFuture.completedFuture(true); } + if (log.isDebugEnabled()) { + log.debug("Topic [{}] does not belong to local cluster [{}]", topicName.toString(), conf.getClusterName()); + } + return pulsarResources.getClusterResources().listAsync() + .thenApply(clusters -> { + return clusters.contains(topicName.getCluster()); + }); } public CompletableFuture checkPermission(TopicName topicName, String role, AuthAction action) { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java index 1a3cf89b59efe..a4ee27af4929a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java @@ -42,6 +42,10 @@ public ClusterResources(MetadataStore store, int operationTimeoutSec) { this.failureDomainResources = new FailureDomainResources(store, FailureDomainImpl.class, operationTimeoutSec); } + public CompletableFuture> listAsync() { + return getChildrenAsync(BASE_CLUSTERS_PATH).thenApply(list -> new HashSet<>(list)); + } + public Set list() throws MetadataStoreException { return new HashSet<>(super.getChildren(BASE_CLUSTERS_PATH)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 04f96ec8d898f..87e2730fd68f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -1310,6 +1310,27 @@ public void testDoNotReplicateSystemTopic() throws Exception { }); } + @Test + public void testLookupAnotherCluster() throws Exception { + log.info("--- Starting ReplicatorTest::testLookupAnotherCluster ---"); + + String namespace = "pulsar/r2/cross-cluster-ns"; + admin1.namespaces().createNamespace(namespace); + final TopicName topicName = TopicName + .get("persistent://" + namespace + "/topic"); + + @Cleanup + PulsarClient client1 = PulsarClient.builder() + .serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + Producer producer = client1.newProducer().topic(topicName.toString()) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + producer.close(); + } + private void checkListContainExpectedTopic(PulsarAdmin admin, String namespace, List expectedTopicList) { // wait non-partitioned topics replicators created finished final List list = new ArrayList<>(); From a9e8b3f5e7bcd689577090ade3d6efa3796fae19 Mon Sep 17 00:00:00 2001 From: feynmanlin Date: Fri, 12 Nov 2021 13:43:50 +0800 Subject: [PATCH 139/823] Even if always compatible is set, Consumers cannot be created (#12721) (cherry picked from commit c3da1452a444c9599cb85562a3faa82ddfdecec8) --- .../schema/SchemaRegistryServiceImpl.java | 3 +++ .../SchemaCompatibilityCheckTest.java | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index 0eff36b54ce94..e1c9b13e60f0d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -270,6 +270,9 @@ public CompletableFuture findSchemaVersion(String schemaId, SchemaData sch @Override public CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, SchemaCompatibilityStrategy strategy) { + if (SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE == strategy) { + return CompletableFuture.completedFuture(null); + } return getSchema(schemaId).thenCompose(existingSchema -> { if (existingSchema != null && !existingSchema.schema.isDeleted()) { if (strategy == SchemaCompatibilityStrategy.BACKWARD diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 02913c628f27f..293f71d5f878f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -395,6 +395,28 @@ public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityS } + @Test + public void testAutoProduceSchemaAlwaysCompatible() throws Exception { + final String tenant = PUBLIC_TENANT; + final String topic = "topic" + randomName(16); + + String namespace = "test-namespace-" + randomName(16); + String topicName = TopicName.get( + TopicDomain.persistent.value(), tenant, namespace, topic).toString(); + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + admin.namespaces().createNamespace(tenant + "/" + namespace, Sets.newHashSet(CLUSTER_NAME)); + + // set ALWAYS_COMPATIBLE + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE); + + Producer producer = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName).create(); + // should not fail + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).subscriptionName("my-sub").topic(topicName).subscribe(); + + producer.close(); + consumer.close(); + } + @Test(dataProvider = "CanReadLastSchemaCompatibilityStrategy") public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { final String tenant = PUBLIC_TENANT; From 1ca6946e4b31ed0adcbd299ddec66369d0ffe4c2 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Mon, 15 Nov 2021 08:45:40 +0800 Subject: [PATCH 140/823] [pulsar-admin] Add --all option to get all version schema of topic (#12535) ### Motivation Add `--all` option to get all version schema of topic when we use CLI `./bin/pulsar-admin schemas get tenant/ns/tp` (cherry picked from commit 2609c787be850a70fc2d222db1f4378511e384d0) --- .../pulsar/admin/cli/PulsarAdminToolTest.java | 65 +++++++++++++++++++ .../test/resources/test_schema_create.json | 4 ++ .../apache/pulsar/admin/cli/CmdSchemas.java | 18 ++++- 3 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 pulsar-client-tools-test/src/test/resources/test_schema_create.json diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 5cdaba27efa3e..300eefc368c88 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -28,10 +28,15 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + +import java.io.File; import java.lang.reflect.Field; +import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -40,6 +45,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.admin.cli.utils.SchemaExtractor; import org.apache.pulsar.client.admin.Bookies; import org.apache.pulsar.client.admin.BrokerStats; import org.apache.pulsar.client.admin.Brokers; @@ -59,6 +65,7 @@ import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; @@ -92,6 +99,7 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; @@ -1597,6 +1605,63 @@ void transactions() throws Exception { verify(transactions).getPendingAckInternalStats("test", "test", false); } + @Test + void schemas() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Schemas schemas = Mockito.mock(Schemas.class); + doReturn(schemas).when(admin).schemas(); + + CmdSchemas cmdSchemas = new CmdSchemas(() -> admin); + cmdSchemas.run(split("get -v 1 persistent://tn1/ns1/tp1")); + verify(schemas).getSchemaInfo("persistent://tn1/ns1/tp1", 1); + + cmdSchemas = new CmdSchemas(() -> admin); + cmdSchemas.run(split("get -a persistent://tn1/ns1/tp1")); + verify(schemas).getAllSchemas("persistent://tn1/ns1/tp1"); + + cmdSchemas = new CmdSchemas(() -> admin); + cmdSchemas.run(split("get persistent://tn1/ns1/tp1")); + verify(schemas).getSchemaInfoWithVersion("persistent://tn1/ns1/tp1"); + + cmdSchemas = new CmdSchemas(() -> admin); + cmdSchemas.run(split("delete persistent://tn1/ns1/tp1")); + verify(schemas).deleteSchema("persistent://tn1/ns1/tp1"); + + cmdSchemas = new CmdSchemas(() -> admin); + String schemaFile = PulsarAdminToolTest.class.getClassLoader() + .getResource("test_schema_create.json").getFile(); + cmdSchemas.run(split("upload -f " + schemaFile + " persistent://tn1/ns1/tp1")); + PostSchemaPayload input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class); + verify(schemas).createSchema("persistent://tn1/ns1/tp1", input); + + cmdSchemas = new CmdSchemas(() -> admin); + String jarFile = PulsarAdminToolTest.class.getClassLoader() + .getResource("dummyexamples.jar").getFile(); + String className = SchemaDemo.class.getName(); + cmdSchemas.run(split("extract -j " + jarFile + " -c " + className + " -t json persistent://tn1/ns1/tp1")); + File file = new File(jarFile); + ClassLoader cl = new URLClassLoader(new URL[]{file.toURI().toURL()}); + Class cls = cl.loadClass(className); + SchemaDefinition schemaDefinition = + SchemaDefinition.builder() + .withPojo(cls) + .withAlwaysAllowNull(true) + .build(); + PostSchemaPayload postSchemaPayload = new PostSchemaPayload(); + postSchemaPayload.setType("JSON"); + postSchemaPayload.setSchema(SchemaExtractor.getJsonSchemaInfo(schemaDefinition)); + postSchemaPayload.setProperties(schemaDefinition.getProperties()); + verify(schemas).createSchema("persistent://tn1/ns1/tp1", postSchemaPayload); + } + + public static class SchemaDemo { + public SchemaDemo() { + } + + public static void main(String[] args) { + } + } + String[] split(String s) { return s.split(" "); } diff --git a/pulsar-client-tools-test/src/test/resources/test_schema_create.json b/pulsar-client-tools-test/src/test/resources/test_schema_create.json new file mode 100644 index 0000000000000..241a985c073f7 --- /dev/null +++ b/pulsar-client-tools-test/src/test/resources/test_schema_create.json @@ -0,0 +1,4 @@ +{ + "type":"json", + "schema":"" +} \ No newline at end of file diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java index 045f64e0f12f5..302f6e2f5c8bc 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java @@ -19,6 +19,7 @@ package org.apache.pulsar.admin.cli; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -48,16 +49,27 @@ private class GetSchema extends CliCommand { @Parameter(description = "persistent://tenant/namespace/topic", required = true) private java.util.List params; - @Parameter(names = { "--version" }, description = "version", required = false) + @Parameter(names = {"-v", "--version"}, description = "version", required = false) private Long version; + @Parameter(names = {"-a", "--all-version"}, description = "all version", required = false) + private boolean all = false; + @Override void run() throws Exception { String topic = validateTopicName(params); - if (version == null) { + if (version != null && all) { + throw new ParameterException("Only one or neither of --version and --all-version can be specified."); + } + if (version == null && !all) { System.out.println(getAdmin().schemas().getSchemaInfoWithVersion(topic)); - } else { + } else if (!all) { + if (version < 0) { + throw new ParameterException("Option --version must be greater than 0, but found " + version); + } System.out.println(getAdmin().schemas().getSchemaInfo(topic, version)); + } else { + print(getAdmin().schemas().getAllSchemas(topic)); } } } From b6ae15796e01d9137066c39d7d7dd6885a49a9d7 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Mon, 15 Nov 2021 08:53:09 +0800 Subject: [PATCH 141/823] remove redundant placeholders (#12717) (cherry picked from commit 4fac93cf0a6a0f41c0674691f108f14af74be1b8) --- .../src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java | 2 +- .../org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 76ce022346fc1..8151a82b5c089 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -300,7 +300,7 @@ private void processPublishMessageResults(List produceMessageResult produceMessageResults.get(index).setMessageId(messageId.toString()); } catch (Exception e) { if (log.isDebugEnabled()) { - log.debug("Fail publish [{}] message with rest produce message request for topic {}: {} ", + log.debug("Fail publish [{}] message with rest produce message request for topic {}", index, topicName); } if (e instanceof BrokerServiceException.TopicNotFoundException) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 2c356fb8e28f5..09dc5643698d3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -1328,7 +1328,7 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa return FutureUtil.failedFuture(new NotSupportedException("not support shrink topic partitions")); } }).exceptionally(throwable -> { - log.warn("[{}] Failed to get partitions for topic to determine if new partitions are added", throwable); + log.warn("Failed to get partitions for topic to determine if new partitions are added", throwable); return null; }); } From 502f1431b5cecf5a19330473a0ebb67918c5b0b1 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Mon, 15 Nov 2021 08:53:21 +0800 Subject: [PATCH 142/823] remove unnecessary string operation (#12719) (cherry picked from commit 4cf4b85a353c726725673201bdc9cce5f5442e53) --- .../java/org/apache/pulsar/broker/cache/BundlesQuotas.java | 2 +- .../org/apache/pulsar/client/admin/internal/TopicsImpl.java | 2 +- .../src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java | 4 ++-- .../java/org/apache/pulsar/admin/cli/PulsarAdminTool.java | 2 +- .../main/java/org/apache/pulsar/common/net/ServiceURI.java | 2 +- .../org/apache/pulsar/testclient/DefaultMessageFormatter.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java index 251203eb12de7..88ddaf15314c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java @@ -73,7 +73,7 @@ public CompletableFuture getResourceQuota(NamespaceBundle bundle) } public CompletableFuture getResourceQuota(String bundle) { - return resourceQuotaCache.get(RESOURCE_QUOTA_ROOT + "/" + bundle.toString()) + return resourceQuotaCache.get(RESOURCE_QUOTA_ROOT + "/" + bundle) .thenCompose(optResourceQuota -> { if (optResourceQuota.isPresent()) { return CompletableFuture.completedFuture(optResourceQuota.get()); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index caf32e4aacf8a..57f36995a8fdf 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -1602,7 +1602,7 @@ private List> getMessagesFromHttpResponse(String topic, Response } else { brokerEntryMetadata = new BrokerEntryMetadata(); if (brokerEntryTimestamp != null) { - brokerEntryMetadata.setBrokerTimestamp(DateFormatter.parse(brokerEntryTimestamp.toString())); + brokerEntryMetadata.setBrokerTimestamp(DateFormatter.parse(brokerEntryTimestamp)); } if (brokerEntryIndex != null) { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java index 302f6e2f5c8bc..9575ec64d1822 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java @@ -139,10 +139,10 @@ void run() throws Exception { .withPojo(cls) .withAlwaysAllowNull(alwaysAllowNull) .build(); - if (type.toLowerCase().equalsIgnoreCase("avro")) { + if (type.equalsIgnoreCase("avro")) { input.setType("AVRO"); input.setSchema(SchemaExtractor.getAvroSchemaInfo(schemaDefinition)); - } else if (type.toLowerCase().equalsIgnoreCase("json")){ + } else if (type.equalsIgnoreCase("json")){ input.setType("JSON"); input.setSchema(SchemaExtractor.getJsonSchemaInfo(schemaDefinition)); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index a41f44438fed5..d0f514d6f1f03 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -317,7 +317,7 @@ public static void main(String[] args) throws Exception { } ++cmdPos; - boolean isLocalRun = cmdPos < args.length && "localrun".equals(args[cmdPos].toLowerCase()); + boolean isLocalRun = cmdPos < args.length && "localrun".equalsIgnoreCase(args[cmdPos]); Function adminFactory; if (isLocalRun) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/net/ServiceURI.java b/pulsar-common/src/main/java/org/apache/pulsar/common/net/ServiceURI.java index 16a070d41865c..f52afbcc886c4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/net/ServiceURI.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/net/ServiceURI.java @@ -219,7 +219,7 @@ private static int getServicePort(String serviceName, String[] serviceInfos) { case BINARY_SERVICE: if (serviceInfos.length == 0) { port = BINARY_PORT; - } else if (serviceInfos.length == 1 && serviceInfos[0].toLowerCase().equals(SSL_SERVICE)) { + } else if (serviceInfos.length == 1 && serviceInfos[0].equalsIgnoreCase(SSL_SERVICE)) { port = BINARY_TLS_PORT; } else { throw new IllegalArgumentException("Invalid pulsar service : " + serviceName + "+" diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java index e619a00027946..bf183a6d24b1d 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java @@ -50,7 +50,7 @@ public byte[] formatMessage(String producerName, long msgId, byte[] message) { break; } if (i != 1) { - size = Float.valueOf(new String(sMessage.substring(idx+1,idx+i))); + size = Float.parseFloat(sMessage.substring(idx + 1, idx + i)); } String sub = sMessage.substring(idx, idx+i+1); From 5d369762fb8776401039405ebba9057fe2c66b0e Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Mon, 15 Nov 2021 08:52:31 +0800 Subject: [PATCH 143/823] [Issue 12723] Fix race condition in PersistentTopic#addReplicationCluster (#12729) See #12723 Add a method org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap#removeNullValue to remove null value in a thread safe way. (cherry picked from commit a3fe00efc4ccba55a0f28fd02b535c6624e3ed0a) --- .../service/persistent/PersistentTopic.java | 6 ++-- .../collections/ConcurrentOpenHashMap.java | 26 +++++++++++++++ .../ConcurrentOpenHashMapTest.java | 32 +++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index c2320717f3549..275c7ea8989ee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1552,7 +1552,7 @@ protected CompletableFuture addReplicationCluster(String remoteCluster, Ma .thenApply(clusterData -> brokerService.getReplicationClient(remoteCluster, clusterData))) .thenAccept(replicationClient -> { - replicators.computeIfAbsent(remoteCluster, r -> { + Replicator replicator = replicators.computeIfAbsent(remoteCluster, r -> { try { return new PersistentReplicator(PersistentTopic.this, cursor, localCluster, remoteCluster, brokerService, (PulsarClientImpl) replicationClient); @@ -1563,8 +1563,8 @@ protected CompletableFuture addReplicationCluster(String remoteCluster, Ma }); // clean up replicator if startup is failed - if (replicators.containsKey(remoteCluster) && replicators.get(remoteCluster) == null) { - replicators.remove(remoteCluster); + if (replicator == null) { + replicators.removeNullValue(remoteCluster); } }); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java index 47927a98eeaf3..2c7eed1b58e03 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -42,6 +43,27 @@ public class ConcurrentOpenHashMap { private static final Object EmptyKey = null; private static final Object DeletedKey = new Object(); + /** + * This object is used to delete empty value in this map. + * EmptyValue.equals(null) = true. + */ + private static final Object EmptyValue = new Object() { + + @SuppressFBWarnings + @Override + public boolean equals(Object obj) { + return obj == null; + } + + /** + * This is just for avoiding spotbugs errors + */ + @Override + public int hashCode() { + return super.hashCode(); + } + }; + private static final float MapFillFactor = 0.66f; private static final int DefaultExpectedItems = 256; @@ -142,6 +164,10 @@ public boolean remove(K key, Object value) { return getSection(h).remove(key, value, (int) h) != null; } + public void removeNullValue(K key) { + remove(key, EmptyValue); + } + private Section getSection(long hash) { // Use 32 msb out of long to get the section final int sectionIdx = (int) (hash >>> 32) & (sections.length - 1); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java index e18012cdf13f2..254be51f29239 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -369,6 +370,37 @@ public boolean equals(Object obj) { assertNull(map.get(t1_b)); } + @Test + public void testNullValue() { + ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(16, 1); + String key = "a"; + assertThrows(NullPointerException.class, () -> map.put(key, null)); + + //put a null value. + assertNull(map.computeIfAbsent(key, k -> null)); + assertEquals(1, map.size()); + assertEquals(1, map.keys().size()); + assertEquals(1, map.values().size()); + assertNull(map.get(key)); + assertFalse(map.containsKey(key)); + + //test remove null value + map.removeNullValue(key); + assertTrue(map.isEmpty()); + assertEquals(0, map.keys().size()); + assertEquals(0, map.values().size()); + assertNull(map.get(key)); + assertFalse(map.containsKey(key)); + + + //test not remove non-null value + map.put(key, "V"); + assertEquals(1, map.size()); + map.removeNullValue(key); + assertEquals(1, map.size()); + + } + static final int Iterations = 1; static final int ReadIterations = 1000; static final int N = 1_000_000; From e35b56173647ba3305ff37deabacd7825e45a395 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 15 Nov 2021 11:32:56 +0800 Subject: [PATCH 144/823] Remove unnecessary boxing unboxing (#12790) (cherry picked from commit 3e6fedf9b69758c13d92c6ff75a7bd779038543a) --- .../java/org/apache/pulsar/tests/FailFastNotifier.java | 2 +- .../pulsar/tests/FastThreadLocalCleanupListener.java | 2 +- .../org/apache/pulsar/tests/MockitoCleanupListener.java | 2 +- .../apache/pulsar/tests/ThreadLeakDetectorListener.java | 4 +--- .../apache/bookkeeper/mledger/impl/ManagedCursorTest.java | 2 +- .../broker/authentication/AuthenticationProviderList.java | 2 +- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 2 +- .../java/org/apache/pulsar/broker/rest/TopicsBase.java | 2 +- .../transaction/buffer/impl/InMemTransactionBuffer.java | 2 +- .../org/apache/pulsar/broker/admin/TopicPoliciesTest.java | 2 +- .../pulsar/client/impl/auth/AuthenticationAthenz.java | 2 +- .../org/apache/pulsar/common/tls/TlsHostnameVerifier.java | 2 +- .../pulsar/policies/data/loadbalancer/BrokerUsage.java | 4 ++-- .../windowing/triggers/WatermarkCountTriggerPolicy.java | 2 +- .../functions/windowing/WindowFunctionExecutorTest.java | 2 +- .../apache/pulsar/functions/utils/FunctionConfigUtils.java | 2 +- .../pulsar/functions/worker/FunctionMetaDataManager.java | 2 +- .../pulsar/functions/worker/rest/api/ComponentImpl.java | 7 +++---- .../apache/pulsar/io/cassandra/CassandraAbstractSink.java | 2 +- .../decoder/primitive/PulsarPrimitiveRowDecoder.java | 6 +++--- .../apache/pulsar/sql/presto/TestPulsarRecordCursor.java | 2 +- .../sql/presto/decoder/primitive/TestPrimitiveDecoder.java | 2 +- .../java/org/apache/pulsar/websocket/ConsumerHandler.java | 2 +- .../jcloud/provider/TieredStorageConfiguration.java | 4 ++-- 24 files changed, 30 insertions(+), 33 deletions(-) diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java index 3ee4ad70a7674..0f001332a98d8 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java @@ -41,7 +41,7 @@ */ public class FailFastNotifier implements IInvokedMethodListener, ITestListener { - private static final boolean FAIL_FAST_ENABLED = Boolean.valueOf( + private static final boolean FAIL_FAST_ENABLED = Boolean.parseBoolean( System.getProperty("testFailFast", "true")); static class FailFastEventsSingleton { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java index cd75bfbd4318e..c40956a9e259e 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java @@ -27,7 +27,7 @@ public class FastThreadLocalCleanupListener extends BetweenTestClassesListenerAdapter { private static final Logger LOG = LoggerFactory.getLogger(FastThreadLocalCleanupListener.class); private static final boolean FAST_THREAD_LOCAL_CLEANUP_ENABLED = - Boolean.valueOf(System.getProperty("testFastThreadLocalCleanup", "true")); + Boolean.parseBoolean(System.getProperty("testFastThreadLocalCleanup", "true")); private static final String FAST_THREAD_LOCAL_CLEANUP_PACKAGE = System.getProperty("testFastThreadLocalCleanupPackage", "org.apache.pulsar"); private static final FastThreadLocalStateCleaner CLEANER = new FastThreadLocalStateCleaner(object -> { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java index 0724a9dc22261..354a55c10e39f 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java @@ -32,7 +32,7 @@ public class MockitoCleanupListener extends BetweenTestClassesListenerAdapter { private static final Logger LOG = LoggerFactory.getLogger(MockitoCleanupListener.class); private static final boolean - MOCKITO_CLEANUP_ENABLED = Boolean.valueOf(System.getProperty("testMockitoCleanup", "true")); + MOCKITO_CLEANUP_ENABLED = Boolean.parseBoolean(System.getProperty("testMockitoCleanup", "true")); @Override protected void onBetweenTestClasses(Class endedTestClass, Class startedTestClass) { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java index ddfebc9a44d9e..163591cf82554 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java @@ -32,9 +32,7 @@ */ public class ThreadLeakDetectorListener extends BetweenTestClassesListenerAdapter { private static final Logger LOG = LoggerFactory.getLogger(ThreadLeakDetectorListener.class); - private static final boolean - THREAD_LEAK_DETECTOR_ENABLED = Boolean.valueOf(System.getProperty("testThreadLeakDetector", - "true")); + private Set capturedThreadKeys; @Override diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 4bfcf944c437e..bbc182719d89e 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -2251,7 +2251,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional c1.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, entry -> { try { - long publishTime = Long.valueOf(new String(entry.getData())); + long publishTime = Long.parseLong(new String(entry.getData())); return publishTime <= timestamp; } catch (Exception e) { log.error("Error de-serializing message for message position find", e); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java index a79fabef3deb2..9ec1c2eb706cc 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java @@ -206,7 +206,7 @@ public boolean authenticateHttpRequest(HttpServletRequest request, HttpServletRe } } ); - return authenticated.booleanValue(); + return authenticated; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index f317e03136977..be76ff1949b17 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -1058,7 +1058,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut existsFutures.put(i, topicResources().persistentTopicExists(topicName.getPartition(i))); } FutureUtil.waitForAll(Lists.newArrayList(existsFutures.values())).thenApply(__ -> - existsFutures.entrySet().stream().filter(e -> e.getValue().join().booleanValue()) + existsFutures.entrySet().stream().filter(e -> e.getValue().join()) .map(item -> topicName.getPartition(item.getKey()).toString()) .collect(Collectors.toList()) ).thenAccept(topics -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 8151a82b5c089..1df2c14bc9c94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -641,7 +641,7 @@ private List buildMessage(ProducerMessages producerMessages, Schema sch } } if (null != message.getEventTime() && !message.getEventTime().isEmpty()) { - messageMetadata.setEventTime(Long.valueOf(message.getEventTime())); + messageMetadata.setEventTime(Long.parseLong(message.getEventTime())); } if (message.isDisableReplication()) { messageMetadata.clearReplicateTo(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java index f66b263812da3..98d572cc0827f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java @@ -194,7 +194,7 @@ public TransactionBufferReader newReader(long sequenceId) throws final SortedMap entriesToRead = new TreeMap<>(); synchronized (entries) { - SortedMap subEntries = entries.tailMap(Long.valueOf(sequenceId)); + SortedMap subEntries = entries.tailMap(sequenceId); subEntries.values().forEach(value -> value.retain()); entriesToRead.putAll(subEntries); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 59494fb799d0c..91de20332e14f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -1397,7 +1397,7 @@ public void testRemovePublishRate() throws Exception { @Test public void testCheckMaxConsumers() throws Exception { - Integer maxProducers = new Integer(-1); + Integer maxProducers = -1; log.info("MaxConsumers: {} will set to the topic: {}", maxProducers, testTopic); try { admin.topics().setMaxConsumers(testTopic, maxProducers); diff --git a/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java b/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java index 1e9588a1df144..447a21ce05645 100644 --- a/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java +++ b/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java @@ -159,7 +159,7 @@ private void setAuthParams(Map authParams) { } this.keyId = authParams.getOrDefault("keyId", "0"); - this.autoPrefetchEnabled = Boolean.valueOf(authParams.getOrDefault("autoPrefetchEnabled", "false")); + this.autoPrefetchEnabled = Boolean.parseBoolean(authParams.getOrDefault("autoPrefetchEnabled", "false")); if (isNotBlank(authParams.get("athenzConfPath"))) { System.setProperty("athenz.athenz_conf", authParams.get("athenzConfPath")); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java b/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java index 9a2964d123801..3ab5b5f63f76c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java @@ -289,7 +289,7 @@ static List getSubjectAltNames(final X509Certificate cert) { if (type != null) { final Object o = entry.get(1); if (o instanceof String) { - result.add(new SubjectName((String) o, type.intValue())); + result.add(new SubjectName((String) o, type)); } else if (o instanceof byte[]) { // TODO ASN.1 DER encoded form } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BrokerUsage.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BrokerUsage.java index 57915029d6f5a..6fd7d60fbd9c3 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BrokerUsage.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BrokerUsage.java @@ -56,13 +56,13 @@ public static BrokerUsage populateFrom(Map metrics) { BrokerUsage brokerUsage = null; if (metrics.containsKey("brk_conn_cnt")) { brokerUsage = new BrokerUsage(); - brokerUsage.connectionCount = ((Long) metrics.get("brk_conn_cnt")).longValue(); + brokerUsage.connectionCount = (Long) metrics.get("brk_conn_cnt"); } if (metrics.containsKey("brk_repl_conn_cnt")) { if (brokerUsage == null) { brokerUsage = new BrokerUsage(); } - brokerUsage.replicationConnectionCount = ((Long) metrics.get("brk_repl_conn_cnt")).longValue(); + brokerUsage.replicationConnectionCount = (Long) metrics.get("brk_repl_conn_cnt"); } return brokerUsage; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkCountTriggerPolicy.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkCountTriggerPolicy.java index 26a9ce737996e..7b60e63e45224 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkCountTriggerPolicy.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkCountTriggerPolicy.java @@ -83,7 +83,7 @@ private void handleWaterMarkEvent(Event waterMarkEvent) { List eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { - evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); + evictionPolicy.setContext(new DefaultEvictionContext(ts, null, (long) count)); handler.onTrigger(); lastProcessedTs = ts; } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WindowFunctionExecutorTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WindowFunctionExecutorTest.java index 9e0973370603f..f3502d667b381 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WindowFunctionExecutorTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WindowFunctionExecutorTest.java @@ -89,7 +89,7 @@ public long extractTimestamp(Long input) { private static class TestWrongTimestampExtractor implements TimestampExtractor { @Override public long extractTimestamp(String input) { - return Long.valueOf(input); + return Long.parseLong(input); } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 539f066a6db10..9b8d8d4f4ed85 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -56,7 +56,7 @@ @Slf4j public class FunctionConfigUtils { - static final Integer MAX_PENDING_ASYNC_REQUESTS_DEFAULT = Integer.valueOf(1000); + static final Integer MAX_PENDING_ASYNC_REQUESTS_DEFAULT = 1000; static final Boolean FORWARD_SOURCE_MESSAGE_PROPERTY_DEFAULT = Boolean.TRUE; private static final ObjectMapper OBJECT_MAPPER = ObjectMapperFactory.create(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java index 5388bd703312f..eddb469ba9273 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java @@ -365,7 +365,7 @@ private void processUncompactedMetaDataTopicMessage(Message message) thr } private void processCompactedMetaDataTopicMessage(Message message) throws IOException { - long version = Long.valueOf(message.getProperty(versionTag)); + long version = Long.parseLong(message.getProperty(versionTag)); String tenant = FunctionCommon.extractTenantFromFullyQualifiedName(message.getKey()); String namespace = FunctionCommon.extractNamespaceFromFullyQualifiedName(message.getKey()); String functionName = FunctionCommon.extractNameFromFullyQualifiedName(message.getKey()); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index e8d67a5c60930..98994c9745144 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -341,10 +341,9 @@ private void deleteStatestoreTableAsync(String namespace, String table) { StorageAdminClient adminClient = worker().getStateStoreAdminClient(); if (adminClient != null) { adminClient.deleteStream(namespace, table).whenComplete((res, throwable) -> { - if ((throwable == null && res.booleanValue()) - || (throwable != null && - (throwable instanceof NamespaceNotFoundException - || throwable instanceof StreamNotFoundException) )) { + if ((throwable == null && res) + || ((throwable instanceof NamespaceNotFoundException + || throwable instanceof StreamNotFoundException))) { log.info("{}/{} table deleted successfully", namespace, table); } else { if (throwable != null) { diff --git a/pulsar-io/cassandra/src/main/java/org/apache/pulsar/io/cassandra/CassandraAbstractSink.java b/pulsar-io/cassandra/src/main/java/org/apache/pulsar/io/cassandra/CassandraAbstractSink.java index 874710afb4860..4cd31380f0078 100644 --- a/pulsar-io/cassandra/src/main/java/org/apache/pulsar/io/cassandra/CassandraAbstractSink.java +++ b/pulsar-io/cassandra/src/main/java/org/apache/pulsar/io/cassandra/CassandraAbstractSink.java @@ -98,7 +98,7 @@ private void createClient(String roots) { String[] hostPort = hosts[i].split(":"); b.addContactPoint(hostPort[0]); if (hostPort.length > 1) { - b.withPort(Integer.valueOf(hostPort[1])); + b.withPort(Integer.parseInt(hostPort[1])); } } cluster = b.build(); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index 5eb2d7f6db613..bc192c81eb012 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -79,12 +79,12 @@ public Optional> decodeRow(ByteBuf primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf((Boolean) value))); } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType) { - primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(value.toString()))); + primitiveColumn.put(columnHandle, longValueProvider(Long.parseLong(value.toString()))); } else if (type instanceof DoubleType) { - primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(value.toString()))); + primitiveColumn.put(columnHandle, doubleValueProvider(Double.parseDouble(value.toString()))); } else if (type instanceof RealType) { primitiveColumn.put(columnHandle, longValueProvider( - Float.floatToIntBits((Float.valueOf(value.toString()))))); + Float.floatToIntBits((Float.parseFloat(value.toString()))))); } else if (type instanceof VarbinaryType) { primitiveColumn.put(columnHandle, bytesValueProvider((byte[]) value)); } else if (type instanceof VarcharType) { diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 8d521834c6a5c..610a1c2e695aa 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -101,7 +101,7 @@ public void testTopics() throws Exception { assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), ((String) fooFunctions.get("field2").apply(count)).getBytes()); columnsSeen.add(fooColumnHandles.get(i).getName()); } else if (fooColumnHandles.get(i).getName().equals("field3")) { - assertEquals(pulsarRecordCursor.getLong(i), Float.floatToIntBits(((Float) fooFunctions.get("field3").apply(count)).floatValue())); + assertEquals(pulsarRecordCursor.getLong(i), Float.floatToIntBits((Float) fooFunctions.get("field3").apply(count))); columnsSeen.add(fooColumnHandles.get(i).getName()); } else if (fooColumnHandles.get(i).getName().equals("field4")) { assertEquals(pulsarRecordCursor.getDouble(i), ((Double) fooFunctions.get("field4").apply(count)).doubleValue()); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java index d01210b47adc5..97d4d055598b5 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java @@ -142,7 +142,7 @@ public void testPrimitiveType() { .copiedBuffer(schemaFloat.encode(floatValue))).get(); checkValue(decodedRowFloat, new PulsarColumnHandle(getPulsarConnectorId().toString(), PRIMITIVE_COLUMN_NAME, REAL, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), Long.valueOf(Float.floatToIntBits(floatValue))); + PulsarColumnHandle.HandleKeyValueType.NONE), Float.floatToIntBits(floatValue)); double doubleValue = 0.22d; SchemaInfo schemaInfoDouble = SchemaInfo.builder().type(SchemaType.DOUBLE).build(); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index 0192188b12549..6dd44cb9b7551 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -107,7 +107,7 @@ public ConsumerHandler(WebSocketService service, HttpServletRequest request, Ser this.numMsgsDelivered = new LongAdder(); this.numBytesDelivered = new LongAdder(); this.numMsgsAcked = new LongAdder(); - this.pullMode = Boolean.valueOf(queryParams.get("pullMode")); + this.pullMode = Boolean.parseBoolean(queryParams.get("pullMode")); try { // checkAuth() and getConsumerConfiguration() should be called after assigning a value to this.subscription diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java index 442980ad336d2..c1054969a4280 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java @@ -244,7 +244,7 @@ public Integer getMaxBlockSizeInBytes() { return Integer.valueOf(configProperties.get(key)); } } - return new Integer(64 * MB); + return 64 * MB; } public Integer getMinBlockSizeInBytes() { @@ -262,7 +262,7 @@ public Integer getReadBufferSizeInBytes() { return Integer.valueOf(configProperties.get(key)); } } - return new Integer(MB); + return MB; } public Integer getWriteBufferSizeInBytes() { From 5277984b2a42e6ace54b86a70195d2a3729a36a3 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 16 Nov 2021 13:16:58 +0800 Subject: [PATCH 145/823] [Pulsar SQL] Handle message null schema version in PulsarRecordCursor (#12809) ### Motivation Currently, if the schema version of the message is null, the Pulsar SQL will encounter an NPE problem. ### Modifications Adjust logic for null schema version in `PulsarRecordCursor`. 1. If the schema type of pulsarSplit is NONE or BYTES, use the BYTES schema. 2. If the schema type of pulsarSplit is BYTEBUFFER, use the BYTEBUFFER schema. 3. If the schema version of the message is null, use the latest schema of the topic. 4. If the schema version of the message is not null, get the specific version schema by PulsarAdmin. 5. If the final schema is null throw a runtime exception. (cherry picked from commit e5619cffce702d9f446c27e69927148e45797b28) --- .../pulsar/sql/presto/PulsarRecordCursor.java | 42 +++++-- .../presto/PulsarSqlSchemaInfoProvider.java | 9 +- .../sql/presto/TestPulsarRecordCursor.java | 107 +++++++++++++++++- 3 files changed, 146 insertions(+), 12 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index f1e2bdb4bb4bf..b1230d3d9c989 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -61,6 +61,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; @@ -479,14 +480,7 @@ public boolean advanceNextPosition() { //start time for deseralizing record metricsTracker.start_RECORD_DESERIALIZE_TIME(); - SchemaInfo schemaInfo = getBytesSchemaInfo(pulsarSplit.getSchemaType(), pulsarSplit.getSchemaName()); - try { - if (schemaInfo == null) { - schemaInfo = schemaInfoProvider.getSchemaByVersion(this.currentMessage.getSchemaVersion()).get(); - } - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } + SchemaInfo schemaInfo = getSchemaInfo(pulsarSplit); Map currentRowValuesMap = new HashMap<>(); @@ -600,6 +594,38 @@ public boolean advanceNextPosition() { return true; } + /** + * Get the schemaInfo of the message. + * + * 1. If the schema type of pulsarSplit is NONE or BYTES, use the BYTES schema. + * 2. If the schema type of pulsarSplit is BYTEBUFFER, use the BYTEBUFFER schema. + * 3. If the schema version of the message is null, use the schema info of pulsarSplit. + * 4. If the schema version of the message is not null, get the specific version schema by PulsarAdmin. + * 5. If the final schema is null throw a runtime exception. + */ + private SchemaInfo getSchemaInfo(PulsarSplit pulsarSplit) { + SchemaInfo schemaInfo = getBytesSchemaInfo(pulsarSplit.getSchemaType(), pulsarSplit.getSchemaName()); + if (schemaInfo != null) { + return schemaInfo; + } + try { + if (this.currentMessage.getSchemaVersion() == null) { + schemaInfo = pulsarSplit.getSchemaInfo(); + } else { + schemaInfo = schemaInfoProvider.getSchemaByVersion(this.currentMessage.getSchemaVersion()).get(); + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + if (schemaInfo == null) { + String schemaVersion = this.currentMessage.getSchemaVersion() == null + ? "null" : BytesSchemaVersion.of(this.currentMessage.getSchemaVersion()).toString(); + throw new RuntimeException("The specific version (" + schemaVersion + ") schema of the table " + + pulsarSplit.getTableName() + " is null"); + } + return schemaInfo; + } + private SchemaInfo getBytesSchemaInfo(SchemaType schemaType, String schemaName) { if (!schemaType.equals(SchemaType.BYTES) && !schemaType.equals(SchemaType.NONE)) { return null; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java index 3a9233c3d872b..828ceef3d44f2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java @@ -102,8 +102,13 @@ private SchemaInfo loadSchema(BytesSchemaVersion bytesSchemaVersion) throws Puls ClassLoader originalContextLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(InjectionManagerFactory.class.getClassLoader()); - return pulsarAdmin.schemas() - .getSchemaInfo(topicName.toString(), ByteBuffer.wrap(bytesSchemaVersion.get()).getLong()); + long version = ByteBuffer.wrap(bytesSchemaVersion.get()).getLong(); + SchemaInfo schemaInfo = pulsarAdmin.schemas().getSchemaInfo(topicName.toString(), version); + if (schemaInfo == null) { + throw new RuntimeException( + "The specific version (" + version + ") schema of the topic " + topicName + " is null"); + } + return schemaInfo; } finally { Thread.currentThread().setContextClassLoader(originalContextLoader); } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 610a1c2e695aa..d60ff20522c3c 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -34,18 +34,31 @@ import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.stats.NullStatsProvider; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.Schemas; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.api.raw.RawMessage; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.common.schema.LongSchemaVersion; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.annotations.Test; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; @@ -56,6 +69,8 @@ import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -65,6 +80,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class TestPulsarRecordCursor extends TestPulsarConnector { @@ -323,9 +339,14 @@ public void run() { MessageMetadata messageMetadata = new MessageMetadata() - .setProducerName("test-producer").setSequenceId(positions.get(topic)) + .setProducerName("test-producer") + .setSequenceId(positions.get(topic)) .setPublishTime(System.currentTimeMillis()); + if (i % 2 == 0) { + messageMetadata.setSchemaVersion(new LongSchemaVersion(1L).bytes()); + } + if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) { messageMetadata .setPartitionKey(new String(schema @@ -380,7 +401,7 @@ public Long answer(InvocationOnMock invocationOnMock) throws Throwable { PulsarSplit split = new PulsarSplit(0, pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), entriesNum, - new String(schema.getSchemaInfo().getSchema()), + new String(schema.getSchemaInfo().getSchema(), "ISO8859-1"), schema.getSchemaInfo().getType(), 0, entriesNum, 0, 0, TupleDomain.all(), @@ -416,4 +437,86 @@ static class Boo { private Double field3; } + @Test + public void testGetSchemaInfo() throws Exception { + String topic = "get-schema-test"; + PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class); + Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName()); + Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default"); + PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class); + Schemas schemas = Mockito.mock(Schemas.class); + Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas); + PulsarConnectorConfig connectorConfig = spy(new PulsarConnectorConfig()); + Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin); + PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor( + new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), + new ManagedLedgerConfig(), null, null)); + + Class clazz = PulsarRecordCursor.class; + Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class); + getSchemaInfo.setAccessible(true); + Field currentMessage = clazz.getDeclaredField("currentMessage"); + currentMessage.setAccessible(true); + RawMessage rawMessage = Mockito.mock(RawMessage.class); + currentMessage.set(pulsarRecordCursor, rawMessage); + + // If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema + Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE); + SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + assertEquals(SchemaType.BYTES, schemaInfo.getType()); + + Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES); + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + assertEquals(SchemaType.BYTES, schemaInfo.getType()); + + Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName()); + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + assertEquals(SchemaType.BYTES, schemaInfo.getType()); + + // If the schemaVersion of the message is not null, try to get the schema. + Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO); + Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes()); + Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))) + .thenReturn(Schema.AVRO(Foo.class).getSchemaInfo()); + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + assertEquals(SchemaType.AVRO, schemaInfo.getType()); + + String schemaTopic = "persistent://public/default/" + topic; + + // If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception. + Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null); + Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null); + try { + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + fail("The message schema version is null and the latest schema is null, should fail."); + } catch (InvocationTargetException e) { + assertTrue(e.getCause() instanceof RuntimeException); + assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null")); + } + + // If the schemaVersion of the message is null, try to get the latest schema. + Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null); + Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo()); + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo); + + // If the specific version schema is null, throw runtime exception. + Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes()); + Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null); + try { + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + fail("The specific version " + 1 + " schema is null, should fail."); + } catch (InvocationTargetException e) { + String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString(); + assertTrue(e.getCause() instanceof RuntimeException); + assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null")); + } + + // Get the specific version schema. + Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes()); + Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo()); + schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); + assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo); + } + } From bd0a40ce06fb4077a9f0e45ad59a9f489d4ddc1e Mon Sep 17 00:00:00 2001 From: xiaolong ran Date: Wed, 17 Nov 2021 17:00:24 +0800 Subject: [PATCH 146/823] Fix znode leakage caused by deleting tenant (#12711) Signed-off-by: xiaolongran Fixes #12710 ### Motivation 1. According to the previous code logic, if the tenant resource is not forcibly deleted, the zk-node resource will be leaked, because under the condition of **!force**, the `clearTenantPersistence` code logic will not be called. ``` protected void internalDeleteTenant(AsyncResponse asyncResponse, String tenant, boolean force) { if (force) { internalDeleteTenantForcefully(asyncResponse, tenant); } else { internalDeleteTenant(asyncResponse, tenant); } } ``` 2. Clear resource of `/namespace/xxx` z-node (cherry picked from commit f68bec4d2a921de00730fce21dd6a78121f7dd3d) --- .../broker/resources/BaseResources.java | 72 +++++++++---- .../resources/LocalPoliciesResources.java | 4 + .../broker/resources/NamespaceResources.java | 70 ++++++++++-- .../broker/admin/impl/NamespacesBase.java | 100 ++++++------------ .../pulsar/broker/admin/impl/TenantsBase.java | 69 +++++------- .../pulsar/broker/admin/AdminApiTest2.java | 90 ++++++++++++++++ 6 files changed, 273 insertions(+), 132 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java index 1e463fad1c954..a6972cd01f4eb 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java @@ -31,6 +31,7 @@ import java.util.function.Function; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -181,37 +182,70 @@ protected static String joinPath(String... parts) { return sb.toString(); } + protected static CompletableFuture deleteRecursiveAsync(BaseResources resources, final String pathRoot) { + PathUtils.validatePath(pathRoot); + CompletableFuture completableFuture = new CompletableFuture<>(); + listSubTreeBFSAsync(resources, pathRoot).whenComplete((tree, ex) -> { + if (ex == null) { + log.debug("Deleting {} with size {}", tree, tree.size()); + + final List> futures = new ArrayList<>(); + for (int i = tree.size() - 1; i >= 0; --i) { + // Delete the leaves first and eventually get rid of the root + futures.add(resources.deleteAsync(tree.get(i))); + } + + FutureUtil.waitForAll(futures).handle((result, exception) -> { + if (exception != null) { + log.error("Failed to remove partitioned topics", exception); + return completableFuture.completeExceptionally(exception.getCause()); + } + return completableFuture.complete(null); + }); + } else { + log.warn("Failed to delete partitioned topics z-node [{}]", pathRoot, ex.getCause()); + } + }); - protected static void deleteRecursive(BaseResources resources, final String pathRoot) throws MetadataStoreException { - PathUtils.validatePath(pathRoot); - List tree = listSubTreeBFS(resources, pathRoot); - log.debug("Deleting {} with size {}", tree, tree.size()); - log.debug("Deleting " + tree.size() + " subnodes "); - for (int i = tree.size() - 1; i >= 0; --i) { - // Delete the leaves first and eventually get rid of the root - resources.delete(tree.get(i)); - } + return completableFuture; } - protected static List listSubTreeBFS(BaseResources resources, final String pathRoot) - throws MetadataStoreException { + protected static CompletableFuture> listSubTreeBFSAsync(BaseResources resources, + final String pathRoot) { Deque queue = new LinkedList<>(); List tree = new ArrayList<>(); queue.add(pathRoot); tree.add(pathRoot); - while (true) { + CompletableFuture> completableFuture = new CompletableFuture<>(); + final List> futures = new ArrayList<>(); + for (int i = 0; i < queue.size(); i++) { String node = queue.pollFirst(); if (node == null) { break; } - List children = resources.getChildren(node); - for (final String child : children) { - final String childPath = node + "/" + child; - queue.add(childPath); - tree.add(childPath); - } + futures.add(resources.getChildrenAsync(node) + .whenComplete((children, ex) -> { + if (ex == null) { + for (final String child : (List) children) { + final String childPath = node + "/" + child; + queue.add(childPath); + tree.add(childPath); + } + } else { + log.warn("Failed to get data error from z-node [{}]", node); + } + })); } - return tree; + + FutureUtil.waitForAll(futures).handle((result, exception) -> { + if (exception != null) { + log.error("Failed to get partitioned topics", exception); + return completableFuture.completeExceptionally(exception.getCause()); + } + return completableFuture.complete(tree); + }); + + return completableFuture; } } \ No newline at end of file diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java index 0023e5fc283eb..29a6a4666e2e4 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java @@ -79,6 +79,10 @@ public void deleteLocalPolicies(NamespaceName ns) throws MetadataStoreException delete(joinPath(LOCAL_POLICIES_ROOT, ns.toString())); } + public CompletableFuture deleteLocalPoliciesAsync(NamespaceName ns) { + return deleteAsync(joinPath(LOCAL_POLICIES_ROOT, ns.toString())); + } + public static boolean isLocalPoliciesPath(String path) { return path.startsWith(LOCAL_POLICIES_ROOT); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index dd33ee831ab73..190d25144daeb 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -26,7 +26,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import lombok.Getter; - import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @@ -39,9 +38,14 @@ import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Getter public class NamespaceResources extends BaseResources { + private static final Logger log = LoggerFactory.getLogger(NamespaceResources.class); + private final IsolationPolicyResources isolationPolicies; private final PartitionedTopicResources partitionedTopicResources; private final MetadataStore configurationStore; @@ -49,6 +53,7 @@ public class NamespaceResources extends BaseResources { private final MetadataCache localPoliciesCache; private static final String POLICIES_READONLY_FLAG_PATH = "/admin/flags/policies-readonly"; + private static final String NAMESPACE_BASE_PATH = "/namespace"; public NamespaceResources(MetadataStore localStore, MetadataStore configurationStore, int operationTimeoutSec) { super(configurationStore, Policies.class, operationTimeoutSec); @@ -97,6 +102,10 @@ public void deletePolicies(NamespaceName ns) throws MetadataStoreException{ delete(joinPath(BASE_POLICIES_PATH, ns.toString())); } + public CompletableFuture deletePoliciesAsync(NamespaceName ns){ + return deleteAsync(joinPath(BASE_POLICIES_PATH, ns.toString())); + } + public Optional getPolicies(NamespaceName ns) throws MetadataStoreException{ return get(joinPath(BASE_POLICIES_PATH, ns.toString())); } @@ -122,6 +131,40 @@ public static boolean pathIsFromNamespace(String path) { && path.substring(BASE_POLICIES_PATH.length() + 1).contains("/"); } + // clear resource of `/namespace/{namespaceName}` for zk-node + public CompletableFuture deleteNamespaceAsync(NamespaceName ns) { + final String namespacePath = joinPath(NAMESPACE_BASE_PATH, ns.toString()); + CompletableFuture future = new CompletableFuture(); + deleteAsync(namespacePath).whenComplete((ignore, ex) -> { + if (ex != null && ex.getCause().getCause() instanceof KeeperException.NoNodeException) { + future.complete(null); + } else if (ex != null) { + future.completeExceptionally(ex); + } else { + future.complete(null); + } + }); + + return future; + } + + // clear resource of `/namespace/{tenant}` for zk-node + public CompletableFuture deleteTenantAsync(String tenant) { + final String tenantPath = joinPath(NAMESPACE_BASE_PATH, tenant); + CompletableFuture future = new CompletableFuture(); + deleteAsync(tenantPath).whenComplete((ignore, ex) -> { + if (ex != null && ex.getCause().getCause() instanceof KeeperException.NoNodeException) { + future.complete(null); + } else if (ex != null) { + future.completeExceptionally(ex); + } else { + future.complete(null); + } + }); + + return future; + } + public static NamespaceName namespaceFromPath(String path) { return NamespaceName.get(path.substring(BASE_POLICIES_PATH.length() + 1)); } @@ -215,12 +258,27 @@ public CompletableFuture deletePartitionedTopicAsync(TopicName tn) { tn.getEncodedLocalName())); } - public void clearPartitionedTopicMetadata(NamespaceName namespaceName) throws MetadataStoreException { + public CompletableFuture clearPartitionedTopicMetadataAsync(NamespaceName namespaceName) { final String globalPartitionedPath = joinPath(PARTITIONED_TOPIC_PATH, namespaceName.toString()); - // check whether partitioned topics metadata node exist - if (exists(globalPartitionedPath)) { - deleteRecursive(this, globalPartitionedPath); - } + + CompletableFuture completableFuture = new CompletableFuture<>(); + + deleteRecursiveAsync(this, globalPartitionedPath) + .thenAccept(ignore -> { + log.info("Clear partitioned topic metadata [{}] success.", namespaceName); + completableFuture.complete(null); + }).exceptionally(ex -> { + if (ex.getCause().getCause() instanceof KeeperException.NoNodeException) { + completableFuture.complete(null); + } else { + log.error("Clear partitioned topic metadata failed."); + completableFuture.completeExceptionally(ex.getCause()); + return null; + } + return null; + }); + + return completableFuture; } } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index dd02f67321942..87c091ae923d9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -298,46 +298,41 @@ protected void internalDeleteNamespace(AsyncResponse asyncResponse, boolean auth } } - try { - namespaceResources().getPartitionedTopicResources().clearPartitionedTopicMetadata(namespaceName); - - try { - pulsar().getPulsarResources().getTopicResources() - .clearDomainPersistence(namespaceName).get(); - pulsar().getPulsarResources().getTopicResources() - .clearNamespacePersistence(namespaceName).get(); - } catch (ExecutionException | InterruptedException e) { - // warn level log here since this failure has no side effect besides left a un-used metadata - // and also will not affect the re-creation of namespace - log.warn("[{}] Failed to remove managed-ledger for {}", clientAppId(), namespaceName, e); - } - - // we have successfully removed all the ownership for the namespace, the policies znode can be deleted - // now - namespaceResources().deletePolicies(namespaceName); - - try { - namespaceResources().deletePolicies(namespaceName); - } catch (NotFoundException e) { - // If the node with the modified information is not there anymore, we're already good - } - - try { - getLocalPolicies().deleteLocalPolicies(namespaceName); - } catch (NotFoundException nne) { - // If the z-node with the modified information is not there anymore, we're already good - } - } catch (Exception e) { - log.error("[{}] Failed to remove owned namespace {} from metadata", clientAppId(), namespaceName, e); - asyncResponse.resume(new RestException(e)); - return null; - } + internalClearZkSources(asyncResponse); - asyncResponse.resume(Response.noContent().build()); return null; }); } + // clear zk-node resources for deleting namespace + protected void internalClearZkSources(AsyncResponse asyncResponse) { + // clear resource of `/namespace/{namespaceName}` for zk-node + namespaceResources().deleteNamespaceAsync(namespaceName) + .thenCompose(ignore -> namespaceResources().getPartitionedTopicResources() + .clearPartitionedTopicMetadataAsync(namespaceName)) + // clear resource for manager-ledger z-node + .thenCompose(ignore -> pulsar().getPulsarResources().getTopicResources() + .clearDomainPersistence(namespaceName)) + .thenCompose(ignore -> pulsar().getPulsarResources().getTopicResources() + .clearNamespacePersistence(namespaceName)) + // we have successfully removed all the ownership for the namespace, the policies + // z-node can be deleted now + .thenCompose(ignore -> namespaceResources().deletePoliciesAsync(namespaceName)) + // clear z-node of local policies + .thenCompose(ignore -> getLocalPolicies().deleteLocalPoliciesAsync(namespaceName)) + .whenComplete((ignore, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to remove namespace or managed-ledger for {}", + clientAppId(), namespaceName, ex); + asyncResponse.resume(new RestException(ex)); + } else { + log.info("[{}] Remove namespace or managed-ledger successfully {}", + clientAppId(), namespaceName); + asyncResponse.resume(Response.noContent().build()); + } + }); + } + @SuppressWarnings("deprecation") protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, boolean authoritative) { validateTenantOperation(namespaceName.getTenant(), TenantOperation.DELETE_NAMESPACE); @@ -474,7 +469,7 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo } } } catch (Exception e) { - log.error("[{}] Failed to remove owned namespace {}", clientAppId(), namespaceName, e); + log.error("[{}] Failed to remove forcefully owned namespace {}", clientAppId(), namespaceName, e); asyncResponse.resume(new RestException(e)); return; } @@ -485,42 +480,15 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo asyncResponse.resume(new RestException((PulsarAdminException) exception.getCause())); return null; } else { - log.error("[{}] Failed to remove owned namespace {}", clientAppId(), namespaceName, exception); + log.error("[{}] Failed to remove forcefully owned namespace {}", + clientAppId(), namespaceName, exception); asyncResponse.resume(new RestException(exception.getCause())); return null; } } - try { - // remove partitioned topics znode - pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .clearPartitionedTopicMetadata(namespaceName); - - try { - pulsar().getPulsarResources().getTopicResources().clearDomainPersistence(namespaceName).get(); - pulsar().getPulsarResources().getTopicResources().clearNamespacePersistence(namespaceName).get(); - } catch (ExecutionException | InterruptedException e) { - // warn level log here since this failure has no side effect besides left a un-used metadata - // and also will not affect the re-creation of namespace - log.warn("[{}] Failed to remove managed-ledger for {}", clientAppId(), namespaceName, e); - } - - // we have successfully removed all the ownership for the namespace, the policies znode can be deleted - // now - namespaceResources().deletePolicies(namespaceName); + internalClearZkSources(asyncResponse); - try { - getLocalPolicies().deleteLocalPolicies(namespaceName); - } catch (NotFoundException nne) { - // If the z-node with the modified information is not there anymore, we're already good - } - } catch (Exception e) { - log.error("[{}] Failed to remove owned namespace {} from ZK", clientAppId(), namespaceName, e); - asyncResponse.resume(new RestException(e)); - return null; - } - - asyncResponse.resume(Response.noContent().build()); return null; }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java index 978e59eb737c0..c5f5402b5916d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.broker.admin.impl; import com.google.common.collect.Lists; @@ -28,7 +29,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -60,8 +60,8 @@ public class TenantsBase extends PulsarWebResource { @GET @ApiOperation(value = "Get the list of existing tenants.", response = String.class, responseContainer = "List") - @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), - @ApiResponse(code = 404, message = "Tenant doesn't exist") }) + @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponse(code = 404, message = "Tenant doesn't exist")}) public void getTenants(@Suspended final AsyncResponse asyncResponse) { final String clientAppId = clientAppId(); try { @@ -86,8 +86,8 @@ public void getTenants(@Suspended final AsyncResponse asyncResponse) { @GET @Path("/{tenant}") @ApiOperation(value = "Get the admin configuration for a given tenant.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), - @ApiResponse(code = 404, message = "Tenant does not exist") }) + @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponse(code = 404, message = "Tenant does not exist")}) public void getTenantAdmin(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "The tenant name") @PathParam("tenant") String tenant) { final String clientAppId = clientAppId(); @@ -112,11 +112,11 @@ public void getTenantAdmin(@Suspended final AsyncResponse asyncResponse, @PUT @Path("/{tenant}") @ApiOperation(value = "Create a new tenant.", notes = "This operation requires Pulsar super-user privileges.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 409, message = "Tenant already exists"), @ApiResponse(code = 412, message = "Tenant name is not valid"), @ApiResponse(code = 412, message = "Clusters can not be empty"), - @ApiResponse(code = 412, message = "Clusters do not exist") }) + @ApiResponse(code = 412, message = "Clusters do not exist")}) public void createTenant(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "The tenant name") @PathParam("tenant") String tenant, @ApiParam(value = "TenantInfo") TenantInfoImpl tenantInfo) { @@ -153,7 +153,7 @@ public void createTenant(@Suspended final AsyncResponse asyncResponse, return; } } - tenantResources().tenantExistsAsync(tenant).thenAccept(exist ->{ + tenantResources().tenantExistsAsync(tenant).thenAccept(exist -> { if (exist) { asyncResponse.resume(new RestException(Status.CONFLICT, "Tenant already exist")); return; @@ -177,12 +177,12 @@ public void createTenant(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}") @ApiOperation(value = "Update the admins for a tenant.", - notes = "This operation requires Pulsar super-user privileges.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + notes = "This operation requires Pulsar super-user privileges.") + @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 404, message = "Tenant does not exist"), @ApiResponse(code = 409, message = "Tenant already exists"), @ApiResponse(code = 412, message = "Clusters can not be empty"), - @ApiResponse(code = 412, message = "Clusters do not exist") }) + @ApiResponse(code = 412, message = "Clusters do not exist")}) public void updateTenant(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "The tenant name") @PathParam("tenant") String tenant, @ApiParam(value = "TenantInfo") TenantInfoImpl newTenantAdmin) { @@ -227,10 +227,10 @@ public void updateTenant(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}") @ApiOperation(value = "Delete a tenant and all namespaces and topics under it.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 404, message = "Tenant does not exist"), @ApiResponse(code = 405, message = "Broker doesn't allow forced deletion of tenants"), - @ApiResponse(code = 409, message = "The tenant still has active namespaces") }) + @ApiResponse(code = 409, message = "The tenant still has active namespaces")}) public void deleteTenant(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") @ApiParam(value = "The tenant name") String tenant, @QueryParam("force") @DefaultValue("false") boolean force) { @@ -258,27 +258,22 @@ protected void internalDeleteTenant(AsyncResponse asyncResponse, String tenant) asyncResponse.resume(new RestException(Status.NOT_FOUND, "Tenant doesn't exist")); return null; } - return hasActiveNamespace(tenant).thenAccept(ns -> { - try { - tenantResources().deleteTenantAsync(tenant) - .thenAccept(t -> { - log.info("[{}] Deleted tenant {}", clientAppId(), tenant); - asyncResponse.resume(Response.noContent().build()); - }).exceptionally(ex -> { - log.error("Failed to delete tenant {}", tenant, ex.getCause()); - asyncResponse.resume(new RestException(ex)); - return null; + + return hasActiveNamespace(tenant) + .thenCompose(ignore -> tenantResources().deleteTenantAsync(tenant)) + .thenCompose(ignore -> pulsar().getPulsarResources().getTopicResources() + .clearTenantPersistence(tenant)) + .thenCompose(ignore -> pulsar().getPulsarResources().getNamespaceResources() + .deleteTenantAsync(tenant)) + .whenComplete((ignore, ex) -> { + if (ex != null) { + log.error("[{}] Failed to delete tenant {}", clientAppId(), tenant, ex); + asyncResponse.resume(new RestException(ex)); + } else { + log.info("[{}] Deleted tenant {}", clientAppId(), tenant); + asyncResponse.resume(Response.noContent().build()); + } }); - log.info("[{}] Deleted tenant {}", clientAppId(), tenant); - } catch (Exception e) { - log.error("[{}] Failed to delete tenant {}", clientAppId(), tenant, e); - asyncResponse.resume(new RestException(e)); - } - }).exceptionally(ex -> { - log.error("Failed to delete tenant due to active namespace {}", tenant, ex.getCause()); - asyncResponse.resume(new RestException(ex)); - return null; - }); }); } @@ -319,14 +314,6 @@ protected void internalDeleteTenantForcefully(AsyncResponse asyncResponse, Strin return null; } - - try { - pulsar().getPulsarResources().getTopicResources().clearTenantPersistence(tenant).get(); - } catch (ExecutionException | InterruptedException e) { - // warn level log here since this failure has no side effect besides left a un-used metadata - // and also will not affect the re-creation of tenant - log.warn("[{}] Failed to remove managed-ledger for {}", clientAppId(), tenant, e); - } // delete tenant normally internalDeleteTenant(asyncResponse, tenant); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java index 33efa2bc08b1d..bc1ab256cfb3d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java @@ -1267,6 +1267,96 @@ public void testPreciseBacklog() throws PulsarClientException, PulsarAdminExcept assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 9); } + @Test + public void testDeleteTenant() throws Exception { + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + + String tenant = "test-tenant-1"; + assertFalse(admin.tenants().getTenants().contains(tenant)); + + // create tenant + admin.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"))); + assertTrue(admin.tenants().getTenants().contains(tenant)); + + // create namespace + String namespace = tenant + "/test-ns-1"; + admin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); + assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList(namespace)); + + // create topic + String topic = namespace + "/test-topic-1"; + admin.topics().createPartitionedTopic(topic, 10); + assertFalse(admin.topics().getList(namespace).isEmpty()); + + try { + admin.namespaces().deleteNamespace(namespace, false); + fail("should have failed due to namespace not empty"); + } catch (PulsarAdminException e) { + // Expected: cannot delete non-empty tenant + } + + // delete topic + admin.topics().deletePartitionedTopic(topic); + assertTrue(admin.topics().getList(namespace).isEmpty()); + + // delete namespace + admin.namespaces().deleteNamespace(namespace, false); + assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace)); + assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty()); + + // delete tenant + admin.tenants().deleteTenant(tenant); + assertFalse(admin.tenants().getTenants().contains(tenant)); + + final String managedLedgersPath = "/managed-ledgers/" + tenant; + assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); + } + + @Test + public void testDeleteNamespace() throws Exception { + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + + String tenant = "test-tenant"; + assertFalse(admin.tenants().getTenants().contains(tenant)); + + // create tenant + admin.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"))); + assertTrue(admin.tenants().getTenants().contains(tenant)); + + // create namespace + String namespace = tenant + "/test-ns"; + admin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); + assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList(namespace)); + + // create topic + String topic = namespace + "/test-topic"; + admin.topics().createPartitionedTopic(topic, 10); + assertFalse(admin.topics().getList(namespace).isEmpty()); + + try { + admin.namespaces().deleteNamespace(namespace, false); + fail("should have failed due to namespace not empty"); + } catch (PulsarAdminException e) { + // Expected: cannot delete non-empty tenant + } + + // delete topic + admin.topics().deletePartitionedTopic(topic); + assertTrue(admin.topics().getList(namespace).isEmpty()); + + // delete namespace + admin.namespaces().deleteNamespace(namespace, false); + assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace)); + assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty()); + + + final String managedLedgersPath = "/managed-ledgers/" + namespace; + assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); + } + + @Test(timeOut = 30000) public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminException, InterruptedException { final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-" + UUID.randomUUID().toString(); From 67d210d2594db69a471905b5d70e23f814ff99f3 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Wed, 17 Nov 2021 16:52:05 +0800 Subject: [PATCH 147/823] Improve exception info for invaild time-related option (#12828) (cherry picked from commit 60f5475740f129d02d3896b31bddf001b9ea6704) --- .../utils/auth/tokens/TokensCliUtils.java | 10 ++++- .../pulsar/admin/cli/CmdNamespaces.java | 39 ++++++++++++++++--- .../pulsar/admin/cli/CmdPersistentTopics.java | 10 ++++- .../apache/pulsar/admin/cli/CmdTopics.java | 32 ++++++++++++--- .../pulsar/admin/cli/CmdTransactions.java | 9 ++++- 5 files changed, 84 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java index c089fa09953d4..47364bc68bdb4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java @@ -22,6 +22,7 @@ import com.beust.jcommander.IUsageFormatter; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.google.common.base.Charsets; import io.jsonwebtoken.Claims; @@ -155,8 +156,13 @@ public void run() throws Exception { Optional optExpiryTime = Optional.empty(); if (expiryTime != null) { - long relativeTimeMillis = TimeUnit.SECONDS - .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(expiryTime)); + long relativeTimeMillis; + try { + relativeTimeMillis = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(expiryTime)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } optExpiryTime = Optional.of(new Date(System.currentTimeMillis() + relativeTimeMillis)); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index a326f82b19d1a..f9f94c379e9ce 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -697,7 +697,12 @@ private class SetRetention extends CliCommand { void run() throws PulsarAdminException { String namespace = validateNamespace(params); long sizeLimit = validateSizeString(limitStr); - long retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); + long retentionTimeInSec; + try { + retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } final int retentionTimeInMin; if (retentionTimeInSec != -1) { @@ -1392,7 +1397,13 @@ private class SetInactiveTopicPolicies extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds(RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); + long maxInactiveDurationInSeconds; + try { + maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds( + RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } if (enableDeleteWhileInactive == disableDeleteWhileInactive) { throw new ParameterException("Need to specify either enable-delete-while-inactive or disable-delete-while-inactive"); @@ -1425,7 +1436,13 @@ private class SetDelayedDelivery extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); + long delayedDeliveryTimeInMills; + try { + delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -1795,7 +1812,13 @@ private class SetOffloadDeletionLag extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - getAdmin().namespaces().setOffloadDeleteLag(namespace, RelativeTimeUtil.parseRelativeTimeInSeconds(lag), + long lagInSec; + try { + lagInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(lag); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } + getAdmin().namespaces().setOffloadDeleteLag(namespace, lagInSec, TimeUnit.SECONDS); } } @@ -2120,7 +2143,13 @@ && maxValueCheck("ReadBufferSize", readBufferSize, Integer.MAX_VALUE)) { Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; if (StringUtils.isNotEmpty(offloadAfterElapsedStr)) { - Long offloadAfterElapsed = TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(offloadAfterElapsedStr)); + Long offloadAfterElapsed; + try { + offloadAfterElapsed = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(offloadAfterElapsedStr)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } if (positiveCheck("OffloadAfterElapsed", offloadAfterElapsed) && maxValueCheck("OffloadAfterElapsed", offloadAfterElapsed, Long.MAX_VALUE)) { offloadAfterElapsedInMillis = offloadAfterElapsed; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java index 759b602cc0605..611c1b08be490 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java @@ -20,6 +20,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.beust.jcommander.converters.CommaParameterSplitter; import com.google.gson.Gson; @@ -528,8 +529,13 @@ void run() throws PulsarAdminException { MessageId messageId = validateMessageIdString(resetMessageIdStr); getPersistentTopics().resetCursor(persistentTopic, subName, messageId); } else if (isNotBlank(resetTimeStr)) { - long resetTimeInMillis = TimeUnit.SECONDS - .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); + long resetTimeInMillis; + try { + resetTimeInMillis = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } // now - go back time long timestamp = System.currentTimeMillis() - resetTimeInMillis; getPersistentTopics().resetCursor(persistentTopic, subName, timestamp); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index d0ef10c85de27..e37954de7419b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -792,8 +792,13 @@ void run() throws PulsarAdminException { getTopics().resetCursor(persistentTopic, subName, messageId); } } else if (isNotBlank(resetTimeStr)) { - long resetTimeInMillis = TimeUnit.SECONDS - .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); + long resetTimeInMillis; + try { + resetTimeInMillis = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } // now - go back time long timestamp = System.currentTimeMillis() - resetTimeInMillis; getTopics().resetCursor(persistentTopic, subName, timestamp); @@ -1280,7 +1285,13 @@ private class SetDelayedDelivery extends CliCommand { @Override void run() throws PulsarAdminException { String topicName = validateTopicName(params); - long delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); + long delayedDeliveryTimeInMills; + try { + delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -1430,7 +1441,12 @@ private class SetRetention extends CliCommand { void run() throws PulsarAdminException { String persistentTopic = validatePersistentTopic(params); long sizeLimit = validateSizeString(limitStr); - long retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); + long retentionTimeInSec; + try { + retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } final int retentionTimeInMin; if (retentionTimeInSec != -1) { @@ -2287,7 +2303,13 @@ private class SetInactiveTopicPolicies extends CliCommand { @Override void run() throws PulsarAdminException { String persistentTopic = validatePersistentTopic(params); - long maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds(RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); + long maxInactiveDurationInSeconds; + try { + maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds( + RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } if (enableDeleteWhileInactive == disableDeleteWhileInactive) { throw new ParameterException("Need to specify either enable-delete-while-inactive or disable-delete-while-inactive"); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java index 7faf006066ca6..e6953817b0d26 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java @@ -19,6 +19,7 @@ package org.apache.pulsar.admin.cli; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -133,8 +134,12 @@ private class GetSlowTransactions extends CliCommand { @Override void run() throws Exception { - long timeout = - TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(timeoutStr)); + long timeout; + try { + timeout = TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(timeoutStr)); + } catch (IllegalArgumentException exception) { + throw new ParameterException(exception.getMessage()); + } if (coordinatorId != null) { print(getAdmin().transactions().getSlowTransactionsByCoordinatorId(coordinatorId, timeout, TimeUnit.MILLISECONDS)); From b9629cd86db4755a445fb8817496a685784c7d03 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 18 Nov 2021 15:28:07 +0800 Subject: [PATCH 148/823] The problem of two exception handling (#12744) whenCompleteAsync has handle exception, don't use exceptionally, otherwise it will be handle twice (cherry picked from commit fc8d50ecf841bd1a4b01fa09720411f6190ee5be) --- .../mledger/impl/EntryCacheImpl.java | 19 ++----------------- .../mledger/impl/EntryCacheManager.java | 3 --- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java index 6a0ac2c650c49..6d54abbf95d32 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java @@ -234,12 +234,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt } finally { ledgerEntries.close(); } - }, ml.getExecutor().chooseThread(ml.getName())).exceptionally(exception->{ - ml.invalidateLedgerHandle(lh); - callback.readEntryFailed(createManagedLedgerException(exception), ctx); - return null; - } - ); + }, ml.getExecutor().chooseThread(ml.getName())); } } @@ -333,17 +328,7 @@ private void asyncReadEntry0(ReadHandle lh, long firstEntry, long lastEntry, boo } finally { ledgerEntries.close(); } - }, ml.getExecutor().chooseThread(ml.getName())).exceptionally(exception->{ - if (exception instanceof BKException - && ((BKException)exception).getCode() == BKException.Code.TooManyRequestsException) { - callback.readEntriesFailed(createManagedLedgerException(exception), ctx); - } else { - ml.invalidateLedgerHandle(lh); - ManagedLedgerException mlException = createManagedLedgerException(exception); - callback.readEntriesFailed(mlException, ctx); - } - return null; - }); + }, ml.getExecutor().chooseThread(ml.getName())); } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java index c87bcb8aa4031..d360bbd6e5cee 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java @@ -215,9 +215,6 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole ml.mbean.addReadEntriesSample(entries.size(), totalSize); callback.readEntriesComplete(entries, ctx); - }).exceptionally(exception -> { - callback.readEntriesFailed(createManagedLedgerException(exception), ctx); - return null; }); } From cd2c4c1de2dcaea74b6e48bfe9b408a57e9be5e8 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 18 Nov 2021 13:44:42 +0800 Subject: [PATCH 149/823] Fix TopicPoliciesCacheNotInitException issue. (#12773) ### Motivation Sometimes, we may get `TopicPoliciesCacheNotInitException` with below stack trace: ``` 15:45:47.020 [pulsar-web-41-3] INFO org.eclipse.jetty.server.RequestLog - 10.0.0.42 - - [10/Nov/2021:15:45:47 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.19+" 1 15:45:51.221 [pulsar-2-15] ERROR org.apache.pulsar.broker.admin.impl.PersistentTopicsBase - [null] Failed to perform getRetention on topic persistent://public/default/UpdateNodeCharts java.lang.RuntimeException: org.apache.pulsar.broker.service.BrokerServiceException$TopicPoliciesCacheNotInitException: Topic policies cache have not init. at org.apache.pulsar.broker.service.TopicPoliciesService.lambda$getTopicPoliciesAsyncWithRetry$0(TopicPoliciesService.java:84) ~[io.streamnative-pulsar-broker-2.8.1.21.jar:2.8.1.21] at org.apache.pulsar.client.util.RetryUtil.executeWithRetry(RetryUtil.java:50) ~[io.streamnative-pulsar-client-original-2.8.1.21.jar:2.8.1.21] at org.apache.pulsar.client.util.RetryUtil.lambda$executeWithRetry$1(RetryUtil.java:63) ~[io.streamnative-pulsar-client-original-2.8.1.21.jar:2.8.1.21] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:829) [?:?] ``` This is because : https://github.com/apache/pulsar/blob/c3da1452a444c9599cb85562a3faa82ddfdecec8/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java#L298-L312 when `reader.readNextAsync()` throws exceptions, the msg will be null which will throw NPE without any catch block. (cherry picked from commit 11298144ac118cda951deffa092ab17110d254b7) --- .../TransactionMetadataStoreService.java | 5 +-- .../SystemTopicBasedTopicPoliciesService.java | 42 ++++++++++++------- ...temTopicBasedTopicPoliciesServiceTest.java | 40 ++++++++++++++++++ .../impl/MLTransactionLogImpl.java | 8 +++- 4 files changed, 76 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 607f05e481673..240c6c9ad7c75 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker; +import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.getMLTransactionLogName; import static org.apache.pulsar.transaction.coordinator.proto.TxnStatus.ABORTING; import static org.apache.pulsar.transaction.coordinator.proto.TxnStatus.COMMITTING; import com.google.common.annotations.VisibleForTesting; @@ -63,7 +64,6 @@ import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.CoordinatorNotFoundException; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.InvalidTxnStatusException; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.TransactionMetadataStoreStateException; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -230,8 +230,7 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc public CompletableFuture openTransactionMetadataStore(TransactionCoordinatorID tcId) { return pulsarService.getBrokerService() - .getManagedLedgerConfig(TopicName.get(MLTransactionLogImpl - .TRANSACTION_LOG_PREFIX + tcId)).thenCompose(v -> { + .getManagedLedgerConfig(getMLTransactionLogName(tcId)).thenCompose(v -> { TransactionTimeoutTracker timeoutTracker = timeoutTrackerFactory.newTracker(tcId); TransactionRecoverTracker recoverTracker = new TransactionRecoverTrackerImpl(TransactionMetadataStoreService.this, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 7055682638fca..4dbffece6023e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -160,6 +160,10 @@ private void notifyListener(Message msg) { @Override public TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesCacheNotInitException { + if (!policyCacheInitMap.containsKey(topicName.getNamespaceObject())) { + NamespaceName namespace = topicName.getNamespaceObject(); + prepareInitPoliciesCache(namespace, new CompletableFuture<>()); + } if (policyCacheInitMap.containsKey(topicName.getNamespaceObject()) && !policyCacheInitMap.get(topicName.getNamespaceObject())) { throw new TopicPoliciesCacheNotInitException(); @@ -198,24 +202,29 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name result.complete(null); } else { ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); - policyCacheInitMap.put(namespace, false); - CompletableFuture> readerCompletableFuture = - creatSystemTopicClientWithRetry(namespace); - readerCaches.put(namespace, readerCompletableFuture); - readerCompletableFuture.whenComplete((reader, ex) -> { - if (ex != null) { - log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); - result.completeExceptionally(ex); - } else { - initPolicesCache(reader, result); - result.thenRun(() -> readMorePolicies(reader)); - } - }); + prepareInitPoliciesCache(namespace, result); } } return result; } + private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture result) { + if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { + CompletableFuture> readerCompletableFuture = + creatSystemTopicClientWithRetry(namespace); + readerCaches.put(namespace, readerCompletableFuture); + readerCompletableFuture.whenComplete((reader, ex) -> { + if (ex != null) { + log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); + result.completeExceptionally(ex); + } else { + initPolicesCache(reader, result); + result.thenRun(() -> readMorePolicies(reader)); + } + }); + } + } + protected CompletableFuture> creatSystemTopicClientWithRetry( NamespaceName namespace) { SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory @@ -284,6 +293,9 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp reader.getSystemTopic().getTopicName(), ex); future.completeExceptionally(ex); readerCaches.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); + policyCacheInitMap.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); + reader.closeAsync(); + return; } if (hasMore) { reader.readNextAsync().whenComplete((msg, e) -> { @@ -292,6 +304,9 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp reader.getSystemTopic().getTopicName(), ex); future.completeExceptionally(e); readerCaches.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); + policyCacheInitMap.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); + reader.closeAsync(); + return; } refreshTopicPoliciesCache(msg); if (log.isDebugEnabled()) { @@ -306,7 +321,6 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } policyCacheInitMap.computeIfPresent( reader.getSystemTopic().getTopicName().getNamespaceObject(), (k, v) -> true); - // replay policy message policiesCache.forEach(((topicName, topicPolicies) -> { if (listeners.get(topicName) != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 49d86993e330f..80f3dc93e86d3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -32,8 +32,12 @@ import java.lang.reflect.Field; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; @@ -279,4 +283,40 @@ public void testCreatSystemTopicClientWithRetry() throws Exception { assertEquals(reader1, reader); } + + @Test + public void testGetTopicPoliciesWithRetry() throws Exception { + Field initMapField = SystemTopicBasedTopicPoliciesService.class.getDeclaredField("policyCacheInitMap"); + initMapField.setAccessible(true); + Map initMap = (Map)initMapField.get(systemTopicBasedTopicPoliciesService); + initMap.remove(NamespaceName.get(NAMESPACE1)); + Field readerCaches = SystemTopicBasedTopicPoliciesService.class.getDeclaredField("readerCaches"); + readerCaches.setAccessible(true); + Map>> readers = (Map)readerCaches.get(systemTopicBasedTopicPoliciesService); + readers.remove(NamespaceName.get(NAMESPACE1)); + Backoff backoff = new BackoffBuilder() + .setInitialTime(500, TimeUnit.MILLISECONDS) + .setMandatoryStop(5000, TimeUnit.MILLISECONDS) + .setMax(1000, TimeUnit.MILLISECONDS) + .create(); + TopicPolicies initPolicy = TopicPolicies.builder() + .maxConsumerPerTopic(10) + .build(); + ScheduledExecutorService executors = Executors.newScheduledThreadPool(1); + executors.schedule(new Runnable() { + @Override + public void run() { + try { + systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC1, initPolicy).get(); + } catch (Exception ignore) {} + } + }, 2000, TimeUnit.MILLISECONDS); + Awaitility.await().untilAsserted(() -> { + Optional topicPolicies = systemTopicBasedTopicPoliciesService.getTopicPoliciesAsyncWithRetry(TOPIC1, backoff, pulsar.getExecutor()).get(); + Assert.assertTrue(topicPolicies.isPresent()); + if (topicPolicies.isPresent()) { + Assert.assertEquals(topicPolicies.get(), initPolicy); + } + }); + } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index f2324afec82ee..c0442753e7a7b 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -73,8 +73,7 @@ public class MLTransactionLogImpl implements TransactionLog { public MLTransactionLogImpl(TransactionCoordinatorID tcID, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig) { - this.topicName = TopicName.get(TopicDomain.persistent.value(), - NamespaceName.SYSTEM_NAMESPACE, TRANSACTION_LOG_PREFIX + tcID.getId()); + this.topicName = getMLTransactionLogName(tcID); this.tcId = tcID.getId(); this.mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); managedLedgerConfig.setManagedLedgerInterceptor(this.mlTransactionLogInterceptor); @@ -83,6 +82,11 @@ public MLTransactionLogImpl(TransactionCoordinatorID tcID, this.entryQueue = new SpscArrayQueue<>(2000); } + public static TopicName getMLTransactionLogName(TransactionCoordinatorID tcID) { + return TopicName.get(TopicDomain.persistent.value(), + NamespaceName.SYSTEM_NAMESPACE, TRANSACTION_LOG_PREFIX + tcID.getId()); + } + @Override public CompletableFuture initialize() { CompletableFuture future = new CompletableFuture<>(); From 012e1b1a5541d0ca17c65efc1f4bb905ad76ea7b Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Thu, 18 Nov 2021 20:37:37 +0800 Subject: [PATCH 150/823] [Issue 12757][broker] add broker config isAllowAutoUpdateSchema (#12786) (cherry picked from commit fa7be236efcc6772e0aac05f25f8d5f3cf0ad741) --- conf/broker.conf | 4 + conf/standalone.conf | 4 + .../pulsar/broker/ServiceConfiguration.java | 8 ++ .../pulsar/broker/service/AbstractTopic.java | 38 +++++---- .../SchemaCompatibilityCheckTest.java | 81 +++++++++++++++++++ .../pulsar/common/policies/data/Policies.java | 2 +- 6 files changed, 121 insertions(+), 16 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 3dd85905055e2..2d7df90c611ef 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -270,6 +270,10 @@ brokerMaxConnections=0 # The maximum number of connections per IP. If it exceeds, new connections are rejected. brokerMaxConnectionsPerIp=0 +# Allow schema to be auto updated at broker level. User can override this by +# 'is_allow_auto_update_schema' of namespace policy. +isAllowAutoUpdateSchemaEnabled=true + # Enable check for minimum allowed client library version clientLibraryVersionCheckEnabled=false diff --git a/conf/standalone.conf b/conf/standalone.conf index 878a852a586e9..577a6ffad42cd 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -176,6 +176,10 @@ defaultNumberOfNamespaceBundles=4 # Using a value of 0, is disabling maxTopicsPerNamespace-limit check. maxTopicsPerNamespace=0 +# Allow schema to be auto updated at broker level. User can override this by +# 'is_allow_auto_update_schema' of namespace policy. +isAllowAutoUpdateSchemaEnabled=true + # Enable check for minimum allowed client library version clientLibraryVersionCheckEnabled=false diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 69a3d35645713..0d578f4da369c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -572,6 +572,14 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private int brokerMaxConnectionsPerIp = 0; + @FieldContext( + category = CATEGORY_POLICIES, + dynamic = true, + doc = "Allow schema to be auto updated at broker level. User can override this by 'is_allow_auto_update_schema'" + + " of namespace policy. This is enabled by default." + ) + private boolean isAllowAutoUpdateSchemaEnabled = true; + @FieldContext( category = CATEGORY_SERVER, dynamic = true, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 9463be9f8fefc..30c0faeab100a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -95,7 +95,7 @@ public abstract class AbstractTopic implements Topic { protected volatile boolean isEncryptionRequired = false; protected volatile SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.FULL; - protected volatile boolean isAllowAutoUpdateSchema = true; + protected volatile Boolean isAllowAutoUpdateSchema; // schema validation enforced flag protected volatile boolean schemaValidationEnforced = false; @@ -328,20 +328,28 @@ public CompletableFuture addSchema(SchemaData schema) { String base = TopicName.get(getName()).getPartitionedTopicName(); String id = TopicName.get(base).getSchemaName(); SchemaRegistryService schemaRegistryService = brokerService.pulsar().getSchemaRegistryService(); - return isAllowAutoUpdateSchema ? schemaRegistryService - .putSchemaIfAbsent(id, schema, schemaCompatibilityStrategy) - : schemaRegistryService.trimDeletedSchemaAndGetList(id).thenCompose(schemaAndMetadataList -> - schemaRegistryService.getSchemaVersionBySchemaData(schemaAndMetadataList, schema) - .thenCompose(schemaVersion -> { - if (schemaVersion == null) { - return FutureUtil - .failedFuture( - new IncompatibleSchemaException( - "Schema not found and schema auto updating is disabled.")); - } else { - return CompletableFuture.completedFuture(schemaVersion); - } - })); + + if (allowAutoUpdateSchema()) { + return schemaRegistryService.putSchemaIfAbsent(id, schema, schemaCompatibilityStrategy); + } else { + return schemaRegistryService.trimDeletedSchemaAndGetList(id).thenCompose(schemaAndMetadataList -> + schemaRegistryService.getSchemaVersionBySchemaData(schemaAndMetadataList, schema) + .thenCompose(schemaVersion -> { + if (schemaVersion == null) { + return FutureUtil.failedFuture(new IncompatibleSchemaException( + "Schema not found and schema auto updating is disabled.")); + } else { + return CompletableFuture.completedFuture(schemaVersion); + } + })); + } + } + + private boolean allowAutoUpdateSchema() { + if (isAllowAutoUpdateSchema == null) { + return brokerService.pulsar().getConfig().isAllowAutoUpdateSchemaEnabled(); + } + return isAllowAutoUpdateSchema; } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 293f71d5f878f..80168b9ae4c9f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -218,6 +218,87 @@ public void testConsumerCompatibilityReadAllCheckTest(SchemaCompatibilityStrateg } } + @Test(dataProvider = "AllCheckSchemaCompatibilityStrategy") + public void testBrokerAllowAutoUpdateSchemaDisabled(SchemaCompatibilityStrategy schemaCompatibilityStrategy) + throws Exception { + + final String tenant = PUBLIC_TENANT; + final String topic = "test-consumer-compatibility"; + String namespace = "test-namespace-" + randomName(16); + String fqtn = TopicName.get( + TopicDomain.persistent.value(), + tenant, + namespace, + topic + ).toString(); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(CLUSTER_NAME) + ); + + assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()), + SchemaCompatibilityStrategy.FULL); + + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); + admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + + + pulsar.getConfig().setAllowAutoUpdateSchemaEnabled(false); + ProducerBuilder producerThreeBuilder = pulsarClient + .newProducer(Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonTwo.class).build())) + .topic(fqtn); + try { + producerThreeBuilder.create(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled.")); + } + + pulsar.getConfig().setAllowAutoUpdateSchemaEnabled(true); + ConsumerBuilder comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO( + SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonTwo.class).build())) + .subscriptionName("test") + .topic(fqtn); + + Producer producer = producerThreeBuilder.create(); + Consumer consumerTwo = comsumerBuilder.subscribe(); + + producer.send(new Schemas.PersonTwo(2, "Lucy")); + Message message = consumerTwo.receive(); + + Schemas.PersonTwo personTwo = message.getValue(); + consumerTwo.acknowledge(message); + + assertEquals(personTwo.getId(), 2); + assertEquals(personTwo.getName(), "Lucy"); + + producer.close(); + consumerTwo.close(); + + pulsar.getConfig().setAllowAutoUpdateSchemaEnabled(false); + + producer = producerThreeBuilder.create(); + consumerTwo = comsumerBuilder.subscribe(); + + producer.send(new Schemas.PersonTwo(2, "Lucy")); + message = consumerTwo.receive(); + + personTwo = message.getValue(); + consumerTwo.acknowledge(message); + + assertEquals(personTwo.getId(), 2); + assertEquals(personTwo.getName(), "Lucy"); + + consumerTwo.close(); + producer.close(); + } + @Test(dataProvider = "AllCheckSchemaCompatibilityStrategy") public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { final String tenant = PUBLIC_TENANT; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 631675fcbf729..d26c61eefe838 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -108,7 +108,7 @@ public class Policies { public SchemaCompatibilityStrategy schema_compatibility_strategy = SchemaCompatibilityStrategy.UNDEFINED; @SuppressWarnings("checkstyle:MemberName") - public boolean is_allow_auto_update_schema = true; + public Boolean is_allow_auto_update_schema = null; @SuppressWarnings("checkstyle:MemberName") public boolean schema_validation_enforced = false; From 28dcdc380c409e09408079c4d34abd724d51bc25 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Thu, 18 Nov 2021 16:04:22 +0800 Subject: [PATCH 151/823] JavaInstanceTest should be AssertEquals (#12836) (cherry picked from commit 4a4c1de6174c41ccf630fd17ea12d033adde9dd9) --- .../apache/pulsar/functions/instance/JavaInstanceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java index 03c0da547bd97..9931ddf4ce839 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java @@ -34,6 +34,7 @@ import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.instance.JavaInstance.AsyncFuncRequest; +import org.testng.Assert; import org.testng.annotations.Test; @Slf4j @@ -226,7 +227,7 @@ public void testAsyncFunctionMaxPending() throws Exception { for (int i = 0; i < 3; i++) { AsyncFuncRequest request = instance.getPendingAsyncRequests().poll(); - assertNotNull(testString + "-lambda", (String) request.getProcessResult().get()); + Assert.assertEquals(request.getProcessResult().get(), testString + "-lambda"); } long endTime = System.currentTimeMillis(); @@ -235,7 +236,6 @@ public void testAsyncFunctionMaxPending() throws Exception { instance.close(); } - @SuppressWarnings("serial") private static class UserException extends Exception { public UserException(String msg) { super(msg); From 1d58b1fd86d79408e543d5d8df3996fd6e1156d7 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Thu, 18 Nov 2021 16:05:23 +0800 Subject: [PATCH 152/823] Add error log when new jetty client (#12840) (cherry picked from commit c18063f44684052ef07f0259084743a7a45e5656) --- .../org/apache/pulsar/proxy/server/AdminProxyHandler.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 13f9372fa1dea..7d3c658a12ac8 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -256,7 +256,7 @@ protected HttpClient newHttpClient() { if (config.isTlsEnabledWithBroker()) { try { - X509Certificate trustCertificates[] = SecurityUtility + X509Certificate[] trustCertificates = SecurityUtility .loadCertificatesFromPemFile(config.getBrokerClientTrustCertsFilePath()); SSLContext sslCtx; @@ -281,6 +281,7 @@ protected HttpClient newHttpClient() { return new JettyHttpClient(contextFactory); } catch (Exception e) { + LOG.error("new jetty http client exception ", e); try { auth.close(); } catch (IOException ioe) { @@ -303,7 +304,7 @@ protected String rewriteTarget(HttpServletRequest request) { boolean isFunctionsRestRequest = false; String requestUri = request.getRequestURI(); - for (String routePrefix: functionRoutes) { + for (String routePrefix : functionRoutes) { if (requestUri.startsWith(routePrefix)) { isFunctionsRestRequest = true; break; @@ -324,7 +325,7 @@ protected String rewriteTarget(HttpServletRequest request) { if (LOG.isDebugEnabled()) { LOG.debug("[{}:{}] Selected active broker is {}", request.getRemoteAddr(), request.getRemotePort(), - url.toString()); + url); } } catch (Exception e) { LOG.warn("[{}:{}] Failed to get next active broker {}", request.getRemoteAddr(), From 18698648d0f62e89262a7cc525f2d6e380afde62 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Fri, 19 Nov 2021 18:38:45 +0800 Subject: [PATCH 153/823] Remove unused listeners if it have no listeners. (#12654) (cherry picked from commit d74af88a6aed5a7da3139a4228ae29f793ec72b2) --- .../SystemTopicBasedTopicPoliciesService.java | 21 +++++++++-- ...temTopicBasedTopicPoliciesServiceTest.java | 36 +++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 4dbffece6023e..902784e0f0e00 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -69,7 +69,8 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic @VisibleForTesting final Map policyCacheInitMap = new ConcurrentHashMap<>(); - private final Map>> listeners = new ConcurrentHashMap<>(); + @VisibleForTesting + final Map>> listeners = new ConcurrentHashMap<>(); public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { this.pulsarService = pulsarService; @@ -472,12 +473,26 @@ Boolean getPoliciesCacheInit(NamespaceName namespaceName) { @Override public void registerListener(TopicName topicName, TopicPolicyListener listener) { - listeners.computeIfAbsent(topicName, k -> Lists.newCopyOnWriteArrayList()).add(listener); + listeners.compute(topicName, (k, topicListeners) -> { + if (topicListeners == null) { + topicListeners = Lists.newCopyOnWriteArrayList(); + } + topicListeners.add(listener); + return topicListeners; + }); } @Override public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { - listeners.computeIfAbsent(topicName, k -> Lists.newCopyOnWriteArrayList()).remove(listener); + listeners.compute(topicName, (k, topicListeners) -> { + if (topicListeners != null){ + topicListeners.remove(listener); + if (topicListeners.isEmpty()) { + topicListeners = null; + } + } + return topicListeners; + }); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 80f3dc93e86d3..be52f9a119f8f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -76,7 +76,6 @@ public class SystemTopicBasedTopicPoliciesServiceTest extends MockedPulsarServic private static final TopicName TOPIC5 = TopicName.get("persistent", NamespaceName.get(NAMESPACE3), "topic-1"); private static final TopicName TOPIC6 = TopicName.get("persistent", NamespaceName.get(NAMESPACE3), "topic-2"); - private NamespaceEventsSystemTopicFactory systemTopicFactory; private SystemTopicBasedTopicPoliciesService systemTopicBasedTopicPoliciesService; @BeforeMethod(alwaysRun = true) @@ -94,6 +93,40 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testConcurrentlyRegisterUnregisterListeners() throws ExecutionException, InterruptedException { + TopicName topicName = TopicName.get("test"); + class TopicPolicyListenerImpl implements TopicPolicyListener { + + @Override + public void onUpdate(TopicPolicies data) { + //no op. + } + } + + CompletableFuture f = CompletableFuture.completedFuture(null).thenRunAsync(() -> { + for (int i = 0; i < 100; i++) { + TopicPolicyListener listener = new TopicPolicyListenerImpl(); + systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); + Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); + Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); + systemTopicBasedTopicPoliciesService.unregisterListener(topicName, listener); + } + }); + + for (int i = 0; i < 100; i++) { + TopicPolicyListener listener = new TopicPolicyListenerImpl(); + systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); + Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); + Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); + systemTopicBasedTopicPoliciesService.unregisterListener(topicName, listener); + } + + f.get(); + //Some system topics will be added to the listeners. Just check if it contains topicName. + Assert.assertFalse(systemTopicBasedTopicPoliciesService.listeners.containsKey(topicName)); + } + @Test public void testGetPolicy() throws ExecutionException, InterruptedException, TopicPoliciesCacheNotInitException { @@ -239,7 +272,6 @@ private void prepareData() throws PulsarAdminException { admin.lookups().lookupTopic(TOPIC4.toString()); admin.lookups().lookupTopic(TOPIC5.toString()); admin.lookups().lookupTopic(TOPIC6.toString()); - systemTopicFactory = new NamespaceEventsSystemTopicFactory(pulsarClient); systemTopicBasedTopicPoliciesService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); } From a0db36fd5c54e05180709f19317c0dfcdbb31063 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 19 Nov 2021 22:56:42 +0800 Subject: [PATCH 154/823] Fix deleting tenants with active namespaces with 500. (#12848) (cherry picked from commit 1646be2e29ed1109cc76b11081e829f30809362d) --- .../apache/pulsar/broker/resources/TenantResources.java | 4 ++-- .../org/apache/pulsar/broker/admin/impl/TenantsBase.java | 6 +++++- .../java/org/apache/pulsar/broker/admin/AdminApiTest.java | 8 ++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TenantResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TenantResources.java index 95820576c50c0..36c88cf3a8b60 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TenantResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TenantResources.java @@ -134,7 +134,7 @@ public CompletableFuture hasActiveNamespace(String tenant) { } if (children != null && !children.isEmpty()) { checkNs.completeExceptionally( - new IllegalStateException("Tenant has active namespace")); + new IllegalStateException("The tenant still has active namespaces")); return; } String namespace = NamespaceName.get(tenant, clusterOrNamespace).toString(); @@ -145,7 +145,7 @@ public CompletableFuture hasActiveNamespace(String tenant) { getAsync(joinPath(BASE_POLICIES_PATH, namespace)).thenApply(data -> { if (data.isPresent()) { checkNs.completeExceptionally(new IllegalStateException( - "Tenant has active namespace")); + "The tenant still has active namespaces")); } else { checkNs.complete(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java index c5f5402b5916d..209a532970191 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java @@ -268,7 +268,11 @@ protected void internalDeleteTenant(AsyncResponse asyncResponse, String tenant) .whenComplete((ignore, ex) -> { if (ex != null) { log.error("[{}] Failed to delete tenant {}", clientAppId(), tenant, ex); - asyncResponse.resume(new RestException(ex)); + if (ex.getCause() instanceof IllegalStateException) { + asyncResponse.resume(new RestException(Status.CONFLICT, ex.getCause())); + } else { + asyncResponse.resume(new RestException(ex)); + } } else { log.info("[{}] Deleted tenant {}", clientAppId(), tenant); asyncResponse.resume(Response.noContent().build()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 2c3f704ab58e6..3e87f13c436f3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -670,6 +670,14 @@ public void properties() throws PulsarAdminException { assertEquals(admin.tenants().getTenantInfo("prop-xyz"), newTenantAdmin); + try { + admin.tenants().deleteTenant("prop-xyz"); + fail("should have failed"); + } catch (PulsarAdminException e) { + assertTrue(e instanceof ConflictException); + assertEquals(e.getStatusCode(), 409); + assertEquals(e.getMessage(), "The tenant still has active namespaces"); + } admin.namespaces().deleteNamespace("prop-xyz/ns1"); admin.tenants().deleteTenant("prop-xyz"); assertEquals(admin.tenants().getTenants(), Lists.newArrayList()); From 68d5af20befbff5b87772efe3041cd2192778b94 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Fri, 19 Nov 2021 22:55:57 +0800 Subject: [PATCH 155/823] [Authorization] Support GET_BACKLOG_SIZE topic op after enable auth (#12850) (cherry picked from commit 5eb43eac8e9d238963f29be4184ac90e95a04523) --- .../authorization/PulsarAuthorizationProvider.java | 1 + .../client/api/AuthorizationProducerConsumerTest.java | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index b50d7de0ab637..61d77a7a8a0a9 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -581,6 +581,7 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, case EXPIRE_MESSAGES: case PEEK_MESSAGES: case RESET_CURSOR: + case GET_BACKLOG_SIZE: case SET_REPLICATED_SUBSCRIPTION_STATUS: isAuthorizedFuture = canConsumeAsync(topicName, role, authData, authData.getSubscription()); break; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 5eeac534780fa..c9d76b025376b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -218,6 +218,13 @@ public void testSubscriberPermission() throws Exception { assertTrue(e.getMessage().startsWith( "Unauthorized to validateTopicOperation for operation [GET_STATS]")); } + try { + sub1Admin.topics().getBacklogSizeByMessageId(topicName, MessageId.earliest); + fail("should have failed with authorization exception"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith( + "Unauthorized to validateTopicOperation for operation")); + } // grant topic consume authorization to the subscriptionRole tenantAdmin.topics().grantPermission(topicName, subscriptionRole, @@ -239,8 +246,10 @@ public void testSubscriberPermission() throws Exception { assertEquals(subscriptions.size(), 2); // now, subscriptionRole have consume authorization on topic, so it will successfully get topic internal stats - PersistentTopicInternalStats internalStats = superAdmin.topics().getInternalStats(topicName, true); + PersistentTopicInternalStats internalStats = sub1Admin.topics().getInternalStats(topicName, true); assertNotNull(internalStats); + Long backlogSize = sub1Admin.topics().getBacklogSizeByMessageId(topicName, MessageId.earliest); + assertEquals(backlogSize.longValue(), 0); // verify tenant is able to perform all subscription-admin api tenantAdmin.topics().skipAllMessages(topicName, subscriptionName); From 3d7fe33d7d52446185d1e928cb158918119a11a6 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 19 Nov 2021 07:35:47 +0800 Subject: [PATCH 156/823] Remove readerCaches and close reader when exception occurs in SystemTopicBasedTopicPoliciesService. (#12873) (cherry picked from commit bcc219b5308379fb354b32eb858b4f54868721d5) --- .../broker/service/SystemTopicBasedTopicPoliciesService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 902784e0f0e00..12ba88e99bb7a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -218,6 +218,8 @@ private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture if (ex != null) { log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); result.completeExceptionally(ex); + readerCaches.remove(namespace); + reader.closeAsync(); } else { initPolicesCache(reader, result); result.thenRun(() -> readMorePolicies(reader)); From f3bdaec1f38142d1e7e624424a924769b7382594 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 20 Nov 2021 00:15:52 +0800 Subject: [PATCH 157/823] [Transaction]stop TC replaying with exception (#12705) When MLTransactionLogImpl replaying, if any ledger was deleted from bookkeeper, or ManagerLedger was fenced, MLTransactionLogImpl will not stop recovering and continue to report the exception. End replaying when there is no ledger to read or the managerLedger is fenced. (cherry picked from commit 06f1a91c05d1e11cd9ce8c85e042224e57495390) --- .../broker/transaction/TransactionTest.java | 81 ++++++++++++++++++- .../impl/MLTransactionLogImpl.java | 10 ++- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index a237314331662..e4975d97cea0a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -21,6 +21,12 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; import static org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore.PENDING_ACK_STORE_SUFFIX; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -28,6 +34,8 @@ import io.netty.buffer.Unpooled; import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -37,6 +45,8 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; @@ -73,6 +83,12 @@ import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; +import org.apache.pulsar.transaction.coordinator.TransactionRecoverTracker; +import org.apache.pulsar.transaction.coordinator.TransactionTimeoutTracker; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogInterceptor; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -411,11 +427,11 @@ public void completed(Exception e, long ledgerId, long entryId) { } @Test - public void testMaxReadPositionForNormalPublish() throws Exception{ + public void testMaxReadPositionForNormalPublish() throws Exception { String topic = "persistent://" + NAMESPACE1 + "/NormalPublish"; admin.topics().createNonPartitionedTopic(topic); PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() - .getTopic(topic, false).get().get(); + .getTopic(topic, false).get().get(); TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); PulsarClient noTxnClient = PulsarClient.builder().enableTransaction(false) @@ -443,7 +459,7 @@ public void testMaxReadPositionForNormalPublish() throws Exception{ .sendTimeout(0, TimeUnit.SECONDS) .create(); - Awaitility.await().untilAsserted(() ->Assert.assertTrue(topicTransactionBuffer.checkIfReady())); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(topicTransactionBuffer.checkIfReady())); //test publishing txn messages will not change maxReadPosition if don`t commit or abort. Transaction transaction = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); @@ -483,5 +499,62 @@ public void testMaxReadPositionForNormalPublish() throws Exception{ Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); - } + } + + @Test + public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ + String topic = NAMESPACE1 + "/testEndTBRecoveringWhenManagerLedgerDisReadable"; + admin.topics().createNonPartitionedTopic(topic); + + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(topic, false).get().get(); + persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true); + Map map = new HashMap<>(); + map.put(MLTransactionLogInterceptor.MAX_LOCAL_TXN_ID, "1"); + persistentTopic.getManagedLedger().setProperties(map); + + ManagedCursor managedCursor = mock(ManagedCursor.class); + doReturn(true).when(managedCursor).hasMoreEntries(); + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), + null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + MLTransactionLogImpl mlTransactionLog = + new MLTransactionLogImpl(new TransactionCoordinatorID(1), null, + persistentTopic.getManagedLedger().getConfig()); + Class mlTransactionLogClass = MLTransactionLogImpl.class; + Field field = mlTransactionLogClass.getDeclaredField("cursor"); + field.setAccessible(true); + field.set(mlTransactionLog, managedCursor); + field = mlTransactionLogClass.getDeclaredField("managedLedger"); + field.setAccessible(true); + field.set(mlTransactionLog, persistentTopic.getManagedLedger()); + + TransactionRecoverTracker transactionRecoverTracker = mock(TransactionRecoverTracker.class); + doNothing().when(transactionRecoverTracker).appendOpenTransactionToTimeoutTracker(); + doNothing().when(transactionRecoverTracker).handleCommittingAndAbortingTransaction(); + TransactionTimeoutTracker timeoutTracker = mock(TransactionTimeoutTracker.class); + doNothing().when(timeoutTracker).start(); + MLTransactionMetadataStore metadataStore1 = + new MLTransactionMetadataStore(new TransactionCoordinatorID(1), + mlTransactionLog, timeoutTracker, transactionRecoverTracker); + + Awaitility.await().untilAsserted(() -> + assertEquals(metadataStore1.getCoordinatorStats().state, "Ready")); + + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + MLTransactionMetadataStore metadataStore2 = + new MLTransactionMetadataStore(new TransactionCoordinatorID(1), + mlTransactionLog, timeoutTracker, transactionRecoverTracker); + Awaitility.await().untilAsserted(() -> + assertEquals(metadataStore2.getCoordinatorStats().state, "Ready")); + } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index c0442753e7a7b..2d11d984b8655 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -281,18 +281,19 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private final AtomicLong outstandingReadsRequests = new AtomicLong(0); + private boolean isReadable = true; boolean fillQueue() { if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); readAsync(100, this); - return true; + return isReadable; } else { return false; } } else { - return true; + return isReadable; } } @@ -313,6 +314,11 @@ public Entry get() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + if (managedLedgerConfig.isAutoSkipNonRecoverableData() + && exception instanceof ManagedLedgerException.NonRecoverableLedgerException + || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { + isReadable = false; + } log.error("Transaction log init fail error!", exception); outstandingReadsRequests.decrementAndGet(); } From 33ff93f401a51f7599f77ffe8d8c6d6851dae317 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 20 Nov 2021 00:20:46 +0800 Subject: [PATCH 158/823] Handle exception double (#12881) Fixes https://github.com/apache/pulsar/pull/12744 ### Motivation The exception here was handled twice, resulting in a null pointer exception. And the position will be updated twice. ### Modifications Keep the exceptionally() part and convert the whenComplete() into thenAccept(). (cherry picked from commit dbac1217ee6b8247ea970e3a70e928259a389244) --- .../mledger/impl/EntryCacheImpl.java | 44 +++++++++---------- .../mledger/impl/EntryCacheManager.java | 13 +++--- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java index 6d54abbf95d32..085923e25ccdf 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheImpl.java @@ -209,14 +209,8 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt manager.mlFactoryMBean.recordCacheHit(cachedEntry.getLength()); callback.readEntryComplete(cachedEntry, ctx); } else { - lh.readAsync(position.getEntryId(), position.getEntryId()).whenCompleteAsync( - (ledgerEntries, exception) -> { - if (exception != null) { - ml.invalidateLedgerHandle(lh); - callback.readEntryFailed(createManagedLedgerException(exception), ctx); - return; - } - + lh.readAsync(position.getEntryId(), position.getEntryId()).thenAcceptAsync( + ledgerEntries -> { try { Iterator iterator = ledgerEntries.iterator(); if (iterator.hasNext()) { @@ -234,7 +228,11 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt } finally { ledgerEntries.close(); } - }, ml.getExecutor().chooseThread(ml.getName())); + }, ml.getExecutor().chooseThread(ml.getName())).exceptionally(exception -> { + ml.invalidateLedgerHandle(lh); + callback.readEntryFailed(createManagedLedgerException(exception), ctx); + return null; + }); } } @@ -292,20 +290,8 @@ private void asyncReadEntry0(ReadHandle lh, long firstEntry, long lastEntry, boo } // Read all the entries from bookkeeper - lh.readAsync(firstEntry, lastEntry).whenCompleteAsync( - (ledgerEntries, exception) -> { - if (exception != null) { - if (exception instanceof BKException - && ((BKException)exception).getCode() == BKException.Code.TooManyRequestsException) { - callback.readEntriesFailed(createManagedLedgerException(exception), ctx); - } else { - ml.invalidateLedgerHandle(lh); - ManagedLedgerException mlException = createManagedLedgerException(exception); - callback.readEntriesFailed(mlException, ctx); - } - return; - } - + lh.readAsync(firstEntry, lastEntry).thenAcceptAsync( + ledgerEntries -> { checkNotNull(ml.getName()); checkNotNull(ml.getExecutor()); @@ -328,7 +314,17 @@ private void asyncReadEntry0(ReadHandle lh, long firstEntry, long lastEntry, boo } finally { ledgerEntries.close(); } - }, ml.getExecutor().chooseThread(ml.getName())); + }, ml.getExecutor().chooseThread(ml.getName())).exceptionally(exception -> { + if (exception instanceof BKException + && ((BKException)exception).getCode() == BKException.Code.TooManyRequestsException) { + callback.readEntriesFailed(createManagedLedgerException(exception), ctx); + } else { + ml.invalidateLedgerHandle(lh); + ManagedLedgerException mlException = createManagedLedgerException(exception); + callback.readEntriesFailed(mlException, ctx); + } + return null; + }); } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java index d360bbd6e5cee..1c0c288e8784d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCacheManager.java @@ -193,12 +193,8 @@ public void invalidateEntriesBeforeTimestamp(long timestamp) { @Override public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boolean isSlowestReader, final ReadEntriesCallback callback, Object ctx) { - lh.readAsync(firstEntry, lastEntry).whenComplete( - (ledgerEntries, exception) -> { - if (exception != null) { - callback.readEntriesFailed(createManagedLedgerException(exception), ctx); - return; - } + lh.readAsync(firstEntry, lastEntry).thenAcceptAsync( + ledgerEntries -> { List entries = Lists.newArrayList(); long totalSize = 0; try { @@ -215,7 +211,10 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole ml.mbean.addReadEntriesSample(entries.size(), totalSize); callback.readEntriesComplete(entries, ctx); - }); + }, ml.getExecutor().chooseThread(ml.getName())).exceptionally(exception -> { + callback.readEntriesFailed(createManagedLedgerException(exception), ctx); + return null; + }); } @Override From 626b1d366d5fd396484cecf220a2eba172c32b94 Mon Sep 17 00:00:00 2001 From: mingyifei <38995810+mingyifei@users.noreply.github.com> Date: Sat, 20 Nov 2021 03:47:47 +0800 Subject: [PATCH 159/823] [pulsar-perf]Support listenerThreads configuration. (#12892) * Support listenerThreads configuration. * Support listenerThreads configuration. * Modified to short option. * Add listenerThreads configuration document. Co-authored-by: mingyifei (cherry picked from commit ec715f280d20910fbeefe9e2bdd436799173a008) --- .../org/apache/pulsar/testclient/PerformanceConsumer.java | 5 +++++ .../java/org/apache/pulsar/testclient/PerformanceReader.java | 5 +++++ site2/docs/reference-cli-tools.md | 2 ++ site2/website-next/docs/reference-cli-tools.md | 2 ++ 4 files changed, 14 insertions(+) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index db69778dc6e57..a220c9fc0fc5c 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -180,6 +180,10 @@ static class Arguments { "used for handling connections to brokers, default is 1 thread") public int ioThreads = 1; + @Parameter(names = {"-lt", "--num-listener-threads"}, description = "Set the number of threads" + + " to be used for message listeners") + public int listenerThreads = 1; + @Parameter(names = {"--batch-index-ack" }, description = "Enable or disable the batch index acknowledgment") public boolean batchIndexAck = false; @@ -340,6 +344,7 @@ public static void main(String[] args) throws Exception { .connectionsPerBroker(arguments.maxConnections) // .statsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS) // .ioThreads(arguments.ioThreads) // + .listenerThreads(arguments.listenerThreads) .enableBusyWait(arguments.enableBusyWait) .tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath); if (isNotBlank(arguments.authPluginClassName)) { diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java index a7f66ad36a3b4..44e555560cc13 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java @@ -134,6 +134,10 @@ static class Arguments { @Parameter(names = {"-ioThreads", "--num-io-threads"}, description = "Set the number of threads to be " + "used for handling connections to brokers, default is 1 thread") public int ioThreads = 1; + + @Parameter(names = {"-lt", "--num-listener-threads"}, description = "Set the number of threads" + + " to be used for message listeners") + public int listenerThreads = 1; } public static void main(String[] args) throws Exception { @@ -252,6 +256,7 @@ public static void main(String[] args) throws Exception { .connectionsPerBroker(arguments.maxConnections) // .statsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS) // .ioThreads(arguments.ioThreads) // + .listenerThreads(arguments.listenerThreads) .enableTls(arguments.useTls) // .tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath); diff --git a/site2/docs/reference-cli-tools.md b/site2/docs/reference-cli-tools.md index 1911fe3217463..f4ae1365d8723 100644 --- a/site2/docs/reference-cli-tools.md +++ b/site2/docs/reference-cli-tools.md @@ -447,6 +447,7 @@ Options |`-mc`, `--max_chunked_msg`|Max pending chunk messages|0| |`-n`, `--num-consumers`|Number of consumers (per topic)|1| |`-ioThreads`, `--num-io-threads`|Set the number of threads to be used for handling connections to brokers|1| +|`-lt`, `--num-listener-threads`|Set the number of threads to be used for message listeners|1| |`-ns`, `--num-subscriptions`|Number of subscriptions (per topic)|1| |`-t`, `--num-topics`|The number of topics|1| |`-pm`, `--pool-messages`|Use the pooled message|true| @@ -541,6 +542,7 @@ Options |`-n`, `--num-messages`|Number of messages to consume in total. If the value is equal to or smaller than 0, it keeps consuming messages.|0| |`-c`, `--max-connections`|Max number of TCP connections to a single broker|100| |`-ioThreads`, `--num-io-threads`|Set the number of threads to be used for handling connections to brokers|1| +|`-lt`, `--num-listener-threads`|Set the number of threads to be used for message listeners|1| |`-t`, `--num-topics`|The number of topics|1| |`-r`, `--rate`|Simulate a slow message reader (rate in msg/s)|0| |`-q`, `--receiver-queue-size`|Size of the receiver queue|1000| diff --git a/site2/website-next/docs/reference-cli-tools.md b/site2/website-next/docs/reference-cli-tools.md index cbdf7173ba927..a2eda14a32c62 100644 --- a/site2/website-next/docs/reference-cli-tools.md +++ b/site2/website-next/docs/reference-cli-tools.md @@ -533,6 +533,7 @@ Options |`-mc`, `--max_chunked_msg`|Max pending chunk messages|0| |`-n`, `--num-consumers`|Number of consumers (per topic)|1| |`-ioThreads`, `--num-io-threads`|Set the number of threads to be used for handling connections to brokers|1| +|`-lt`, `--num-listener-threads`|Set the number of threads to be used for message listeners|1| |`-ns`, `--num-subscriptions`|Number of subscriptions (per topic)|1| |`-t`, `--num-topics`|The number of topics|1| |`-pm`, `--pool-messages`|Use the pooled message|true| @@ -633,6 +634,7 @@ Options |`-n`, `--num-messages`|Number of messages to consume in total. If the value is equal to or smaller than 0, it keeps consuming messages.|0| |`-c`, `--max-connections`|Max number of TCP connections to a single broker|100| |`-ioThreads`, `--num-io-threads`|Set the number of threads to be used for handling connections to brokers|1| +|`-lt`, `--num-listener-threads`|Set the number of threads to be used for message listeners|1| |`-t`, `--num-topics`|The number of topics|1| |`-r`, `--rate`|Simulate a slow message reader (rate in msg/s)|0| |`-q`, `--receiver-queue-size`|Size of the receiver queue|1000| From fbadbd07b1fb335ae8db3d4a64ece4fdaf31dbad Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Sun, 21 Nov 2021 12:49:15 +0800 Subject: [PATCH 160/823] [Issue 12897] Fix flaky test in testReplicatorProducerName (#12898) (cherry picked from commit 78c2981a8bc0af4ee883357b99a40e83be8ffa4b) --- .../apache/pulsar/broker/service/ReplicatorTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 87e2730fd68f5..488fbfef0235c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -730,10 +730,12 @@ public void testReplicatorProducerName() throws Exception { }); Optional topic = pulsar2.getBrokerService().getTopicReference(topicName); assertTrue(topic.isPresent()); - Set remoteClusters = topic.get().getProducers().values().stream() - .map(org.apache.pulsar.broker.service.Producer::getRemoteCluster) - .collect(Collectors.toSet()); - assertTrue(remoteClusters.contains("r1")); + Awaitility.await().untilAsserted(() -> { + Set remoteClusters = topic.get().getProducers().values().stream() + .map(org.apache.pulsar.broker.service.Producer::getRemoteCluster) + .collect(Collectors.toSet()); + assertTrue(remoteClusters.contains("r1")); + }); } @Test(priority = 5, timeOut = 30000) From 4b789236c8d7ed1212c7f2007e082f5157bd9df1 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 22 Nov 2021 16:23:18 +0800 Subject: [PATCH 161/823] [Transaction] Add transaction perf (#11933) Master Issue: #https://github.com/apache/pulsar/issues/11765 ### Motivation - When opening a transaction, what is the delay and rate of pulsar sending and consuming messages? - How does the granularity of the transaction affect the rate at which pulsar produces or consumes messages - Is the rate of abort and commit transactions different - Try to keep the functions of ordinary produce perfand consume perf as much as possible ### Modifications Add PerformanceTransaction.Class - According to the number of topics set by the user, each topic starts a thread to execute a number of transactions asynchronously. - If there is a preset transaction value, the production or consumption of this topic is terminated after the corresponding number of transactions is executed, and the production and consumption of all topics are summed up. After the consumption ends, the main thread ends. - Each consumer can have multiple subscriptions, but each subscription has only one consumer. In order to better compare the rate of production and consumption. (cherry picked from commit 793a8b95637f534733d2b93b6eedade1399a13a4) --- bin/pulsar-perf | 3 + .../testclient/CmdGenerateDocumentation.java | 1 + .../testclient/PerformanceConsumer.java | 252 +++++-- .../testclient/PerformanceProducer.java | 160 +++- .../testclient/PerformanceTransaction.java | 707 ++++++++++++++++++ .../testclient/utils/PerformanceUtils.java | 59 ++ .../PerformanceTransactionTest.java | 240 ++++++ 7 files changed, 1358 insertions(+), 64 deletions(-) create mode 100644 pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java create mode 100644 pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java create mode 100644 pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java diff --git a/bin/pulsar-perf b/bin/pulsar-perf index a2f90b0d7e7cc..cef0cf3c0ee7b 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -87,6 +87,7 @@ Usage: pulsar-perf where command is one of: produce Run a producer consume Run a consumer + transaction Run a transaction repeatedly read Run a topic reader websocket-producer Run a websocket producer @@ -164,6 +165,8 @@ if [ "$COMMAND" == "produce" ]; then exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceProducer --conf-file $PULSAR_PERFTEST_CONF "$@" elif [ "$COMMAND" == "consume" ]; then exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceConsumer --conf-file $PULSAR_PERFTEST_CONF "$@" +elif [ "$COMMAND" == "transaction" ]; then + exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceTransaction --conf-file $PULSAR_PERFTEST_CONF "$@" elif [ "$COMMAND" == "read" ]; then exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceReader --conf-file $PULSAR_PERFTEST_CONF "$@" elif [ "$COMMAND" == "monitor-brokers" ]; then diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java index c316fc4c204c5..29c38aca7944d 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java @@ -64,6 +64,7 @@ public static void main(String[] args) throws Exception { Map> cmdClassMap = new LinkedHashMap<>(); cmdClassMap.put("produce", Class.forName("org.apache.pulsar.testclient.PerformanceProducer$Arguments")); cmdClassMap.put("consume", Class.forName("org.apache.pulsar.testclient.PerformanceConsumer$Arguments")); + cmdClassMap.put("transaction", Class.forName("org.apache.pulsar.testclient.PerformanceTransaction$Arguments")); cmdClassMap.put("read", Class.forName("org.apache.pulsar.testclient.PerformanceReader$Arguments")); cmdClassMap.put("monitor-brokers", Class.forName("org.apache.pulsar.testclient.BrokerMonitor$Arguments")); cmdClassMap.put("simulation-client", Class.forName("org.apache.pulsar.testclient.LoadSimulationClient$MainArguments")); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index a220c9fc0fc5c..134ffdd414c59 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -20,6 +20,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.testclient.utils.PerformanceUtils.buildTransaction; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -34,7 +35,10 @@ import java.util.List; import java.util.Properties; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import org.HdrHistogram.Histogram; @@ -48,6 +52,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; @@ -67,8 +72,19 @@ public class PerformanceConsumer { private static final LongAdder totalMessagesReceived = new LongAdder(); private static final LongAdder totalBytesReceived = new LongAdder(); - private static Recorder recorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); - private static Recorder cumulativeRecorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); + private static final LongAdder totalNumTxnOpenFail = new LongAdder(); + private static final LongAdder totalNumTxnOpenSuccess = new LongAdder(); + + private static final LongAdder totalMessageAck = new LongAdder(); + private static final LongAdder totalMessageAckFailed = new LongAdder(); + private static final LongAdder messageAck = new LongAdder(); + + private static final LongAdder totalEndTxnOpFailNum = new LongAdder(); + private static final LongAdder totalEndTxnOpSuccessNum = new LongAdder(); + private static final LongAdder numTxnOpSuccess = new LongAdder(); + + private static final Recorder recorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); + private static final Recorder cumulativeRecorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); @Parameters(commandDescription = "Test pulsar consumer performance.") static class Arguments { @@ -94,7 +110,7 @@ static class Arguments { @Parameter(names = { "-s", "--subscriber-name" }, description = "Subscriber name prefix", hidden = true) public String subscriberName; - @Parameter(names = { "-ss", "--subscriptions" }, description = "A list of subscriptions to consume on (e.g. sub1,sub2)") + @Parameter(names = { "-ss", "--subscriptions" }, description = "A list of subscriptions to consume (for example, sub1,sub2)") public List subscriptions = Collections.singletonList("sub"); @Parameter(names = { "-st", "--subscription-type" }, description = "Subscription type") @@ -177,7 +193,7 @@ static class Arguments { public long testTime = 0; @Parameter(names = {"-ioThreads", "--num-io-threads"}, description = "Set the number of threads to be " + - "used for handling connections to brokers, default is 1 thread") + "used for handling connections to brokers. The default value is 1.") public int ioThreads = 1; @Parameter(names = {"-lt", "--num-listener-threads"}, description = "Set the number of threads" @@ -193,6 +209,26 @@ static class Arguments { @Parameter(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client") public boolean enableBusyWait = false; + @Parameter(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," + + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") + public long transactionTimeout = 10; + + @Parameter(names = {"-nmt", "--numMessage-perTransaction"}, + description = "The number of messages acknowledged by a transaction. " + + "(After --txn-enable setting to true, -numMessage-perTransaction takes effect") + public int numMessagesPerTransaction = 50; + + @Parameter(names = {"-txn", "--txn-enable"}, description = "Enable or disable the transaction") + public boolean isEnableTransaction = false; + + @Parameter(names = {"-ntxn"}, description = "The number of opened transactions, 0 means keeping open." + + "(After --txn-enable setting to true, -ntxn takes effect.)") + public long totalNumTxn = 0; + + @Parameter(names = {"-abort"}, description = "Abort the transaction. (After --txn-enable " + + "setting to true, -abort takes effect)") + public boolean isAbortTransaction = false; + @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") public String histogramFile = null; } @@ -303,43 +339,9 @@ public static void main(String[] args) throws Exception { final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; long startTime = System.nanoTime(); long testEndTime = startTime + (long) (arguments.testTime * 1e9); - MessageListener listener = (consumer, msg) -> { - if (arguments.testTime > 0) { - if (System.nanoTime() > testEndTime) { - log.info("------------- DONE (reached the maximum duration: [{} seconds] of consumption) --------------", arguments.testTime); - printAggregatedStats(); - PerfClientUtils.exit(0); - } - } - if (arguments.numMessages > 0 && totalMessagesReceived.sum() >= arguments.numMessages) { - log.info("------------- DONE (reached the maximum number: [{}] of consumption) --------------", arguments.numMessages); - printAggregatedStats(); - PerfClientUtils.exit(0); - } - messagesReceived.increment(); - bytesReceived.add(msg.size()); - - totalMessagesReceived.increment(); - totalBytesReceived.add(msg.size()); - - if (limiter != null) { - limiter.acquire(); - } - - long latencyMillis = System.currentTimeMillis() - msg.getPublishTime(); - if (latencyMillis >= 0) { - recorder.recordValue(latencyMillis); - cumulativeRecorder.recordValue(latencyMillis); - } - - consumer.acknowledgeAsync(msg); - - if(arguments.poolMessages) { - msg.release(); - } - }; ClientBuilder clientBuilder = PulsarClient.builder() // + .enableTransaction(arguments.isEnableTransaction) .serviceUrl(arguments.serviceURL) // .connectionsPerBroker(arguments.maxConnections) // .statsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS) // @@ -358,9 +360,121 @@ public static void main(String[] args) throws Exception { if (isNotBlank(arguments.listenerName)) { clientBuilder.listenerName(arguments.listenerName); } - PulsarClient pulsarClient = clientBuilder.build(); + AtomicReference atomicReference = buildTransaction(pulsarClient, arguments.isEnableTransaction, + arguments.transactionTimeout); + + AtomicLong messageAckedCount = new AtomicLong(); + Semaphore messageReceiveLimiter = new Semaphore(arguments.numMessagesPerTransaction); + Thread thread = Thread.currentThread(); + MessageListener listener = (consumer, msg) -> { + if (arguments.testTime > 0) { + if (System.nanoTime() > testEndTime) { + log.info("------------------- DONE -----------------------"); + printAggregatedStats(); + PerfClientUtils.exit(0); + thread.interrupt(); + } + } + if (arguments.totalNumTxn > 0) { + if (totalEndTxnOpFailNum.sum() + totalEndTxnOpSuccessNum.sum() >= arguments.totalNumTxn) { + log.info("------------------- DONE -----------------------"); + printAggregatedStats(); + PerfClientUtils.exit(0); + thread.interrupt(); + } + } + messagesReceived.increment(); + bytesReceived.add(msg.size()); + + totalMessagesReceived.increment(); + totalBytesReceived.add(msg.size()); + + if (limiter != null) { + limiter.acquire(); + } + + long latencyMillis = System.currentTimeMillis() - msg.getPublishTime(); + if (latencyMillis >= 0) { + recorder.recordValue(latencyMillis); + cumulativeRecorder.recordValue(latencyMillis); + } + if (arguments.isEnableTransaction) { + try { + messageReceiveLimiter.acquire(); + }catch (InterruptedException e){ + log.error("Got error: ", e); + } + consumer.acknowledgeAsync(msg.getMessageId(), atomicReference.get()).thenRun(() -> { + totalMessageAck.increment(); + messageAck.increment(); + }).exceptionally(throwable ->{ + log.error("Ack message {} failed with exception", msg, throwable); + totalMessageAckFailed.increment(); + return null; + }); + } else { + consumer.acknowledgeAsync(msg).thenRun(()->{ + totalMessageAck.increment(); + messageAck.increment(); + } + ).exceptionally(throwable ->{ + log.error("Ack message {} failed with exception", msg, throwable); + totalMessageAckFailed.increment(); + return null; + } + ); + } + if (arguments.poolMessages) { + msg.release(); + } + if (arguments.isEnableTransaction + && messageAckedCount.incrementAndGet() == arguments.numMessagesPerTransaction) { + Transaction transaction = atomicReference.get(); + if (!arguments.isAbortTransaction) { + transaction.commit() + .thenRun(() -> { + totalEndTxnOpSuccessNum.increment(); + numTxnOpSuccess.increment(); + }) + .exceptionally(exception -> { + log.error("Commit transaction failed with exception : ", exception); + totalEndTxnOpFailNum.increment(); + return null; + }); + } else { + transaction.abort().thenRun(() -> { + log.info("Abort transaction {}", transaction.getTxnID().toString()); + totalEndTxnOpSuccessNum.increment(); + numTxnOpSuccess.increment(); + }).exceptionally(exception -> { + log.error("Commit transaction {} failed with exception", + transaction.getTxnID().toString(), + exception); + totalEndTxnOpFailNum.increment(); + return null; + }); + } + while (true) { + try { + Transaction newTransaction = pulsarClient.newTransaction() + .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .build().get(); + atomicReference.compareAndSet(transaction, newTransaction); + totalNumTxnOpenSuccess.increment(); + messageAckedCount.set(0); + messageReceiveLimiter.release(arguments.numMessagesPerTransaction); + break; + } catch (Exception e) { + log.error("Failed to new transaction with exception:", e); + totalNumTxnOpenFail.increment(); + } + } + } + + }; + List>> futures = Lists.newArrayList(); ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.BYTEBUFFER) // .messageListener(listener) // @@ -398,18 +512,16 @@ public static void main(String[] args) throws Exception { } } } - for (Future> future : futures) { future.get(); } - log.info("Start receiving from {} consumers per subscription on {} topics", arguments.numConsumers, arguments.numTopics); long start = System.nanoTime(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { - printAggregatedThroughput(start); + printAggregatedThroughput(start, arguments); printAggregatedStats(); })); @@ -443,11 +555,27 @@ public static void main(String[] args) throws Exception { long total = totalMessagesReceived.sum(); double rate = messagesReceived.sumThenReset() / elapsed; double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024; - + double rateAck = messageAck.sumThenReset() / elapsed; + long totalTxnOpSuccessNum = 0; + long totalTxnOpFailNum = 0; + double rateOpenTxn = 0; reportHistogram = recorder.getIntervalHistogram(reportHistogram); + if (arguments.isEnableTransaction) { + totalTxnOpSuccessNum = totalEndTxnOpSuccessNum.sum(); + totalTxnOpFailNum = totalEndTxnOpFailNum.sum(); + rateOpenTxn = numTxnOpSuccess.sumThenReset() / elapsed; + log.info("--- Transaction: {} transaction end successfully --- {} transaction end failed " + + "--- {} Txn/s --- AckRate: {} msg/s", + totalTxnOpSuccessNum, + totalTxnOpFailNum, + dec.format(rateOpenTxn), + dec.format(rateAck)); + } log.info( - "Throughput received: {} msg --- {} msg/s -- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", + "Throughput received: {} msg --- {} msg/s -- {} Mbit/s " + + "--- Latency: mean: {} ms - med: {} " + + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", intFormat.format(total), dec.format(rate), dec.format(throughput), dec.format(reportHistogram.getMean()), reportHistogram.getValueAtPercentile(50), reportHistogram.getValueAtPercentile(95), @@ -465,15 +593,41 @@ public static void main(String[] args) throws Exception { pulsarClient.close(); } - private static void printAggregatedThroughput(long start) { + private static void printAggregatedThroughput(long start, Arguments arguments) { double elapsed = (System.nanoTime() - start) / 1e9; double rate = totalMessagesReceived.sum() / elapsed; double throughput = totalBytesReceived.sum() / elapsed * 8 / 1024 / 1024; + long totalEndTxnSuccess = 0; + long totalEndTxnFail = 0; + long numTransactionOpenFailed = 0; + long numTransactionOpenSuccess = 0; + long totalnumMessageAckFailed = 0; + double rateAck = totalMessageAck.sum() / elapsed; + double rateOpenTxn = 0; + if (arguments.isEnableTransaction) { + totalEndTxnSuccess = totalEndTxnOpSuccessNum.sum(); + totalEndTxnFail = totalEndTxnOpFailNum.sum(); + rateOpenTxn = (totalEndTxnSuccess + totalEndTxnFail) / elapsed; + totalnumMessageAckFailed = totalMessageAckFailed.sum(); + numTransactionOpenFailed = totalNumTxnOpenFail.sum(); + numTransactionOpenSuccess = totalNumTxnOpenSuccess.sum(); + log.info("-- Transaction: {} transaction end successfully --- {} transaction end failed " + + "--- {} transaction open successfully --- {} transaction open failed " + + "--- {} Txn/s ", + totalEndTxnSuccess, + totalEndTxnFail, + numTransactionOpenSuccess, + numTransactionOpenFailed, + dec.format(rateOpenTxn)); + } log.info( - "Aggregated throughput stats --- {} records received --- {} msg/s --- {} Mbit/s", - totalMessagesReceived, + "Aggregated throughput stats --- {} records received --- {} msg/s --- {} Mbit/s" + + "--- AckRate: {} msg/s --- ack failed {} msg", + totalMessagesReceived.sum(), dec.format(rate), - dec.format(throughput)); + dec.format(throughput), + rateAck, + totalnumMessageAckFailed); } private static void printAggregatedStats() { diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index fee96a8a8ee67..cfd5568cecc2c 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -49,7 +49,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import org.HdrHistogram.Histogram; @@ -70,7 +74,9 @@ import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES; +import static org.apache.pulsar.testclient.utils.PerformanceUtils.buildTransaction; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; @@ -88,11 +94,18 @@ public class PerformanceProducer { private static final LongAdder messagesFailed = new LongAdder(); private static final LongAdder bytesSent = new LongAdder(); + private static final LongAdder totalNumTxnOpenTxnFail = new LongAdder(); + private static final LongAdder totalNumTxnOpenTxnSuccess = new LongAdder(); + private static final LongAdder totalMessagesSent = new LongAdder(); private static final LongAdder totalBytesSent = new LongAdder(); - private static Recorder recorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); - private static Recorder cumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + private static final Recorder recorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + private static final Recorder cumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + + private static final LongAdder totalEndTxnOpSuccessNum = new LongAdder(); + private static final LongAdder totalEndTxnOpFailNum = new LongAdder(); + private static final LongAdder numTxnOpSuccess = new LongAdder(); private static IMessageFormatter messageFormatter = null; @@ -238,7 +251,7 @@ static class Arguments { public String messageKeyGenerationMode = null; @Parameter(names = {"-ioThreads", "--num-io-threads"}, description = "Set the number of threads to be " + - "used for handling connections to brokers, default is 1 thread") + "used for handling connections to brokers. The default value is 1.") public int ioThreads = 1; @Parameter(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client") @@ -254,6 +267,22 @@ static class Arguments { @Parameter(names = {"-fc", "--format-class"}, description="Custom Formatter class name") public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; + @Parameter(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," + + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") + public long transactionTimeout = 10; + + @Parameter(names = {"-nmt", "--numMessage-perTransaction"}, + description = "The number of messages sent by a transaction. " + + "(After --txn-enable setting to true, -nmt takes effect)") + public int numMessagesPerTransaction = 50; + + @Parameter(names = {"-txn", "--txn-enable"}, description = "Enable or disable the transaction") + public boolean isEnableTransaction = false; + + @Parameter(names = {"-abort"}, description = "Abort the transaction. (After --txn-enable " + + "setting to true, -abort takes effect)") + public boolean isAbortTransaction = false; + @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") public String histogramFile = null; } @@ -376,7 +405,7 @@ public static void main(String[] args) throws Exception { long start = System.nanoTime(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { - printAggregatedThroughput(start); + printAggregatedThroughput(start, arguments); printAggregatedStats(); })); @@ -467,14 +496,27 @@ public static void main(String[] args) throws Exception { long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; long total = totalMessagesSent.sum(); + long totalTxnOpSuccess = 0; + long totalTxnOpFail = 0; + double rateOpenTxn = 0; double rate = messagesSent.sumThenReset() / elapsed; double failureRate = messagesFailed.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); + if (arguments.isEnableTransaction) { + totalTxnOpSuccess = totalEndTxnOpSuccessNum.sum(); + totalTxnOpFail = totalEndTxnOpFailNum.sum(); + rateOpenTxn = numTxnOpSuccess.sumThenReset() / elapsed; + log.info("--- Transaction : {} transaction end successfully ---{} transaction end failed " + + "--- {} Txn/s", + totalTxnOpSuccess, totalTxnOpFail, totalFormat.format(rateOpenTxn)); + } log.info( - "Throughput produced: {} msg --- {} msg/s --- {} Mbit/s --- failure {} msg/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", + "Throughput produced: {} msg --- {} msg/s --- {} Mbit/s --- failure {} msg/s " + + "--- Latency: mean: " + + "{} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", intFormat.format(total), throughputFormat.format(rate), throughputFormat.format(throughput), throughputFormat.format(failureRate), @@ -520,6 +562,7 @@ private static void runProducer(int producerId, List>> futures = Lists.newArrayList(); ClientBuilder clientBuilder = PulsarClient.builder() // + .enableTransaction(arguments.isEnableTransaction)// .serviceUrl(arguments.serviceURL) // .connectionsPerBroker(arguments.maxConnections) // .ioThreads(arguments.ioThreads) // @@ -549,6 +592,9 @@ private static void runProducer(int producerId, // enable round robin message routing if it is a partitioned topic .messageRoutingMode(MessageRoutingMode.RoundRobinPartition); + if (arguments.isEnableTransaction) { + producerBuilder.sendTimeout(0, TimeUnit.SECONDS); + } if (arguments.producerName != null) { String producerName = String.format("%s%s%d", arguments.producerName, arguments.separator, producerId); producerBuilder.producerName(producerName); @@ -613,6 +659,10 @@ private static void runProducer(int producerId, } // Send messages on all topics/producers long totalSent = 0; + AtomicReference transactionAtomicReference = buildTransaction(client, + arguments.isEnableTransaction, arguments.transactionTimeout); + AtomicLong numMessageSend = new AtomicLong(0); + Semaphore numMsgPerTxnLimit = new Semaphore(arguments.numMessagesPerTransaction); while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { @@ -635,7 +685,8 @@ private static void runProducer(int producerId, } } rateLimiter.acquire(); - + //if transaction is disable, transaction will be null. + Transaction transaction = transactionAtomicReference.get(); final long sendTime = System.nanoTime(); byte[] payloadData; @@ -650,10 +701,22 @@ private static void runProducer(int producerId, } else { payloadData = payloadBytes; } - - TypedMessageBuilder messageBuilder = producer.newMessage() - .value(payloadData); - if (arguments.delay >0) { + TypedMessageBuilder messageBuilder; + if (arguments.isEnableTransaction) { + if (arguments.numMessagesPerTransaction> 0) { + try{ + numMsgPerTxnLimit.acquire(); + }catch (InterruptedException exception){ + log.error("Get exception: ", exception); + } + } + messageBuilder = producer.newMessage(transaction) + .value(payloadData); + } else { + messageBuilder = producer.newMessage() + .value(payloadData); + } + if (arguments.delay > 0) { messageBuilder.deliverAfter(arguments.delay, TimeUnit.SECONDS); } //generate msg key @@ -662,8 +725,8 @@ private static void runProducer(int producerId, } else if (msgKeyMode == MessageKeyGenerationMode.autoIncrement) { messageBuilder.key(String.valueOf(totalSent)); } + PulsarClient pulsarClient = client; messageBuilder.sendAsync().thenRun(() -> { - messagesSent.increment(); bytesSent.add(payloadData.length); totalMessagesSent.increment(); @@ -681,13 +744,58 @@ private static void runProducer(int producerId, if (ex.getCause() instanceof ArrayIndexOutOfBoundsException) { return null; } - log.warn("Write error on message", ex); + log.warn("Write message error with exception", ex); messagesFailed.increment(); if (arguments.exitOnFailure) { PerfClientUtils.exit(-1); } return null; }); + if (arguments.isEnableTransaction + && numMessageSend.incrementAndGet() == arguments.numMessagesPerTransaction) { + if (!arguments.isAbortTransaction) { + transaction.commit() + .thenRun(() -> { + log.info("Committed transaction {}", + transaction.getTxnID().toString()); + totalEndTxnOpSuccessNum.increment(); + numTxnOpSuccess.increment(); + }) + .exceptionally(exception -> { + log.error("Commit transaction failed with exception : ", + exception); + totalEndTxnOpFailNum.increment(); + return null; + }); + } else { + transaction.abort().thenRun(() -> { + log.info("Abort transaction {}", transaction.getTxnID().toString()); + totalEndTxnOpSuccessNum.increment(); + numTxnOpSuccess.increment(); + }).exceptionally(exception -> { + log.error("Commit transaction {} failed with exception", + transaction.getTxnID().toString(), + exception); + totalEndTxnOpFailNum.increment(); + return null; + }); + } + while(true) { + try { + Transaction newTransaction = pulsarClient.newTransaction() + .withTransactionTimeout(arguments.transactionTimeout, + TimeUnit.SECONDS).build().get(); + transactionAtomicReference.compareAndSet(transaction, newTransaction); + numMessageSend.set(0); + numMsgPerTxnLimit.release(arguments.numMessagesPerTransaction); + totalNumTxnOpenTxnSuccess.increment(); + break; + }catch (Exception e){ + totalNumTxnOpenTxnFail.increment(); + log.error("Failed to new transaction with exception: ", e); + } + } + } } } } catch (Throwable t) { @@ -696,6 +804,7 @@ private static void runProducer(int producerId, if (null != client) { try { client.close(); + PerfClientUtils.exit(-1); } catch (PulsarClientException e) { log.error("Failed to close test client", e); } @@ -703,13 +812,34 @@ private static void runProducer(int producerId, } } - private static void printAggregatedThroughput(long start) { + private static void printAggregatedThroughput(long start, Arguments arguments) { double elapsed = (System.nanoTime() - start) / 1e9; double rate = totalMessagesSent.sum() / elapsed; double throughput = totalBytesSent.sum() / elapsed / 1024 / 1024 * 8; + long totalTxnSuccess = 0; + long totalTxnFail = 0; + double rateOpenTxn = 0; + long numTransactionOpenFailed = 0; + long numTransactionOpenSuccess = 0; + + if (arguments.isEnableTransaction) { + totalTxnSuccess = totalEndTxnOpSuccessNum.sum(); + totalTxnFail = totalEndTxnOpFailNum.sum(); + rateOpenTxn = elapsed / (totalTxnFail + totalTxnSuccess); + numTransactionOpenFailed = totalNumTxnOpenTxnFail.sum(); + numTransactionOpenSuccess = totalNumTxnOpenTxnSuccess.sum(); + log.info("--- Transaction : {} transaction end successfully --- {} transaction end failed " + + "--- {} transaction open successfully --- {} transaction open failed " + + "--- {} Txn/s", + totalTxnSuccess, + totalTxnFail, + numTransactionOpenSuccess, + numTransactionOpenFailed, + totalFormat.format(rateOpenTxn)); + } log.info( - "Aggregated throughput stats --- {} records sent --- {} msg/s --- {} Mbit/s", - totalMessagesSent, + "Aggregated throughput stats --- {} records sent --- {} msg/s --- {} Mbit/s ", + totalMessagesSent.sum(), totalFormat.format(rate), totalFormat.format(throughput)); } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java new file mode 100644 index 0000000000000..49d441e1f4cdd --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java @@ -0,0 +1,707 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.pulsar.testclient.utils.PerformanceUtils.buildTransaction; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.common.collect.Lists; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.text.DecimalFormat; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; +import org.HdrHistogram.Histogram; +import org.HdrHistogram.HistogramLogWriter; +import org.HdrHistogram.Recorder; +import org.apache.curator.shaded.com.google.common.util.concurrent.RateLimiter; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PerformanceTransaction { + + + private static final LongAdder totalNumEndTxnOpFailed = new LongAdder(); + private static final LongAdder totalNumEndTxnOpSuccess = new LongAdder(); + private static final LongAdder numTxnOpSuccess = new LongAdder(); + private static final LongAdder totalNumTxnOpenTxnFail = new LongAdder(); + private static final LongAdder totalNumTxnOpenTxnSuccess = new LongAdder(); + + private static final LongAdder numMessagesAckFailed = new LongAdder(); + private static final LongAdder numMessagesAckSuccess = new LongAdder(); + private static final LongAdder numMessagesSendFailed = new LongAdder(); + private static final LongAdder numMessagesSendSuccess = new LongAdder(); + + private static final Recorder messageAckRecorder = + new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + private static final Recorder messageAckCumulativeRecorder = + new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + + private static final Recorder messageSendRecorder = + new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + private static final Recorder messageSendRCumulativeRecorder = + new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + + + @Parameters(commandDescription = "Test pulsar transaction performance.") + static class Arguments { + + @Parameter(names = {"-h", "--help"}, description = "Help message", help = true) + boolean help; + + @Parameter(names = {"--conf-file"}, description = "Configuration file") + public String confFile; + + @Parameter(names = "--topics-c", description = "All topics that need ack for a transaction", required = + true) + public List consumerTopic = Collections.singletonList("test-consume"); + + @Parameter(names = "--topics-p", description = "All topics that need produce for a transaction", + required = true) + public List producerTopic = Collections.singletonList("test-produce"); + + @Parameter(names = {"-threads", "--num-test-threads"}, description = "Number of test threads." + + "This thread is for a new transaction to ack messages from consumer topics and produce message to " + + "producer topics, and then commit or abort this transaction. " + + "Increasing the number of threads increases the parallelism of the performance test, " + + "thereby increasing the intensity of the stress test.") + public int numTestThreads = 1; + + @Parameter(names = {"-au", "--admin-url"}, description = "Pulsar Admin URL") + public String adminURL; + + @Parameter(names = {"-u", "--service-url"}, description = "Pulsar Service URL") + public String serviceURL; + + @Parameter(names = {"-np", + "--partitions"}, description = "Create partitioned topics with a given number of partitions, 0 means" + + "not trying to create a topic") + public Integer partitions = null; + + @Parameter(names = {"-c", + "--max-connections"}, description = "Max number of TCP connections to a single broker") + public int maxConnections = 100; + + @Parameter(names = {"-time", + "--test-duration"}, description = "Test duration (in second). 0 means keeping publishing") + public long testTime = 0; + + @Parameter(names = {"-ioThreads", "--num-io-threads"}, description = "Set the number of threads to be " + + "used for handling connections to brokers. The default value is 1.") + public int ioThreads = 1; + + @Parameter(names = {"-ss", + "--subscriptions"}, description = "A list of subscriptions to consume (for example, sub1,sub2)") + public List subscriptions = Collections.singletonList("sub"); + + @Parameter(names = {"-ns", "--num-subscriptions"}, description = "Number of subscriptions (per topic)") + public int numSubscriptions = 1; + + @Parameter(names = {"-sp", "--subscription-position"}, description = "Subscription position") + private SubscriptionInitialPosition subscriptionInitialPosition = SubscriptionInitialPosition.Earliest; + + @Parameter(names = {"-st", "--subscription-type"}, description = "Subscription type") + public SubscriptionType subscriptionType = SubscriptionType.Shared; + + @Parameter(names = {"-q", "--receiver-queue-size"}, description = "Size of the receiver queue") + public int receiverQueueSize = 1000; + + @Parameter(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," + + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") + public long transactionTimeout = 5; + + @Parameter(names = {"-ntxn", + "--number-txn"}, description = "Set the number of transaction. 0 means keeping open." + + "If transaction disabled, it means the number of tasks. The task or transaction produces or " + + "consumes a specified number of messages.") + public long numTransactions = 0; + + @Parameter(names = {"-nmp", "--numMessage-perTransaction-produce"}, + description = "Set the number of messages produced in a transaction." + + "If transaction disabled, it means the number of messages produced in a task.") + public int numMessagesProducedPerTransaction = 1; + + @Parameter(names = {"-nmc", "--numMessage-perTransaction-consume"}, + description = "Set the number of messages consumed in a transaction." + + "If transaction disabled, it means the number of messages consumed in a task.") + public int numMessagesReceivedPerTransaction = 1; + + @Parameter(names = {"--txn-disable"}, description = "Disable transaction") + public boolean isDisableTransaction = false; + + @Parameter(names = {"-abort"}, description = "Abort the transaction. (After --txn-disEnable " + + "setting to false, -abort takes effect)") + public boolean isAbortTransaction = false; + + @Parameter(names = "-txnRate", description = "Set the rate of opened transaction or task. 0 means no limit") + public int openTxnRate = 0; + } + + public static void main(String[] args) + throws IOException, PulsarAdminException, ExecutionException, InterruptedException { + final Arguments arguments = new Arguments(); + JCommander jc = new JCommander(arguments); + jc.setProgramName("pulsar-perf transaction"); + + try { + jc.parse(args); + } catch (ParameterException e) { + System.out.println(e.getMessage()); + jc.usage(); + PerfClientUtils.exit(-1); + } + + if (arguments.help) { + jc.usage(); + PerfClientUtils.exit(-1); + } + + + if (arguments.confFile != null) { + Properties prop = new Properties(System.getProperties()); + prop.load(new FileInputStream(arguments.confFile)); + + if (arguments.serviceURL == null) { + arguments.serviceURL = prop.getProperty("brokerServiceUrl"); + } + + if (arguments.serviceURL == null) { + arguments.serviceURL = prop.getProperty("webServiceUrl"); + } + + // fallback to previous-version serviceUrl property to maintain backward-compatibility + if (arguments.serviceURL == null) { + arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); + } + + if (arguments.adminURL == null) { + arguments.adminURL = prop.getProperty("webServiceUrl"); + } + if (arguments.adminURL == null) { + arguments.adminURL = prop.getProperty("adminURL", "http://localhost:8080/"); + } + } + + + // Dump config variables + PerfClientUtils.printJVMInformation(log); + + ObjectMapper m = new ObjectMapper(); + ObjectWriter w = m.writerWithDefaultPrettyPrinter(); + log.info("Starting Pulsar perf transaction with config: {}", w.writeValueAsString(arguments)); + + final byte[] payloadBytes = new byte[1024]; + Random random = new Random(0); + for (int i = 0; i < payloadBytes.length; ++i) { + payloadBytes[i] = (byte) (random.nextInt(26) + 65); + } + if (arguments.partitions != null) { + PulsarAdminBuilder clientBuilder = PulsarAdmin.builder() + .serviceHttpUrl(arguments.adminURL); + try (PulsarAdmin client = clientBuilder.build()) { + for (String topic : arguments.producerTopic) { + log.info("Creating produce partitioned topic {} with {} partitions", topic, arguments.partitions); + try { + client.topics().createPartitionedTopic(topic, arguments.partitions); + } catch (PulsarAdminException.ConflictException alreadyExists) { + if (log.isDebugEnabled()) { + log.debug("Topic {} already exists: {}", topic, alreadyExists); + } + PartitionedTopicMetadata partitionedTopicMetadata = + client.topics().getPartitionedTopicMetadata(topic); + if (partitionedTopicMetadata.partitions != arguments.partitions) { + log.error( + "Topic {} already exists but it has a wrong number of partitions: {}, expecting {}", + topic, partitionedTopicMetadata.partitions, arguments.partitions); + PerfClientUtils.exit(-1); + } + } + } + } + } + + PulsarClient client = + PulsarClient.builder().enableTransaction(!arguments.isDisableTransaction) + .serviceUrl(arguments.serviceURL) + .connectionsPerBroker(arguments.maxConnections) + .statsInterval(0, TimeUnit.SECONDS) + .ioThreads(arguments.ioThreads) + .build(); + + ExecutorService executorService = new ThreadPoolExecutor(arguments.numTestThreads, + arguments.numTestThreads, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue()); + + + long startTime = System.nanoTime(); + long testEndTime = startTime + (long) (arguments.testTime * 1e9); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + if (!arguments.isDisableTransaction) { + printTxnAggregatedThroughput(startTime); + } else { + printAggregatedThroughput(startTime); + } + printAggregatedStats(); + })); + + // start perf test + AtomicBoolean executing = new AtomicBoolean(true); + + RateLimiter rateLimiter = arguments.openTxnRate > 0 + ? RateLimiter.create(arguments.openTxnRate) + : null; + for(int i = 0; i < arguments.numTestThreads; i++) { + executorService.submit(() -> { + //The producer and consumer clients are built in advance, and then this thread is + //responsible for the production and consumption tasks of the transaction through the loop. + //A thread may perform tasks of multiple transactions in a traversing manner. + List> producers = null; + List>> consumers = null; + try { + producers = buildProducers(client, arguments); + consumers = buildConsumer(client, arguments); + } catch (Exception e) { + log.error("Failed to build Producer/Consumer with exception : ", e); + executorService.shutdownNow(); + PerfClientUtils.exit(-1); + } + AtomicReference atomicReference = buildTransaction(client, + !arguments.isDisableTransaction, arguments.transactionTimeout); + //The while loop has no break, and finally ends the execution through the shutdownNow of + //the executorService + while (true) { + if (arguments.numTransactions > 0) { + if (totalNumTxnOpenTxnFail.sum() + + totalNumTxnOpenTxnSuccess.sum() >= arguments.numTransactions) { + if (totalNumEndTxnOpFailed.sum() + + totalNumEndTxnOpSuccess.sum() < arguments.numTransactions ) { + continue; + } + log.info("------------------- DONE -----------------------"); + executing.compareAndSet(true, false); + executorService.shutdownNow(); + PerfClientUtils.exit(0); + break; + } + } + if (arguments.testTime > 0) { + if (System.nanoTime() > testEndTime) { + log.info("------------------- DONE -----------------------"); + executing.compareAndSet(true, false); + executorService.shutdownNow(); + PerfClientUtils.exit(0); + break; + } + } + Transaction transaction = atomicReference.get(); + for (List> subscriptions : consumers) { + for (Consumer consumer : subscriptions) { + for (int j = 0; j < arguments.numMessagesReceivedPerTransaction; j++) { + Message message = null; + try { + message = consumer.receive(); + } catch (PulsarClientException e) { + log.error("Receive message failed", e); + executorService.shutdownNow(); + PerfClientUtils.exit(-1); + } + long receiveTime = System.nanoTime(); + if (!arguments.isDisableTransaction) { + consumer.acknowledgeAsync(message.getMessageId(), transaction) + .thenRun(() -> { + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - receiveTime); + messageAckRecorder.recordValue(latencyMicros); + messageAckCumulativeRecorder.recordValue(latencyMicros); + numMessagesAckSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && !executing.get()) { + return null; + } + log.error( + "Ack message failed with transaction {} throw exception", + transaction, exception); + numMessagesAckFailed.increment(); + return null; + }); + } else { + consumer.acknowledgeAsync(message).thenRun(() -> { + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - receiveTime); + messageAckRecorder.recordValue(latencyMicros); + messageAckCumulativeRecorder.recordValue(latencyMicros); + numMessagesAckSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && !executing.get()) { + return null; + } + log.error( + "Ack message failed with transaction {} throw exception", + transaction, exception); + numMessagesAckFailed.increment(); + return null; + }); + } + } + } + } + + for(Producer producer : producers){ + for (int j = 0; j < arguments.numMessagesProducedPerTransaction; j++) { + long sendTime = System.nanoTime(); + if (!arguments.isDisableTransaction) { + producer.newMessage(transaction).value(payloadBytes) + .sendAsync().thenRun(() -> { + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - sendTime); + messageSendRecorder.recordValue(latencyMicros); + messageSendRCumulativeRecorder.recordValue(latencyMicros); + numMessagesSendSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && ! executing.get()) { + return null; + } + log.error("Send transaction message failed with exception : ", exception); + numMessagesSendFailed.increment(); + return null; + }); + } else { + producer.newMessage().value(payloadBytes) + .sendAsync().thenRun(() -> { + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - sendTime); + messageSendRecorder.recordValue(latencyMicros); + messageSendRCumulativeRecorder.recordValue(latencyMicros); + numMessagesSendSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && ! executing.get()) { + return null; + } + log.error("Send message failed with exception : ", exception); + numMessagesSendFailed.increment(); + return null; + }); + } + } + } + + if (rateLimiter != null){ + rateLimiter.tryAcquire(); + } + if (!arguments.isDisableTransaction) { + if (!arguments.isAbortTransaction) { + transaction.commit() + .thenRun(() -> { + numTxnOpSuccess.increment(); + totalNumEndTxnOpSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && ! executing.get()) { + return null; + } + log.error("Commit transaction {} failed with exception", + transaction.getTxnID().toString(), + exception); + totalNumEndTxnOpFailed.increment(); + return null; + }); + } else { + transaction.abort().thenRun(() -> { + numTxnOpSuccess.increment(); + totalNumEndTxnOpSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && ! executing.get()) { + return null; + } + log.error("Commit transaction {} failed with exception", + transaction.getTxnID().toString(), + exception); + totalNumEndTxnOpFailed.increment(); + return null; + }); + } + while (true) { + try{ + Transaction newTransaction = client.newTransaction() + .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .build() + .get(); + atomicReference.compareAndSet(transaction, newTransaction); + totalNumTxnOpenTxnSuccess.increment(); + break; + }catch (Exception throwable){ + if (throwable instanceof InterruptedException && !executing.get()) { + break; + } + log.error("Failed to new transaction with exception: ", throwable); + totalNumTxnOpenTxnFail.increment(); + } + } + } else { + totalNumTxnOpenTxnSuccess.increment(); + totalNumEndTxnOpSuccess.increment(); + numTxnOpSuccess.increment(); + } + } + }); + } + + + + // Print report stats + long oldTime = System.nanoTime(); + + Histogram reportSendHistogram = null; + Histogram reportAckHistogram = null; + + String statsFileName = "perf-transaction-" + System.currentTimeMillis() + ".hgrm"; + log.info("Dumping latency stats to {}", statsFileName); + + PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); + HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); + + // Some log header bits + histogramLogWriter.outputLogFormatVersion(); + histogramLogWriter.outputLegend(); + + while (executing.get()) { + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + break; + } + long now = System.nanoTime(); + double elapsed = (now - oldTime) / 1e9; + long total = totalNumEndTxnOpFailed.sum() + totalNumTxnOpenTxnSuccess.sum(); + double rate = numTxnOpSuccess.sumThenReset() / elapsed; + reportSendHistogram = messageSendRecorder.getIntervalHistogram(reportSendHistogram); + reportAckHistogram = messageAckRecorder.getIntervalHistogram(reportAckHistogram); + String txnOrTaskLog = !arguments.isDisableTransaction + ? "Throughput transaction: {} transaction executes --- {} transaction/s" + : "Throughput task: {} task executes --- {} task/s"; + log.info( + txnOrTaskLog + " ---send Latency: mean: {} ms - med: {} " + + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}" + "---ack Latency: " + + "mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", + intFormat.format(total), + dec.format(rate), + dec.format(reportSendHistogram.getMean() / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(50) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(95) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99.9) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99.99) / 1000.0), + dec.format(reportSendHistogram.getMaxValue() / 1000.0), + dec.format(reportAckHistogram.getMean() / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(50) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(95) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99.9) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99.99) / 1000.0), + dec.format(reportAckHistogram.getMaxValue() / 1000.0)); + + histogramLogWriter.outputIntervalHistogram(reportSendHistogram); + histogramLogWriter.outputIntervalHistogram(reportAckHistogram); + reportSendHistogram.reset(); + reportAckHistogram.reset(); + + oldTime = now; + } + + + } + + + private static void printTxnAggregatedThroughput(long start) { + double elapsed = (System.nanoTime() - start) / 1e9; + long numTransactionEndFailed = totalNumEndTxnOpFailed.sum(); + long numTransactionEndSuccess = totalNumEndTxnOpSuccess.sum(); + long total = numTransactionEndFailed + numTransactionEndSuccess; + double rate = total / elapsed; + long numMessageAckFailed = numMessagesAckFailed.sum(); + long numMessageAckSuccess = numMessagesAckSuccess.sum(); + long numMessageSendFailed = numMessagesSendFailed.sum(); + long numMessageSendSuccess = numMessagesSendSuccess.sum(); + long numTransactionOpenFailed = totalNumTxnOpenTxnFail.sum(); + long numTransactionOpenSuccess = totalNumTxnOpenTxnSuccess.sum(); + + log.info( + "Aggregated throughput stats --- {} transaction executed --- {} transaction/s " + + " --- {} transaction open successfully --- {} transaction open failed" + + " --- {} transaction end successfully --- {} transaction end failed" + + "--- {} message ack failed --- {} message send failed" + + "--- {} message ack success --- {} message send success ", + total, + dec.format(rate), + numTransactionOpenSuccess, + numTransactionOpenFailed, + numTransactionEndSuccess, + numTransactionEndFailed, + numMessageAckFailed, + numMessageSendFailed, + numMessageAckSuccess, + numMessageSendSuccess); + + } + + private static void printAggregatedThroughput(long start) { + double elapsed = (System.nanoTime() - start) / 1e9; + long total = totalNumEndTxnOpFailed.sum() + totalNumEndTxnOpSuccess.sum(); + double rate = total / elapsed; + long numMessageAckFailed = numMessagesAckFailed.sum(); + long numMessageAckSuccess = numMessagesAckSuccess.sum(); + long numMessageSendFailed = numMessagesSendFailed.sum(); + long numMessageSendSuccess = numMessagesSendSuccess.sum(); + log.info( + "Aggregated throughput stats --- {} task executed --- {} task/s " + + "--- {} message ack failed --- {} message send failed" + + "--- {} message ack success --- {} message send success ", + total, + totalFormat.format(rate), + numMessageAckFailed, + numMessageSendFailed, + numMessageAckSuccess, + numMessageSendSuccess); + } + + private static void printAggregatedStats() { + Histogram reportAckHistogram = messageAckCumulativeRecorder.getIntervalHistogram(); + Histogram reportSendHistogram = messageSendRCumulativeRecorder.getIntervalHistogram(); + log.info( + "Messages ack aggregated latency stats --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - " + + "99.9pct: {} - " + + "99.99pct: {} - 99.999pct: {} - Max: {}", + dec.format(reportAckHistogram.getMean() / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(50) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(95) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99.9) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99.99) / 1000.0), + dec.format(reportAckHistogram.getValueAtPercentile(99.999) / 1000.0), + dec.format(reportAckHistogram.getMaxValue() / 1000.0)); + log.info( + "Messages send aggregated latency stats --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - " + + "99.9pct: {} - " + + "99.99pct: {} - 99.999pct: {} - Max: {}", + dec.format(reportSendHistogram.getMean() / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(50) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(95) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99.9) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99.99) / 1000.0), + dec.format(reportSendHistogram.getValueAtPercentile(99.999) / 1000.0), + dec.format(reportSendHistogram.getMaxValue() / 1000.0)); + } + + + + static final DecimalFormat dec = new PaddingDecimalFormat("0.000", 7); + static final DecimalFormat intFormat = new PaddingDecimalFormat("0", 7); + static final DecimalFormat totalFormat = new DecimalFormat("0.000"); + private static final Logger log = LoggerFactory.getLogger(PerformanceProducer.class); + + + private static List>> buildConsumer(PulsarClient client, Arguments arguments) + throws ExecutionException, InterruptedException { + ConsumerBuilder consumerBuilder = client.newConsumer(Schema.BYTES) // + .subscriptionType(arguments.subscriptionType) + .receiverQueueSize(arguments.receiverQueueSize) + .subscriptionInitialPosition(arguments.subscriptionInitialPosition); + + Iterator consumerTopicsIterator = arguments.consumerTopic.iterator(); + List>> consumers = Lists.newArrayListWithCapacity(arguments.consumerTopic.size()); + while(consumerTopicsIterator.hasNext()){ + String topic = consumerTopicsIterator.next(); + final List> subscriptions = Lists.newArrayListWithCapacity(arguments.numSubscriptions); + final List>> subscriptionFutures = + Lists.newArrayListWithCapacity(arguments.numSubscriptions); + log.info("Create subscriptions for topic {}", topic); + for (int j = 0; j < arguments.numSubscriptions; j++) { + String subscriberName = arguments.subscriptions.get(j); + subscriptionFutures + .add(consumerBuilder.clone().topic(topic).subscriptionName(subscriberName) + .subscribeAsync()); + } + for (Future> future : subscriptionFutures) { + subscriptions.add(future.get()); + } + consumers.add(subscriptions); + } + return consumers; + } + + private static List> buildProducers(PulsarClient client, Arguments arguments) + throws ExecutionException, InterruptedException { + + ProducerBuilder producerBuilder = client.newProducer(Schema.BYTES) + .sendTimeout(0, TimeUnit.SECONDS); + + final List>> producerFutures = Lists.newArrayList(); + Iterator produceTopicsIterator = arguments.producerTopic.iterator(); + while(produceTopicsIterator.hasNext()){ + String topic = produceTopicsIterator.next(); + log.info("Create producer for topic {}", topic); + producerFutures.add(producerBuilder.clone().topic(topic).createAsync()); + } + final List> producers = Lists.newArrayListWithCapacity(producerFutures.size()); + + for (Future> future : producerFutures) { + producers.add(future.get()); + } + return producers; + } + +} diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java new file mode 100644 index 0000000000000..ded11315070ed --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient.utils; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.testclient.PerformanceProducer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PerformanceUtils { + + private static final Logger log = LoggerFactory.getLogger(PerformanceProducer.class); + + public static AtomicReference buildTransaction(PulsarClient pulsarClient, boolean isEnableTransaction, + long transactionTimeout) { + + AtomicLong numBuildTxnFailed = new AtomicLong(); + if (isEnableTransaction) { + while(true) { + AtomicReference atomicReference = null; + try { + atomicReference = new AtomicReference(pulsarClient.newTransaction() + .withTransactionTimeout(transactionTimeout, TimeUnit.SECONDS).build().get()); + } catch (Exception e) { + numBuildTxnFailed.incrementAndGet(); + if (numBuildTxnFailed.get()%10 == 0) { + log.error("Failed to new a transaction with {} times", numBuildTxnFailed.get(), e); + } + } + if (atomicReference != null && atomicReference.get() != null) { + log.info("After {} failures, the transaction was created successfully for the first time", + numBuildTxnFailed.get()); + return atomicReference; + } + } + } + return new AtomicReference<>(null); + } +} diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java new file mode 100644 index 0000000000000..e04ea049080dc --- /dev/null +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import com.google.common.collect.Sets; +import java.net.URL; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.testng.Assert.fail; + +@Slf4j +public class PerformanceTransactionTest extends MockedPulsarServiceBaseTest { + private final String testTenant = "pulsar"; + private final String testNamespace = "perf"; + private final String myNamespace = testTenant + "/" + testNamespace; + private final String testTopic = "persistent://" + myNamespace + "/test-"; + private final AtomicInteger lastExitCode = new AtomicInteger(0); + + @BeforeMethod + @Override + protected void setup() throws Exception { + ServiceConfiguration serviceConfiguration = getDefaultConf(); + serviceConfiguration.setSystemTopicEnabled(true); + serviceConfiguration.setTransactionCoordinatorEnabled(true); + super.internalSetup(serviceConfiguration); + PerfClientUtils.setExitProcedure(code -> { + log.error("JVM exit code is {}", code); + if (code != 0) { + throw new RuntimeException("JVM should exit with code " + code); + } + }); + // Setup namespaces + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 1); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + int exitCode = lastExitCode.get(); + if (exitCode != 0) { + fail("Unexpected JVM exit code "+exitCode); + } + } + + @Test + public void testTxnPerf() throws Exception { + String argString = "--topics-c %s --topics-p %s -threads 1 -ntxn 50 -u %s -ss %s -np 1 -au %s"; + String testConsumeTopic = testTopic + UUID.randomUUID().toString(); + String testProduceTopic = testTopic + UUID.randomUUID().toString(); + String testSub = "testSub"; + admin.topics().createPartitionedTopic(testConsumeTopic, 1); + String args = String.format(argString, testConsumeTopic, testProduceTopic, + pulsar.getBrokerServiceUrl(), testSub, new URL(pulsar.getWebServiceAddress())); + + PulsarClient pulsarClient = PulsarClient.builder() + .enableTransaction(true) + .serviceUrl(pulsar.getBrokerServiceUrl()) + .connectionsPerBroker(100) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + Producer produceToConsumeTopic = pulsarClient.newProducer(Schema.BYTES) + .producerName("perf-transaction-producer") + .sendTimeout(0, TimeUnit.SECONDS) + .topic(testConsumeTopic) + .create(); + pulsarClient.newConsumer(Schema.BYTES) + .consumerName("perf-transaction-consumeVerify") + .topic(testConsumeTopic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName(testSub + "pre") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + CountDownLatch countDownLatch = new CountDownLatch(50); + for (int i = 0; i < 50 + ; i++) { + produceToConsumeTopic.newMessage().value(("testConsume " + i).getBytes()).sendAsync().thenRun(() -> { + countDownLatch.countDown(); + }); + } + + countDownLatch.await(); + + Thread thread = new Thread(() -> { + try { + PerformanceTransaction.main(args.split(" ")); + } catch (Exception e) { + e.printStackTrace(); + } + }); + thread.start(); + thread.join(); + Consumer consumeFromConsumeTopic = pulsarClient.newConsumer(Schema.BYTES) + .consumerName("perf-transaction-consumeVerify") + .topic(testConsumeTopic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName(testSub) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + Consumer consumeFromProduceTopic = pulsarClient.newConsumer(Schema.BYTES) + .consumerName("perf-transaction-produceVerify") + .topic(testProduceTopic) + .subscriptionName(testSub) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + for (int i = 0; i < 50; i++) { + Message message = consumeFromProduceTopic.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(message); + consumeFromProduceTopic.acknowledge(message); + } + Message message = consumeFromConsumeTopic.receive(2, TimeUnit.SECONDS); + Assert.assertNull(message); + message = consumeFromProduceTopic.receive(2, TimeUnit.SECONDS); + Assert.assertNull(message); + } + + + @Test + public void testProduceTxnMessage() throws InterruptedException, PulsarClientException { + String argString = "%s -r 10 -u %s -m %d -txn"; + String topic = testTopic + UUID.randomUUID(); + int totalMessage = 100; + String args = String.format(argString, topic, pulsar.getBrokerServiceUrl(), totalMessage); + pulsarClient.newConsumer().subscriptionName("subName" + "pre").topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Exclusive) + .enableBatchIndexAcknowledgment(false) + .subscribe(); + Thread thread = new Thread(() -> { + try { + log.info(""); + PerformanceProducer.main(args.split(" ")); + } catch (Exception e) { + e.printStackTrace(); + } + }); + thread.start(); + thread.join(); + Consumer consumer = pulsarClient.newConsumer().subscriptionName("subName").topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Exclusive) + .enableBatchIndexAcknowledgment(false) + .subscribe(); + for (int i = 0; i < totalMessage; i++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(message); + consumer.acknowledge(message); + } + Message message = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNull(message); + } + + @Test + public void testConsumeTxnMessage() throws InterruptedException, PulsarClientException, ExecutionException { + String argString = "%s -r 10 -u %s -txn -ss %s -st %s -sp %s -ntxn %d"; + String subName = "sub"; + String topic = testTopic + UUID.randomUUID(); + String args = String.format(argString, topic, pulsar.getBrokerServiceUrl(), subName, + SubscriptionType.Exclusive, SubscriptionInitialPosition.Earliest, 10); + Producer producer = pulsarClient.newProducer().topic(topic).sendTimeout(0, TimeUnit.SECONDS) + .create(); + pulsarClient.newConsumer(Schema.BYTES) + .consumerName("perf-transaction-consumeVerify") + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName(subName + "pre") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + for (int i = 0; i < 505; i++) { + producer.newMessage().send(); + } + Thread thread = new Thread(() -> { + try { + log.info(""); + PerformanceConsumer.main(args.split(" ")); + } catch (Exception e) { + e.printStackTrace(); + } + }); + thread.start(); + thread.join(); + Consumer consumer = pulsarClient.newConsumer().subscriptionName(subName).topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Exclusive) + .enableBatchIndexAcknowledgment(false) + .subscribe(); + for (int i = 0; i < 5; i++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(message); + } + Message message = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNull(message); + } + +} From 4954c09e87be2672ffe04f3a18487c7507484ed7 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 22 Nov 2021 04:45:52 -0500 Subject: [PATCH 162/823] [Java Client] Avoid IllegalStateException in ClientCnx debug logs (#12899) (cherry picked from commit 32b697d3adb18ecc9992f6dfbf9ac13158649af3) --- .../java/org/apache/pulsar/client/impl/ClientCnx.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index e2386a91436dc..cae1594219eed 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -543,7 +543,9 @@ protected void handleProducerSuccess(CommandProducerSuccess success) { @Override protected void handleLookupResponse(CommandLookupTopicResponse lookupResult) { if (log.isDebugEnabled()) { - log.debug("Received Broker lookup response: {}", lookupResult.getResponse()); + CommandLookupTopicResponse.LookupType response = + lookupResult.hasResponse() ? lookupResult.getResponse() : null; + log.debug("Received Broker lookup response: {} {}", lookupResult.getRequestId(), response); } long requestId = lookupResult.getRequestId(); @@ -579,7 +581,11 @@ protected void handleLookupResponse(CommandLookupTopicResponse lookupResult) { @Override protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse lookupResult) { if (log.isDebugEnabled()) { - log.debug("Received Broker Partition response: {}", lookupResult.getPartitions()); + CommandPartitionedTopicMetadataResponse.LookupType response = + lookupResult.hasResponse() ? lookupResult.getResponse() : null; + int partitions = lookupResult.hasPartitions() ? lookupResult.getPartitions() : -1; + log.debug("Received Broker Partition response: {} {} {}", lookupResult.getRequestId(), response, + partitions); } long requestId = lookupResult.getRequestId(); From 96238e0eb1fcde8a8b064d5e710e76b0bfa5d758 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Mon, 22 Nov 2021 09:57:46 +0800 Subject: [PATCH 163/823] [broker] Fix topic policy listener deleted by mistake. (#12904) ### Motivation Here is the current way of dealing topic policy listeners in PersistentTopic, for example topic name is "A", with 3 partitions. - Register: call TopicPoliciesService.registerListener("A", listener), for all 3 partitions of topic "A". - Clean: call TopicPoliciesService.clean("A-partition-x"), here is the problem it will delete all listeners of all partitions of topic "A", if any partition is closed. This means, if we calls `admin.topics().unload("A-partition-0")`, "A-partition-1" and "A-partition-2" will not be able to receive topic policy update callbacks any more. A detailed case is designed in the new unit test `testListenerCleanupByPartition`. ### Modifications With previous optimization of #12654 , now we can use `org.apache.pulsar.broker.service.TopicPoliciesService#unregisterListener` to do the clean up. (cherry picked from commit a0c96a08de83de6ce51fffd06e907833f075bbca) --- .../service/persistent/PersistentTopic.java | 20 +++++++++------- ...temTopicBasedTopicPoliciesServiceTest.java | 24 +++++++++++++++++++ .../pulsar/common/naming/TopicName.java | 8 +++++++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 275c7ea8989ee..f4adb2a4b0fc2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1156,8 +1156,7 @@ public void deleteLedgerComplete(Object ctx) { subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close); - brokerService.pulsar().getTopicPoliciesService() - .clean(TopicName.get(topic)); + unregisterTopicPolicyListener(); log.info("[{}] Topic deleted", topic); deleteFuture.complete(null); @@ -1263,7 +1262,7 @@ public void closeComplete(Object ctx) { subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close); - brokerService.pulsar().getTopicPoliciesService().clean(TopicName.get(topic)); + unregisterTopicPolicyListener(); log.info("[{}] Topic closed", topic); closeFuture.complete(null); }) @@ -3148,13 +3147,16 @@ private void initializeTopicSubscribeRateLimiterIfNeeded(Optional private void registerTopicPolicyListener() { if (brokerService.pulsar().getConfig().isSystemTopicEnabled() && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { - TopicName topicName = TopicName.get(topic); - TopicName cloneTopicName = topicName; - if (topicName.isPartitioned()) { - cloneTopicName = TopicName.get(topicName.getPartitionedTopicName()); - } + brokerService.getPulsar().getTopicPoliciesService() + .registerListener(TopicName.getPartitionedTopicName(topic), this); + } + } - brokerService.getPulsar().getTopicPoliciesService().registerListener(cloneTopicName, this); + private void unregisterTopicPolicyListener() { + if (brokerService.pulsar().getConfig().isSystemTopicEnabled() + && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { + brokerService.getPulsar().getTopicPoliciesService() + .unregisterListener(TopicName.getPartitionedTopicName(topic), this); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index be52f9a119f8f..9a489cf593984 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -259,6 +259,30 @@ public void testCacheCleanup() throws Exception { assertNull(listMap.get(topicName)); } + @Test + public void testListenerCleanupByPartition() throws Exception { + final String topic = "persistent://" + NAMESPACE1 + "/test" + UUID.randomUUID(); + TopicName topicName = TopicName.get(topic); + admin.topics().createPartitionedTopic(topic, 3); + pulsarClient.newProducer().topic(topic).create().close(); + + Map>> listMap = + systemTopicBasedTopicPoliciesService.getListeners(); + Awaitility.await().untilAsserted(() -> { + // all 3 topic partition have registered the topic policy listeners. + assertEquals(listMap.get(topicName).size(), 3); + }); + + admin.topics().unload(topicName.getPartition(0).toString()); + assertEquals(listMap.get(topicName).size(), 2); + admin.topics().unload(topicName.getPartition(1).toString()); + assertEquals(listMap.get(topicName).size(), 1); + admin.topics().unload(topicName.getPartition(2).toString()); + assertNull(listMap.get(topicName)); + } + + + private void prepareData() throws PulsarAdminException { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); admin.tenants().createTenant("system-topic", diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index 3729697c9e36b..67bc92acff3d9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -90,6 +90,14 @@ public static TopicName get(String topic) { } } + public static TopicName getPartitionedTopicName(String topic) { + TopicName topicName = TopicName.get(topic); + if (topicName.isPartitioned()) { + return TopicName.get(topicName.getPartitionedTopicName()); + } + return topicName; + } + public static boolean isValid(String topic) { try { get(topic); From ca6540ed9272c881de533405ccfce945b9d4d96d Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Wed, 24 Nov 2021 08:37:18 +0800 Subject: [PATCH 164/823] [Broker] Correct param of delete method for v1 topic (#12936) (cherry picked from commit e6d9df81c446870107dbb8d8e454b11b71cc9255) --- .../broker/admin/v1/PersistentTopics.java | 10 ++- .../org/apache/pulsar/schema/SchemaTest.java | 90 +++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 9ef87f4d7d7b5..c0c4b486dd782 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -264,10 +264,11 @@ public void deletePartitionedTopic(@Suspended final AsyncResponse asyncResponse, @PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("force") @DefaultValue("false") boolean force, - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("deleteSchema") @DefaultValue("false") boolean deleteSchema) { try { validateTopicName(property, cluster, namespace, encodedTopic); - internalDeletePartitionedTopic(asyncResponse, authoritative, force, false); + internalDeletePartitionedTopic(asyncResponse, authoritative, force, deleteSchema); } catch (WebApplicationException wae) { asyncResponse.resume(wae); } catch (Exception e) { @@ -302,9 +303,10 @@ public void unloadTopic(@Suspended final AsyncResponse asyncResponse, @PathParam public void deleteTopic(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("force") @DefaultValue("false") boolean force, - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("deleteSchema") @DefaultValue("false") boolean deleteSchema) { validateTopicName(property, cluster, namespace, encodedTopic); - internalDeleteTopic(authoritative, force); + internalDeleteTopic(authoritative, force, deleteSchema); } @GET diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index db56fbce5a19b..45f12aba0b0dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -724,6 +724,96 @@ public void testDeleteTopicAndSchema() throws Exception { } } + @Test + public void testDeleteTopicAndSchemaForV1() throws Exception { + final String tenant = PUBLIC_TENANT; + final String cluster = CLUSTER_NAME; + final String namespace = "test-namespace-" + randomName(16); + final String topicOne = "not-partitioned-topic"; + final String topic2 = "persistent://" + tenant + "/" + cluster + "/" + namespace + "/partitioned-topic"; + + // persistent, not-partitioned v1/topic + final String topic1 = TopicName.get( + TopicDomain.persistent.value(), + tenant, + cluster, + namespace, + topicOne).toString(); + + // persistent, partitioned v1/topic + admin.topics().createPartitionedTopic(topic2, 1); + + @Cleanup + Producer p1_1 = pulsarClient.newProducer(Schema.JSON(Schemas.PersonOne.class)) + .topic(topic1) + .create(); + + @Cleanup + Producer p1_2 = pulsarClient.newProducer(Schema.JSON(Schemas.PersonThree.class)) + .topic(topic1) + .create(); + @Cleanup + Producer p2_1 = pulsarClient.newProducer(Schema.JSON(Schemas.PersonThree.class)) + .topic(topic2) + .create(); + + List> schemaFutures1 = + this.getPulsar().getSchemaRegistryService().getAllSchemas(TopicName.get(topic1).getSchemaName()).get(); + FutureUtil.waitForAll(schemaFutures1).get(); + List schemas1 = schemaFutures1.stream().map(future -> { + try { + return future.get(); + } catch (Exception e) { + return null; + } + }).collect(Collectors.toList()); + assertEquals(schemas1.size(), 2); + for (SchemaRegistry.SchemaAndMetadata schema : schemas1) { + assertNotNull(schema); + } + + List> schemaFutures2 = + this.getPulsar().getSchemaRegistryService().getAllSchemas(TopicName.get(topic2).getSchemaName()).get(); + FutureUtil.waitForAll(schemaFutures2).get(); + List schemas2 = schemaFutures2.stream().map(future -> { + try { + return future.get(); + } catch (Exception e) { + return null; + } + }).collect(Collectors.toList()); + assertEquals(schemas2.size(), 1); + for (SchemaRegistry.SchemaAndMetadata schema : schemas2) { + assertNotNull(schema); + } + + // not-force and not-delete-schema when delete topic + try { + admin.topics().delete(topic1, false, false); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith("Topic has active producers/subscriptions")); + } + assertEquals(this.getPulsar().getSchemaRegistryService() + .trimDeletedSchemaAndGetList(TopicName.get(topic1).getSchemaName()).get().size(), 2); + try { + admin.topics().deletePartitionedTopic(topic2, false, false); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith("Topic has active producers/subscriptions")); + } + assertEquals(this.getPulsar().getSchemaRegistryService() + .trimDeletedSchemaAndGetList(TopicName.get(topic2).getSchemaName()).get().size(), 1); + + // force and delete-schema when delete topic + admin.topics().delete(topic1, true, true); + assertEquals(this.getPulsar().getSchemaRegistryService() + .trimDeletedSchemaAndGetList(TopicName.get(topic1).getSchemaName()).get().size(), 0); + admin.topics().deletePartitionedTopic(topic2, true, true); + assertEquals(this.getPulsar().getSchemaRegistryService() + .trimDeletedSchemaAndGetList(TopicName.get(topic2).getSchemaName()).get().size(), 0); + } + @Test public void testProducerMultipleSchemaMessages() throws Exception { final String tenant = PUBLIC_TENANT; From 29e12dd2b4cb069912d0d06fbbd00853d55c5b80 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 23 Nov 2021 19:28:15 +0200 Subject: [PATCH 165/823] [CI] Replace deprecated "adopt" distribution with "temurin" (#12945) - see https://github.com/actions/setup-java#supported-distributions for more details" (cherry picked from commit 60ef5e983e5e8956bd0b602b5741bd6255c6258a) --- .github/workflows/ci-build-macos.yaml | 2 +- .github/workflows/ci-cpp.yaml | 2 +- .github/workflows/ci-integration-backwards-compatibility.yaml | 2 +- .github/workflows/ci-integration-cli.yaml | 2 +- .github/workflows/ci-integration-function.yaml | 2 +- .github/workflows/ci-integration-messaging.yaml | 2 +- .github/workflows/ci-integration-process.yaml | 2 +- .github/workflows/ci-integration-pulsar-io-ora.yaml | 2 +- .github/workflows/ci-integration-pulsar-io.yaml | 2 +- .github/workflows/ci-integration-schema.yaml | 2 +- .github/workflows/ci-integration-sql.yaml | 2 +- .github/workflows/ci-integration-standalone.yaml | 2 +- .github/workflows/ci-integration-thread.yaml | 2 +- .github/workflows/ci-integration-tiered-filesystem.yaml | 2 +- .github/workflows/ci-integration-tiered-jcloud.yaml | 2 +- .github/workflows/ci-integration-transaction.yaml | 2 +- .github/workflows/ci-license.yaml | 2 +- .github/workflows/ci-maven-cache-update.yaml | 2 +- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- .github/workflows/ci-pulsar-website-build.yaml | 2 +- .github/workflows/ci-shade-test.yaml | 2 +- .github/workflows/ci-unit-broker-broker-gp1.yaml | 2 +- .github/workflows/ci-unit-broker-broker-gp2.yaml | 2 +- .github/workflows/ci-unit-broker-client-api.yaml | 2 +- .github/workflows/ci-unit-broker-client-impl.yaml | 2 +- .github/workflows/ci-unit-broker-jdk8.yaml | 2 +- .github/workflows/ci-unit-broker-other.yaml | 2 +- .github/workflows/ci-unit-proxy.yaml | 2 +- .github/workflows/ci-unit.yaml | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml index 0236ab3c6fab5..0d128c4422c03 100644 --- a/.github/workflows/ci-build-macos.yaml +++ b/.github/workflows/ci-build-macos.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml index 70f5f6f917bd4..ed823792ded8c 100644 --- a/.github/workflows/ci-cpp.yaml +++ b/.github/workflows/ci-cpp.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml index 3b13b63d38da4..59efbabe45a80 100644 --- a/.github/workflows/ci-integration-backwards-compatibility.yaml +++ b/.github/workflows/ci-integration-backwards-compatibility.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml index cf51dcc761103..7fb9b950c8208 100644 --- a/.github/workflows/ci-integration-cli.yaml +++ b/.github/workflows/ci-integration-cli.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-function.yaml b/.github/workflows/ci-integration-function.yaml index 52b6d8f972a5d..1e7e5005f996e 100644 --- a/.github/workflows/ci-integration-function.yaml +++ b/.github/workflows/ci-integration-function.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml index 0a81e4a487997..68eeda8aed350 100644 --- a/.github/workflows/ci-integration-messaging.yaml +++ b/.github/workflows/ci-integration-messaging.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml index a5523c15f2a2e..10d3b5ebd2def 100644 --- a/.github/workflows/ci-integration-process.yaml +++ b/.github/workflows/ci-integration-process.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml index 8bb1cba5db3d0..33b5d11af8329 100644 --- a/.github/workflows/ci-integration-pulsar-io-ora.yaml +++ b/.github/workflows/ci-integration-pulsar-io-ora.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-pulsar-io.yaml b/.github/workflows/ci-integration-pulsar-io.yaml index 538e94e284247..9b811f2b8c540 100644 --- a/.github/workflows/ci-integration-pulsar-io.yaml +++ b/.github/workflows/ci-integration-pulsar-io.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml index edeec555b8bc7..84c02c39471a7 100644 --- a/.github/workflows/ci-integration-schema.yaml +++ b/.github/workflows/ci-integration-schema.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml index 53c996874583a..6b5d7ba0077bc 100644 --- a/.github/workflows/ci-integration-sql.yaml +++ b/.github/workflows/ci-integration-sql.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml index dc3577677f742..17499c4741ff3 100644 --- a/.github/workflows/ci-integration-standalone.yaml +++ b/.github/workflows/ci-integration-standalone.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml index 0420baec17970..2133a53727521 100644 --- a/.github/workflows/ci-integration-thread.yaml +++ b/.github/workflows/ci-integration-thread.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml index 89ed30de58aff..413838faecc6e 100644 --- a/.github/workflows/ci-integration-tiered-filesystem.yaml +++ b/.github/workflows/ci-integration-tiered-filesystem.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml index 3cf661a5a162f..5ba8d357dd154 100644 --- a/.github/workflows/ci-integration-tiered-jcloud.yaml +++ b/.github/workflows/ci-integration-tiered-jcloud.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml index 547066db495f1..cd2624865856f 100644 --- a/.github/workflows/ci-integration-transaction.yaml +++ b/.github/workflows/ci-integration-transaction.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml index f706a1cd73bfb..ae60426d482ad 100644 --- a/.github/workflows/ci-license.yaml +++ b/.github/workflows/ci-license.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index b04d2860c93a8..755f2afdf1b26 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -102,7 +102,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: Download dependencies diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 3e951275addcb..301dd25b1bcc9 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -53,7 +53,7 @@ jobs: - name: Set up JDK 11 uses: actions/setup-java@v2 with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: run install by skip tests diff --git a/.github/workflows/ci-pulsar-website-build.yaml b/.github/workflows/ci-pulsar-website-build.yaml index 58d67ad3e0cbc..1c8658c0f4e5e 100644 --- a/.github/workflows/ci-pulsar-website-build.yaml +++ b/.github/workflows/ci-pulsar-website-build.yaml @@ -53,7 +53,7 @@ jobs: - name: Set up JDK 11 uses: actions/setup-java@v2 with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml index 8789bf7dfed44..1aecf70b17c99 100644 --- a/.github/workflows/ci-shade-test.yaml +++ b/.github/workflows/ci-shade-test.yaml @@ -69,7 +69,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: clean disk diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml index 6defee0cdc2e3..71204e5ff8a6e 100644 --- a/.github/workflows/ci-unit-broker-broker-gp1.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp1.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: build modules diff --git a/.github/workflows/ci-unit-broker-broker-gp2.yaml b/.github/workflows/ci-unit-broker-broker-gp2.yaml index 821157c369387..6433dd10f0b2b 100644 --- a/.github/workflows/ci-unit-broker-broker-gp2.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp2.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: build modules diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml index 5bcefdd0529d9..699caf0779d43 100644 --- a/.github/workflows/ci-unit-broker-client-api.yaml +++ b/.github/workflows/ci-unit-broker-client-api.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: build modules diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml index 21003a14288f4..6841b5782acbb 100644 --- a/.github/workflows/ci-unit-broker-client-impl.yaml +++ b/.github/workflows/ci-unit-broker-client-impl.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: build modules diff --git a/.github/workflows/ci-unit-broker-jdk8.yaml b/.github/workflows/ci-unit-broker-jdk8.yaml index ce56ca2b92d70..e0460854b4375 100644 --- a/.github/workflows/ci-unit-broker-jdk8.yaml +++ b/.github/workflows/ci-unit-broker-jdk8.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 8 - name: build modules diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml index 50a87b5124ab6..5826244751e28 100644 --- a/.github/workflows/ci-unit-broker-other.yaml +++ b/.github/workflows/ci-unit-broker-other.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: build modules diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml index bc2cfb9d14679..2ffed49586cde 100644 --- a/.github/workflows/ci-unit-proxy.yaml +++ b/.github/workflows/ci-unit-proxy.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: build modules pulsar-proxy diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml index 37bdfafb0c641..fd553b0700632 100644 --- a/.github/workflows/ci-unit.yaml +++ b/.github/workflows/ci-unit.yaml @@ -68,7 +68,7 @@ jobs: uses: actions/setup-java@v2 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} with: - distribution: 'adopt' + distribution: 'temurin' java-version: 11 - name: run unit test 'OTHER' From f13e9e324f9664463bb7499978a38ba0c7d99431 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 25 Nov 2021 15:29:24 +0200 Subject: [PATCH 166/823] Use JDK default security provider when Conscrypt isn't available (#12938) - fixes issue with ARM64 platform where Conscrypt isn't available (cherry picked from commit 4f2d52edfbb53f043ed5640ccde694b60707eea3) --- .../apache/pulsar/common/util/SecurityUtility.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index faefc8adcd78e..db8f861c789a3 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -116,6 +116,16 @@ public static Provider getProvider() { } private static Provider loadConscryptProvider() { + Class conscryptClazz; + + try { + conscryptClazz = Class.forName("org.conscrypt.Conscrypt"); + conscryptClazz.getMethod("checkAvailability").invoke(null); + } catch (Throwable e) { + log.warn("Conscrypt isn't available. Using JDK default security provider.", e); + return null; + } + Provider provider; try { provider = (Provider) Class.forName(CONSCRYPT_PROVIDER_CLASS).getDeclaredConstructor().newInstance(); @@ -143,7 +153,6 @@ private static Provider loadConscryptProvider() { // contains the workaround. try { HostnameVerifier hostnameVerifier = new TlsHostnameVerifier(); - Class conscryptClazz = Class.forName("org.conscrypt.Conscrypt"); Object wrappedHostnameVerifier = conscryptClazz .getMethod("wrapHostnameVerifier", new Class[]{HostnameVerifier.class}).invoke(null, hostnameVerifier); From 13b07fdb8ef3a2f2f02d29c619a0f712cf9df15a Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Wed, 24 Nov 2021 18:31:45 -0800 Subject: [PATCH 167/823] [function] pulsar admin exposes secrets for function (#12950) ### Motivation Fixes #12834 ### Modifications add the `--secrets` argument into `pulsar-admin functions create/update/localrun` command (cherry picked from commit 9c0cee959e0f215a9a33adca98948f3a5fdf2dfa) --- .../org/apache/pulsar/admin/cli/CmdFunctions.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 0664ca8fcfc6e..2efef9982fe6d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -37,6 +37,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Type; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -319,6 +320,8 @@ abstract class FunctionDetailsCommand extends BaseCommand { protected Integer maxMessageRetries; @Parameter(names = "--custom-runtime-options", description = "A string that encodes options to customize the runtime, see docs for configured runtime for details") protected String customRuntimeOptions; + @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates how the secret is fetched by the underlying secrets provider") + protected String secretsString; @Parameter(names = "--dead-letter-topic", description = "The topic where messages that are not processed successfully are sent to") protected String deadLetterTopic; protected FunctionConfig functionConfig; @@ -520,6 +523,15 @@ void processArguments() throws Exception { functionConfig.setCustomRuntimeOptions(customRuntimeOptions); } + if (secretsString != null) { + Type type = new TypeToken>() {}.getType(); + Map secretsMap = new Gson().fromJson(secretsString, type); + if (secretsMap == null) { + secretsMap = Collections.emptyMap(); + } + functionConfig.setSecrets(secretsMap); + } + // window configs WindowConfig windowConfig = functionConfig.getWindowConfig(); if (null != windowLengthCount) { From 8113c17ff68cad0f0ffa48fd2f31383699d3b355 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 25 Nov 2021 10:29:41 +0800 Subject: [PATCH 168/823] Fix wrong isEmpty method of ConcurrentOpenLongPairRangeSet (#12953) (cherry picked from commit 6cc5cff0f051cf50c84edb4bb67034cb978d3646) --- .../collections/ConcurrentOpenLongPairRangeSet.java | 12 +++++------- .../ConcurrentOpenLongPairRangeSetTest.java | 13 +++++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 174f318aa1b61..7ae6c7e96625a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -152,14 +152,12 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - AtomicBoolean isEmpty = new AtomicBoolean(false); - rangeBitSetMap.forEach((key, val) -> { - if (!isEmpty.get()) { - return; + for (BitSet rangeBitSet : rangeBitSetMap.values()) { + if (!rangeBitSet.isEmpty()) { + return false; } - isEmpty.set(val.isEmpty()); - }); - return isEmpty.get(); + } + return true; } @Override diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java index f57b75d52e10c..90c23ae0dd01f 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java @@ -19,7 +19,9 @@ package org.apache.pulsar.common.util.collections; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import java.util.List; import java.util.Set; @@ -37,6 +39,17 @@ public class ConcurrentOpenLongPairRangeSetTest { static final LongPairConsumer consumer = (key, value) -> new LongPair(key, value); + @Test + public void testIsEmpty() { + ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + assertTrue(set.isEmpty()); + // lowerValueOpen and upperValue are both -1 so that an empty set will be added + set.addOpenClosed(0, -1, 0, -1); + assertTrue(set.isEmpty()); + set.addOpenClosed(1, 1, 1, 5); + assertFalse(set.isEmpty()); + } + @Test public void testAddForSameKey() { ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); From a5c34667e1eb293a928efded16d7de73c0add6ea Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Fri, 26 Nov 2021 11:43:13 +0900 Subject: [PATCH 169/823] [c++] Define and expose PULSAR_VERSION macro (#12769) * Define and expose PULSAR_VERSION macro * Change default value of _PULSAR_VERSION_INTERNAL_ to unknown (cherry picked from commit 2dcf7e924e3d393a23fca3522f539e95c5ae4d62) --- pulsar-client-cpp/.gitignore | 3 ++ pulsar-client-cpp/CMakeLists.txt | 4 +++ .../Version.h => include/pulsar/c/version.h} | 8 ++--- pulsar-client-cpp/lib/CMakeLists.txt | 2 +- pulsar-client-cpp/lib/Commands.cc | 6 ++-- pulsar-client-cpp/lib/HTTPLookupService.cc | 2 +- pulsar-client-cpp/lib/HTTPLookupService.h | 2 +- pulsar-client-cpp/lib/VersionInternal.h | 26 ++++++++++++++ pulsar-client-cpp/templates/Version.h.in | 28 +++++++++++++++ pulsar-client-cpp/tests/VersionTest.cc | 29 +++++++++++++++ src/gen-pulsar-version-macro.py | 35 +++++++++++++++++++ 11 files changed, 133 insertions(+), 12 deletions(-) rename pulsar-client-cpp/{lib/Version.h => include/pulsar/c/version.h} (85%) create mode 100644 pulsar-client-cpp/lib/VersionInternal.h create mode 100644 pulsar-client-cpp/templates/Version.h.in create mode 100644 pulsar-client-cpp/tests/VersionTest.cc create mode 100755 src/gen-pulsar-version-macro.py diff --git a/pulsar-client-cpp/.gitignore b/pulsar-client-cpp/.gitignore index 0f3d36de7ac20..f2a623dfad71a 100644 --- a/pulsar-client-cpp/.gitignore +++ b/pulsar-client-cpp/.gitignore @@ -47,6 +47,9 @@ lib*.so* /perf/perfConsumer /system-test/SystemTest +# Files generated from templates by CMAKE +include/pulsar/Version.h + # IDE generated files .csettings .cproject diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 7c95791544898..8521dbf0e6ec3 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -21,6 +21,10 @@ cmake_minimum_required(VERSION 3.4) project (pulsar-cpp) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") +execute_process(COMMAND python ${CMAKE_SOURCE_DIR}/../src/gen-pulsar-version-macro.py OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE PVM) +set(PVM_COMMENT "This is generated from Version.h.in by CMAKE. DO NOT EDIT DIRECTLY") +configure_file(templates/Version.h.in include/pulsar/Version.h @ONLY) + if (VCPKG_TRIPLET) message(STATUS "Use vcpkg, triplet is ${VCPKG_TRIPLET}") set(CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}") diff --git a/pulsar-client-cpp/lib/Version.h b/pulsar-client-cpp/include/pulsar/c/version.h similarity index 85% rename from pulsar-client-cpp/lib/Version.h rename to pulsar-client-cpp/include/pulsar/c/version.h index a274f47bb8888..ab63c8a708991 100644 --- a/pulsar-client-cpp/lib/Version.h +++ b/pulsar-client-cpp/include/pulsar/c/version.h @@ -16,11 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -#ifndef LIB_VERSION_H_ -#define LIB_VERSION_H_ -#ifndef _PULSAR_VERSION_ -#define _PULSAR_VERSION_ "1.17" -#endif +#pragma once -#endif /* LIB_VERSION_H_ */ +#include diff --git a/pulsar-client-cpp/lib/CMakeLists.txt b/pulsar-client-cpp/lib/CMakeLists.txt index 6e970fa3c5277..ee9214eec8af5 100644 --- a/pulsar-client-cpp/lib/CMakeLists.txt +++ b/pulsar-client-cpp/lib/CMakeLists.txt @@ -20,7 +20,7 @@ file(GLOB PULSAR_SOURCES *.cc *.h lz4/*.cc lz4/*.h checksum/*.cc checksum/*.h stats/*.cc stats/*.h c/*.cc c/*.h auth/*.cc auth/*.h auth/athenz/*.cc auth/athenz/*.h) execute_process(COMMAND python ${CMAKE_SOURCE_DIR}/../src/get-project-version.py OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE PV) -set (CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -D_PULSAR_VERSION_=\\\"${PV}\\\"") +set (CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -D_PULSAR_VERSION_INTERNAL_=\\\"${PV}\\\"") if (NOT PROTOC_PATH) set(PROTOC_PATH protoc) diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc index 54c8c65f0c71b..1094efb8eee35 100644 --- a/pulsar-client-cpp/lib/Commands.cc +++ b/pulsar-client-cpp/lib/Commands.cc @@ -18,7 +18,7 @@ */ #include "Commands.h" #include "MessageImpl.h" -#include "Version.h" +#include "VersionInternal.h" #include "pulsar/MessageBuilder.h" #include "LogUtils.h" #include "PulsarApi.pb.h" @@ -215,7 +215,7 @@ SharedBuffer Commands::newConnect(const AuthenticationPtr& authentication, const BaseCommand cmd; cmd.set_type(BaseCommand::CONNECT); CommandConnect* connect = cmd.mutable_connect(); - connect->set_client_version(_PULSAR_VERSION_); + connect->set_client_version(_PULSAR_VERSION_INTERNAL_); connect->set_auth_method_name(authentication->getAuthMethodName()); connect->set_protocol_version(ProtocolVersion_MAX); @@ -243,7 +243,7 @@ SharedBuffer Commands::newAuthResponse(const AuthenticationPtr& authentication, BaseCommand cmd; cmd.set_type(BaseCommand::AUTH_RESPONSE); CommandAuthResponse* authResponse = cmd.mutable_authresponse(); - authResponse->set_client_version(_PULSAR_VERSION_); + authResponse->set_client_version(_PULSAR_VERSION_INTERNAL_); AuthData* authData = authResponse->mutable_response(); authData->set_auth_method_name(authentication->getAuthMethodName()); diff --git a/pulsar-client-cpp/lib/HTTPLookupService.cc b/pulsar-client-cpp/lib/HTTPLookupService.cc index a54a4c1f4f770..d377171b83c52 100644 --- a/pulsar-client-cpp/lib/HTTPLookupService.cc +++ b/pulsar-client-cpp/lib/HTTPLookupService.cc @@ -151,7 +151,7 @@ void HTTPLookupService::handleNamespaceTopicsHTTPRequest(NamespaceTopicsPromise Result HTTPLookupService::sendHTTPRequest(const std::string completeUrl, std::string &responseData) { CURL *handle; CURLcode res; - std::string version = std::string("Pulsar-CPP-v") + _PULSAR_VERSION_; + std::string version = std::string("Pulsar-CPP-v") + _PULSAR_VERSION_INTERNAL_; handle = curl_easy_init(); if (!handle) { diff --git a/pulsar-client-cpp/lib/HTTPLookupService.h b/pulsar-client-cpp/lib/HTTPLookupService.h index 166a14a03b957..eb7365447afad 100644 --- a/pulsar-client-cpp/lib/HTTPLookupService.h +++ b/pulsar-client-cpp/lib/HTTPLookupService.h @@ -22,7 +22,7 @@ #include #include #include -#include +#include namespace pulsar { class HTTPLookupService : public LookupService, public std::enable_shared_from_this { diff --git a/pulsar-client-cpp/lib/VersionInternal.h b/pulsar-client-cpp/lib/VersionInternal.h new file mode 100644 index 0000000000000..c2560352692dd --- /dev/null +++ b/pulsar-client-cpp/lib/VersionInternal.h @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef LIB_VERSION_INTERNAL_H_ +#define LIB_VERSION_INTERNAL_H_ + +#ifndef _PULSAR_VERSION_INTERNAL_ +#define _PULSAR_VERSION_INTERNAL_ "unknown" +#endif + +#endif /* LIB_VERSION_INTERNAL_H_ */ diff --git a/pulsar-client-cpp/templates/Version.h.in b/pulsar-client-cpp/templates/Version.h.in new file mode 100644 index 0000000000000..d52121ac8c19e --- /dev/null +++ b/pulsar-client-cpp/templates/Version.h.in @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * @PVM_COMMENT@ + */ +#ifndef PULSAR_VERSION_H_ +#define PULSAR_VERSION_H_ + +#define PULSAR_VERSION @PVM@ + +#endif /* PULSAR_VERSION_H_ */ diff --git a/pulsar-client-cpp/tests/VersionTest.cc b/pulsar-client-cpp/tests/VersionTest.cc new file mode 100644 index 0000000000000..57e1e78376200 --- /dev/null +++ b/pulsar-client-cpp/tests/VersionTest.cc @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include + +TEST(VersionTest, testMacro) { +#ifdef PULSAR_VERSION + ASSERT_GE(PULSAR_VERSION, 2000000); + ASSERT_LE(PULSAR_VERSION, 999999999); +#else + FAIL(); +#endif +} diff --git a/src/gen-pulsar-version-macro.py b/src/gen-pulsar-version-macro.py new file mode 100755 index 0000000000000..f32df91772f74 --- /dev/null +++ b/src/gen-pulsar-version-macro.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import xml.etree.ElementTree as ET +import re +from os.path import dirname, realpath, join + +# Derive the POM path from the current script location +TOP_LEVEL_PATH = dirname(dirname(realpath(__file__))) +POM_PATH = join(TOP_LEVEL_PATH, 'pom.xml') + +root = ET.XML(open(POM_PATH).read()) +m = re.search(r'^(\d+)\.(\d+)\.(\d+)', root.find('{http://maven.apache.org/POM/4.0.0}version').text) + +version_macro = 0 +for i in range(3): + version_macro += int(m.group(3 - i)) * (1000 ** i) +print(version_macro) From 73863f8fb87d33f9d90191dfc2aae7d30fd3b676 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Fri, 26 Nov 2021 11:23:12 +0800 Subject: [PATCH 170/823] Fix NPE in checkSubscriptionTypesEnable (#12961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NPE may appear in `PersistentTopic.checkSubscriptionTypesEnable`. The `topicPolicies.getSubscriptionTypesEnabled()` may be null. The problem occurred when upgrading from pulsar version 2.7 to 2.8. We added `SubscriptionTypesEnable` in 2.8: https://github.com/apache/pulsar/pull/9401. When we initialize a topic's topicPolicy in pulasr 2.7, and then get the topic's topicPolicy in version 2.8, the `SubscriptionTypesEnable` will be null. This leads to the problem. Here are steps to reproduce: * Start broker 2.7 * Create a topic and init the topic policy * Upgrade broker to 2.8 * We will get the `SubscriptionTypesEnable` with the value of null ```sh ➜ pulsar-281 bin/pulsar-admin topics get-subscription-types-enabled my-topic null ``` * When we consume from that topic, the issue occurs ``` Caused by: java.lang.NullPointerException at org.apache.pulsar.broker.service.persistent.PersistentTopic.checkSubscriptionTypesEnable(PersistentTopic.java:3206) ~[io.streamnative-pulsar-broker-2.8.1.21.jar:2.8.1.21] at org.apache.pulsar.broker.service.persistent.PersistentTopic.subscribe(PersistentTopic.java:688) ~[io.streamnative-pulsar-broker-2.8.1.21.jar:2.8.1.21] at org.apache.pulsar.broker.service.ServerCnx.lambda$null$13(ServerCnx.java:1029) ~[io.streamnative-pulsar-broker-2.8.1.21.jar:2.8.1.21] at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1106) ~[?:?] ... 31 more ``` * Use `CollectionUtils.isEmpty` to determine if the list is empty to fix the problem of throwing NPE. (cherry picked from commit 877bf3a34e72336c05706ffd48826e4347606196) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index f4adb2a4b0fc2..46c9dfdb1a50a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -72,6 +72,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.net.BookieId; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; @@ -3198,7 +3199,7 @@ public boolean checkSubscriptionTypesEnable(SubType subType) throws Exception { if (topicPolicies == null) { return checkNsAndBrokerSubscriptionTypesEnable(topicName, subType); } else { - if (topicPolicies.getSubscriptionTypesEnabled().isEmpty()) { + if (CollectionUtils.isEmpty(topicPolicies.getSubscriptionTypesEnabled())) { return checkNsAndBrokerSubscriptionTypesEnable(topicName, subType); } return topicPolicies.getSubscriptionTypesEnabled().contains(subType); From da973ca0a1e1221b46db546790e8469e630125c5 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Fri, 26 Nov 2021 20:08:51 -0600 Subject: [PATCH 171/823] feat(cli): support autorecovery service in pulsar cli (#12985) ### Motivation Autorecovery service will be shutdown if the zk session expired and then will lead the bk service shutdown together. So, in the production environment, it is recommand to deploy autorecovery service seperately but currently pulsar doesn't support it. ### Modifications Added the autorecovery service in pulsar cli Added the autorecovery service in pulsar-daemon cli (cherry picked from commit f192209af58e86a2c6da4130956b172b2d1ddc13) --- bin/pulsar | 4 ++++ bin/pulsar-daemon | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/bin/pulsar b/bin/pulsar index 3ec65f3233609..0640a94333bd4 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -140,6 +140,7 @@ where command is one of: sql-worker Run a sql worker server sql Run sql CLI standalone Run a broker server with local bookies and local zookeeper + autorecovery Run an autorecovery service initialize-cluster-metadata One-time metadata initialization delete-cluster-metadata Delete a cluster's metadata @@ -343,6 +344,9 @@ elif [ $COMMAND == "functions-worker" ]; then elif [ $COMMAND == "standalone" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"} exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarStandaloneStarter --config $PULSAR_STANDALONE_CONF $@ +elif [ ${COMMAND} == "autorecovery" ]; then + PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-autorecovery.log"} + exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.replication.AutoRecoveryMain --conf $PULSAR_BOOKKEEPER_CONF $@ elif [ $COMMAND == "initialize-cluster-metadata" ]; then exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup $@ elif [ $COMMAND == "delete-cluster-metadata" ]; then diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon index f018bc4b66154..8bbc62806be3b 100755 --- a/bin/pulsar-daemon +++ b/bin/pulsar-daemon @@ -30,6 +30,7 @@ where command is one of: functions-worker Run a functions worker server standalone Run a standalone Pulsar service proxy Run a Proxy Pulsar service + autorecovery Run an autorecovery service where argument is one of: -force (accepted only with stop command): Decides whether to stop the server forcefully if not stopped by normal shutdown @@ -102,6 +103,9 @@ case $command in (proxy) echo "doing $startStop $command ..." ;; + (autorecovery) + echo "doing $startStop $command ..." + ;; (*) echo "Error: unknown service name $command" usage From 71ba8ef2afda1a71663d5b2e7f415ba73dc3ea2c Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Mon, 29 Nov 2021 12:22:55 +0800 Subject: [PATCH 172/823] Fix update ledger list to znode version mismatch failed, ledger not delete (#12015) ### Motivation When Zookeeper throws `Failed to update ledger list. z-node version mismatch. Closing managed ledger` exception when update ZNode list, it will not clean up the created ledger, which will lead to the new created ledger not be indexed to the topic managedLedger list, and can't be cleanup as topic retention. What's more, it will cause ZNode number increase in Zookeeper if the `z-node version mismatch` exception keeping throw out. The exception list as follow: ``` 10:44:29.017 [main-EventThread] INFO org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [test/test/persistent/test_v1-partition-4] Created new ledger 67311140 10:44:29.018 [bookkeeper-ml-workers-OrderedExecutor-2-0] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [test/test/persistent/test_v1-partition-4] Failed to update ledger list. z-node version mismatch. Closing managed ledger 10:44:29.018 [bookkeeper-ml-workers-OrderedExecutor-2-0] INFO org.apache.pulsar.broker.service.Producer - Disconnecting producer: Producer{topic=PersistentTopic{topic=persistent://test/test/test_v1-partition-4}, client=/10.1.2.3:38938, producerName=pulsar-101-1123, producerId=20} ``` ### Modification 1. When updating ZNode list failed, delete the created ledger from broker cache and BookKeeper, regardless of whether the exception type is BadVersionException or not. (cherry picked from commit e7b0e3dbc5b6a05e955dc2cad034de0487463150) --- .../mledger/impl/ManagedLedgerImpl.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index caff2960e1c37..7f4d2913471da 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1438,22 +1438,7 @@ public void operationComplete(Void v, Stat stat) { @Override public void operationFailed(MetaStoreException e) { - if (e instanceof BadVersionException) { - synchronized (ManagedLedgerImpl.this) { - log.error( - "[{}] Failed to update ledger list. z-node version mismatch. Closing managed ledger", - name); - STATE_UPDATER.set(ManagedLedgerImpl.this, State.Fenced); - // Return ManagedLedgerFencedException to addFailed callback - // to indicate that the ledger is now fenced and topic needs to be closed - clearPendingAddEntries(new ManagedLedgerFencedException(e)); - // Do not need to unlock ledgersListMutex here because we are going to close to topic anyways - return; - } - } - log.warn("[{}] Error updating meta data with the new list of ledgers: {}", name, e.getMessage()); - // Remove the ledger, since we failed to update the list ledgers.remove(lh.getId()); mbean.startDataLedgerDeleteOp(); @@ -1465,6 +1450,21 @@ public void operationFailed(MetaStoreException e) { } }, null); + if (e instanceof BadVersionException) { + synchronized (ManagedLedgerImpl.this) { + log.error( + "[{}] Failed to update ledger list. z-node version mismatch. Closing managed ledger", + name); + lastLedgerCreationFailureTimestamp = clock.millis(); + STATE_UPDATER.set(ManagedLedgerImpl.this, State.Fenced); + // Return ManagedLedgerFencedException to addFailed callback + // to indicate that the ledger is now fenced and topic needs to be closed + clearPendingAddEntries(new ManagedLedgerFencedException(e)); + // Do not need to unlock ledgersListMutex here because we are going to close to topic anyways + return; + } + } + metadataMutex.unlock(); synchronized (ManagedLedgerImpl.this) { From 6b9d02cb0a488dfe84359b02785973e2e8a2f105 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 30 Nov 2021 03:51:54 +0800 Subject: [PATCH 173/823] optimize if statement (#12741) Co-authored-by: gavingaozhangmin (cherry picked from commit f3641a3fa89686705ebe4bf5ea9936af3003d465) --- .../java/org/apache/pulsar/admin/cli/CmdFunctions.java | 6 +++--- .../main/java/org/apache/pulsar/admin/cli/CmdSinks.java | 6 +++--- .../main/java/org/apache/pulsar/admin/cli/CmdSources.java | 6 +++--- .../java/org/apache/pulsar/client/impl/ConsumerBase.java | 7 ++----- .../apache/pulsar/client/impl/schema/ByteBufSchema.java | 6 +----- .../org/apache/pulsar/common/tls/TlsHostnameVerifier.java | 4 +--- .../apache/pulsar/functions/worker/FunctionActioner.java | 7 ++----- .../java/org/apache/pulsar/io/flume/FlumeConnector.java | 5 +---- .../org/apache/pulsar/testclient/PerformanceReader.java | 2 +- 9 files changed, 17 insertions(+), 32 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 2efef9982fe6d..90ebf689605a2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -711,13 +711,13 @@ private void mergeArgs() { if (isBlank(clientAuthParams) && !isBlank(DEPRECATED_clientAuthParams)) { clientAuthParams = DEPRECATED_clientAuthParams; } - if (useTls == false && DEPRECATED_useTls != null) { + if (!useTls && DEPRECATED_useTls != null) { useTls = DEPRECATED_useTls; } - if (tlsAllowInsecureConnection == false && DEPRECATED_tlsAllowInsecureConnection != null) { + if (!tlsAllowInsecureConnection && DEPRECATED_tlsAllowInsecureConnection != null) { tlsAllowInsecureConnection = DEPRECATED_tlsAllowInsecureConnection; } - if (tlsHostNameVerificationEnabled == false && DEPRECATED_tlsHostNameVerificationEnabled != null) { + if (!tlsHostNameVerificationEnabled && DEPRECATED_tlsHostNameVerificationEnabled != null) { tlsHostNameVerificationEnabled = DEPRECATED_tlsHostNameVerificationEnabled; } if (isBlank(tlsTrustCertFilePath) && !isBlank(DEPRECATED_tlsTrustCertFilePath)) { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index a4bf82eb3be3e..0035ff48f89ab 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -191,13 +191,13 @@ private void mergeArgs() { if (isBlank(clientAuthParams) && !isBlank(DEPRECATED_clientAuthParams)) { clientAuthParams = DEPRECATED_clientAuthParams; } - if (useTls == false && DEPRECATED_useTls != null) { + if (!useTls && DEPRECATED_useTls != null) { useTls = DEPRECATED_useTls; } - if (tlsAllowInsecureConnection == false && DEPRECATED_tlsAllowInsecureConnection != null) { + if (!tlsAllowInsecureConnection && DEPRECATED_tlsAllowInsecureConnection != null) { tlsAllowInsecureConnection = DEPRECATED_tlsAllowInsecureConnection; } - if (tlsHostNameVerificationEnabled == false && DEPRECATED_tlsHostNameVerificationEnabled != null) { + if (!tlsHostNameVerificationEnabled && DEPRECATED_tlsHostNameVerificationEnabled != null) { tlsHostNameVerificationEnabled = DEPRECATED_tlsHostNameVerificationEnabled; } if (isBlank(tlsTrustCertFilePath) && !isBlank(DEPRECATED_tlsTrustCertFilePath)) { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java index 6d836e2ce6bf2..f78feb001a334 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java @@ -190,13 +190,13 @@ private void mergeArgs() { if (isBlank(clientAuthParams) && !isBlank(DEPRECATED_clientAuthParams)) { clientAuthParams = DEPRECATED_clientAuthParams; } - if (useTls == false && DEPRECATED_useTls != null) { + if (!useTls && DEPRECATED_useTls != null) { useTls = DEPRECATED_useTls; } - if (tlsAllowInsecureConnection == false && DEPRECATED_tlsAllowInsecureConnection != null) { + if (!tlsAllowInsecureConnection && DEPRECATED_tlsAllowInsecureConnection != null) { tlsAllowInsecureConnection = DEPRECATED_tlsAllowInsecureConnection; } - if (tlsHostNameVerificationEnabled == false && DEPRECATED_tlsHostNameVerificationEnabled != null) { + if (!tlsHostNameVerificationEnabled && DEPRECATED_tlsHostNameVerificationEnabled != null) { tlsHostNameVerificationEnabled = DEPRECATED_tlsHostNameVerificationEnabled; } if (isBlank(tlsTrustCertFilePath) && !isBlank(DEPRECATED_tlsTrustCertFilePath)) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 9312df5368255..1251593d959c3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -211,13 +211,10 @@ protected boolean hasNextPendingReceive() { protected CompletableFuture> nextPendingReceive() { CompletableFuture> receivedFuture; - while (true) { + do { receivedFuture = pendingReceives.poll(); // skip done futures (cancelling a future could mark it done) - if (receivedFuture == null || !receivedFuture.isDone()) { - break; - } - } + } while (receivedFuture != null && receivedFuture.isDone()); return receivedFuture; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufSchema.java index ce68298be2b49..7665d96ee727d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufSchema.java @@ -64,11 +64,7 @@ public ByteBuf decode(byte[] bytes) { @Override public ByteBuf decode(ByteBuf byteBuf) { - if (null == byteBuf) { - return null; - } else { - return byteBuf; - } + return byteBuf; } @Override diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java b/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java index 3ab5b5f63f76c..0735013ad7f64 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/tls/TlsHostnameVerifier.java @@ -206,9 +206,7 @@ private static boolean matchIdentity(final String host, final String identity, if (strict) { final String remainder = host.substring( prefix.length(), host.length() - suffix.length()); - if (remainder.contains(".")) { - return false; - } + return !remainder.contains("."); } return true; } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 6497e1547896e..5fa554097257a 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -204,14 +204,11 @@ private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaDa } File tempPkgFile; - while (true) { + do { tempPkgFile = new File( pkgDir, pkgFile.getName() + "." + instanceId + "." + UUID.randomUUID().toString()); - if (!tempPkgFile.exists() && tempPkgFile.createNewFile()) { - break; - } - } + } while (tempPkgFile.exists() || !tempPkgFile.createNewFile()); String pkgLocationPath = functionMetaData.getPackageLocation().getPackagePath(); boolean downloadFromHttp = isPkgUrlProvided && pkgLocationPath.startsWith(HTTP); log.info("{}/{}/{} Function package file {} will be downloaded from {}", tempPkgFile, details.getTenant(), diff --git a/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/FlumeConnector.java b/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/FlumeConnector.java index 29a98478533ed..66eb2d61094da 100644 --- a/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/FlumeConnector.java +++ b/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/FlumeConnector.java @@ -43,10 +43,7 @@ public void StartConnector(FlumeConfig flumeConfig) throws Exception { SSLUtil.initGlobalSSLParameters(); String agentName = flumeConfig.getName(); boolean reload = !flumeConfig.getNoReloadConf(); - boolean isZkConfigured = false; - if (flumeConfig.getZkConnString().length() > 0) { - isZkConfigured = true; - } + boolean isZkConfigured = flumeConfig.getZkConnString().length() > 0; if (isZkConfigured) { // get options String zkConnectionStr = flumeConfig.getZkConnString(); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java index 44e555560cc13..d18c76a2f8b4f 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java @@ -199,7 +199,7 @@ public static void main(String[] args) throws Exception { arguments.authParams = prop.getProperty("authParams", null); } - if (arguments.useTls == false) { + if (!arguments.useTls) { arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls")); } From 5a84ae708d61d9b1d0b450227211a81d5ecb33f9 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Mon, 29 Nov 2021 10:54:48 -0800 Subject: [PATCH 174/823] Don't create AvroData for each KafkaSourceRecord (#12859) (cherry picked from commit 792a77ebf841c8cae5ffe8a0cba5bbf1ed64f83c) --- .../apache/pulsar/io/kafka/connect/KafkaConnectSource.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index f84aa64cc46c3..5d30e95acefc9 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -69,15 +69,12 @@ public synchronized KafkaSourceRecord processSourceRecord(final SourceRecord src return record; } - private static Map PROPERTIES = Collections.emptyMap(); - private static Optional RECORD_SEQUENCE = Optional.empty(); - private static long FLUSH_TIMEOUT_MS = 2000; + private static final AvroData avroData = new AvroData(1000); private class KafkaSourceRecord extends AbstractKafkaSourceRecord> implements KVRecord { KafkaSourceRecord(SourceRecord srcRecord) { super(srcRecord); - AvroData avroData = new AvroData(1000); byte[] keyBytes = keyConverter.fromConnectData( srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); From ab57a6a60854ca6dc225e53323adedc3c1322716 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 30 Nov 2021 01:46:53 +0800 Subject: [PATCH 175/823] Use sendAsync instead of send when produce message to retry topic. (#12946) * Use sendAsync instead of send when produce message to retry letter topic. * add exception handler. (cherry picked from commit 09cc1d6aa91422c71601664dac8d94ba574beb7b) --- .../org/apache/pulsar/client/impl/ConsumerImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1b8d19cd7f66e..ff0b826f8d48b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -590,9 +590,9 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a propertiesMap.put(RetryMessageUtil.SYSTEM_PROPERTY_RECONSUMETIMES, String.valueOf(reconsumetimes)); propertiesMap.put(RetryMessageUtil.SYSTEM_PROPERTY_DELAY_TIME, String.valueOf(unit.toMillis(delayTime))); + MessageId finalMessageId = messageId; if (reconsumetimes > this.deadLetterPolicy.getMaxRedeliverCount() && StringUtils.isNotBlank(deadLetterPolicy.getDeadLetterTopic())) { initDeadLetterProducerIfNeeded(); - MessageId finalMessageId = messageId; deadLetterProducer.thenAccept(dlqProducer -> { TypedMessageBuilder typedMessageBuilderNew = dlqProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(retryMessage.getReaderSchema().get())) @@ -624,8 +624,12 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a if (message.hasKey()) { typedMessageBuilderNew.key(message.getKey()); } - typedMessageBuilderNew.send(); - return doAcknowledge(messageId, ackType, properties, null); + typedMessageBuilderNew.sendAsync() + .thenAccept(__ -> doAcknowledge(finalMessageId, ackType, properties, null).thenAccept(v -> result.complete(null))) + .exceptionally(ex -> { + result.completeExceptionally(ex); + return null; + }); } } catch (Exception e) { log.error("Send to retry letter topic exception with topic: {}, messageId: {}", retryLetterProducer.getTopic(), messageId, e); From db1a34cfe708c182b9fa54515e0a26c3c042163b Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 30 Nov 2021 01:32:22 +0800 Subject: [PATCH 176/823] fix fix-13004 Race condition in ResourceLockImpl#revalidate (#13006) (cherry picked from commit bb2c9345bc42d61090897757a9f7435c138c8ee3) --- .../coordination/impl/ResourceLockImpl.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java index 677ace7218ede..ec9d3c42904b4 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java @@ -44,6 +44,7 @@ public class ResourceLockImpl implements ResourceLock { private long version; private final CompletableFuture expiredFuture; private boolean revalidateAfterReconnection = false; + private CompletableFuture revalidateFuture; private enum State { Init, @@ -144,6 +145,9 @@ synchronized CompletableFuture acquire(T newValue) { // Simple operation of acquiring the lock with no retries, or checking for the lock content private CompletableFuture acquireWithNoRevalidation(T newValue) { + if (log.isDebugEnabled()) { + log.debug("acquireWithNoRevalidation,newValue={},version={}", newValue, version); + } byte[] payload; try { payload = serde.serialize(path, newValue); @@ -212,6 +216,30 @@ synchronized CompletableFuture revalidateIfNeededAfterReconnection() { } synchronized CompletableFuture revalidate(T newValue) { + if (revalidateFuture == null || revalidateFuture.isDone()) { + revalidateFuture = doRevalidate(newValue); + } else { + if (log.isDebugEnabled()) { + log.debug("Previous revalidating is not finished while revalidate newValue={}, value={}, version={}", + newValue, value, version); + } + CompletableFuture newFuture = new CompletableFuture<>(); + revalidateFuture.whenComplete((unused, throwable) -> { + doRevalidate(newValue).thenRun(() -> newFuture.complete(null)) + .exceptionally(throwable1 -> { + newFuture.completeExceptionally(throwable1); + return null; + }); + }); + revalidateFuture = newFuture; + } + return revalidateFuture; + } + + private synchronized CompletableFuture doRevalidate(T newValue) { + if (log.isDebugEnabled()) { + log.debug("doRevalidate with newValue={}, version={}", newValue, version); + } return store.get(path) .thenCompose(optGetResult -> { if (!optGetResult.isPresent()) { From 8e689260abf47fa48520da6d36bdc8252bf008dc Mon Sep 17 00:00:00 2001 From: Aloys Date: Tue, 30 Nov 2021 14:38:17 +0800 Subject: [PATCH 177/823] add missed import (#13037) ### Motivation The current master branch was broken and can't be compiled successfully due to there missing an import for Sets. https://github.com/apache/pulsar/blob/master/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java#L220 ### Modifications add missed import (cherry picked from commit bebfc772456e689b14f6d10a6231165993e8ab14) --- .../src/main/java/org/apache/pulsar/admin/cli/CliCommand.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java index 2f1cf72db813a..d91a880b3926d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; +import com.google.common.collect.Sets; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.impl.MessageIdImpl; From ea79a8b75218bd6176b25b8197016a11a3f54351 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Tue, 30 Nov 2021 23:29:30 -0600 Subject: [PATCH 178/823] fix(functions): missing runtime set in GoInstanceConfig (#13031) * fix(functions): missing runtime set in GoInstanceConfig Signed-off-by: Eric Shen * fix ci ut Signed-off-by: Eric Shen * fix test ci Signed-off-by: Eric Shen * rollback some change in function-go Signed-off-by: Eric Shen (cherry picked from commit aa992e843581b65c854a0f97353f68ab0170b576) --- .../java/org/apache/pulsar/functions/runtime/RuntimeUtils.java | 3 +++ .../org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java | 3 ++- .../functions/runtime/kubernetes/KubernetesRuntimeTest.java | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 4acbd353685b2..4e4e2dcc77785 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -173,6 +173,9 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig, if (instanceConfig.getFunctionDetails().getProcessingGuarantees() != null) { goInstanceConfig.setProcessingGuarantees(instanceConfig.getFunctionDetails().getProcessingGuaranteesValue()); } + if (instanceConfig.getFunctionDetails().getRuntime() != null) { + goInstanceConfig.setRuntime(instanceConfig.getFunctionDetails().getRuntimeValue()); + } if (instanceConfig.getFunctionDetails().getSecretsMap() != null) { goInstanceConfig.setSecretsMap(instanceConfig.getFunctionDetails().getSecretsMap()); } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java index f8bbbc4a883af..bc00776c78ebd 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java @@ -99,6 +99,7 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException { .setName("go-func") .setLogTopic("go-func-log") .setProcessingGuarantees(Function.ProcessingGuarantees.ATLEAST_ONCE) + .setRuntime(Function.FunctionDetails.Runtime.GO) .setSecretsMap(secretsMap.toJSONString()) .setParallelism(1) .setSource(sources) @@ -137,7 +138,7 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException { Assert.assertEquals(goInstanceConfig.get("autoAck"), true); Assert.assertEquals(goInstanceConfig.get("regexPatternSubscription"), false); Assert.assertEquals(goInstanceConfig.get("pulsarServiceURL"), "pulsar://localhost:6650"); - Assert.assertEquals(goInstanceConfig.get("runtime"), 0); + Assert.assertEquals(goInstanceConfig.get("runtime"), 3); Assert.assertEquals(goInstanceConfig.get("cpu"), 2.0); Assert.assertEquals(goInstanceConfig.get("funcID"), "func-7734"); Assert.assertEquals(goInstanceConfig.get("funcVersion"), "1.0.0"); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 7f6c36acc6104..b295cf8a72a80 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -876,7 +876,7 @@ private void verifyGolangInstance(InstanceConfig config) throws Exception { assertEquals(goInstanceConfig.get("autoAck"), false); assertEquals(goInstanceConfig.get("regexPatternSubscription"), false); assertEquals(goInstanceConfig.get("pulsarServiceURL"), pulsarServiceUrl); - assertEquals(goInstanceConfig.get("runtime"), 0); + assertEquals(goInstanceConfig.get("runtime"), 3); assertEquals(goInstanceConfig.get("cpu"), 1.0); assertEquals(goInstanceConfig.get("funcVersion"), "1.0"); assertEquals(goInstanceConfig.get("disk"), 10000); From dfcdacf40ecacec45e56a00d577866b014f7fa67 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 30 Nov 2021 19:28:58 +0200 Subject: [PATCH 179/823] Don't attempt to delete pending ack store unless transactions are enabled (#13041) (cherry picked from commit 46720247d9a06daae9f8eae7740887c92406b2c3) --- .../service/persistent/PersistentTopic.java | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 46c9dfdb1a50a..053d72d6d5d0b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -979,27 +979,32 @@ public CompletableFuture createSubscription(String subscriptionNam @Override public CompletableFuture unsubscribe(String subscriptionName) { CompletableFuture unsubscribeFuture = new CompletableFuture<>(); - getBrokerService().getManagedLedgerFactory().asyncDelete(TopicName.get(MLPendingAckStore - .getTransactionPendingAckStoreSuffix(topic, - Codec.encode(subscriptionName))).getPersistenceNamingEncoding(), - new AsyncCallbacks.DeleteLedgerCallback() { - @Override - public void deleteLedgerComplete(Object ctx) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); - } - @Override - public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { - if (exception instanceof MetadataNotFoundException) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); - return; - } + if (brokerService.pulsar().getConfiguration().isTransactionCoordinatorEnabled()) { + getBrokerService().getManagedLedgerFactory().asyncDelete(TopicName.get(MLPendingAckStore + .getTransactionPendingAckStoreSuffix(topic, + Codec.encode(subscriptionName))).getPersistenceNamingEncoding(), + new AsyncCallbacks.DeleteLedgerCallback() { + @Override + public void deleteLedgerComplete(Object ctx) { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + } - unsubscribeFuture.completeExceptionally(exception); - log.error("[{}][{}] Error deleting subscription pending ack store", - topic, subscriptionName, exception); - } - }, null); + @Override + public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { + if (exception instanceof MetadataNotFoundException) { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + return; + } + + unsubscribeFuture.completeExceptionally(exception); + log.error("[{}][{}] Error deleting subscription pending ack store", + topic, subscriptionName, exception); + } + }, null); + } else { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + } return unsubscribeFuture; } From 76b424f35ee8ff390608b366e49e64dc79a82112 Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Wed, 1 Dec 2021 18:33:56 -0800 Subject: [PATCH 180/823] pulsar admin exposes secret for source and sink (#13059) ### Motivation Follow-up fix of #12950 for #12834 It turns out the Source and Sink doesn't inherit from Function cmd, so we need to add the api separately. ### Modifications add the `--secrets` argument into `pulsar-admin [source|sink] create/update/localrun` command (cherry picked from commit e888c2980f61428650779a8d23fe707bb61a31a1) --- .../java/org/apache/pulsar/admin/cli/CmdSinks.java | 12 ++++++++++++ .../org/apache/pulsar/admin/cli/CmdSources.java | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index 0035ff48f89ab..5d00627e3b418 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -39,6 +39,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Type; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -359,6 +360,8 @@ abstract class SinkDetailsCommand extends BaseCommand { protected Long negativeAckRedeliveryDelayMs; @Parameter(names = "--custom-runtime-options", description = "A string that encodes options to customize the runtime, see docs for configured runtime for details") protected String customRuntimeOptions; + @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates how the secret is fetched by the underlying secrets provider") + protected String secretsString; protected SinkConfig sinkConfig; @@ -524,6 +527,15 @@ void processArguments() throws Exception { sinkConfig.setCustomRuntimeOptions(customRuntimeOptions); } + if (secretsString != null) { + Type type = new TypeToken>() {}.getType(); + Map secretsMap = new Gson().fromJson(secretsString, type); + if (secretsMap == null) { + secretsMap = Collections.emptyMap(); + } + sinkConfig.setSecrets(secretsMap); + } + // check if configs are valid validateSinkConfigs(sinkConfig); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java index f78feb001a334..1eedf654923a3 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Type; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -339,6 +340,8 @@ abstract class SourceDetailsCommand extends BaseCommand { protected String batchSourceConfigString; @Parameter(names = "--custom-runtime-options", description = "A string that encodes options to customize the runtime, see docs for configured runtime for details") protected String customRuntimeOptions; + @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates how the secret is fetched by the underlying secrets provider") + protected String secretsString; protected SourceConfig sourceConfig; @@ -463,6 +466,16 @@ void processArguments() throws Exception { if (customRuntimeOptions != null) { sourceConfig.setCustomRuntimeOptions(customRuntimeOptions); } + + if (secretsString != null) { + Type type = new TypeToken>() {}.getType(); + Map secretsMap = new Gson().fromJson(secretsString, type); + if (secretsMap == null) { + secretsMap = Collections.emptyMap(); + } + sourceConfig.setSecrets(secretsMap); + } + // check if source configs are valid validateSourceConfigs(sourceConfig); } From 112d9ee8ab0563816bb5775b2ca7ccd3cc3b8bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 2 Dec 2021 16:27:33 +0100 Subject: [PATCH 181/823] [ElasticSearch Sink] Correct @FieldDoc defaultValue for some fields: primaryFields,maxRetries,indexNumberOfReplicas,createIndexIfNeeded (#12697) (cherry picked from commit 5baa2e0e16c0487cf2f160a739f5cb18dc420fe6) --- .../pulsar/io/elasticsearch/ElasticSearchConfig.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java index 7dbfd0388a4b4..dc6d0d415a047 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java @@ -79,14 +79,14 @@ public class ElasticSearchConfig implements Serializable { @FieldDoc( required = false, - defaultValue = "true", + defaultValue = "false", help = "Create the index if it does not exist" ) private boolean createIndexIfNeeded = false; @FieldDoc( required = false, - defaultValue = "1", + defaultValue = "0", help = "The number of replicas of the index" ) private int indexNumberOfReplicas = 0; @@ -109,7 +109,7 @@ public class ElasticSearchConfig implements Serializable { @FieldDoc( required = false, - defaultValue = "-1", + defaultValue = "1", help = "The maximum number of retries for elasticsearch requests. Use -1 to disable it." ) private int maxRetries = 1; @@ -216,7 +216,7 @@ public class ElasticSearchConfig implements Serializable { @FieldDoc( required = false, - defaultValue = "id", + defaultValue = "", help = "The comma separated ordered list of field names used to build the Elasticsearch document _id from the record value. If this list is a singleton, the field is converted as a string. If this list has 2 or more fields, the generated _id is a string representation of a JSON array of the field values." ) private String primaryFields = ""; From c5da572ea40bdd20f1e5ada6a3ad8a6a60183e89 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 6 Dec 2021 22:00:38 +0800 Subject: [PATCH 182/823] [Transaction] Add a check for uninitialized PendingAck (#13088) ### Motivation We shoud not generate the statistics of a uninitialized PendingAck,and we should check if it is initialized when we get it by `getStoreManageLedger()`. ### Modifications Shoud not generate the statistics of a uninitialized PendingAck Add check if it is initialized when we get it by `getStoreManageLedger()`. (cherry picked from commit 591b4e80a7652ed608c04b769052744179473f0a) --- .../persistent/PersistentSubscription.java | 4 ++ .../prometheus/TransactionAggregator.java | 4 +- .../pendingack/PendingAckHandle.java | 5 ++ .../impl/PendingAckHandleDisabled.java | 5 ++ .../pendingack/impl/PendingAckHandleImpl.java | 7 +- .../broker/stats/TransactionMetricsTest.java | 68 +++++++++++++++++++ 6 files changed, 91 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index acfd9ee192cb5..8d75ea7b9a3ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -1219,5 +1219,9 @@ public CompletableFuture getPendingAckManageLedger() { } } + public boolean checkIfPendingAckStoreInit() { + return this.pendingAckHandle.checkIfPendingAckStoreInit(); + } + private static final Logger log = LoggerFactory.getLogger(PersistentSubscription.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java index 142ec48ae956a..65399d4effce3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java @@ -64,7 +64,9 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, topic.getSubscriptions().values().forEach(subscription -> { try { localManageLedgerStats.get().reset(); - if (!checkTopicIsEventsNames(TopicName.get(subscription.getTopic().getName()))) { + if (!checkTopicIsEventsNames(TopicName.get(subscription.getTopic().getName())) + && subscription instanceof PersistentSubscription + && ((PersistentSubscription) subscription).checkIfPendingAckStoreInit()) { ManagedLedger managedLedger = ((PersistentSubscription) subscription) .getPendingAckManageLedger().get(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index 3664c5d046f6d..dc64cbe3b16a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -159,4 +159,9 @@ CompletableFuture commitTxn(TxnID txnID, Map properties, */ CompletableFuture close(); + /** + * Check if the PendingAckStore is init. + * @return if the PendingAckStore is init. + */ + boolean checkIfPendingAckStoreInit(); } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index cf6b5c82366a7..634655e4ae7bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -99,4 +99,9 @@ public TransactionPendingAckStats getStats() { public CompletableFuture close() { return CompletableFuture.completedFuture(null); } + + @Override + public boolean checkIfPendingAckStoreInit() { + return false; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 78bab966b92e9..d92793a933e09 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -923,7 +923,7 @@ public CompletableFuture close() { } public CompletableFuture getStoreManageLedger() { - if (this.pendingAckStoreFuture.isDone()) { + if (this.pendingAckStoreFuture != null && this.pendingAckStoreFuture.isDone()) { return this.pendingAckStoreFuture.thenCompose(pendingAckStore -> { if (pendingAckStore instanceof MLPendingAckStore) { return ((MLPendingAckStore) pendingAckStore).getManagedLedger(); @@ -937,6 +937,11 @@ public CompletableFuture getStoreManageLedger() { } } + @Override + public boolean checkIfPendingAckStoreInit() { + return this.pendingAckStoreFuture != null && this.pendingAckStoreFuture.isDone(); + } + protected void handleCacheRequest() { while (true) { Runnable runnable = acceptQueue.poll(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java index cb8e4305ddf1e..6a4b5c401a1dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java @@ -40,6 +40,7 @@ import org.apache.pulsar.transaction.coordinator.TransactionSubscription; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -271,6 +272,73 @@ public void testManagedLedgerMetrics() throws Exception{ assertEquals(metric.size(), 2); } + @Test + public void testManagedLedgerMetricsWhenPendingAckNotInit() throws Exception{ + String ns1 = "prop/ns-abc1"; + admin.namespaces().createNamespace(ns1); + String topic = "persistent://" + ns1 + "/testManagedLedgerMetricsWhenPendingAckNotInit"; + String subName = "test_managed_ledger_metrics"; + String subName2 = "test_pending_ack_no_init"; + admin.topics().createNonPartitionedTopic(topic); + admin.lookups().lookupTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()); + TransactionCoordinatorID transactionCoordinatorIDOne = TransactionCoordinatorID.get(0); + pulsar.getTransactionMetadataStoreService().handleTcClientConnect(transactionCoordinatorIDOne).get(); + admin.topics().createSubscription(topic, subName, MessageId.earliest); + admin.topics().createSubscription(topic, subName2, MessageId.earliest); + + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> + pulsar.getTransactionMetadataStoreService().getStores().size() == 1); + + pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString()).enableTransaction(true).build(); + + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .receiverQueueSize(10) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + + Transaction transaction = + pulsarClient.newTransaction().withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); + producer.send("hello pulsar".getBytes()); + consumer.acknowledgeAsync(consumer.receive().getMessageId(), transaction).get(); + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + String metricsStr = statsOut.toString(); + + Multimap metrics = parseMetrics(metricsStr); + + Collection metric = metrics.get("pulsar_storage_size"); + checkManagedLedgerMetrics(subName, 32, metric); + //No statistics of the pendingAck are generated when the pendingAck is not initialized. + for (PrometheusMetricsTest.Metric metric1 : metric) { + if (metric1.tags.containsValue(subName2)) { + Assert.fail(); + } + } + + consumer = pulsarClient.newConsumer() + .topic(topic) + .receiverQueueSize(10) + .subscriptionName(subName2) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + transaction = + pulsarClient.newTransaction().withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); + consumer.acknowledgeAsync(consumer.receive().getMessageId(), transaction).get(); + + statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + metricsStr = statsOut.toString(); + metrics = parseMetrics(metricsStr); + metric = metrics.get("pulsar_storage_size"); + checkManagedLedgerMetrics(subName2, 32, metric); + } + private void checkManagedLedgerMetrics(String tag, double value, Collection metrics) { boolean exist = false; for (PrometheusMetricsTest.Metric metric1 : metrics) { From 6dea985c58b11d806e2a1f8659ae84bce5550932 Mon Sep 17 00:00:00 2001 From: Zhanpeng Wu Date: Tue, 7 Dec 2021 21:31:10 +0800 Subject: [PATCH 183/823] Fix flaky test BrokerServiceLookupTest.testModularLoadManagerSplitBundle (#13159) Co-authored-by: wuzhanpeng (cherry picked from commit 4b319f38256d586bf179ac8df9f401709b128b15) --- .../pulsar/client/api/BrokerServiceLookupTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 251ccd3eda3bd..6628cf64c6a93 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -654,15 +654,18 @@ public void testModularLoadManagerSplitBundle() throws Exception { conf2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); conf2.setZookeeperServers("localhost:2181"); conf2.setConfigurationStoreServers("localhost:3181"); - - @Cleanup - PulsarService pulsar2 = startBroker(conf2); + conf2.setLoadBalancerAutoBundleSplitEnabled(true); + conf2.setLoadBalancerAutoUnloadSplitBundlesEnabled(true); + conf2.setLoadBalancerNamespaceBundleMaxTopics(1); // configure broker-1 with ModularLoadManager stopBroker(); conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); startBroker(); + @Cleanup + PulsarService pulsar2 = startBroker(conf2); + pulsar.getLoadManager().get().writeLoadReportOnZookeeper(); pulsar2.getLoadManager().get().writeLoadReportOnZookeeper(); @@ -732,9 +735,6 @@ public void testModularLoadManagerSplitBundle() throws Exception { .getLoadManager().get()).getLoadManager(); updateAllMethod.invoke(loadManager); - conf2.setLoadBalancerAutoBundleSplitEnabled(true); - conf2.setLoadBalancerAutoUnloadSplitBundlesEnabled(true); - conf2.setLoadBalancerNamespaceBundleMaxTopics(1); loadManager.checkNamespaceBundleSplit(); // (6) Broker-2 should get the watch and update bundle cache From 0e0de02ca80d19048065c5f5d94905b289b539ec Mon Sep 17 00:00:00 2001 From: Travis Sturzl Date: Tue, 7 Dec 2021 10:08:50 -0700 Subject: [PATCH 184/823] [Issue #12486][Python Client]JsonSchema encoding is not idempotent (#12490) * fix JsonSchema, copy data out to prevent modifying the reference object, check keys before deleting them * add unit test, but cannot test due to compilation failure for cpp lib (cherry picked from commit 2df53da3126d8e0471aa96cadc102254fff286f5) --- .../python/pulsar/schema/schema.py | 15 +- pulsar-client-cpp/python/pulsar_test.py | 787 +++++++++--------- 2 files changed, 402 insertions(+), 400 deletions(-) diff --git a/pulsar-client-cpp/python/pulsar/schema/schema.py b/pulsar-client-cpp/python/pulsar/schema/schema.py index 083efc353596b..349087ed75e03 100644 --- a/pulsar-client-cpp/python/pulsar/schema/schema.py +++ b/pulsar-client-cpp/python/pulsar/schema/schema.py @@ -85,11 +85,16 @@ def _get_serialized_value(self, o): def encode(self, obj): self._validate_object_type(obj) - del obj.__dict__['_default'] - del obj.__dict__['_required'] - del obj.__dict__['_required_default'] - - return json.dumps(obj.__dict__, default=self._get_serialized_value, indent=True).encode('utf-8') + # Copy the dict of the object as to not modify the provided object via the reference provided + data = obj.__dict__.copy() + if '_default' in data: + del data['_default'] + if '_required' in data: + del data['_required'] + if '_required_default' in data: + del data['_required_default'] + + return json.dumps(data, default=self._get_serialized_value, indent=True).encode('utf-8') def decode(self, data): return self._record_cls(**json.loads(data)) diff --git a/pulsar-client-cpp/python/pulsar_test.py b/pulsar-client-cpp/python/pulsar_test.py index 8db53bdaf6e59..fd3656b1eedb2 100755 --- a/pulsar-client-cpp/python/pulsar_test.py +++ b/pulsar-client-cpp/python/pulsar_test.py @@ -25,10 +25,19 @@ import pulsar import uuid from datetime import timedelta -from pulsar import Client, MessageId, \ - CompressionType, ConsumerType, PartitionsRoutingMode, \ - AuthenticationTLS, Authentication, AuthenticationToken, InitialPosition, \ - CryptoKeyReader +from pulsar import ( + Client, + MessageId, + CompressionType, + ConsumerType, + PartitionsRoutingMode, + AuthenticationTLS, + Authentication, + AuthenticationToken, + InitialPosition, + CryptoKeyReader, +) +from pulsar.schema import JsonSchema, Record, Integer from _pulsar import ProducerConfiguration, ConsumerConfiguration @@ -46,19 +55,19 @@ def doHttpPost(url, data): req = Request(url, data.encode()) - req.add_header('Content-Type', 'application/json') + req.add_header("Content-Type", "application/json") urlopen(req) def doHttpPut(url, data): try: req = Request(url, data.encode()) - req.add_header('Content-Type', 'application/json') - req.get_method = lambda: 'PUT' + req.add_header("Content-Type", "application/json") + req.get_method = lambda: "PUT" urlopen(req) except Exception as ex: # ignore conflicts exception to have test idempotency - if '409' in str(ex): + if "409" in str(ex): pass else: raise ex @@ -66,16 +75,21 @@ def doHttpPut(url, data): def doHttpGet(url): req = Request(url) - req.add_header('Accept', 'application/json') + req.add_header("Accept", "application/json") return urlopen(req).read() +class TestRecord(Record): + a = Integer() + b = Integer() + + class PulsarTest(TestCase): - serviceUrl = 'pulsar://localhost:6650' - adminUrl = 'http://localhost:8080' + serviceUrl = "pulsar://localhost:6650" + adminUrl = "http://localhost:8080" - serviceUrlTls = 'pulsar+ssl://localhost:6651' + serviceUrlTls = "pulsar+ssl://localhost:6651" def test_producer_config(self): conf = ProducerConfiguration() @@ -95,7 +109,7 @@ def test_consumer_config(self): conf.consumer_type(ConsumerType.Shared) self.assertEqual(conf.consumer_type(), ConsumerType.Shared) - self.assertEqual(conf.consumer_name(), '') + self.assertEqual(conf.consumer_name(), "") conf.consumer_name("my-name") self.assertEqual(conf.consumer_name(), "my-name") @@ -105,8 +119,8 @@ def test_consumer_config(self): def test_connect_error(self): with self.assertRaises(pulsar.ConnectError): - client = Client('fakeServiceUrl') - client.create_producer('connect-error-topic') + client = Client("fakeServiceUrl") + client.create_producer("connect-error-topic") client.close() def test_exception_inheritance(self): @@ -115,23 +129,23 @@ def test_exception_inheritance(self): def test_simple_producer(self): client = Client(self.serviceUrl) - producer = client.create_producer('my-python-topic') - producer.send(b'hello') + producer = client.create_producer("my-python-topic") + producer.send(b"hello") producer.close() client.close() def test_producer_send_async(self): client = Client(self.serviceUrl) - producer = client.create_producer('my-python-topic') + producer = client.create_producer("my-python-topic") sent_messages = [] def send_callback(producer, msg): sent_messages.append(msg) - producer.send_async(b'hello', send_callback) - producer.send_async(b'hello', send_callback) - producer.send_async(b'hello', send_callback) + producer.send_async(b"hello", send_callback) + producer.send_async(b"hello", send_callback) + producer.send_async(b"hello", send_callback) i = 0 while len(sent_messages) < 3 and i < 100: @@ -142,28 +156,26 @@ def send_callback(producer, msg): def test_producer_send(self): client = Client(self.serviceUrl) - topic = 'test_producer_send' + topic = "test_producer_send" producer = client.create_producer(topic) - consumer = client.subscribe(topic, 'sub-name') - msg_id = producer.send(b'hello') - print('send to {}'.format(msg_id)) + consumer = client.subscribe(topic, "sub-name") + msg_id = producer.send(b"hello") + print("send to {}".format(msg_id)) msg = consumer.receive(TM) consumer.acknowledge(msg) - print('receive from {}'.format(msg.message_id())) + print("receive from {}".format(msg.message_id())) self.assertEqual(msg_id, msg.message_id()) client.close() def test_producer_consumer(self): client = Client(self.serviceUrl) - consumer = client.subscribe('my-python-topic-producer-consumer', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('my-python-topic-producer-consumer') - producer.send(b'hello') + consumer = client.subscribe("my-python-topic-producer-consumer", "my-sub", consumer_type=ConsumerType.Shared) + producer = client.create_producer("my-python-topic-producer-consumer") + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) @@ -173,12 +185,14 @@ def test_producer_consumer(self): def test_redelivery_count(self): client = Client(self.serviceUrl) - consumer = client.subscribe('my-python-topic-redelivery-count', - 'my-sub', - consumer_type=ConsumerType.Shared, - negative_ack_redelivery_delay_ms=500) - producer = client.create_producer('my-python-topic-redelivery-count') - producer.send(b'hello') + consumer = client.subscribe( + "my-python-topic-redelivery-count", + "my-sub", + consumer_type=ConsumerType.Shared, + negative_ack_redelivery_delay_ms=500, + ) + producer = client.create_producer("my-python-topic-redelivery-count") + producer.send(b"hello") redelivery_count = 0 for i in range(4): @@ -188,7 +202,7 @@ def test_redelivery_count(self): redelivery_count = msg.redelivery_count() self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") self.assertEqual(3, redelivery_count) consumer.unsubscribe() producer.close() @@ -196,12 +210,10 @@ def test_redelivery_count(self): def test_deliver_at(self): client = Client(self.serviceUrl) - consumer = client.subscribe('my-python-topic-deliver-at', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('my-python-topic-deliver-at') + consumer = client.subscribe("my-python-topic-deliver-at", "my-sub", consumer_type=ConsumerType.Shared) + producer = client.create_producer("my-python-topic-deliver-at") # Delay message in 1.1s - producer.send(b'hello', deliver_at=int(round(time.time() * 1000)) + 1100) + producer.send(b"hello", deliver_at=int(round(time.time() * 1000)) + 1100) # Message should not be available in the next second with self.assertRaises(pulsar.Timeout): @@ -210,19 +222,17 @@ def test_deliver_at(self): # Message should be published now msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") consumer.unsubscribe() producer.close() client.close() def test_deliver_after(self): client = Client(self.serviceUrl) - consumer = client.subscribe('my-python-topic-deliver-after', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('my-python-topic-deliver-after') + consumer = client.subscribe("my-python-topic-deliver-after", "my-sub", consumer_type=ConsumerType.Shared) + producer = client.create_producer("my-python-topic-deliver-after") # Delay message in 1.1s - producer.send(b'hello', deliver_after=timedelta(milliseconds=1100)) + producer.send(b"hello", deliver_after=timedelta(milliseconds=1100)) # Message should not be available in the next second with self.assertRaises(pulsar.Timeout): @@ -231,33 +241,35 @@ def test_deliver_after(self): # Message should be published in the next 500ms msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") consumer.unsubscribe() producer.close() client.close() def test_consumer_initial_position(self): client = Client(self.serviceUrl) - producer = client.create_producer('consumer-initial-position') + producer = client.create_producer("consumer-initial-position") # Sending 5 messages before consumer creation. # These should be received with initial_position set to Earliest but not with Latest. for i in range(5): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) - consumer = client.subscribe('consumer-initial-position', - 'my-sub', - consumer_type=ConsumerType.Shared, - initial_position=InitialPosition.Earliest) + consumer = client.subscribe( + "consumer-initial-position", + "my-sub", + consumer_type=ConsumerType.Shared, + initial_position=InitialPosition.Earliest, + ) # Sending 5 other messages that should be received regardless of the initial_position. for i in range(5, 10): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) for i in range(10): msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) with self.assertRaises(pulsar.Timeout): consumer.receive(100) @@ -267,65 +279,59 @@ def test_consumer_initial_position(self): def test_consumer_queue_size_is_zero(self): client = Client(self.serviceUrl) - consumer = client.subscribe('my-python-topic-consumer-init-queue-size-is-zero', - 'my-sub', - consumer_type=ConsumerType.Shared, - receiver_queue_size=0, - initial_position=InitialPosition.Earliest) - producer = client.create_producer('my-python-topic-consumer-init-queue-size-is-zero') - producer.send(b'hello') + consumer = client.subscribe( + "my-python-topic-consumer-init-queue-size-is-zero", + "my-sub", + consumer_type=ConsumerType.Shared, + receiver_queue_size=0, + initial_position=InitialPosition.Earliest, + ) + producer = client.create_producer("my-python-topic-consumer-init-queue-size-is-zero") + producer.send(b"hello") time.sleep(0.1) msg = consumer.receive() self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") consumer.unsubscribe() client.close() def test_message_properties(self): client = Client(self.serviceUrl) - topic = 'my-python-test-message-properties' - consumer = client.subscribe(topic=topic, - subscription_name='my-subscription', - schema=pulsar.schema.StringSchema()) - producer = client.create_producer(topic=topic, - schema=pulsar.schema.StringSchema()) - producer.send('hello', - properties={ - 'a': '1', - 'b': '2' - }) + topic = "my-python-test-message-properties" + consumer = client.subscribe( + topic=topic, subscription_name="my-subscription", schema=pulsar.schema.StringSchema() + ) + producer = client.create_producer(topic=topic, schema=pulsar.schema.StringSchema()) + producer.send("hello", properties={"a": "1", "b": "2"}) msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.value(), 'hello') - self.assertEqual(msg.properties(), { - 'a': '1', - 'b': '2' - }) + self.assertEqual(msg.value(), "hello") + self.assertEqual(msg.properties(), {"a": "1", "b": "2"}) consumer.unsubscribe() client.close() def test_tls_auth(self): - certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/' + certs_dir = "/pulsar/pulsar-broker/src/test/resources/authentication/tls/" if not os.path.exists(certs_dir): certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/" - client = Client(self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + 'cacert.pem', - tls_allow_insecure_connection=False, - authentication=AuthenticationTLS(certs_dir + 'client-cert.pem', certs_dir + 'client-key.pem')) - - topic = 'my-python-topic-tls-auth-' + str(time.time()) - consumer = client.subscribe(topic, - 'my-sub', - consumer_type=ConsumerType.Shared) + client = Client( + self.serviceUrlTls, + tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_allow_insecure_connection=False, + authentication=AuthenticationTLS(certs_dir + "client-cert.pem", certs_dir + "client-key.pem"), + ) + + topic = "my-python-topic-tls-auth-" + str(time.time()) + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) - producer.send(b'hello') + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) @@ -333,27 +339,27 @@ def test_tls_auth(self): client.close() def test_tls_auth2(self): - certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/' + certs_dir = "/pulsar/pulsar-broker/src/test/resources/authentication/tls/" if not os.path.exists(certs_dir): certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/" authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls" authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir) - client = Client(self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + 'cacert.pem', - tls_allow_insecure_connection=False, - authentication=Authentication(authPlugin, authParams)) + client = Client( + self.serviceUrlTls, + tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_allow_insecure_connection=False, + authentication=Authentication(authPlugin, authParams), + ) - topic = 'my-python-topic-tls-auth-2-' + str(time.time()) - consumer = client.subscribe(topic, - 'my-sub', - consumer_type=ConsumerType.Shared) + topic = "my-python-topic-tls-auth-2-" + str(time.time()) + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) - producer.send(b'hello') + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) @@ -365,25 +371,25 @@ def test_encryption(self): privateKeyPath = "/pulsar/pulsar-broker/src/test/resources/certificate/private-key.client-rsa.pem" crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath) client = Client(self.serviceUrl) - topic = 'my-python-test-end-to-end-encryption' - consumer = client.subscribe(topic=topic, - subscription_name='my-subscription', - crypto_key_reader=crypto_key_reader) - producer = client.create_producer(topic=topic, - encryption_key="client-rsa.pem", - crypto_key_reader=crypto_key_reader) - reader = client.create_reader(topic=topic, - start_message_id=MessageId.earliest, - crypto_key_reader=crypto_key_reader) - producer.send(b'hello') + topic = "my-python-test-end-to-end-encryption" + consumer = client.subscribe( + topic=topic, subscription_name="my-subscription", crypto_key_reader=crypto_key_reader + ) + producer = client.create_producer( + topic=topic, encryption_key="client-rsa.pem", crypto_key_reader=crypto_key_reader + ) + reader = client.create_reader( + topic=topic, start_message_id=MessageId.earliest, crypto_key_reader=crypto_key_reader + ) + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.value(), b'hello') + self.assertEqual(msg.value(), b"hello") consumer.unsubscribe() msg = reader.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): reader.read_next(100) @@ -393,27 +399,27 @@ def test_encryption(self): client.close() def test_tls_auth3(self): - certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/' + certs_dir = "/pulsar/pulsar-broker/src/test/resources/authentication/tls/" if not os.path.exists(certs_dir): certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/" authPlugin = "tls" authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir) - client = Client(self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + 'cacert.pem', - tls_allow_insecure_connection=False, - authentication=Authentication(authPlugin, authParams)) + client = Client( + self.serviceUrlTls, + tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_allow_insecure_connection=False, + authentication=Authentication(authPlugin, authParams), + ) - topic = 'my-python-topic-tls-auth-3-' + str(time.time()) - consumer = client.subscribe(topic, - 'my-sub', - consumer_type=ConsumerType.Shared) + topic = "my-python-topic-tls-auth-3-" + str(time.time()) + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) - producer.send(b'hello') + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) @@ -421,20 +427,20 @@ def test_tls_auth3(self): client.close() def test_auth_junk_params(self): - certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/' + certs_dir = "/pulsar/pulsar-broker/src/test/resources/authentication/tls/" if not os.path.exists(certs_dir): certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/" authPlugin = "someoldjunk.so" authParams = "blah" - client = Client(self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + 'cacert.pem', - tls_allow_insecure_connection=False, - authentication=Authentication(authPlugin, authParams)) + client = Client( + self.serviceUrlTls, + tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_allow_insecure_connection=False, + authentication=Authentication(authPlugin, authParams), + ) with self.assertRaises(pulsar.ConnectError): - client.subscribe('my-python-topic-auth-junk-params', - 'my-sub', - consumer_type=ConsumerType.Shared) + client.subscribe("my-python-topic-auth-junk-params", "my-sub", consumer_type=ConsumerType.Shared) def test_message_listener(self): client = Client(self.serviceUrl) @@ -446,14 +452,13 @@ def listener(consumer, msg): received_messages.append(msg) consumer.acknowledge(msg) - client.subscribe('my-python-topic-listener', - 'my-sub', - consumer_type=ConsumerType.Exclusive, - message_listener=listener) - producer = client.create_producer('my-python-topic-listener') - producer.send(b'hello-1') - producer.send(b'hello-2') - producer.send(b'hello-3') + client.subscribe( + "my-python-topic-listener", "my-sub", consumer_type=ConsumerType.Exclusive, message_listener=listener + ) + producer = client.create_producer("my-python-topic-listener") + producer.send(b"hello-1") + producer.send(b"hello-2") + producer.send(b"hello-3") time.sleep(0.1) self.assertEqual(len(received_messages), 3) @@ -464,15 +469,14 @@ def listener(consumer, msg): def test_reader_simple(self): client = Client(self.serviceUrl) - reader = client.create_reader('my-python-topic-reader-simple', - MessageId.earliest) + reader = client.create_reader("my-python-topic-reader-simple", MessageId.earliest) - producer = client.create_producer('my-python-topic-reader-simple') - producer.send(b'hello') + producer = client.create_producer("my-python-topic-reader-simple") + producer.send(b"hello") msg = reader.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): reader.read_next(100) @@ -482,21 +486,20 @@ def test_reader_simple(self): def test_reader_on_last_message(self): client = Client(self.serviceUrl) - producer = client.create_producer('my-python-topic-reader-on-last-message') + producer = client.create_producer("my-python-topic-reader-on-last-message") for i in range(10): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) - reader = client.create_reader('my-python-topic-reader-on-last-message', - MessageId.latest) + reader = client.create_reader("my-python-topic-reader-on-last-message", MessageId.latest) for i in range(10, 20): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) for i in range(10, 20): msg = reader.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) reader.close() client.close() @@ -504,26 +507,21 @@ def test_reader_on_last_message(self): def test_reader_on_specific_message(self): num_of_msgs = 10 client = Client(self.serviceUrl) - producer = client.create_producer( - 'my-python-topic-reader-on-specific-message') + producer = client.create_producer("my-python-topic-reader-on-specific-message") for i in range(num_of_msgs): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) - reader1 = client.create_reader( - 'my-python-topic-reader-on-specific-message', - MessageId.earliest) + reader1 = client.create_reader("my-python-topic-reader-on-specific-message", MessageId.earliest) - for i in range(num_of_msgs//2): + for i in range(num_of_msgs // 2): msg = reader1.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) last_msg_id = msg.message_id() last_msg_idx = i - reader2 = client.create_reader( - 'my-python-topic-reader-on-specific-message', - last_msg_id) + reader2 = client.create_reader("my-python-topic-reader-on-specific-message", last_msg_id) # The reset would be effectively done on the next position relative to reset. # When available, we should test this behaviour with `startMessageIdInclusive` opt. @@ -531,7 +529,7 @@ def test_reader_on_specific_message(self): for i in range(from_msg_idx, num_of_msgs): msg = reader2.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) reader1.close() reader2.close() @@ -540,32 +538,29 @@ def test_reader_on_specific_message(self): def test_reader_on_specific_message_with_batches(self): client = Client(self.serviceUrl) producer = client.create_producer( - 'my-python-topic-reader-on-specific-message-with-batches', + "my-python-topic-reader-on-specific-message-with-batches", batching_enabled=True, - batching_max_publish_delay_ms=1000) + batching_max_publish_delay_ms=1000, + ) for i in range(10): - producer.send_async(b'hello-%d' % i, None) + producer.send_async(b"hello-%d" % i, None) # Send one sync message to make sure everything was published - producer.send(b'hello-10') + producer.send(b"hello-10") - reader1 = client.create_reader( - 'my-python-topic-reader-on-specific-message-with-batches', - MessageId.earliest) + reader1 = client.create_reader("my-python-topic-reader-on-specific-message-with-batches", MessageId.earliest) for i in range(5): msg = reader1.read_next(TM) last_msg_id = msg.message_id() - reader2 = client.create_reader( - 'my-python-topic-reader-on-specific-message-with-batches', - last_msg_id) + reader2 = client.create_reader("my-python-topic-reader-on-specific-message-with-batches", last_msg_id) for i in range(5, 11): msg = reader2.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) reader1.close() reader2.close() @@ -573,60 +568,56 @@ def test_reader_on_specific_message_with_batches(self): def test_producer_sequence_after_reconnection(self): # Enable deduplication on namespace - doHttpPost(self.adminUrl + '/admin/v2/namespaces/public/default/deduplication', - 'true') + doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "true") client = Client(self.serviceUrl) - topic = 'my-python-test-producer-sequence-after-reconnection-' \ - + str(time.time()) + topic = "my-python-test-producer-sequence-after-reconnection-" + str(time.time()) - producer = client.create_producer(topic, producer_name='my-producer-name') + producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), -1) for i in range(10): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) self.assertEqual(producer.last_sequence_id(), i) producer.close() - producer = client.create_producer(topic, producer_name='my-producer-name') + producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), 9) for i in range(10, 20): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) self.assertEqual(producer.last_sequence_id(), i) client.close() - doHttpPost(self.adminUrl + '/admin/v2/namespaces/public/default/deduplication', - 'false') + doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "false") def test_producer_deduplication(self): # Enable deduplication on namespace - doHttpPost(self.adminUrl + '/admin/v2/namespaces/public/default/deduplication', - 'true') + doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "true") client = Client(self.serviceUrl) - topic = 'my-python-test-producer-deduplication-' + str(time.time()) + topic = "my-python-test-producer-deduplication-" + str(time.time()) - producer = client.create_producer(topic, producer_name='my-producer-name') + producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), -1) - consumer = client.subscribe(topic, 'my-sub') + consumer = client.subscribe(topic, "my-sub") - producer.send(b'hello-0', sequence_id=0) - producer.send(b'hello-1', sequence_id=1) - producer.send(b'hello-2', sequence_id=2) + producer.send(b"hello-0", sequence_id=0) + producer.send(b"hello-1", sequence_id=1) + producer.send(b"hello-2", sequence_id=2) self.assertEqual(producer.last_sequence_id(), 2) # Repeat the messages and verify they're not received by consumer - producer.send(b'hello-1', sequence_id=1) - producer.send(b'hello-2', sequence_id=2) + producer.send(b"hello-1", sequence_id=1) + producer.send(b"hello-2", sequence_id=2) self.assertEqual(producer.last_sequence_id(), 2) for i in range(3): msg = consumer.receive(TM) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): @@ -634,12 +625,12 @@ def test_producer_deduplication(self): producer.close() - producer = client.create_producer(topic, producer_name='my-producer-name') + producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), 2) # Repeat the messages and verify they're not received by consumer - producer.send(b'hello-1', sequence_id=1) - producer.send(b'hello-2', sequence_id=2) + producer.send(b"hello-1", sequence_id=1) + producer.send(b"hello-2", sequence_id=2) self.assertEqual(producer.last_sequence_id(), 2) with self.assertRaises(pulsar.Timeout): @@ -647,32 +638,32 @@ def test_producer_deduplication(self): client.close() - doHttpPost(self.adminUrl + '/admin/v2/namespaces/public/default/deduplication', - 'false') + doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "false") def test_producer_routing_mode(self): client = Client(self.serviceUrl) - producer = client.create_producer('my-python-test-producer', - message_routing_mode=PartitionsRoutingMode.UseSinglePartition) - producer.send(b'test') + producer = client.create_producer( + "my-python-test-producer", message_routing_mode=PartitionsRoutingMode.UseSinglePartition + ) + producer.send(b"test") client.close() def test_message_argument_errors(self): client = Client(self.serviceUrl) - topic = 'my-python-test-producer' + topic = "my-python-test-producer" producer = client.create_producer(topic) - content = 'test'.encode('utf-8') + content = "test".encode("utf-8") self._check_type_error(lambda: producer.send(5)) - self._check_value_error(lambda: producer.send(content, properties='test')) + self._check_value_error(lambda: producer.send(content, properties="test")) self._check_value_error(lambda: producer.send(content, partition_key=5)) - self._check_value_error(lambda: producer.send(content, sequence_id='test')) + self._check_value_error(lambda: producer.send(content, sequence_id="test")) self._check_value_error(lambda: producer.send(content, replication_clusters=5)) - self._check_value_error(lambda: producer.send(content, disable_replication='test')) - self._check_value_error(lambda: producer.send(content, event_timestamp='test')) - self._check_value_error(lambda: producer.send(content, deliver_at='test')) - self._check_value_error(lambda: producer.send(content, deliver_after='test')) + self._check_value_error(lambda: producer.send(content, disable_replication="test")) + self._check_value_error(lambda: producer.send(content, event_timestamp="test")) + self._check_value_error(lambda: producer.send(content, deliver_at="test")) + self._check_value_error(lambda: producer.send(content, deliver_after="test")) client.close() def test_client_argument_errors(self): @@ -692,75 +683,75 @@ def test_producer_argument_errors(self): self._check_value_error(lambda: client.create_producer(None)) - topic = 'my-python-test-producer' + topic = "my-python-test-producer" self._check_value_error(lambda: client.create_producer(topic, producer_name=5)) - self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id='test')) - self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis='test')) + self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id="test")) + self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis="test")) self._check_value_error(lambda: client.create_producer(topic, compression_type=None)) - self._check_value_error(lambda: client.create_producer(topic, max_pending_messages='test')) - self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full='test')) - self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test')) - self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test')) - self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes='test')) - self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms='test')) + self._check_value_error(lambda: client.create_producer(topic, max_pending_messages="test")) + self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full="test")) + self._check_value_error(lambda: client.create_producer(topic, batching_enabled="test")) + self._check_value_error(lambda: client.create_producer(topic, batching_enabled="test")) + self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes="test")) + self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms="test")) client.close() def test_consumer_argument_errors(self): client = Client(self.serviceUrl) - topic = 'my-python-test-producer' - sub_name = 'my-sub-name' + topic = "my-python-test-producer" + sub_name = "my-sub-name" self._check_value_error(lambda: client.subscribe(None, sub_name)) self._check_value_error(lambda: client.subscribe(topic, None)) self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_type=None)) - self._check_value_error(lambda: client.subscribe(topic, sub_name, receiver_queue_size='test')) + self._check_value_error(lambda: client.subscribe(topic, sub_name, receiver_queue_size="test")) self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_name=5)) - self._check_value_error(lambda: client.subscribe(topic, sub_name, unacked_messages_timeout_ms='test')) - self._check_value_error(lambda: client.subscribe(topic, sub_name, broker_consumer_stats_cache_time_ms='test')) + self._check_value_error(lambda: client.subscribe(topic, sub_name, unacked_messages_timeout_ms="test")) + self._check_value_error(lambda: client.subscribe(topic, sub_name, broker_consumer_stats_cache_time_ms="test")) client.close() def test_reader_argument_errors(self): client = Client(self.serviceUrl) - topic = 'my-python-test-producer' + topic = "my-python-test-producer" # This should not raise exception client.create_reader(topic, MessageId.earliest) self._check_value_error(lambda: client.create_reader(None, MessageId.earliest)) self._check_value_error(lambda: client.create_reader(topic, None)) - self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test')) + self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size="test")) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5)) client.close() def test_publish_compact_and_consume(self): client = Client(self.serviceUrl) - topic = 'compaction_%s' % (uuid.uuid4()) - producer = client.create_producer(topic, producer_name='my-producer-name', batching_enabled=False) + topic = "compaction_%s" % (uuid.uuid4()) + producer = client.create_producer(topic, producer_name="my-producer-name", batching_enabled=False) self.assertEqual(producer.last_sequence_id(), -1) - consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True) + consumer = client.subscribe(topic, "my-sub1", is_read_compacted=True) consumer.close() - consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False) + consumer2 = client.subscribe(topic, "my-sub2", is_read_compacted=False) # producer create 2 messages with same key. - producer.send(b'hello-0', partition_key='key0') - producer.send(b'hello-1', partition_key='key0') + producer.send(b"hello-0", partition_key="key0") + producer.send(b"hello-1", partition_key="key0") producer.close() # issue compact command, and wait success - url='%s/admin/v2/persistent/public/default/%s/compaction' % (self.adminUrl, topic) - doHttpPut(url, '') + url = "%s/admin/v2/persistent/public/default/%s/compaction" % (self.adminUrl, topic) + doHttpPut(url, "") while True: - s=doHttpGet(url).decode('utf-8') - if 'RUNNING' in s: + s = doHttpGet(url).decode("utf-8") + if "RUNNING" in s: print(s) print("Compact still running") time.sleep(0.2) else: print(s) print("Compact Complete now") - self.assertTrue('SUCCESS' in s) + self.assertTrue("SUCCESS" in s) break # after compaction completes the compacted ledger is recorded @@ -772,87 +763,84 @@ def test_publish_compact_and_consume(self): time.sleep(1.0) # after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key. - consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True) + consumer1 = client.subscribe(topic, "my-sub1", is_read_compacted=True) msg0 = consumer1.receive(TM) - self.assertEqual(msg0.data(), b'hello-1') + self.assertEqual(msg0.data(), b"hello-1") consumer1.acknowledge(msg0) consumer1.close() # ditto for reader reader1 = client.create_reader(topic, MessageId.earliest, is_read_compacted=True) msg0 = reader1.read_next(TM) - self.assertEqual(msg0.data(), b'hello-1') + self.assertEqual(msg0.data(), b"hello-1") reader1.close() # after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key. msg0 = consumer2.receive(TM) - self.assertEqual(msg0.data(), b'hello-0') + self.assertEqual(msg0.data(), b"hello-0") consumer2.acknowledge(msg0) msg1 = consumer2.receive(TM) - self.assertEqual(msg1.data(), b'hello-1') + self.assertEqual(msg1.data(), b"hello-1") consumer2.acknowledge(msg1) consumer2.close() # ditto for reader reader2 = client.create_reader(topic, MessageId.earliest, is_read_compacted=False) msg0 = reader2.read_next(TM) - self.assertEqual(msg0.data(), b'hello-0') + self.assertEqual(msg0.data(), b"hello-0") msg1 = reader2.read_next(TM) - self.assertEqual(msg1.data(), b'hello-1') + self.assertEqual(msg1.data(), b"hello-1") reader2.close() client.close() def test_reader_has_message_available(self): # create client, producer, reader client = Client(self.serviceUrl) - producer = client.create_producer('my-python-topic-reader-has-message-available') - reader = client.create_reader('my-python-topic-reader-has-message-available', - MessageId.latest) + producer = client.create_producer("my-python-topic-reader-has-message-available") + reader = client.create_reader("my-python-topic-reader-has-message-available", MessageId.latest) # before produce data, expected not has message available - self.assertFalse(reader.has_message_available()); + self.assertFalse(reader.has_message_available()) for i in range(10): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) # produced data, expected has message available - self.assertTrue(reader.has_message_available()); + self.assertTrue(reader.has_message_available()) for i in range(10): msg = reader.read_next(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) # consumed all data, expected not has message available - self.assertFalse(reader.has_message_available()); + self.assertFalse(reader.has_message_available()) for i in range(10, 20): - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) # produced data again, expected has message available - self.assertTrue(reader.has_message_available()); + self.assertTrue(reader.has_message_available()) reader.close() producer.close() client.close() def test_seek(self): client = Client(self.serviceUrl) - topic = 'my-python-topic-seek-' + str(time.time()) - consumer = client.subscribe(topic, - 'my-sub', - consumer_type=ConsumerType.Shared) + topic = "my-python-topic-seek-" + str(time.time()) + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) for i in range(100): if i > 0: time.sleep(0.02) - producer.send(b'hello-%d' % i) + producer.send(b"hello-%d" % i) ids = [] timestamps = [] for i in range(100): msg = consumer.receive(TM) - self.assertEqual(msg.data(), b'hello-%d' % i) + self.assertEqual(msg.data(), b"hello-%d" % i) ids.append(msg.message_id()) timestamps.append(msg.publish_timestamp()) consumer.acknowledge(msg) @@ -861,19 +849,19 @@ def test_seek(self): consumer.seek(MessageId.earliest) time.sleep(0.5) msg = consumer.receive(TM) - self.assertEqual(msg.data(), b'hello-0') + self.assertEqual(msg.data(), b"hello-0") # seek on messageId consumer.seek(ids[50]) time.sleep(0.5) msg = consumer.receive(TM) - self.assertEqual(msg.data(), b'hello-50') + self.assertEqual(msg.data(), b"hello-50") # ditto, but seek on timestamp consumer.seek(timestamps[42]) time.sleep(0.5) msg = consumer.receive(TM) - self.assertEqual(msg.data(), b'hello-42') + self.assertEqual(msg.data(), b"hello-42") # repeat with reader reader = client.create_reader(topic, MessageId.latest) @@ -884,25 +872,25 @@ def test_seek(self): reader.seek(MessageId.earliest) time.sleep(0.5) msg = reader.read_next(TM) - self.assertEqual(msg.data(), b'hello-0') + self.assertEqual(msg.data(), b"hello-0") msg = reader.read_next(TM) - self.assertEqual(msg.data(), b'hello-1') + self.assertEqual(msg.data(), b"hello-1") # seek on messageId reader.seek(ids[33]) time.sleep(0.5) msg = reader.read_next(TM) - self.assertEqual(msg.data(), b'hello-33') + self.assertEqual(msg.data(), b"hello-33") msg = reader.read_next(TM) - self.assertEqual(msg.data(), b'hello-34') + self.assertEqual(msg.data(), b"hello-34") # seek on timestamp reader.seek(timestamps[79]) time.sleep(0.5) msg = reader.read_next(TM) - self.assertEqual(msg.data(), b'hello-79') + self.assertEqual(msg.data(), b"hello-79") msg = reader.read_next(TM) - self.assertEqual(msg.data(), b'hello-80') + self.assertEqual(msg.data(), b"hello-80") reader.close() client.close() @@ -915,15 +903,13 @@ def test_v2_topics_http(self): def _v2_topics(self, url): client = Client(url) - consumer = client.subscribe('my-v2-topic-producer-consumer', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('my-v2-topic-producer-consumer') - producer.send(b'hello') + consumer = client.subscribe("my-v2-topic-producer-consumer", "my-sub", consumer_type=ConsumerType.Shared) + producer = client.create_producer("my-v2-topic-producer-consumer") + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): @@ -933,38 +919,35 @@ def _v2_topics(self, url): def test_topics_consumer(self): client = Client(self.serviceUrl) - topic1 = 'persistent://public/default/my-python-topics-consumer-1' - topic2 = 'persistent://public/default/my-python-topics-consumer-2' - topic3 = 'persistent://public/default-2/my-python-topics-consumer-3' # topic from different namespace + topic1 = "persistent://public/default/my-python-topics-consumer-1" + topic2 = "persistent://public/default/my-python-topics-consumer-2" + topic3 = "persistent://public/default-2/my-python-topics-consumer-3" # topic from different namespace topics = [topic1, topic2, topic3] - url1 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-topics-consumer-1/partitions' - url2 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-topics-consumer-2/partitions' - url3 = self.adminUrl + '/admin/v2/persistent/public/default-2/my-python-topics-consumer-3/partitions' + url1 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-topics-consumer-1/partitions" + url2 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-topics-consumer-2/partitions" + url3 = self.adminUrl + "/admin/v2/persistent/public/default-2/my-python-topics-consumer-3/partitions" - doHttpPut(url1, '2') - doHttpPut(url2, '3') - doHttpPut(url3, '4') + doHttpPut(url1, "2") + doHttpPut(url2, "3") + doHttpPut(url3, "4") producer1 = client.create_producer(topic1) producer2 = client.create_producer(topic2) producer3 = client.create_producer(topic3) - consumer = client.subscribe(topics, - 'my-topics-consumer-sub', - consumer_type=ConsumerType.Shared, - receiver_queue_size=10 - ) + consumer = client.subscribe( + topics, "my-topics-consumer-sub", consumer_type=ConsumerType.Shared, receiver_queue_size=10 + ) for i in range(100): - producer1.send(b'hello-1-%d' % i) + producer1.send(b"hello-1-%d" % i) for i in range(100): - producer2.send(b'hello-2-%d' % i) + producer2.send(b"hello-2-%d" % i) for i in range(100): - producer3.send(b'hello-3-%d' % i) - + producer3.send(b"hello-3-%d" % i) for i in range(300): msg = consumer.receive(TM) @@ -976,45 +959,46 @@ def test_topics_consumer(self): def test_topics_pattern_consumer(self): import re + client = Client(self.serviceUrl) - topics_pattern = 'persistent://public/default/my-python-pattern-consumer.*' + topics_pattern = "persistent://public/default/my-python-pattern-consumer.*" - topic1 = 'persistent://public/default/my-python-pattern-consumer-1' - topic2 = 'persistent://public/default/my-python-pattern-consumer-2' - topic3 = 'persistent://public/default/my-python-pattern-consumer-3' + topic1 = "persistent://public/default/my-python-pattern-consumer-1" + topic2 = "persistent://public/default/my-python-pattern-consumer-2" + topic3 = "persistent://public/default/my-python-pattern-consumer-3" - url1 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-pattern-consumer-1/partitions' - url2 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-pattern-consumer-2/partitions' - url3 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-pattern-consumer-3/partitions' + url1 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-pattern-consumer-1/partitions" + url2 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-pattern-consumer-2/partitions" + url3 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-pattern-consumer-3/partitions" - doHttpPut(url1, '2') - doHttpPut(url2, '3') - doHttpPut(url3, '4') + doHttpPut(url1, "2") + doHttpPut(url2, "3") + doHttpPut(url3, "4") producer1 = client.create_producer(topic1) producer2 = client.create_producer(topic2) producer3 = client.create_producer(topic3) - consumer = client.subscribe(re.compile(topics_pattern), - 'my-pattern-consumer-sub', - consumer_type = ConsumerType.Shared, - receiver_queue_size = 10, - pattern_auto_discovery_period = 1 - ) + consumer = client.subscribe( + re.compile(topics_pattern), + "my-pattern-consumer-sub", + consumer_type=ConsumerType.Shared, + receiver_queue_size=10, + pattern_auto_discovery_period=1, + ) # wait enough time to trigger auto discovery time.sleep(2) for i in range(100): - producer1.send(b'hello-1-%d' % i) + producer1.send(b"hello-1-%d" % i) for i in range(100): - producer2.send(b'hello-2-%d' % i) + producer2.send(b"hello-2-%d" % i) for i in range(100): - producer3.send(b'hello-3-%d' % i) - + producer3.send(b"hello-3-%d" % i) for i in range(300): msg = consumer.receive(TM) @@ -1033,70 +1017,72 @@ def test_message_id(self): def test_get_topics_partitions(self): client = Client(self.serviceUrl) - topic_partitioned = 'persistent://public/default/test_get_topics_partitions' - topic_non_partitioned = 'persistent://public/default/test_get_topics_not-partitioned' - - url1 = self.adminUrl + '/admin/v2/persistent/public/default/test_get_topics_partitions/partitions' - doHttpPut(url1, '3') - - self.assertEqual(client.get_topic_partitions(topic_partitioned), - ['persistent://public/default/test_get_topics_partitions-partition-0', - 'persistent://public/default/test_get_topics_partitions-partition-1', - 'persistent://public/default/test_get_topics_partitions-partition-2']) + topic_partitioned = "persistent://public/default/test_get_topics_partitions" + topic_non_partitioned = "persistent://public/default/test_get_topics_not-partitioned" + + url1 = self.adminUrl + "/admin/v2/persistent/public/default/test_get_topics_partitions/partitions" + doHttpPut(url1, "3") + + self.assertEqual( + client.get_topic_partitions(topic_partitioned), + [ + "persistent://public/default/test_get_topics_partitions-partition-0", + "persistent://public/default/test_get_topics_partitions-partition-1", + "persistent://public/default/test_get_topics_partitions-partition-2", + ], + ) - self.assertEqual(client.get_topic_partitions(topic_non_partitioned), - [topic_non_partitioned]) + self.assertEqual(client.get_topic_partitions(topic_non_partitioned), [topic_non_partitioned]) client.close() def test_token_auth(self): - with open('/tmp/pulsar-test-data/tokens/token.txt') as tf: + with open("/tmp/pulsar-test-data/tokens/token.txt") as tf: token = tf.read().strip() # Use adminUrl to test both HTTP request and binary protocol - client = Client(self.adminUrl, - authentication=AuthenticationToken(token)) + client = Client(self.adminUrl, authentication=AuthenticationToken(token)) - consumer = client.subscribe('persistent://private/auth/my-python-topic-token-auth', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('persistent://private/auth/my-python-topic-token-auth') - producer.send(b'hello') + consumer = client.subscribe( + "persistent://private/auth/my-python-topic-token-auth", "my-sub", consumer_type=ConsumerType.Shared + ) + producer = client.create_producer("persistent://private/auth/my-python-topic-token-auth") + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") client.close() def test_token_auth_supplier(self): def read_token(): - with open('/tmp/pulsar-test-data/tokens/token.txt') as tf: + with open("/tmp/pulsar-test-data/tokens/token.txt") as tf: return tf.read().strip() - client = Client(self.serviceUrl, - authentication=AuthenticationToken(read_token)) - consumer = client.subscribe('persistent://private/auth/my-python-topic-token-auth', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('persistent://private/auth/my-python-topic-token-auth') - producer.send(b'hello') + client = Client(self.serviceUrl, authentication=AuthenticationToken(read_token)) + consumer = client.subscribe( + "persistent://private/auth/my-python-topic-token-auth", "my-sub", consumer_type=ConsumerType.Shared + ) + producer = client.create_producer("persistent://private/auth/my-python-topic-token-auth") + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") client.close() def test_producer_consumer_zstd(self): client = Client(self.serviceUrl) - consumer = client.subscribe('my-python-topic-producer-consumer-zstd', - 'my-sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('my-python-topic-producer-consumer-zstd', - compression_type=CompressionType.ZSTD) - producer.send(b'hello') + consumer = client.subscribe( + "my-python-topic-producer-consumer-zstd", "my-sub", consumer_type=ConsumerType.Shared + ) + producer = client.create_producer( + "my-python-topic-producer-consumer-zstd", compression_type=CompressionType.ZSTD + ) + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) - self.assertEqual(msg.data(), b'hello') + self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) @@ -1107,41 +1093,46 @@ def test_producer_consumer_zstd(self): def test_client_reference_deleted(self): def get_producer(): cl = Client(self.serviceUrl) - return cl.create_producer(topic='foobar') + return cl.create_producer(topic="foobar") producer = get_producer() - producer.send(b'test_payload') + producer.send(b"test_payload") ##### def test_get_topic_name(self): client = Client(self.serviceUrl) - consumer = client.subscribe('persistent://public/default/topic_name_test', - 'topic_name_test_sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('persistent://public/default/topic_name_test') - producer.send(b'hello') + consumer = client.subscribe( + "persistent://public/default/topic_name_test", "topic_name_test_sub", consumer_type=ConsumerType.Shared + ) + producer = client.create_producer("persistent://public/default/topic_name_test") + producer.send(b"hello") msg = consumer.receive(TM) - self.assertEqual(msg.topic_name(), 'persistent://public/default/topic_name_test') + self.assertEqual(msg.topic_name(), "persistent://public/default/topic_name_test") client.close() def test_get_partitioned_topic_name(self): client = Client(self.serviceUrl) - url1 = self.adminUrl + '/admin/v2/persistent/public/default/partitioned_topic_name_test/partitions' - doHttpPut(url1, '3') - - partitions = ['persistent://public/default/partitioned_topic_name_test-partition-0', - 'persistent://public/default/partitioned_topic_name_test-partition-1', - 'persistent://public/default/partitioned_topic_name_test-partition-2'] - self.assertEqual(client.get_topic_partitions('persistent://public/default/partitioned_topic_name_test'), - partitions) + url1 = self.adminUrl + "/admin/v2/persistent/public/default/partitioned_topic_name_test/partitions" + doHttpPut(url1, "3") + + partitions = [ + "persistent://public/default/partitioned_topic_name_test-partition-0", + "persistent://public/default/partitioned_topic_name_test-partition-1", + "persistent://public/default/partitioned_topic_name_test-partition-2", + ] + self.assertEqual( + client.get_topic_partitions("persistent://public/default/partitioned_topic_name_test"), partitions + ) - consumer = client.subscribe('persistent://public/default/partitioned_topic_name_test', - 'partitioned_topic_name_test_sub', - consumer_type=ConsumerType.Shared) - producer = client.create_producer('persistent://public/default/partitioned_topic_name_test') - producer.send(b'hello') + consumer = client.subscribe( + "persistent://public/default/partitioned_topic_name_test", + "partitioned_topic_name_test_sub", + consumer_type=ConsumerType.Shared, + ) + producer = client.create_producer("persistent://public/default/partitioned_topic_name_test") + producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg.topic_name() in partitions) @@ -1149,12 +1140,12 @@ def test_get_partitioned_topic_name(self): def test_shutdown_client(self): client = Client(self.serviceUrl) - producer = client.create_producer('persistent://public/default/partitioned_topic_name_test') - producer.send(b'hello') + producer = client.create_producer("persistent://public/default/partitioned_topic_name_test") + producer.send(b"hello") client.shutdown() try: - producer.send(b'hello') + producer.send(b"hello") self.assertTrue(False) except pulsar.PulsarException: # Expected @@ -1162,14 +1153,12 @@ def test_shutdown_client(self): def test_negative_acks(self): client = Client(self.serviceUrl) - consumer = client.subscribe('test_negative_acks', - 'test', - schema=pulsar.schema.StringSchema(), - negative_ack_redelivery_delay_ms=1000) - producer = client.create_producer('test_negative_acks', - schema=pulsar.schema.StringSchema()) + consumer = client.subscribe( + "test_negative_acks", "test", schema=pulsar.schema.StringSchema(), negative_ack_redelivery_delay_ms=1000 + ) + producer = client.create_producer("test_negative_acks", schema=pulsar.schema.StringSchema()) for i in range(10): - producer.send_async('hello-%d' % i, callback=None) + producer.send_async("hello-%d" % i, callback=None) producer.flush() @@ -1189,20 +1178,28 @@ def test_negative_acks(self): def test_connect_timeout(self): client = pulsar.Client( - service_url='pulsar://192.0.2.1:1234', - connection_timeout_ms=1000, # 1 second + service_url="pulsar://192.0.2.1:1234", + connection_timeout_ms=1000, # 1 second ) t1 = time.time() try: - producer = client.create_producer('test_connect_timeout') - self.fail('create_producer should not succeed') + producer = client.create_producer("test_connect_timeout") + self.fail("create_producer should not succeed") except pulsar.ConnectError as expected: - print('expected error: {} when create producer'.format(expected)) + print("expected error: {} when create producer".format(expected)) t2 = time.time() self.assertGreater(t2 - t1, 1.0) - self.assertLess(t2 - t1, 1.5) # 1.5 seconds is long enough + self.assertLess(t2 - t1, 1.5) # 1.5 seconds is long enough client.close() + def test_json_schema_encode(self): + schema = JsonSchema(TestRecord) + record = TestRecord(a=1, b=2) + # Ensure that encoding a JsonSchema more than once works and produces the same result + first_encode = schema.encode(record) + second_encode = schema.encode(record) + self.assertEqual(first_encode, second_encode) + def _check_value_error(self, fun): with self.assertRaises(ValueError): fun() @@ -1212,5 +1209,5 @@ def _check_type_error(self, fun): fun() -if __name__ == '__main__': +if __name__ == "__main__": main() From b0d7960e1f562f47269305efe6f8ae5727fa2ac6 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 7 Dec 2021 18:21:42 +0200 Subject: [PATCH 185/823] [Proxy] Fix issue when Proxy fails to start and logs about an uncaught exception (#13171) * [Proxy] Print stacktrace in uncaught exception handler * [Proxy] Fix IllegalStateException: Insufficient configured threads - happens at startup since the proxy client consumes 50% of number of available CPU cores for selectors (cherry picked from commit 3986be6ebe93c1a6ebfa3c3e731b45a6d17948d2) --- .../org/apache/pulsar/proxy/server/AdminProxyHandler.java | 7 +++++-- .../apache/pulsar/proxy/server/ProxyServiceStarter.java | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 7d3c658a12ac8..853eb0bf0bc3a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -53,6 +53,7 @@ import org.eclipse.jetty.client.RedirectProtocolHandler; import org.eclipse.jetty.client.api.ContentProvider; import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.proxy.ProxyServlet; import org.eclipse.jetty.util.HttpCookieStore; @@ -209,12 +210,14 @@ protected ByteBuffer onRead(byte[] buffer, int offset, int length) { } private static class JettyHttpClient extends HttpClient { + private static final int NUMBER_OF_SELECTOR_THREADS = 1; + public JettyHttpClient() { - super(); + super(new HttpClientTransportOverHTTP(NUMBER_OF_SELECTOR_THREADS), null); } public JettyHttpClient(SslContextFactory sslContextFactory) { - super(sslContextFactory); + super(new HttpClientTransportOverHTTP(NUMBER_OF_SELECTOR_THREADS), sslContextFactory); } /** diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index 235927c98c5e2..c16844c1effa0 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -108,6 +108,7 @@ public ProxyServiceStarter(String[] args) throws Exception { FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern()); Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> { System.out.println(String.format("%s [%s] error Uncaught exception in thread %s: %s", dateFormat.format(new Date()), thread.getContextClassLoader(), thread.getName(), exception.getMessage())); + exception.printStackTrace(System.out); }); JCommander jcommander = new JCommander(); From 7cce3f9b2c76c90953b943d9c27a45d5abdff59b Mon Sep 17 00:00:00 2001 From: Zhanpeng Wu Date: Fri, 10 Dec 2021 17:45:45 +0800 Subject: [PATCH 186/823] Update cursor last active timestamp when reseting cursor (#13166) Resolves #13165 ### Modifications 1. trigger last active time update after resetting cursor 2. add related test case (cherry picked from commit 26342996f8336dd4f63634d8962b6fef11087485) --- .../mledger/impl/ManagedCursorImpl.java | 2 +- .../mledger/impl/ManagedCursorTest.java | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index f13fa932fd2bd..521ec7ac18f84 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1103,7 +1103,7 @@ public void operationComplete() { } } callback.resetComplete(newPosition); - + updateLastActive(); } @Override diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index bbc182719d89e..2f80bc85b5821 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -26,6 +26,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -770,6 +771,47 @@ public void resetFailed(ManagedLedgerException exception, Object ctx) { ledger.close(); } + @Test(timeOut = 20000) + void testLastActiveAfterResetCursor() throws Exception { + ManagedLedger ledger = factory.open("test_cursor_ledger"); + ManagedCursor cursor = ledger.openCursor("tla"); + + PositionImpl lastPosition = null; + for (int i = 0; i < 3; i++) { + lastPosition = (PositionImpl) ledger.addEntry("dummy-entry".getBytes(Encoding)); + } + + final AtomicBoolean moveStatus = new AtomicBoolean(false); + CountDownLatch countDownLatch = new CountDownLatch(1); + + long lastActive = cursor.getLastActive(); + + cursor.asyncResetCursor(lastPosition, new AsyncCallbacks.ResetCursorCallback() { + @Override + public void resetComplete(Object ctx) { + moveStatus.set(true); + countDownLatch.countDown(); + } + + @Override + public void resetFailed(ManagedLedgerException exception, Object ctx) { + moveStatus.set(false); + countDownLatch.countDown(); + } + }); + + countDownLatch.await(); + assertTrue(moveStatus.get()); + + assertNotNull(lastPosition); + assertEquals(lastPosition, cursor.getReadPosition()); + + assertNotEquals(lastActive, cursor.getLastActive()); + + cursor.close(); + ledger.close(); + } + @Test(timeOut = 20000) void seekPosition() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); From 8d8da1bbf69723ddde7044db80ebeed0f79b5901 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 10 Dec 2021 01:25:05 +0800 Subject: [PATCH 187/823] Fix in macOS cmake might find error boost-python libs path (#13193) (cherry picked from commit fdfea8af64a1a9e5c259238ca9ec681ec41fb108) --- pulsar-client-cpp/python/CMakeLists.txt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pulsar-client-cpp/python/CMakeLists.txt b/pulsar-client-cpp/python/CMakeLists.txt index f7d40699a86e9..30631cd8a5047 100644 --- a/pulsar-client-cpp/python/CMakeLists.txt +++ b/pulsar-client-cpp/python/CMakeLists.txt @@ -72,11 +72,18 @@ set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY} ${Boost_PYTHON39_LIBRARY}) if (APPLE) - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} - ${Boost_PYTHON27-MT_LIBRARY_RELEASE} - ${Boost_PYTHON37-MT_LIBRARY_RELEASE} - ${Boost_PYTHON38-MT_LIBRARY_RELEASE} - ${Boost_PYTHON39-MT_LIBRARY_RELEASE}) + if (Boost_PYTHON27-MT_LIBRARY_RELEASE) + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON27-MT_LIBRARY_RELEASE}) + endif () + if (Boost_PYTHON37-MT_LIBRARY_RELEASE) + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON37-MT_LIBRARY_RELEASE}) + endif () + if (Boost_PYTHON38-MT_LIBRARY_RELEASE) + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON38-MT_LIBRARY_RELEASE}) + endif () + if (Boost_PYTHON39-MT_LIBRARY_RELEASE) + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON39-MT_LIBRARY_RELEASE}) + endif () endif() message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}") From 95c2ab34677c2b3a4418242d504b734b8c91967b Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 10 Dec 2021 16:41:49 +0800 Subject: [PATCH 188/823] [Transaction] Fix transaction sequenceId generate error. (#13209) (cherry picked from commit 0994254c24be4aec8fef810bcdbf62e06453fe53) --- .../broker/transaction/TransactionTest.java | 9 ++- .../impl/MLTransactionLogImpl.java | 41 ----------- .../impl/MLTransactionLogInterceptor.java | 63 +++++++++++++--- .../impl/MLTransactionMetadataStore.java | 24 +++---- .../MLTransactionMetadataStoreProvider.java | 6 +- .../MLTransactionMetadataStoreTest.java | 72 +++++++++++++------ 6 files changed, 125 insertions(+), 90 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index e4975d97cea0a..07af3044f8afb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -521,7 +521,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ null); return null; }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); - + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + persistentTopic.getManagedLedger().getConfig().setManagedLedgerInterceptor(mlTransactionLogInterceptor); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(new TransactionCoordinatorID(1), null, persistentTopic.getManagedLedger().getConfig()); @@ -540,7 +541,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ doNothing().when(timeoutTracker).start(); MLTransactionMetadataStore metadataStore1 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), - mlTransactionLog, timeoutTracker, transactionRecoverTracker); + mlTransactionLog, timeoutTracker, transactionRecoverTracker, + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore1.getCoordinatorStats().state, "Ready")); @@ -553,7 +555,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionMetadataStore metadataStore2 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), - mlTransactionLog, timeoutTracker, transactionRecoverTracker); + mlTransactionLog, timeoutTracker, transactionRecoverTracker, + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore2.getCoordinatorStats().state, "Ready")); } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index 2d11d984b8655..e154bb840d7c6 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -68,15 +68,11 @@ public class MLTransactionLogImpl implements TransactionLog { private final TopicName topicName; - private final MLTransactionLogInterceptor mlTransactionLogInterceptor; - public MLTransactionLogImpl(TransactionCoordinatorID tcID, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig) { this.topicName = getMLTransactionLogName(tcID); this.tcId = tcID.getId(); - this.mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(this.mlTransactionLogInterceptor); this.managedLedgerFactory = managedLedgerFactory; this.managedLedgerConfig = managedLedgerConfig; this.entryQueue = new SpscArrayQueue<>(2000); @@ -161,7 +157,6 @@ public CompletableFuture append(TransactionMetadataEntry transactionMe @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { buf.release(); - mlTransactionLogInterceptor.setMaxLocalTxnId(transactionMetadataEntry.getMaxLocalTxnId()); completableFuture.complete(position); } @@ -242,42 +237,6 @@ public void start() { } } - public CompletableFuture getMaxLocalTxnId() { - - CompletableFuture completableFuture = new CompletableFuture<>(); - PositionImpl position = (PositionImpl) managedLedger.getLastConfirmedEntry(); - - if (position != null && position.getEntryId() != -1 - && ((ManagedLedgerImpl) managedLedger).ledgerExists(position.getLedgerId())) { - ((ManagedLedgerImpl) this.managedLedger).asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - TransactionMetadataEntry lastConfirmEntry = new TransactionMetadataEntry(); - ByteBuf buffer = entry.getDataBuffer(); - lastConfirmEntry.parseFrom(buffer, buffer.readableBytes()); - completableFuture.complete(lastConfirmEntry.getMaxLocalTxnId()); - } - - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - log.error("[{}] MLTransactionLog recover MaxLocalTxnId fail!", topicName, exception); - completableFuture.completeExceptionally(exception); - } - }, null); - } else if (managedLedger.getProperties() - .get(MLTransactionLogInterceptor.MAX_LOCAL_TXN_ID) != null) { - completableFuture.complete(Long.parseLong(managedLedger.getProperties() - .get(MLTransactionLogInterceptor.MAX_LOCAL_TXN_ID))); - } else { - log.error("[{}] MLTransactionLog recover MaxLocalTxnId fail! " - + "not found MaxLocalTxnId in managedLedger and properties", topicName); - completableFuture.completeExceptionally(new ManagedLedgerException(topicName - + "MLTransactionLog recover MaxLocalTxnId fail! " - + "not found MaxLocalTxnId in managedLedger and properties")); - } - return completableFuture; - } - class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private final AtomicLong outstandingReadsRequests = new AtomicLong(0); diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java index e97b104e273e1..68add4a74e36c 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java @@ -18,14 +18,18 @@ */ package org.apache.pulsar.transaction.coordinator.impl; +import io.netty.buffer.ByteBuf; +import lombok.Getter; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.api.LedgerEntry; import org.apache.bookkeeper.mledger.impl.OpAddEntry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; +import org.apache.pulsar.transaction.coordinator.proto.TransactionMetadataEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; /** * Store max sequenceID in ManagedLedger properties, in order to recover transaction log. @@ -33,31 +37,68 @@ public class MLTransactionLogInterceptor implements ManagedLedgerInterceptor { private static final Logger log = LoggerFactory.getLogger(MLTransactionLogInterceptor.class); + private static final long TC_ID_NOT_USED = -1L; public static final String MAX_LOCAL_TXN_ID = "max_local_txn_id"; - - private volatile long maxLocalTxnId = -1; + @Getter + private final AtomicLong sequenceId = new AtomicLong(TC_ID_NOT_USED); @Override public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { - return null; + return op; } + // When all of ledger have been deleted, we will generate sequenceId from managedLedger properties @Override public void onManagedLedgerPropertiesInitialize(Map propertiesMap) { + if (propertiesMap == null || propertiesMap.size() == 0) { + return; + } + if (propertiesMap.containsKey(MAX_LOCAL_TXN_ID)) { + sequenceId.set(Long.parseLong(propertiesMap.get(MAX_LOCAL_TXN_ID))); + } } + // When we don't roll over ledger, we can init sequenceId from the getLastAddConfirmed transaction metadata entry @Override - public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle ledgerHandle) { - return CompletableFuture.completedFuture(null); + public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) { + CompletableFuture promise = new CompletableFuture<>(); + if (lh.getLastAddConfirmed() >= 0) { + lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> { + if (ex != null) { + log.error("[{}] Read last entry error.", name, ex); + promise.completeExceptionally(ex); + } else { + if (entries != null) { + try { + LedgerEntry ledgerEntry = entries.getEntry(lh.getLastAddConfirmed()); + if (ledgerEntry != null) { + TransactionMetadataEntry lastConfirmEntry = new TransactionMetadataEntry(); + ByteBuf buffer = ledgerEntry.getEntryBuffer(); + lastConfirmEntry.parseFrom(buffer, buffer.readableBytes()); + this.sequenceId.set(lastConfirmEntry.getMaxLocalTxnId()); + } + entries.close(); + promise.complete(null); + } catch (Exception e) { + log.error("[{}] Failed to recover the tc sequenceId from the last add confirmed entry.", + name, e); + promise.completeExceptionally(e); + } + } else { + promise.complete(null); + } + } + }); + } else { + promise.complete(null); + } + return promise; } + // roll over ledger will update sequenceId to managedLedger properties @Override public void onUpdateManagedLedgerInfo(Map propertiesMap) { - propertiesMap.put(MAX_LOCAL_TXN_ID, maxLocalTxnId + ""); - } - - protected void setMaxLocalTxnId(long maxLocalTxnId) { - this.maxLocalTxnId = maxLocalTxnId; + propertiesMap.put(MAX_LOCAL_TXN_ID, sequenceId.get() + ""); } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index 05faaad1aa002..6ef4f171b41e8 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -60,9 +60,8 @@ public class MLTransactionMetadataStore private static final Logger log = LoggerFactory.getLogger(MLTransactionMetadataStore.class); private final TransactionCoordinatorID tcID; - private final AtomicLong sequenceId = new AtomicLong(TC_ID_NOT_USED); + private final AtomicLong sequenceId; private final MLTransactionLogImpl transactionLog; - private static final long TC_ID_NOT_USED = -1L; private final ConcurrentSkipListMap>> txnMetaMap = new ConcurrentSkipListMap<>(); private final TransactionTimeoutTracker timeoutTracker; private final TransactionMetadataStoreStats transactionMetadataStoreStats; @@ -75,8 +74,10 @@ public class MLTransactionMetadataStore public MLTransactionMetadataStore(TransactionCoordinatorID tcID, MLTransactionLogImpl mlTransactionLog, TransactionTimeoutTracker timeoutTracker, - TransactionRecoverTracker recoverTracker) { + TransactionRecoverTracker recoverTracker, + AtomicLong sequenceId) { super(State.None); + this.sequenceId = sequenceId; this.tcID = tcID; this.transactionLog = mlTransactionLog; this.timeoutTracker = timeoutTracker; @@ -96,16 +97,13 @@ public MLTransactionMetadataStore(TransactionCoordinatorID tcID, @Override public void replayComplete() { - mlTransactionLog.getMaxLocalTxnId().thenAccept(id -> { - recoverTracker.appendOpenTransactionToTimeoutTracker(); - sequenceId.set(id); - if (!changeToReadyState()) { - log.error("Managed ledger transaction metadata store change state error when replay complete"); - } else { - recoverTracker.handleCommittingAndAbortingTransaction(); - timeoutTracker.start(); - } - }); + recoverTracker.appendOpenTransactionToTimeoutTracker(); + if (!changeToReadyState()) { + log.error("Managed ledger transaction metadata store change state error when replay complete"); + } else { + recoverTracker.handleCommittingAndAbortingTransaction(); + timeoutTracker.start(); + } } @Override diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java index 36b19585de931..3f20cbc75cfbe 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java @@ -45,10 +45,14 @@ public CompletableFuture openStore(TransactionCoordina ManagedLedgerConfig managedLedgerConfig, TransactionTimeoutTracker timeoutTracker, TransactionRecoverTracker recoverTracker) { + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(new MLTransactionLogInterceptor()); MLTransactionLogImpl txnLog = new MLTransactionLogImpl(transactionCoordinatorId, managedLedgerFactory, managedLedgerConfig); + // MLTransactionLogInterceptor will init sequenceId and update the sequenceId to managedLedger properties. return txnLog.initialize().thenApply(__ -> - new MLTransactionMetadataStore(transactionCoordinatorId, txnLog, timeoutTracker, recoverTracker)); + new MLTransactionMetadataStore(transactionCoordinatorId, txnLog, timeoutTracker, + recoverTracker, mlTransactionLogInterceptor.getSequenceId())); } } \ No newline at end of file diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java index e9b3e0c08d122..03aa1be03cfea 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java @@ -31,6 +31,7 @@ import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.TransactionNotFoundException; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogInterceptor; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.apache.pulsar.transaction.coordinator.test.MockedBookKeeperTestCase; @@ -41,6 +42,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -65,12 +67,16 @@ public void testTransactionOperation() throws Exception { @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, - new ManagedLedgerConfig()); + managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); int checkReplayRetryCount = 0; while (true) { checkReplayRetryCount++; @@ -122,13 +128,13 @@ public void testTransactionOperation() throws Exception { } } - @DataProvider(name = "isUseManagedLedger") + @DataProvider(name = "isUseManagedLedgerProperties") public Object[][] versions() { return new Object[][] { { true }, { false } }; } - @Test(dataProvider = "isUseManagedLedger") - public void testRecoverSequenceId(boolean isUseManagedLedger) throws Exception { + @Test(dataProvider = "isUseManagedLedgerProperties") + public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws Exception { ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); factoryConf.setMaxCacheSize(0); @@ -136,18 +142,21 @@ public void testRecoverSequenceId(boolean isUseManagedLedger) throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); managedLedgerConfig.setMaxEntriesPerLedger(3); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().until(transactionMetadataStore::checkIfReady); TxnID txnID = transactionMetadataStore.newTransaction(20000).get(); transactionMetadataStore.updateTxnStatus(txnID, TxnStatus.COMMITTING, TxnStatus.OPEN, false).get(); - if (isUseManagedLedger) { + if (isUseManagedLedgerProperties) { transactionMetadataStore.updateTxnStatus(txnID, TxnStatus.COMMITTED, TxnStatus.COMMITTING, false).get(); } assertEquals(txnID.getLeastSigBits(), 0); @@ -155,16 +164,20 @@ public void testRecoverSequenceId(boolean isUseManagedLedger) throws Exception { field.setAccessible(true); ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) field.get(mlTransactionLog); Position position = managedLedger.getLastConfirmedEntry(); - - if (isUseManagedLedger) { + if (isUseManagedLedgerProperties) { Awaitility.await().until(() -> { managedLedger.rollCurrentLedgerIfFull(); return !managedLedger.ledgerExists(position.getLedgerId()); }); } + mlTransactionLog.closeAsync().get(); + mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, + managedLedgerConfig); + mlTransactionLog.initialize().join(); transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().until(transactionMetadataStore::checkIfReady); txnID = transactionMetadataStore.newTransaction(100000).get(); @@ -181,12 +194,15 @@ public void testInitTransactionReader() throws Exception { TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); managedLedgerConfig.setMaxEntriesPerLedger(2); + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); int checkReplayRetryCount = 0; while (true) { if (checkReplayRetryCount > 3) { @@ -224,11 +240,12 @@ public void testInitTransactionReader() throws Exception { transactionMetadataStore.closeAsync(); MLTransactionLogImpl txnLog2 = new MLTransactionLogImpl(transactionCoordinatorID, factory, - new ManagedLedgerConfig()); + managedLedgerConfig); txnLog2.initialize().join(); MLTransactionMetadataStore transactionMetadataStoreTest = new MLTransactionMetadataStore(transactionCoordinatorID, - txnLog2, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + txnLog2, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); while (true) { if (checkReplayRetryCount > 6) { @@ -288,12 +305,16 @@ public void testDeleteLog() throws Exception { @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, - new ManagedLedgerConfig()); + managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); int checkReplayRetryCount = 0; while (true) { if (checkReplayRetryCount > 3) { @@ -351,12 +372,16 @@ public void testRecoverWhenDeleteFromCursor() throws Exception { @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, - new ManagedLedgerConfig()); + managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().until(transactionMetadataStore::checkIfReady); @@ -370,11 +395,12 @@ public void testRecoverWhenDeleteFromCursor() throws Exception { transactionMetadataStore.updateTxnStatus(txnID2, TxnStatus.ABORTED, TxnStatus.ABORTING, false).get(); mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, - new ManagedLedgerConfig()); + managedLedgerConfig); mlTransactionLog.initialize().join(); transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().until(transactionMetadataStore::checkIfReady); } @@ -387,12 +413,16 @@ public void testManageLedgerWriteFailState() throws Exception { @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, - new ManagedLedgerConfig()); + managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl()); + new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + mlTransactionLogInterceptor.getSequenceId()); Awaitility.await().until(transactionMetadataStore::checkIfReady); transactionMetadataStore.newTransaction(5000).get(); From 2457ae984ed3bd7887c2639fb294df8f2439e7e2 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 9 Dec 2021 18:11:50 -0600 Subject: [PATCH 189/823] [Broker] Optimize ManagedLedger Ledger Ownership Check (#13222) (cherry picked from commit 02b8de039df38fa95656e7ca9b5c0babd25ff021) --- .../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 7f4d2913471da..cec106c7e66a0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1825,21 +1825,19 @@ public void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Ob if (log.isDebugEnabled()) { log.debug("[{}] Reading entry ledger {}: {}", name, position.getLedgerId(), position.getEntryId()); } - if (!ledgers.containsKey(position.getLedgerId())) { - log.error("[{}] Failed to get message with ledger {}:{} the ledgerId does not belong to this topic " - + "or has been deleted.", name, position.getLedgerId(), position.getEntryId()); - callback.readEntryFailed(new ManagedLedgerException.NonRecoverableLedgerException("Message not found, " - + "the ledgerId does not belong to this topic or has been deleted"), ctx); - return; - } if (position.getLedgerId() == currentLedger.getId()) { asyncReadEntry(currentLedger, position, callback, ctx); - } else { + } else if (ledgers.containsKey(position.getLedgerId())) { getLedgerHandle(position.getLedgerId()).thenAccept(ledger -> asyncReadEntry(ledger, position, callback, ctx)).exceptionally(ex -> { log.error("[{}] Error opening ledger for reading at position {} - {}", name, position, ex.getMessage()); callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx); return null; }); + } else { + log.error("[{}] Failed to get message with ledger {}:{} the ledgerId does not belong to this topic " + + "or has been deleted.", name, position.getLedgerId(), position.getEntryId()); + callback.readEntryFailed(new ManagedLedgerException.NonRecoverableLedgerException("Message not found, " + + "the ledgerId does not belong to this topic or has been deleted"), ctx); } } From f89de87cb1ae672103e1ef4035bbc92826613406 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Sat, 11 Dec 2021 12:31:50 +0800 Subject: [PATCH 190/823] Use current resourceUsage value as historyUsage when leader change in ThresholdShedder (#13136) ### Motivation Fix #13119 ### Modification 1. User current resourceUsage value as historyUsage value when leader change in ThresholdShedder to speed up getting the actual historyUsage value. (cherry picked from commit 6d9d24d50db5418ddbb845d2c7a2be2b9ac72893) --- .../loadbalance/impl/ThresholdShedder.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index 9c89be92c4cc5..3e103261812c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -51,13 +51,9 @@ */ public class ThresholdShedder implements LoadSheddingStrategy { private static final Logger log = LoggerFactory.getLogger(ThresholdShedder.class); - private final Multimap selectedBundlesCache = ArrayListMultimap.create(); - private static final double ADDITIONAL_THRESHOLD_PERCENT_MARGIN = 0.05; - private static final double MB = 1024 * 1024; - private final Map brokerAvgResourceUsage = new HashMap<>(); @Override @@ -153,25 +149,27 @@ private double getBrokerAvgUsage(final LoadData loadData, final double historyPe for (Map.Entry entry : loadData.getBrokerData().entrySet()) { LocalBrokerData localBrokerData = entry.getValue().getLocalData(); String broker = entry.getKey(); - updateAvgResourceUsage(broker, localBrokerData, historyPercentage, conf); - totalUsage += brokerAvgResourceUsage.getOrDefault(broker, 0.0); + totalUsage += updateAvgResourceUsage(broker, localBrokerData, historyPercentage, conf); totalBrokers++; } return totalBrokers > 0 ? totalUsage / totalBrokers : 0; } - private void updateAvgResourceUsage(String broker, LocalBrokerData localBrokerData, final double historyPercentage, - final ServiceConfiguration conf) { - double historyUsage = - brokerAvgResourceUsage.getOrDefault(broker, 0.0); - historyUsage = historyUsage * historyPercentage - + (1 - historyPercentage) * localBrokerData.getMaxResourceUsageWithWeight( + private double updateAvgResourceUsage(String broker, LocalBrokerData localBrokerData, + final double historyPercentage, final ServiceConfiguration conf) { + Double historyUsage = + brokerAvgResourceUsage.get(broker); + double resourceUsage = localBrokerData.getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), conf.getLoadBalancerBandwithInResourceWeight(), conf.getLoadBalancerBandwithOutResourceWeight()); + historyUsage = historyUsage == null + ? resourceUsage : historyUsage * historyPercentage + (1 - historyPercentage) * resourceUsage; + brokerAvgResourceUsage.put(broker, historyUsage); + return historyUsage; } } From 9daba91d8f66627963ada9041699cfee496e8d20 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 11 Dec 2021 15:20:49 +0800 Subject: [PATCH 191/823] Fix when deleting topic with NotFoundException, do not return to client (#13203) (cherry picked from commit bd68b6f05f9749328701c59bdaf3cddda2254d39) --- .../pulsar/broker/admin/AdminResource.java | 8 +++++ .../admin/impl/PersistentTopicsBase.java | 5 ++- .../broker/admin/PersistentTopicsTest.java | 32 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index f3a94d222fdca..9045462f193ed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -35,6 +35,7 @@ import javax.ws.rs.core.Response.Status; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.systopic.SystemTopicClient; @@ -61,6 +62,7 @@ import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyExistsException; import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException; @@ -772,6 +774,12 @@ protected void checkNotNull(Object o, String errorMessage) { } } + protected boolean isManagedLedgerNotFoundException(Exception e) { + Throwable cause = e.getCause(); + return cause instanceof ManagedLedgerException.MetadataNotFoundException + || cause instanceof MetadataStoreException.NotFoundException; + } + protected void checkArgument(boolean b, String errorMessage) { if (!b) { throw new RestException(Status.BAD_REQUEST, errorMessage); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index be76ff1949b17..5a084bf7e10b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -56,7 +56,6 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; @@ -302,7 +301,7 @@ protected void internalDeleteTopicForcefully(boolean authoritative, boolean dele try { pulsar().getBrokerService().deleteTopic(topicName.toString(), true, deleteSchema).get(); } catch (Exception e) { - if (e.getCause() instanceof MetadataNotFoundException) { + if (isManagedLedgerNotFoundException(e)) { log.info("[{}] Topic was already not existing {}", clientAppId(), topicName, e); } else { log.error("[{}] Failed to delete topic forcefully {}", clientAppId(), topicName, e); @@ -1021,7 +1020,7 @@ protected void internalDeleteTopic(boolean authoritative, boolean deleteSchema) log.error("[{}] Failed to delete topic {}", clientAppId(), topicName, t); if (t instanceof TopicBusyException) { throw new RestException(Status.PRECONDITION_FAILED, "Topic has active producers/subscriptions"); - } else if (t instanceof MetadataNotFoundException) { + } else if (isManagedLedgerNotFoundException(e)) { throw new RestException(Status.NOT_FOUND, "Topic not found"); } else { throw new RestException(t); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 6e5ef99efe893..e41db381f4084 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -54,6 +54,7 @@ import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TopicResources; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -78,6 +79,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.zookeeper.KeeperException; import org.mockito.ArgumentCaptor; import org.powermock.reflect.Whitebox; @@ -1084,4 +1086,34 @@ public void onSendAcknowledgement(Producer producer, Message message, MessageId Assert.assertTrue(admin.topics().getMessageIdByTimestamp(topicName, publish2 + 1) .compareTo(id2) > 0); } + + @Test + public void testDeleteTopic() throws Exception { + final String topicName = "topic-1"; + BrokerService brokerService = spy(pulsar.getBrokerService()); + doReturn(brokerService).when(pulsar).getBrokerService(); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, false); + CompletableFuture deleteTopicFuture = new CompletableFuture<>(); + deleteTopicFuture.completeExceptionally(new MetadataStoreException.NotFoundException()); + doReturn(deleteTopicFuture).when(brokerService).deleteTopic(anyString(), anyBoolean(), anyBoolean()); + persistentTopics.deleteTopic(testTenant, testNamespace, topicName, true, true, true); + // + CompletableFuture deleteTopicFuture2 = new CompletableFuture<>(); + deleteTopicFuture2.completeExceptionally(new MetadataStoreException("test exception")); + doReturn(deleteTopicFuture2).when(brokerService).deleteTopic(anyString(), anyBoolean(), anyBoolean()); + try { + persistentTopics.deleteTopic(testTenant, testNamespace, topicName, true, true, true); + } catch (Exception e) { + Assert.assertTrue(e instanceof RestException); + } + // + CompletableFuture deleteTopicFuture3 = new CompletableFuture<>(); + deleteTopicFuture3.completeExceptionally(new MetadataStoreException.NotFoundException()); + doReturn(deleteTopicFuture3).when(brokerService).deleteTopic(anyString(), anyBoolean(), anyBoolean()); + try { + persistentTopics.deleteTopic(testTenant, testNamespace, topicName, false, true, true); + } catch (RestException e) { + Assert.assertEquals(e.getResponse().getStatus(), 404); + } + } } From d9da67710a4cc623cc080244ca91fceef3ea426a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sat, 11 Dec 2021 00:09:19 +0800 Subject: [PATCH 192/823] Fix MessagePayloadContextImpl not recycled (#13233) (cherry picked from commit 0ce155ea3420306a5f9c9d3ff22f4c0d92e5ec71) --- .../org/apache/pulsar/client/impl/MessagePayloadContextImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java index aa6cab80db2bc..f21900387b623 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java @@ -88,6 +88,7 @@ public void recycle() { ackBitSet.recycle(); ackBitSet = null; } + recyclerHandle.recycle(this); } @Override From ac932d545064ee59325edd4bfe0b294eae92ad37 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Sat, 11 Dec 2021 17:02:27 +0800 Subject: [PATCH 193/823] [Transaction] Fix generate transactionId some comment. (#13234) (cherry picked from commit 4ecb874b71e70b23f2f5310317b3c741007ed61b) --- .../ManagedLedgerInterceptorImpl.java | 4 +- .../broker/transaction/TransactionTest.java | 12 ++--- .../impl/MLTransactionMetadataStore.java | 19 ++++---- .../MLTransactionMetadataStoreProvider.java | 6 +-- ... => MLTransactionSequenceIdGenerator.java} | 15 +++++-- .../MLTransactionMetadataStoreTest.java | 45 +++++++++---------- 6 files changed, 52 insertions(+), 49 deletions(-) rename pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/{MLTransactionLogInterceptor.java => MLTransactionSequenceIdGenerator.java} (93%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index bbab84ba1f093..424797fa52a51 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -35,11 +35,8 @@ public class ManagedLedgerInterceptorImpl implements ManagedLedgerInterceptor { private static final Logger log = LoggerFactory.getLogger(ManagedLedgerInterceptorImpl.class); private static final String INDEX = "index"; - - private final Set brokerEntryMetadataInterceptors; - public ManagedLedgerInterceptorImpl(Set brokerEntryMetadataInterceptors) { this.brokerEntryMetadataInterceptors = brokerEntryMetadataInterceptors; } @@ -108,6 +105,7 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, entries.close(); promise.complete(null); } catch (Exception e) { + entries.close(); log.error("[{}] Failed to recover the index generator from the last add confirmed entry.", name, e); promise.completeExceptionally(e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 07af3044f8afb..3d0c406f07c3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -87,7 +87,7 @@ import org.apache.pulsar.transaction.coordinator.TransactionRecoverTracker; import org.apache.pulsar.transaction.coordinator.TransactionTimeoutTracker; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogInterceptor; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; import org.testng.Assert; @@ -510,7 +510,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ .getTopic(topic, false).get().get(); persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true); Map map = new HashMap<>(); - map.put(MLTransactionLogInterceptor.MAX_LOCAL_TXN_ID, "1"); + map.put(MLTransactionSequenceIdGenerator.MAX_LOCAL_TXN_ID, "1"); persistentTopic.getManagedLedger().setProperties(map); ManagedCursor managedCursor = mock(ManagedCursor.class); @@ -521,8 +521,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ null); return null; }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - persistentTopic.getManagedLedger().getConfig().setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + persistentTopic.getManagedLedger().getConfig().setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(new TransactionCoordinatorID(1), null, persistentTopic.getManagedLedger().getConfig()); @@ -542,7 +542,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionMetadataStore metadataStore1 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), mlTransactionLog, timeoutTracker, transactionRecoverTracker, - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore1.getCoordinatorStats().state, "Ready")); @@ -556,7 +556,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionMetadataStore metadataStore2 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), mlTransactionLog, timeoutTracker, transactionRecoverTracker, - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore2.getCoordinatorStats().state, "Ready")); } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index 6ef4f171b41e8..a71d2030ebf2b 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -24,7 +24,6 @@ import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; @@ -60,7 +59,6 @@ public class MLTransactionMetadataStore private static final Logger log = LoggerFactory.getLogger(MLTransactionMetadataStore.class); private final TransactionCoordinatorID tcID; - private final AtomicLong sequenceId; private final MLTransactionLogImpl transactionLog; private final ConcurrentSkipListMap>> txnMetaMap = new ConcurrentSkipListMap<>(); private final TransactionTimeoutTracker timeoutTracker; @@ -70,14 +68,15 @@ public class MLTransactionMetadataStore private final LongAdder abortedTransactionCount; private final LongAdder transactionTimeoutCount; private final LongAdder appendLogCount; + private final MLTransactionSequenceIdGenerator sequenceIdGenerator; public MLTransactionMetadataStore(TransactionCoordinatorID tcID, MLTransactionLogImpl mlTransactionLog, TransactionTimeoutTracker timeoutTracker, TransactionRecoverTracker recoverTracker, - AtomicLong sequenceId) { + MLTransactionSequenceIdGenerator sequenceIdGenerator) { super(State.None); - this.sequenceId = sequenceId; + this.sequenceIdGenerator = sequenceIdGenerator; this.tcID = tcID; this.transactionLog = mlTransactionLog; this.timeoutTracker = timeoutTracker; @@ -204,7 +203,7 @@ public synchronized CompletableFuture newTransaction(long timeOut) { } long mostSigBits = tcID.getId(); - long leastSigBits = sequenceId.incrementAndGet(); + long leastSigBits = sequenceIdGenerator.generateSequenceId(); TxnID txnID = new TxnID(mostSigBits, leastSigBits); long currentTimeMillis = System.currentTimeMillis(); TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() @@ -214,7 +213,7 @@ public synchronized CompletableFuture newTransaction(long timeOut) { .setTimeoutMs(timeOut) .setMetadataOp(TransactionMetadataEntry.TransactionMetadataOp.NEW) .setLastModificationTime(currentTimeMillis) - .setMaxLocalTxnId(sequenceId.get()); + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); return transactionLog.append(transactionMetadataEntry) .thenCompose(position -> { appendLogCount.increment(); @@ -243,7 +242,7 @@ public synchronized CompletableFuture addProducedPartitionToTxn(TxnID txnI .setMetadataOp(TransactionMetadataOp.ADD_PARTITION) .addAllPartitions(partitions) .setLastModificationTime(System.currentTimeMillis()) - .setMaxLocalTxnId(sequenceId.get()); + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); return transactionLog.append(transactionMetadataEntry) .thenCompose(position -> { @@ -280,7 +279,7 @@ public synchronized CompletableFuture addAckedPartitionToTxn(TxnID txnID, .setMetadataOp(TransactionMetadataOp.ADD_SUBSCRIPTION) .addAllSubscriptions(txnSubscriptionToSubscription(txnSubscriptions)) .setLastModificationTime(System.currentTimeMillis()) - .setMaxLocalTxnId(sequenceId.get()); + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); return transactionLog.append(transactionMetadataEntry) .thenCompose(position -> { @@ -321,7 +320,7 @@ public synchronized CompletableFuture updateTxnStatus(TxnID txnID, TxnStat .setMetadataOp(TransactionMetadataOp.UPDATE) .setLastModificationTime(System.currentTimeMillis()) .setNewStatus(newStatus) - .setMaxLocalTxnId(sequenceId.get()); + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); return transactionLog.append(transactionMetadataEntry).thenCompose(position -> { appendLogCount.increment(); @@ -378,7 +377,7 @@ public TransactionCoordinatorStats getCoordinatorStats() { TransactionCoordinatorStats transactionCoordinatorstats = new TransactionCoordinatorStats(); transactionCoordinatorstats.setLowWaterMark(getLowWaterMark()); transactionCoordinatorstats.setState(getState().name()); - transactionCoordinatorstats.setLeastSigBits(sequenceId.get()); + transactionCoordinatorstats.setLeastSigBits(sequenceIdGenerator.getCurrentSequenceId()); return transactionCoordinatorstats; } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java index 3f20cbc75cfbe..f0c32d2648207 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java @@ -45,14 +45,14 @@ public CompletableFuture openStore(TransactionCoordina ManagedLedgerConfig managedLedgerConfig, TransactionTimeoutTracker timeoutTracker, TransactionRecoverTracker recoverTracker) { - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(new MLTransactionLogInterceptor()); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl txnLog = new MLTransactionLogImpl(transactionCoordinatorId, managedLedgerFactory, managedLedgerConfig); // MLTransactionLogInterceptor will init sequenceId and update the sequenceId to managedLedger properties. return txnLog.initialize().thenApply(__ -> new MLTransactionMetadataStore(transactionCoordinatorId, txnLog, timeoutTracker, - recoverTracker, mlTransactionLogInterceptor.getSequenceId())); + recoverTracker, mlTransactionSequenceIdGenerator)); } } \ No newline at end of file diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java similarity index 93% rename from pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java rename to pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java index 68add4a74e36c..c68997b1c0524 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogInterceptor.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java @@ -19,7 +19,6 @@ package org.apache.pulsar.transaction.coordinator.impl; import io.netty.buffer.ByteBuf; -import lombok.Getter; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.LedgerEntry; import org.apache.bookkeeper.mledger.impl.OpAddEntry; @@ -34,12 +33,11 @@ /** * Store max sequenceID in ManagedLedger properties, in order to recover transaction log. */ -public class MLTransactionLogInterceptor implements ManagedLedgerInterceptor { +public class MLTransactionSequenceIdGenerator implements ManagedLedgerInterceptor { - private static final Logger log = LoggerFactory.getLogger(MLTransactionLogInterceptor.class); + private static final Logger log = LoggerFactory.getLogger(MLTransactionSequenceIdGenerator.class); private static final long TC_ID_NOT_USED = -1L; public static final String MAX_LOCAL_TXN_ID = "max_local_txn_id"; - @Getter private final AtomicLong sequenceId = new AtomicLong(TC_ID_NOT_USED); @Override @@ -81,6 +79,7 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, entries.close(); promise.complete(null); } catch (Exception e) { + entries.close(); log.error("[{}] Failed to recover the tc sequenceId from the last add confirmed entry.", name, e); promise.completeExceptionally(e); @@ -101,4 +100,12 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, public void onUpdateManagedLedgerInfo(Map propertiesMap) { propertiesMap.put(MAX_LOCAL_TXN_ID, sequenceId.get() + ""); } + + long generateSequenceId() { + return sequenceId.incrementAndGet(); + } + + long getCurrentSequenceId() { + return sequenceId.get(); + } } diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java index 03aa1be03cfea..7fa3c08300162 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java @@ -31,7 +31,7 @@ import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.TransactionNotFoundException; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogInterceptor; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.apache.pulsar.transaction.coordinator.test.MockedBookKeeperTestCase; @@ -42,7 +42,6 @@ import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -68,15 +67,15 @@ public void testTransactionOperation() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); int checkReplayRetryCount = 0; while (true) { checkReplayRetryCount++; @@ -142,8 +141,8 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); managedLedgerConfig.setMaxEntriesPerLedger(3); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); @@ -151,7 +150,7 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().until(transactionMetadataStore::checkIfReady); TxnID txnID = transactionMetadataStore.newTransaction(20000).get(); @@ -177,7 +176,7 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().until(transactionMetadataStore::checkIfReady); txnID = transactionMetadataStore.newTransaction(100000).get(); @@ -194,15 +193,15 @@ public void testInitTransactionReader() throws Exception { TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); managedLedgerConfig.setMaxEntriesPerLedger(2); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); int checkReplayRetryCount = 0; while (true) { if (checkReplayRetryCount > 3) { @@ -245,7 +244,7 @@ public void testInitTransactionReader() throws Exception { MLTransactionMetadataStore transactionMetadataStoreTest = new MLTransactionMetadataStore(transactionCoordinatorID, txnLog2, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); while (true) { if (checkReplayRetryCount > 6) { @@ -306,15 +305,15 @@ public void testDeleteLog() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); int checkReplayRetryCount = 0; while (true) { if (checkReplayRetryCount > 3) { @@ -373,15 +372,15 @@ public void testRecoverWhenDeleteFromCursor() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().until(transactionMetadataStore::checkIfReady); @@ -400,7 +399,7 @@ public void testRecoverWhenDeleteFromCursor() throws Exception { transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().until(transactionMetadataStore::checkIfReady); } @@ -414,15 +413,15 @@ public void testManageLedgerWriteFailState() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); TransactionCoordinatorID transactionCoordinatorID = new TransactionCoordinatorID(1); ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - MLTransactionLogInterceptor mlTransactionLogInterceptor = new MLTransactionLogInterceptor(); - managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionLogInterceptor); + MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); + managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionLogInterceptor.getSequenceId()); + mlTransactionSequenceIdGenerator); Awaitility.await().until(transactionMetadataStore::checkIfReady); transactionMetadataStore.newTransaction(5000).get(); From 8fa7139ade8cefe2ec1fcd6873204546ed75d9df Mon Sep 17 00:00:00 2001 From: Travis Sturzl Date: Wed, 10 Nov 2021 05:17:39 -0700 Subject: [PATCH 194/823] [Issue #12485][Python Client] cannot use any values that evaluates to False (#12489) (cherry picked from commit aa59f753590ce9e6c0a7cddd1b19a89e5ef539ee) --- pulsar-client-cpp/python/pulsar/schema/definition.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-client-cpp/python/pulsar/schema/definition.py b/pulsar-client-cpp/python/pulsar/schema/definition.py index 9b6c86106cefd..a7a235b25a6e3 100644 --- a/pulsar-client-cpp/python/pulsar/schema/definition.py +++ b/pulsar-client-cpp/python/pulsar/schema/definition.py @@ -184,7 +184,7 @@ def python_type(self): return self.__class__ def validate_type(self, name, val): - if not val and not self._required: + if val is None and not self._required: return self.default() if not isinstance(val, self.__class__): @@ -219,7 +219,7 @@ def python_type(self): pass def validate_type(self, name, val): - if not val and not self._required: + if val is None and not self._required: return self.default() if type(val) != self.python_type(): @@ -350,7 +350,7 @@ def python_type(self): def validate_type(self, name, val): t = type(val) - if not val and not self._required: + if val is None and not self._required: return self.default() if not (t is str or t.__name__ == 'unicode'): From c81aec3a16139f8023c8e8021e3ce74c974212ba Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 10 Nov 2021 06:03:06 +0200 Subject: [PATCH 195/823] [Config] Add readWorkerThreadsThrottlingEnabled to conf/bookkeeper.conf (#12666) - https://github.com/apache/bookkeeper/pull/2646 added "Auto-throttle read operations" which is enabled by default (cherry picked from commit fc6d6dadaf77766189d0731196646d4c79874c8c) --- conf/bookkeeper.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf index 2fdb0fd1ac9b2..f26abe086e4f0 100644 --- a/conf/bookkeeper.conf +++ b/conf/bookkeeper.conf @@ -166,6 +166,11 @@ maxPendingReadRequestsPerThread=2500 # avoid the executor queue to grow indefinitely maxPendingAddRequestsPerThread=10000 +# Use auto-throttling of the read-worker threads. This is done +# to ensure the bookie is not using unlimited amount of memory +# to respond to read-requests. +readWorkerThreadsThrottlingEnabled=true + # Option to enable busy-wait settings. Default is false. # WARNING: This option will enable spin-waiting on executors and IO threads in order to reduce latency during # context switches. The spinning will consume 100% CPU even when bookie is not doing any work. It is recommended to From 757c121f6e2a31a040038d39576d244e4b3f28f2 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Wed, 10 Nov 2021 20:44:58 +0800 Subject: [PATCH 196/823] Some depdency in integration tests scope should be test (#12696) (cherry picked from commit b27a7169e8538430b8b31fe39e2615a8236be436) --- tests/integration/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index bfaa069eb8903..dcbaa8ff2b753 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -148,12 +148,14 @@ org.elasticsearch.client elasticsearch-rest-high-level-client + test com.rabbitmq amqp-client ${rabbitmq-client.version} + test From 9db66c31370c0b2a693f92ea8fba6d02e4a9bdd4 Mon Sep 17 00:00:00 2001 From: ran Date: Mon, 13 Dec 2021 11:20:47 +0800 Subject: [PATCH 197/823] [Pulsar SQL] Support query chunked messages feature in Pulsar SQL (#12720) ### Motivation Currently, the Pulsar SQL didn't support query chunked messages. ### Modifications Add a chunked message map in `PulsarRecordCursor` to maintain incomplete chunked messages, if one chunked message was received completely, it will be offered in the message queue to wait for deserialization. (cherry picked from commit 93b74b5498ced9e75512c593e6e5a9f5a6c8f26b) --- .../pulsar/client/impl/ProducerImpl.java | 9 + .../pulsar/common/api/raw/RawMessage.java | 29 +++ .../pulsar/common/api/raw/RawMessageImpl.java | 44 ++++ .../pulsar/sql/presto/PulsarRecordCursor.java | 141 ++++++++++-- .../sql/presto/TestReadChunkedMessages.java | 214 ++++++++++++++++++ 5 files changed, 424 insertions(+), 13 deletions(-) create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index cc978a11ee90c..a3ca38689a9a1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -463,7 +463,16 @@ public void sendAsync(Message message, SendCallback callback) { sequenceId = msgMetadata.getSequenceId(); } String uuid = totalChunks > 1 ? String.format("%s-%d", producerName, sequenceId) : null; + byte[] schemaVersion = totalChunks > 1 && msg.getMessageBuilder().hasSchemaVersion() ? + msg.getMessageBuilder().getSchemaVersion() : null; for (int chunkId = 0; chunkId < totalChunks; chunkId++) { + // Need to reset the schemaVersion, because the schemaVersion is based on a ByteBuf object in + // `MessageMetadata`, if we want to re-serialize the `SEND` command using a same `MessageMetadata`, + // we need to reset the ByteBuf of the schemaVersion in `MessageMetadata`, I think we need to + // reset `ByteBuf` objects in `MessageMetadata` after call the method `MessageMetadata#writeTo()`. + if (chunkId > 0 && schemaVersion != null) { + msg.getMessageBuilder().setSchemaVersion(schemaVersion); + } serializeAndSendMessage(msg, payload, sequenceId, uuid, chunkId, totalChunks, readStartIndex, ClientCnx.getMaxMessageSize(), compressedPayload, compressed, compressedPayload.readableBytes(), uncompressedSize, callback); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java index d093628ccd105..483b5a30a0abd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java @@ -121,4 +121,33 @@ public interface RawMessage { * @return true if the key is base64 encoded, false otherwise */ boolean hasBase64EncodedKey(); + + /** + * Get uuid of chunked message. + * + * @return uuid + */ + String getUUID(); + + /** + * Get chunkId of chunked message. + * + * @return chunkId + */ + int getChunkId(); + + /** + * Get chunk num of chunked message. + * + * @return chunk num + */ + int getNumChunksFromMsg(); + + /** + * Get chunk message total size in bytes. + * + * @return chunked message total size in bytes + */ + int getTotalChunkMsgSize(); + } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessageImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessageImpl.java index defc1b496b043..3aa0cbc34b710 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessageImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessageImpl.java @@ -81,6 +81,14 @@ public static RawMessage get(ReferenceCountedMessageMetadata msgMetadata, return msg; } + public RawMessage updatePayloadForChunkedMessage(ByteBuf chunkedTotalPayload) { + if (!msgMetadata.getMetadata().hasNumChunksFromMsg() || msgMetadata.getMetadata().getNumChunksFromMsg() <= 1) { + throw new RuntimeException("The update payload operation only support multi chunked messages."); + } + payload = chunkedTotalPayload; + return this; + } + @Override public Map getProperties() { if (singleMessageMetadata != null && singleMessageMetadata.getPropertiesCount() > 0) { @@ -170,6 +178,42 @@ public boolean hasBase64EncodedKey() { return msgMetadata.getMetadata().isPartitionKeyB64Encoded(); } + @Override + public String getUUID() { + if (msgMetadata.getMetadata().hasUuid()) { + return msgMetadata.getMetadata().getUuid(); + } else { + return null; + } + } + + @Override + public int getChunkId() { + if (msgMetadata.getMetadata().hasChunkId()) { + return msgMetadata.getMetadata().getChunkId(); + } else { + return -1; + } + } + + @Override + public int getNumChunksFromMsg() { + if (msgMetadata.getMetadata().hasNumChunksFromMsg()) { + return msgMetadata.getMetadata().getNumChunksFromMsg(); + } else { + return -1; + } + } + + @Override + public int getTotalChunkMsgSize() { + if (msgMetadata.getMetadata().hasTotalChunkMsgSize()) { + return msgMetadata.getMetadata().getTotalChunkMsgSize(); + } else { + return -1; + } + } + public int getBatchSize() { return msgMetadata.getMetadata().getNumMessagesInBatch(); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index b1230d3d9c989..558b87bcd661d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -29,6 +29,9 @@ import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.util.Recycler; +import io.netty.util.ReferenceCountUtil; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.block.Block; @@ -58,6 +61,8 @@ import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; import org.apache.pulsar.common.api.raw.MessageParser; import org.apache.pulsar.common.api.raw.RawMessage; +import org.apache.pulsar.common.api.raw.RawMessageIdImpl; +import org.apache.pulsar.common.api.raw.RawMessageImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; @@ -66,6 +71,7 @@ import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.sql.presto.util.CacheSizeAllocator; import org.apache.pulsar.sql.presto.util.NoStrictCacheSizeAllocator; import org.apache.pulsar.sql.presto.util.NullCacheSizeAllocator; @@ -112,6 +118,8 @@ public class PulsarRecordCursor implements RecordCursor { PulsarDispatchingRowDecoderFactory decoderFactory; + protected ConcurrentOpenHashMap chunkedMessagesMap = new ConcurrentOpenHashMap<>(); + private static final Logger log = Logger.get(PulsarRecordCursor.class); public PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, @@ -265,7 +273,8 @@ public void accept(Entry entry) { metricsTracker.register_BYTES_READ(bytes); // check if we have processed all entries in this split - if (((PositionImpl) entry.getPosition()).compareTo(pulsarSplit.getEndPosition()) >= 0) { + // and no incomplete chunked messages exist + if (entryExceedSplitEndPosition(entry) && chunkedMessagesMap.isEmpty()) { return; } @@ -279,15 +288,25 @@ public void accept(Entry entry) { // start time for message queue read metricsTracker.start_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); - while (true) { - if (!haveAvailableCacheSize( - messageQueueCacheSizeAllocator, messageQueue) - || !messageQueue.offer(message)) { - Thread.sleep(1); - } else { - messageQueueCacheSizeAllocator.allocate( - message.getData().readableBytes()); - break; + if (message.getNumChunksFromMsg() > 1) { + message = processChunkedMessages(message); + } else if (entryExceedSplitEndPosition(entry)) { + // skip no chunk or no multi chunk message + // that exceed split end position + message.release(); + message = null; + } + if (message != null) { + while (true) { + if (!haveAvailableCacheSize( + messageQueueCacheSizeAllocator, messageQueue) + || !messageQueue.offer(message)) { + Thread.sleep(1); + } else { + messageQueueCacheSizeAllocator.allocate( + message.getData().readableBytes()); + break; + } } } @@ -328,6 +347,10 @@ public void accept(Entry entry) { } } + private boolean entryExceedSplitEndPosition(Entry entry) { + return ((PositionImpl) entry.getPosition()).compareTo(pulsarSplit.getEndPosition()) >= 0; + } + @VisibleForTesting class ReadEntries implements AsyncCallbacks.ReadEntriesCallback { @@ -341,8 +364,9 @@ class ReadEntries implements AsyncCallbacks.ReadEntriesCallback { public void run() { if (outstandingReadsRequests.get() > 0) { - if (!cursor.hasMoreEntries() || ((PositionImpl) cursor.getReadPosition()) - .compareTo(pulsarSplit.getEndPosition()) >= 0) { + if (!cursor.hasMoreEntries() || + (((PositionImpl) cursor.getReadPosition()).compareTo(pulsarSplit.getEndPosition()) >= 0 + && chunkedMessagesMap.isEmpty())) { isDone = true; } else { @@ -408,7 +432,7 @@ public Entry get() { public boolean hasFinished() { return messageQueue.isEmpty() && isDone && outstandingReadsRequests.get() >= 1 - && splitSize <= entriesProcessed; + && splitSize <= entriesProcessed && chunkedMessagesMap.isEmpty(); } @Override @@ -732,4 +756,95 @@ private void initEntryCacheSizeAllocator(PulsarConnectorConfig connectorConfig) } } + private RawMessage processChunkedMessages(RawMessage message) { + final String uuid = message.getUUID(); + final int chunkId = message.getChunkId(); + final int totalChunkMsgSize = message.getTotalChunkMsgSize(); + final int numChunks = message.getNumChunksFromMsg(); + + RawMessageIdImpl rawMessageId = (RawMessageIdImpl) message.getMessageId(); + if (rawMessageId.getLedgerId() > pulsarSplit.getEndPositionLedgerId() + && !chunkedMessagesMap.containsKey(uuid)) { + // If the message is out of the split range, we only care about the incomplete chunked messages. + message.release(); + return null; + } + if (chunkId == 0) { + ByteBuf chunkedMsgBuffer = Unpooled.directBuffer(totalChunkMsgSize, totalChunkMsgSize); + chunkedMessagesMap.computeIfAbsent(uuid, (key) -> ChunkedMessageCtx.get(numChunks, chunkedMsgBuffer)); + } + + ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(uuid); + if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null + || chunkId != (chunkedMsgCtx.lastChunkedMessageId + 1) || chunkId >= numChunks) { + // Means we lost the first chunk, it will happen when the beginning chunk didn't belong to this split. + log.info("Received unexpected chunk. messageId: %s, last-chunk-id: %s chunkId: %s, totalChunks: %s", + message.getMessageId(), + (chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), chunkId, + numChunks); + if (chunkedMsgCtx != null) { + if (chunkedMsgCtx.chunkedMsgBuffer != null) { + ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer); + } + chunkedMsgCtx.recycle(); + } + chunkedMessagesMap.remove(uuid); + message.release(); + return null; + } + + // append the chunked payload and update lastChunkedMessage-id + chunkedMsgCtx.chunkedMsgBuffer.writeBytes(message.getData()); + chunkedMsgCtx.lastChunkedMessageId = chunkId; + + // if final chunk is not received yet then release payload and return + if (chunkId != (numChunks - 1)) { + message.release(); + return null; + } + + if (log.isDebugEnabled()) { + log.debug("Chunked message completed. chunkId: %s, totalChunks: %s, msgId: %s, sequenceId: %s", + chunkId, numChunks, rawMessageId, message.getSequenceId()); + } + chunkedMessagesMap.remove(uuid); + ByteBuf unCompressedPayload = chunkedMsgCtx.chunkedMsgBuffer; + chunkedMsgCtx.recycle(); + // The chunked message complete, we use the entire payload to instead of the last chunk payload. + return ((RawMessageImpl) message).updatePayloadForChunkedMessage(unCompressedPayload); + } + + static class ChunkedMessageCtx { + + protected int totalChunks = -1; + protected ByteBuf chunkedMsgBuffer; + protected int lastChunkedMessageId = -1; + + static ChunkedMessageCtx get(int numChunksFromMsg, ByteBuf chunkedMsgBuffer) { + ChunkedMessageCtx ctx = RECYCLER.get(); + ctx.totalChunks = numChunksFromMsg; + ctx.chunkedMsgBuffer = chunkedMsgBuffer; + return ctx; + } + + private final Recycler.Handle recyclerHandle; + + private ChunkedMessageCtx(Recycler.Handle recyclerHandle) { + this.recyclerHandle = recyclerHandle; + } + + private static final Recycler RECYCLER = new Recycler() { + protected ChunkedMessageCtx newObject(Recycler.Handle handle) { + return new ChunkedMessageCtx(handle); + } + }; + + public void recycle() { + this.totalChunks = -1; + this.chunkedMsgBuffer = null; + this.lastChunkedMessageId = -1; + recyclerHandle.recycle(this); + } + } + } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java new file mode 100644 index 0000000000000..0a02dc308251c --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto; + +import com.google.common.collect.Sets; +import io.prestosql.spi.connector.ConnectorContext; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.testing.TestingConnectorContext; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.stats.NullStatsProvider; +import org.apache.commons.lang3.RandomUtils; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Test read chunked messages. + */ +@Test +@Slf4j +public class TestReadChunkedMessages extends MockedPulsarServiceBaseTest { + + private final static int MAX_MESSAGE_SIZE = 1024 * 1024; + + @EqualsAndHashCode + @Data + static class Movie { + private String name; + private Long publishTime; + private byte[] binaryData; + } + + @EqualsAndHashCode + @Data + static class MovieMessage { + private Movie movie; + private String messageId; + } + + @BeforeClass + @Override + protected void setup() throws Exception { + conf.setMaxMessageSize(MAX_MESSAGE_SIZE); + conf.setManagedLedgerMaxEntriesPerLedger(5); + conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0); + internalSetup(); + + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + + // so that clients can test short names + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + internalCleanup(); + } + + @Test + public void queryTest() throws Exception { + String topic = "chunk-topic"; + TopicName topicName = TopicName.get(topic); + int messageCnt = 20; + Set messageSet = prepareChunkedData(topic, messageCnt); + SchemaInfo schemaInfo = Schema.AVRO(Movie.class).getSchemaInfo(); + + PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); + connectorConfig.setWebServiceUrl(pulsar.getWebServiceAddress()); + PulsarSplitManager pulsarSplitManager = new PulsarSplitManager(new PulsarConnectorId("1"), connectorConfig); + Collection splits = pulsarSplitManager.getSplitsForTopic( + topicName.getPersistenceNamingEncoding(), + pulsar.getManagedLedgerFactory(), + new ManagedLedgerConfig(), + 3, + new PulsarTableHandle("1", topicName.getNamespace(), topic, topic), + schemaInfo, + topic, + TupleDomain.all(), + null); + + List columnHandleList = TestPulsarConnector.getColumnColumnHandles( + topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, true); + ConnectorContext prestoConnectorContext = new TestingConnectorContext(); + + for (PulsarSplit split : splits) { + queryAndCheck(columnHandleList, split, connectorConfig, prestoConnectorContext, messageSet); + } + Assert.assertTrue(messageSet.isEmpty()); + } + + private Set prepareChunkedData(String topic, int messageCnt) throws PulsarClientException, InterruptedException { + pulsarClient.newConsumer(Schema.AVRO(Movie.class)) + .topic(topic) + .subscriptionName("sub") + .subscribe() + .close(); + Producer producer = pulsarClient.newProducer(Schema.AVRO(Movie.class)) + .topic(topic) + .enableBatching(false) + .enableChunking(true) + .create(); + Set messageSet = new LinkedHashSet<>(); + CountDownLatch countDownLatch = new CountDownLatch(messageCnt); + for (int i = 0; i < messageCnt; i++) { + final double dataTimes = (i % 5) * 0.5; + byte[] movieBinaryData = RandomUtils.nextBytes((int) (MAX_MESSAGE_SIZE * dataTimes)); + final int length = movieBinaryData.length; + final int index = i; + + Movie movie = new Movie(); + movie.setName("movie-" + i); + movie.setPublishTime(System.currentTimeMillis()); + movie.setBinaryData(movieBinaryData); + producer.newMessage().value(movie).sendAsync() + .whenComplete((msgId, throwable) -> { + if (throwable != null) { + log.error("Failed to produce message.", throwable); + countDownLatch.countDown(); + return; + } + MovieMessage movieMessage = new MovieMessage(); + movieMessage.setMovie(movie); + MessageIdImpl messageId = (MessageIdImpl) msgId; + movieMessage.setMessageId("(" + messageId.getLedgerId() + "," + messageId.getEntryId() + ",0)"); + messageSet.add(movieMessage); + countDownLatch.countDown(); + }); + } + countDownLatch.await(); + Assert.assertEquals(messageCnt, messageSet.size()); + producer.close(); + return messageSet; + } + + private void queryAndCheck(List columnHandleList, + PulsarSplit split, + PulsarConnectorConfig connectorConfig, + ConnectorContext prestoConnectorContext, + Set messageSet) { + PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor( + columnHandleList, split, connectorConfig, pulsar.getManagedLedgerFactory(), + new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), + new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager())); + + AtomicInteger receiveMsgCnt = new AtomicInteger(messageSet.size()); + while (pulsarRecordCursor.advanceNextPosition()) { + Movie movie = new Movie(); + MovieMessage movieMessage = new MovieMessage(); + movieMessage.setMovie(movie); + for (int i = 0; i < columnHandleList.size(); i++) { + switch (columnHandleList.get(i).getName()) { + case "binaryData": + movie.setBinaryData(pulsarRecordCursor.getSlice(i).getBytes()); + break; + case "name": + movie.setName(new String(pulsarRecordCursor.getSlice(i).getBytes())); + break; + case "publishTime": + movie.setPublishTime(pulsarRecordCursor.getLong(i)); + break; + case "__message_id__": + movieMessage.setMessageId(new String(pulsarRecordCursor.getSlice(i).getBytes())); + default: + // do nothing + break; + } + } + + Assert.assertTrue(messageSet.contains(movieMessage)); + messageSet.remove(movieMessage); + receiveMsgCnt.decrementAndGet(); + } + } + +} From 49caf878e1487dda6da509380c651366c0b8f0c7 Mon Sep 17 00:00:00 2001 From: Zhanpeng Wu Date: Mon, 13 Dec 2021 19:44:55 +0800 Subject: [PATCH 198/823] fix shedding heartbeat ns (#13208) Related to #12252 I found that the problem mentioned in #12252 has not been solved, because the `HEARTBEAT_NAMESPACE_PATTERN` pattern needs a namespace as input, but what actually provides is the full name of the bundle. 1. fix the parttern matching problem 2. add a test case for it This change is already covered by existing tests. (cherry picked from commit 78e3d8f7d872746db962be36ad3de49dac1ef015) --- .../apache/pulsar/broker/loadbalance/LoadData.java | 10 ++++++++++ .../broker/loadbalance/impl/OverloadShedder.java | 8 ++------ .../broker/loadbalance/impl/ThresholdShedder.java | 6 +----- .../pulsar/broker/namespace/NamespaceService.java | 6 ++++++ .../apache/pulsar/common/naming/NamespaceBundle.java | 12 ++++++++++++ .../broker/namespace/NamespaceServiceTest.java | 8 ++++++++ 6 files changed, 39 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java index a469c5c24ddb9..4243420391993 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java @@ -20,8 +20,11 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import org.apache.pulsar.broker.BrokerData; import org.apache.pulsar.broker.BundleData; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.NamespaceBundle; /** * This class represents all data that could be relevant when making a load management decision. @@ -59,6 +62,13 @@ public Map getBundleData() { return bundleData; } + public Map getBundleDataForLoadShedding() { + return bundleData.entrySet().stream() + .filter(e -> !NamespaceService.isSystemServiceNamespace( + NamespaceBundle.getBundleNamespace(e.getKey()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + public Map getRecentlyUnloadedBundles() { return recentlyUnloadedBundles; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java index 3f33fa353c2ab..985ed6fd5f81e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.impl; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_PATTERN; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_PATTERN_V2; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import java.util.Map; @@ -102,10 +100,8 @@ public Multimap findBundlesForUnloading(final LoadData loadData, // Sort bundles by throughput, then pick the biggest N which combined // make up for at least the minimum throughput to offload - loadData.getBundleData().entrySet().stream() - .filter(e -> !HEARTBEAT_NAMESPACE_PATTERN.matcher(e.getKey()).matches() - && !HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(e.getKey()).matches() - && localData.getBundles().contains(e.getKey())) + loadData.getBundleDataForLoadShedding().entrySet().stream() + .filter(e -> localData.getBundles().contains(e.getKey())) .map((e) -> { // Map to throughput value // Consider short-term byte rate to address system resource burden diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index 3e103261812c8..afca7084617a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.impl; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_PATTERN; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_PATTERN_V2; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import java.util.HashMap; @@ -105,9 +103,7 @@ public Multimap findBundlesForUnloading(final LoadData loadData, MutableBoolean atLeastOneBundleSelected = new MutableBoolean(false); if (localData.getBundles().size() > 1) { - loadData.getBundleData().entrySet().stream() - .filter(e -> !HEARTBEAT_NAMESPACE_PATTERN.matcher(e.getKey()).matches() - && !HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(e.getKey()).matches()) + loadData.getBundleDataForLoadShedding().entrySet().stream() .map((e) -> { String bundle = e.getKey(); BundleData bundleData = e.getValue(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index f6cba9c7129a5..8f6bfb8d651b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1349,6 +1349,12 @@ public static String getSLAMonitorBrokerName(ServiceUnitId ns) { } } + public static boolean isSystemServiceNamespace(String namespace) { + return HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches() + || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches() + || SLA_NAMESPACE_PATTERN.matcher(namespace).matches(); + } + public boolean registerSLANamespace() throws PulsarServerException { boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(host, config), false); if (isNameSpaceRegistered) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java index 1531095c32212..98dcb93e7d3db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java @@ -152,6 +152,18 @@ public static String getBundleRange(String namespaceBundle) { return namespaceBundle.substring(namespaceBundle.lastIndexOf('/') + 1); } + public static String getBundleNamespace(String namespaceBundle) { + int index = namespaceBundle.lastIndexOf('/'); + if (index != -1) { + try { + return NamespaceName.get(namespaceBundle.substring(0, index)).toString(); + } catch (Exception e) { + // return itself if meets invalid format + } + } + return namespaceBundle; + } + public NamespaceBundleFactory getNamespaceBundleFactory() { return factory; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index 8d35cd316b876..d45dcc2d660a8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -539,6 +539,14 @@ public void testSplitLargestBundle() throws Exception { } } + @Test + public void testHeartbeatNamespaceMatch() throws Exception { + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), conf); + NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getNamespaceBundleFactory().getFullBundle(namespaceName); + assertTrue(NamespaceService.isSystemServiceNamespace( + NamespaceBundle.getBundleNamespace(namespaceBundle.toString()))); + } + @SuppressWarnings("unchecked") private Pair> splitBundles(NamespaceBundleFactory utilityFactory, NamespaceName nsname, NamespaceBundles bundles, NamespaceBundle targetBundle) throws Exception { From 1510cf627d7e90c656d908296ad2f2cd7a66fa39 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 13 Dec 2021 13:09:18 +0800 Subject: [PATCH 199/823] [Transaction] Fix performance (#13253) ### Motivation There is one omission and several irregular log formats. ### Modification 1. messagesSent.increment() 2. log format (cherry picked from commit c531c1ca0d74e0c771786ffb9930611c82a0aefe) --- .../pulsar/testclient/PerformanceConsumer.java | 4 ++-- .../pulsar/testclient/PerformanceProducer.java | 3 ++- .../pulsar/testclient/PerformanceTransaction.java | 14 +++++++------- .../testclient/PerformanceTransactionTest.java | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 134ffdd414c59..9c15a0ec0e845 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -573,7 +573,7 @@ public static void main(String[] args) throws Exception { dec.format(rateAck)); } log.info( - "Throughput received: {} msg --- {} msg/s -- {} Mbit/s " + "Throughput received: {} msg --- {} msg/s --- {} Mbit/s " + "--- Latency: mean: {} ms - med: {} " + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", intFormat.format(total), @@ -622,7 +622,7 @@ private static void printAggregatedThroughput(long start, Arguments arguments) { } log.info( "Aggregated throughput stats --- {} records received --- {} msg/s --- {} Mbit/s" - + "--- AckRate: {} msg/s --- ack failed {} msg", + + " --- AckRate: {} msg/s --- ack failed {} msg", totalMessagesReceived.sum(), dec.format(rate), dec.format(throughput), diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index cfd5568cecc2c..20eb8f34233c8 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -509,7 +509,7 @@ public static void main(String[] args) throws Exception { totalTxnOpSuccess = totalEndTxnOpSuccessNum.sum(); totalTxnOpFail = totalEndTxnOpFailNum.sum(); rateOpenTxn = numTxnOpSuccess.sumThenReset() / elapsed; - log.info("--- Transaction : {} transaction end successfully ---{} transaction end failed " + log.info("--- Transaction : {} transaction end successfully --- {} transaction end failed " + "--- {} Txn/s", totalTxnOpSuccess, totalTxnOpFail, totalFormat.format(rateOpenTxn)); } @@ -728,6 +728,7 @@ private static void runProducer(int producerId, PulsarClient pulsarClient = client; messageBuilder.sendAsync().thenRun(() -> { bytesSent.add(payloadData.length); + messagesSent.increment(); totalMessagesSent.increment(); totalBytesSent.add(payloadData.length); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java index 49d441e1f4cdd..5127f85496685 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java @@ -533,8 +533,8 @@ public static void main(String[] args) ? "Throughput transaction: {} transaction executes --- {} transaction/s" : "Throughput task: {} task executes --- {} task/s"; log.info( - txnOrTaskLog + " ---send Latency: mean: {} ms - med: {} " - + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}" + "---ack Latency: " + txnOrTaskLog + " --- send Latency: mean: {} ms - med: {} " + + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}" + " --- ack Latency: " + "mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", intFormat.format(total), dec.format(rate), @@ -582,8 +582,8 @@ private static void printTxnAggregatedThroughput(long start) { "Aggregated throughput stats --- {} transaction executed --- {} transaction/s " + " --- {} transaction open successfully --- {} transaction open failed" + " --- {} transaction end successfully --- {} transaction end failed" - + "--- {} message ack failed --- {} message send failed" - + "--- {} message ack success --- {} message send success ", + + " --- {} message ack failed --- {} message send failed" + + " --- {} message ack success --- {} message send success ", total, dec.format(rate), numTransactionOpenSuccess, @@ -606,9 +606,9 @@ private static void printAggregatedThroughput(long start) { long numMessageSendFailed = numMessagesSendFailed.sum(); long numMessageSendSuccess = numMessagesSendSuccess.sum(); log.info( - "Aggregated throughput stats --- {} task executed --- {} task/s " - + "--- {} message ack failed --- {} message send failed" - + "--- {} message ack success --- {} message send success ", + "Aggregated throughput stats --- {} task executed --- {} task/s" + + " --- {} message ack failed --- {} message send failed" + + " --- {} message ack success --- {} message send success", total, totalFormat.format(rate), numMessageAckFailed, diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java index e04ea049080dc..c5e62f74cc6e3 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java @@ -212,7 +212,7 @@ public void testConsumeTxnMessage() throws InterruptedException, PulsarClientExc .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); for (int i = 0; i < 505; i++) { - producer.newMessage().send(); + producer.newMessage().value("messages for test transaction consumer".getBytes()).send(); } Thread thread = new Thread(() -> { try { From bff7916441e9ed89d20f25664088290cf856cf5a Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 14 Dec 2021 21:15:57 +0800 Subject: [PATCH 200/823] [Transaction]No TransactionCoordinatorNotFound, but automatic reconnect (#13135) ### Motivation and Modification We should not throw the following exceptions to the user to deal with. 1. `TransactionCoordinatorNotFound` or `ManagerLedgerFenceException` --- we should retry the operation and reconnect to TC 2. `TransactionMetaStoreHandler` was connecting ---- add the operation into `pendingRequests`, and executed the requests in `pendingRequests` when the connected completely. 3. The complexity of concurrent operations is too high. For operations in a TransactionMetaStoreHandler, consider using single-threaded operations --- use `internalPinnedExecutor` (cherry picked from commit 56323e4a5b70c3008706515acd871ba0571ec1eb) --- .../TransactionMetadataStoreService.java | 8 +- .../pulsar/broker/service/ServerCnx.java | 68 +-- .../impl}/TransactionClientConnectTest.java | 218 +++---- .../impl/TransactionMetaStoreHandler.java | 577 ++++++++++++------ 4 files changed, 504 insertions(+), 367 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/{broker/transaction => client/impl}/TransactionClientConnectTest.java (50%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 240c6c9ad7c75..3f3e8d83d937d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -356,7 +356,7 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); } - completableFuture.completeExceptionally(e); + completableFuture.completeExceptionally(e.getCause()); return null; })).exceptionally(e -> { if (!isRetryableException(e.getCause())) { @@ -371,7 +371,7 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); } - completableFuture.completeExceptionally(e); + completableFuture.completeExceptionally(e.getCause()); return null; }); } else { @@ -391,7 +391,7 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea LOG.error("EndTxnInTransactionBuffer fail! TxnId : {}, " + "TxnAction : {}", txnID, txnAction, e); } - completableFuture.completeExceptionally(e); + completableFuture.completeExceptionally(e.getCause()); return null; }); } else { @@ -409,7 +409,7 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, isTimeout), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); } - completableFuture.completeExceptionally(e); + completableFuture.completeExceptionally(e.getCause()); return null; }); return completableFuture; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 7762c225910a9..30ce9c11073af 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2003,7 +2003,26 @@ private boolean checkTransactionEnableAndSenError(long requestId) { return true; } } + private Throwable handleTxnException(Throwable ex, String op, long requestId) { + if (ex instanceof CoordinatorException.CoordinatorNotFoundException || ex != null + && ex.getCause() instanceof CoordinatorException.CoordinatorNotFoundException) { + if (log.isDebugEnabled()) { + log.debug("The Coordinator was not found for the request {}", op); + } + return ex; + } + if (ex instanceof ManagedLedgerException.ManagedLedgerFencedException || ex != null + && ex.getCause() instanceof ManagedLedgerException.ManagedLedgerFencedException) { + if (log.isDebugEnabled()) { + log.debug("Throw a CoordinatorNotFoundException to client " + + "with the message got from a ManagedLedgerFencedException for the request {}", op); + } + return new CoordinatorException.CoordinatorNotFoundException(ex.getMessage()); + } + log.error("Send response error for {} request {}.", op, requestId, ex); + return ex; + } @Override protected void handleNewTxn(CommandNewTxn command) { final long requestId = command.getRequestId(); @@ -2028,9 +2047,7 @@ protected void handleNewTxn(CommandNewTxn command) { ctx.writeAndFlush(Commands.newTxnResponse(requestId, txnID.getLeastSigBits(), txnID.getMostSigBits())); } else { - if (log.isDebugEnabled()) { - log.debug("Send response error for new txn request {}", requestId, ex); - } + ex = handleTxnException(ex, BaseCommand.Type.NEW_TXN.name(), requestId); ctx.writeAndFlush(Commands.newTxnResponse(requestId, tcId.getId(), BrokerServiceException.getClientErrorCode(ex), ex.getMessage())); @@ -2066,19 +2083,11 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { ctx.writeAndFlush(Commands.newAddPartitionToTxnResponse(requestId, txnID.getLeastSigBits(), txnID.getMostSigBits())); } else { - if (log.isDebugEnabled()) { - log.debug("Send response error for add published partition to txn request {}", requestId, - ex); - } + ex = handleTxnException(ex, BaseCommand.Type.ADD_PARTITION_TO_TXN.name(), requestId); - if (ex instanceof CoordinatorException.CoordinatorNotFoundException) { - ctx.writeAndFlush(Commands.newAddPartitionToTxnResponse(requestId, txnID.getMostSigBits(), - BrokerServiceException.getClientErrorCode(ex), ex.getMessage())); - } else { - ctx.writeAndFlush(Commands.newAddPartitionToTxnResponse(requestId, txnID.getMostSigBits(), - BrokerServiceException.getClientErrorCode(ex.getCause()), - ex.getCause().getMessage())); - } + ctx.writeAndFlush(Commands.newAddPartitionToTxnResponse(requestId, txnID.getMostSigBits(), + BrokerServiceException.getClientErrorCode(ex), + ex.getMessage())); transactionMetadataStoreService.handleOpFail(ex, tcId); } })); @@ -2105,16 +2114,10 @@ protected void handleEndTxn(CommandEndTxn command) { ctx.writeAndFlush(Commands.newEndTxnResponse(requestId, txnID.getLeastSigBits(), txnID.getMostSigBits())); } else { - log.error("Send response error for end txn request.", ex); + ex = handleTxnException(ex, BaseCommand.Type.END_TXN.name(), requestId); + ctx.writeAndFlush(Commands.newEndTxnResponse(requestId, txnID.getMostSigBits(), + BrokerServiceException.getClientErrorCode(ex), ex.getMessage())); - if (ex instanceof CoordinatorException.CoordinatorNotFoundException) { - ctx.writeAndFlush(Commands.newEndTxnResponse(requestId, txnID.getMostSigBits(), - BrokerServiceException.getClientErrorCode(ex), ex.getMessage())); - } else { - ctx.writeAndFlush(Commands.newEndTxnResponse(requestId, txnID.getMostSigBits(), - BrokerServiceException.getClientErrorCode(ex.getCause()), - ex.getCause().getMessage())); - } transactionMetadataStoreService.handleOpFail(ex, tcId); } }); @@ -2325,20 +2328,11 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) { txnID.getLeastSigBits(), txnID.getMostSigBits())); log.info("handle add partition to txn finish."); } else { - if (log.isDebugEnabled()) { - log.debug("Send response error for add published partition to txn request {}", - requestId, ex); - } + ex = handleTxnException(ex, BaseCommand.Type.ADD_SUBSCRIPTION_TO_TXN.name(), requestId); - if (ex instanceof CoordinatorException.CoordinatorNotFoundException) { - ctx.writeAndFlush(Commands.newAddSubscriptionToTxnResponse(requestId, - txnID.getMostSigBits(), BrokerServiceException.getClientErrorCode(ex), - ex.getMessage())); - } else { - ctx.writeAndFlush(Commands.newAddSubscriptionToTxnResponse(requestId, - txnID.getMostSigBits(), BrokerServiceException.getClientErrorCode(ex.getCause()), - ex.getCause().getMessage())); - } + ctx.writeAndFlush(Commands.newAddSubscriptionToTxnResponse(requestId, + txnID.getMostSigBits(), BrokerServiceException.getClientErrorCode(ex), + ex.getMessage())); transactionMetadataStoreService.handleOpFail(ex, tcId); } })); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionClientConnectTest.java similarity index 50% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionClientConnectTest.java index a51eae82f006e..7fb924f49b885 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionClientConnectTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionClientConnectTest.java @@ -16,19 +16,28 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.transaction; - +package org.apache.pulsar.client.impl; + +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.TransactionMetadataStoreService; +import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.client.impl.PulsarClientImpl; -import org.apache.pulsar.client.impl.TransactionMetaStoreHandler; import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -39,10 +48,11 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertFalse; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import static org.testng.FileAssert.fail; +@Slf4j public class TransactionClientConnectTest extends TransactionTestBase { private static final String RECONNECT_TOPIC = NAMESPACE1 + "/txn-client-reconnect-test"; @@ -60,142 +70,69 @@ protected void cleanup() { @Test public void testTransactionNewReconnect() throws Exception { - start(); - - // when throw CoordinatorNotFoundException client will reconnect tc - try { - pulsarClient.newTransaction() - .withTransactionTimeout(200, TimeUnit.MILLISECONDS).build().get(); - fail(); - } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); - } - reconnect(); - - fence(getPulsarServiceList().get(0).getTransactionMetadataStoreService()); - - // tc fence will remove this tc and reopen - try { - pulsarClient.newTransaction() - .withTransactionTimeout(200, TimeUnit.MILLISECONDS).build().get(); - fail(); - } catch (ExecutionException e) { - assertEquals(e.getCause().getMessage(), - "org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerFencedException: " + - "java.lang.Exception: Attempted to use a fenced managed ledger"); - } - - reconnect(); + Callable> callable = () -> pulsarClient.newTransaction() + .withTransactionTimeout(200, TimeUnit.MILLISECONDS).build(); + tryCommandReconnect(callable, callable); } @Test public void testTransactionAddSubscriptionToTxnAsyncReconnect() throws Exception { TransactionCoordinatorClientImpl transactionCoordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient(); - start(); + Callable> callable = () -> transactionCoordinatorClient + .addSubscriptionToTxnAsync(new TxnID(0, 0), "test", "test"); + tryCommandReconnect(callable, callable); + } + public void tryCommandReconnect(Callable> callable1, Callable> callable2) + throws Exception { + start(); try { - transactionCoordinatorClient.addSubscriptionToTxnAsync(new TxnID(0, 0), "test", "test").get(); - fail(); + callable1.call().get(); } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); + assertFalse(e.getCause() instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); + waitToReady(); + callable1.call().get(); } - - reconnect(); fence(getPulsarServiceList().get(0).getTransactionMetadataStoreService()); + CompletableFuture completableFuture = callable2.call(); try { - transactionCoordinatorClient.addSubscriptionToTxnAsync(new TxnID(0, 0), "test", "test").get(); - fail(); + completableFuture.get(3, TimeUnit.SECONDS); + } catch (TimeoutException ignore) { } catch (ExecutionException e) { - if (e.getCause() instanceof TransactionCoordinatorClientException.TransactionNotFoundException) { - assertEquals(e.getCause().getMessage(), "The transaction with this txdID `(0,0)`not found "); - } else { - assertEquals(e.getCause().getMessage(), "java.lang.Exception: Attempted to use a fenced managed ledger"); - } + Assert.assertFalse(e.getCause() + instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); } - reconnect(); + + unFence(getPulsarServiceList().get(0).getTransactionMetadataStoreService()); + completableFuture.get(); } @Test public void testTransactionAbortToTxnAsyncReconnect() throws Exception { TransactionCoordinatorClientImpl transactionCoordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient(); - start(); - - try { - transactionCoordinatorClient.abortAsync(new TxnID(0, 0)).get(); - fail(); - } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); - } - - reconnect(); - fence(getPulsarServiceList().get(0).getTransactionMetadataStoreService()); - try { - transactionCoordinatorClient.abortAsync(new TxnID(0, 0)).get(); - fail(); - } catch (ExecutionException e) { - if (e.getCause() instanceof TransactionCoordinatorClientException.TransactionNotFoundException) { - assertEquals(e.getCause().getMessage(), "The transaction with this txdID `(0,0)`not found "); - } else { - assertEquals(e.getCause().getMessage(), "java.lang.Exception: Attempted to use a fenced managed ledger"); - } - } - reconnect(); + Callable> callable1 = () -> transactionCoordinatorClient.abortAsync(new TxnID(0, + 0)); + Callable> callable2 = () -> transactionCoordinatorClient.abortAsync(new TxnID(0, + 1)); + tryCommandReconnect(callable1, callable2); } @Test public void testTransactionCommitToTxnAsyncReconnect() throws Exception { TransactionCoordinatorClientImpl transactionCoordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient(); - start(); - - try { - transactionCoordinatorClient.commitAsync(new TxnID(0, 0)).get(); - fail(); - } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); - } - - reconnect(); - fence(getPulsarServiceList().get(0).getTransactionMetadataStoreService()); - try { - transactionCoordinatorClient.commitAsync(new TxnID(0, 0)).get(); - fail(); - } catch (ExecutionException e) { - if (e.getCause() instanceof TransactionCoordinatorClientException.TransactionNotFoundException) { - assertEquals(e.getCause().getMessage(), "The transaction with this txdID `(0,0)`not found "); - } else { - assertEquals(e.getCause().getMessage(), "java.lang.Exception: Attempted to use a fenced managed ledger"); - } - } - reconnect(); + Callable> callable1 = () -> transactionCoordinatorClient.commitAsync(new TxnID(0, + 0)); + Callable> callable2 = () -> transactionCoordinatorClient.commitAsync(new TxnID(0, + 1)); + tryCommandReconnect(callable1, callable2); } @Test public void testTransactionAddPublishPartitionToTxnReconnect() throws Exception { TransactionCoordinatorClientImpl transactionCoordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient(); - start(); - - try { - transactionCoordinatorClient.addPublishPartitionToTxnAsync(new TxnID(0, 0), - Collections.singletonList("test")).get(); - fail(); - } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof TransactionCoordinatorClientException.CoordinatorNotFoundException); - } - - reconnect(); - fence(getPulsarServiceList().get(0).getTransactionMetadataStoreService()); - try { - transactionCoordinatorClient.addPublishPartitionToTxnAsync(new TxnID(0, 0), - Collections.singletonList("test")).get(); - fail(); - } catch (ExecutionException e) { - if (e.getCause() instanceof TransactionCoordinatorClientException.TransactionNotFoundException) { - assertEquals(e.getCause().getMessage(), "The transaction with this txdID `(0,0)`not found "); - } else { - assertEquals(e.getCause().getMessage(), "java.lang.Exception: Attempted to use a fenced managed ledger"); - } - } - reconnect(); + Callable> callable = () -> transactionCoordinatorClient.addPublishPartitionToTxnAsync(new TxnID(0, 0), + Collections.singletonList("test")); + tryCommandReconnect(callable, callable); } @Test @@ -209,7 +146,11 @@ public void testPulsarClientCloseThenCloseTcClient() throws Exception { for (TransactionMetaStoreHandler handler : handlers) { handler.newTransactionAsync(10, TimeUnit.SECONDS).get(); } - pulsarClient.close(); + for (TransactionMetaStoreHandler handler : handlers) { + Field stateField = HandlerState.class.getDeclaredField("state"); + stateField.setAccessible(true); + stateField.set(handler, HandlerState.State.Closed); + } for (TransactionMetaStoreHandler handler : handlers) { Method method = TransactionMetaStoreHandler.class.getMethod("getConnectHandleState"); method.setAccessible(true); @@ -225,21 +166,14 @@ public void testPulsarClientCloseThenCloseTcClient() throws Exception { public void start() throws Exception { // wait transaction coordinator init success - Awaitility.await().until(() -> { - try { - pulsarClient.newTransaction() - .withTransactionTimeout(200, TimeUnit.MILLISECONDS).build().get(); - } catch (Exception e) { - return false; - } - return true; - }); pulsarClient.newTransaction() - .withTransactionTimeout(200, TimeUnit.MILLISECONDS).build().get(); + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); TransactionMetadataStoreService transactionMetadataStoreService = getPulsarServiceList().get(0).getTransactionMetadataStoreService(); - // remove transaction metadata store + // remove transaction metadap0-ta store transactionMetadataStoreService.removeTransactionMetadataStore(TransactionCoordinatorID.get(0)).get(); } @@ -250,15 +184,35 @@ public void fence(TransactionMetadataStoreService transactionMetadataStoreServic field.set(((MLTransactionMetadataStore) transactionMetadataStoreService.getStores() .get(TransactionCoordinatorID.get(0))).getManagedLedger(), ManagedLedgerImpl.State.Fenced); } + public void unFence(TransactionMetadataStoreService transactionMetadataStoreService) throws Exception { + Field field = ManagedLedgerImpl.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(((MLTransactionMetadataStore) transactionMetadataStoreService.getStores() + .get(TransactionCoordinatorID.get(0))).getManagedLedger(), ManagedLedgerImpl.State.LedgerOpened); + } - public void reconnect() { - //reconnect + public void waitToReady() throws Exception{ + TransactionMetadataStoreService transactionMetadataStoreService = + getPulsarServiceList().get(0).getTransactionMetadataStoreService(); + Class transactionMetadataStoreServiceClass = + TransactionMetadataStoreService.class; + Field field1 = + transactionMetadataStoreServiceClass.getDeclaredField("stores"); + field1.setAccessible(true); + Map stores = + (Map) field1 + .get(transactionMetadataStoreService); Awaitility.await().until(() -> { - try { - pulsarClient.newTransaction() - .withTransactionTimeout(200, TimeUnit.MILLISECONDS).build().get(); - } catch (Exception e) { - return false; + for (TransactionMetadataStore transactionMetadataStore : stores.values()) { + Class transactionMetadataStoreStateClass = + TransactionMetadataStoreState.class; + Field field = transactionMetadataStoreStateClass.getDeclaredField("state"); + field.setAccessible(true); + TransactionMetadataStoreState.State state = + (TransactionMetadataStoreState.State) field.get(transactionMetadataStore); + if (!state.equals(TransactionMetadataStoreState.State.Ready)) { + return false; + } } return true; }); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index ba6ee50d99fd2..b2b756a1ee516 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -23,7 +23,9 @@ import io.netty.util.Recycler; import io.netty.util.ReferenceCountUtil; import io.netty.util.Timeout; +import io.netty.util.Timer; import io.netty.util.TimerTask; +import java.util.concurrent.ExecutorService; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; @@ -61,6 +63,9 @@ public class TransactionMetaStoreHandler extends HandlerState implements Connect new ConcurrentLongHashMap<>(16, 1); private final ConcurrentLinkedQueue timeoutQueue; + protected final Timer timer; + private final ExecutorService internalPinnedExecutor; + private static class RequestTime { final long creationTimeMs; final long requestId; @@ -96,6 +101,8 @@ public TransactionMetaStoreHandler(long transactionCoordinatorId, PulsarClientIm this); this.connectFuture = connectFuture; this.connectionHandler.grabCnx(); + this.timer = pulsarClient.timer(); + internalPinnedExecutor = pulsarClient.getInternalExecutorService(); } @Override @@ -109,64 +116,73 @@ public void connectionFailed(PulsarClientException exception) { @Override public void connectionOpened(ClientCnx cnx) { - LOG.info("Transaction meta handler with transaction coordinator id {} connection opened.", - transactionCoordinatorId); - - if (getState() == State.Closing || getState() == State.Closed) { - setState(State.Closed); - failPendingRequest(); - this.pendingRequests.clear(); - return; - } - - connectionHandler.setClientCnx(cnx); - cnx.registerTransactionMetaStoreHandler(transactionCoordinatorId, this); - - // if broker protocol version < 19, don't send TcClientConnectRequest to broker. - if (cnx.getRemoteEndpointProtocolVersion() > ProtocolVersion.v18.getValue()) { - long requestId = client.newRequestId(); - ByteBuf request = Commands.newTcClientConnectRequest(transactionCoordinatorId, requestId); + internalPinnedExecutor.execute(() -> { + LOG.info("Transaction meta handler with transaction coordinator id {} connection opened.", + transactionCoordinatorId); + + if (getState() == State.Closing || getState() == State.Closed) { + setState(State.Closed); + failPendingRequest(); + this.pendingRequests.clear(); + return; + } - cnx.sendRequestWithId(request, requestId).thenRun(() -> { - LOG.info("Transaction coordinator client connect success! tcId : {}", transactionCoordinatorId); + connectionHandler.setClientCnx(cnx); + cnx.registerTransactionMetaStoreHandler(transactionCoordinatorId, this); + + // if broker protocol version < 19, don't send TcClientConnectRequest to broker. + if (cnx.getRemoteEndpointProtocolVersion() > ProtocolVersion.v18.getValue()) { + long requestId = client.newRequestId(); + ByteBuf request = Commands.newTcClientConnectRequest(transactionCoordinatorId, requestId); + + cnx.sendRequestWithId(request, requestId).thenRun(() -> { + internalPinnedExecutor.execute(() -> { + LOG.info("Transaction coordinator client connect success! tcId : {}", transactionCoordinatorId); + if (!changeToReadyState()) { + setState(State.Closed); + cnx.channel().close(); + } + + if (!this.connectFuture.isDone()) { + this.connectFuture.complete(null); + } + this.connectionHandler.resetBackoff(); + pendingRequests.forEach((requestID, opBase) -> checkStateAndSendRequest(opBase)); + }); + }).exceptionally((e) -> { + internalPinnedExecutor.execute(() -> { + LOG.error("Transaction coordinator client connect fail! tcId : {}", + transactionCoordinatorId, e.getCause()); + if (getState() == State.Closing || getState() == State.Closed + || e.getCause() instanceof PulsarClientException.NotAllowedException) { + setState(State.Closed); + cnx.channel().close(); + } else { + connectionHandler.reconnectLater(e.getCause()); + } + }); + return null; + }); + } else { if (!changeToReadyState()) { - setState(State.Closed); - cnx.channel().close(); - } - - if (!this.connectFuture.isDone()) { - this.connectFuture.complete(null); - } - this.connectionHandler.resetBackoff(); - }).exceptionally((e) -> { - LOG.error("Transaction coordinator client connect fail! tcId : {}", - transactionCoordinatorId, e.getCause()); - if (getState() == State.Closing || getState() == State.Closed - || e.getCause() instanceof PulsarClientException.NotAllowedException) { - setState(State.Closed); cnx.channel().close(); - } else { - connectionHandler.reconnectLater(e.getCause()); } - return null; - }); - } else { - if (!changeToReadyState()) { - cnx.channel().close(); + this.connectFuture.complete(null); } - this.connectFuture.complete(null); - } + }); } private void failPendingRequest() { - pendingRequests.keys().forEach(k -> { - OpBase op = pendingRequests.remove(k); - if (op != null && !op.callback.isDone()) { - op.callback.completeExceptionally(new PulsarClientException.AlreadyClosedException( - "Could not get response from transaction meta store when " + - "the transaction meta store has already close.")); - onResponse(op); - } + internalPinnedExecutor.execute(() -> { + pendingRequests.keys().forEach(k -> { + OpBase op = pendingRequests.remove(k); + if (op != null && !op.callback.isDone()) { + op.callback.completeExceptionally(new PulsarClientException.AlreadyClosedException( + "Could not get response from transaction meta store when " + + "the transaction meta store has already close.")); + onResponse(op); + } + }); }); } @@ -175,42 +191,76 @@ public CompletableFuture newTransactionAsync(long timeout, TimeUnit unit) LOG.debug("New transaction with timeout in ms {}", unit.toMillis(timeout)); } CompletableFuture callback = new CompletableFuture<>(); - if (!canSendRequest(callback)) { return callback; } long requestId = client.newRequestId(); ByteBuf cmd = Commands.newTxn(transactionCoordinatorId, requestId, unit.toMillis(timeout)); - OpForTxnIdCallBack op = OpForTxnIdCallBack.create(cmd, callback); - pendingRequests.put(requestId, op); - timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - cmd.retain(); - cnx().ctx().writeAndFlush(cmd, cnx().ctx().voidPromise()); + OpForTxnIdCallBack op = OpForTxnIdCallBack.create(cmd, callback, client); + internalPinnedExecutor.execute(() -> { + pendingRequests.put(requestId, op); + timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); + checkStateAndSendRequest(op); + }); return callback; } void handleNewTxnResponse(CommandNewTxnResponse response) { - OpForTxnIdCallBack op = (OpForTxnIdCallBack) pendingRequests.remove(response.getRequestId()); - if (op == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Got new txn response for timeout {} - {}", response.getTxnidMostBits(), - response.getTxnidLeastBits()); + boolean hasError = response.hasError(); + ServerError error; + String message; + if (hasError) { + error = response.getError(); + message = response.getMessage(); + } else { + error = null; + message = null; + } + TxnID txnID = new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits()); + long requestId = response.getRequestId(); + internalPinnedExecutor.execute(() -> { + OpForTxnIdCallBack op = (OpForTxnIdCallBack) pendingRequests.remove(requestId); + if (op == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Got new txn response for timeout {} - {}", txnID.getMostSigBits(), + txnID.getLeastSigBits()); + } + return; } - return; - } - if (!response.hasError()) { - TxnID txnID = new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits()); - if (LOG.isDebugEnabled()) { - LOG.debug("Got new txn response {} for request {}", txnID, response.getRequestId()); + if (!hasError) { + if (LOG.isDebugEnabled()) { + LOG.debug("Got new txn response {} for request {}", txnID, requestId); + } + op.callback.complete(txnID); + } else { + if (checkIfNeedRetryByError(error, message, op)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Get a response for the {} request {} error " + + "TransactionCoordinatorNotFound and try it again", + BaseCommand.Type.NEW_TXN.name(), requestId); + } + pendingRequests.put(requestId, op); + timer.newTimeout(timeout -> { + internalPinnedExecutor.execute(() -> { + if (!pendingRequests.containsKey(requestId)) { + if (LOG.isDebugEnabled()) { + LOG.debug("The request {} already timeout", requestId); + } + return; + } + checkStateAndSendRequest(op); + }); + } + , op.backoff.next(), TimeUnit.MILLISECONDS); + return; + } + LOG.error("Got {} for request {} error {}", BaseCommand.Type.NEW_TXN.name(), + requestId, error); } - op.callback.complete(txnID); - } else { - LOG.error("Got new txn for request {} error {}", response.getRequestId(), response.getError()); - handleTransactionFailOp(response.getError(), response.getMessage(), op); - } - onResponse(op); + onResponse(op); + }); } public CompletableFuture addPublishPartitionToTxnAsync(TxnID txnID, List partitions) { @@ -218,42 +268,80 @@ public CompletableFuture addPublishPartitionToTxnAsync(TxnID txnID, List callback = new CompletableFuture<>(); - if (!canSendRequest(callback)) { return callback; } long requestId = client.newRequestId(); ByteBuf cmd = Commands.newAddPartitionToTxn( requestId, txnID.getLeastSigBits(), txnID.getMostSigBits(), partitions); - OpForVoidCallBack op = OpForVoidCallBack.create(cmd, callback); - pendingRequests.put(requestId, op); - timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - cmd.retain(); - cnx().ctx().writeAndFlush(cmd, cnx().ctx().voidPromise()); + OpForVoidCallBack op = OpForVoidCallBack + .create(cmd, callback, client); + internalPinnedExecutor.execute(() -> { + pendingRequests.put(requestId, op); + timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); + checkStateAndSendRequest(op); + }); + return callback; } void handleAddPublishPartitionToTxnResponse(CommandAddPartitionToTxnResponse response) { - OpForVoidCallBack op = (OpForVoidCallBack) pendingRequests.remove(response.getRequestId()); - if (op == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Got add publish partition to txn response for timeout {} - {}", response.getTxnidMostBits(), - response.getTxnidLeastBits()); + boolean hasError = response.hasError(); + ServerError error; + String message; + if (hasError) { + error = response.getError(); + message = response.getMessage(); + } else { + error = null; + message = null; + } + TxnID txnID = new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits()); + long requestId = response.getRequestId(); + internalPinnedExecutor.execute(() -> { + OpForVoidCallBack op = (OpForVoidCallBack) pendingRequests.remove(requestId); + if (op == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Got add publish partition to txn response for timeout {} - {}", txnID.getMostSigBits(), + txnID.getLeastSigBits()); + } + return; } - return; - } - if (!response.hasError()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Add publish partition for request {} success.", response.getRequestId()); + if (!hasError) { + if (LOG.isDebugEnabled()) { + LOG.debug("Add publish partition for request {} success.", requestId); + } + op.callback.complete(null); + } else { + if (checkIfNeedRetryByError(error, message, op)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Get a response for the {} request {} " + + " error TransactionCoordinatorNotFound and try it again", + BaseCommand.Type.ADD_PARTITION_TO_TXN.name(), requestId); + } + pendingRequests.put(requestId, op); + timer.newTimeout(timeout -> { + internalPinnedExecutor.execute(() -> { + if (!pendingRequests.containsKey(requestId)) { + if (LOG.isDebugEnabled()) { + LOG.debug("The request {} already timeout", requestId); + } + return; + } + checkStateAndSendRequest(op); + }); + } + , op.backoff.next(), TimeUnit.MILLISECONDS); + return; + } + LOG.error("{} for request {} error {} with txnID {}.", BaseCommand.Type.ADD_PARTITION_TO_TXN.name(), + requestId, error, txnID); + } - op.callback.complete(null); - } else { - LOG.error("Add publish partition for request {} error {}.", response.getRequestId(), response.getError()); - handleTransactionFailOp(response.getError(), response.getMessage(), op); - } - onResponse(op); + onResponse(op); + }); } public CompletableFuture addSubscriptionToTxn(TxnID txnID, List subscriptionList) { @@ -262,41 +350,76 @@ public CompletableFuture addSubscriptionToTxn(TxnID txnID, List callback = new CompletableFuture<>(); - if (!canSendRequest(callback)) { return callback; } long requestId = client.newRequestId(); ByteBuf cmd = Commands.newAddSubscriptionToTxn( requestId, txnID.getLeastSigBits(), txnID.getMostSigBits(), subscriptionList); - OpForVoidCallBack op = OpForVoidCallBack.create(cmd, callback); - pendingRequests.put(requestId, op); - timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - cmd.retain(); - cnx().ctx().writeAndFlush(cmd, cnx().ctx().voidPromise()); + OpForVoidCallBack op = OpForVoidCallBack.create(cmd, callback, client); + internalPinnedExecutor.execute(() -> { + pendingRequests.put(requestId, op); + timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); + checkStateAndSendRequest(op); + }); return callback; } public void handleAddSubscriptionToTxnResponse(CommandAddSubscriptionToTxnResponse response) { - OpForVoidCallBack op = (OpForVoidCallBack) pendingRequests.remove(response.getRequestId()); - if (op == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Add subscription to txn timeout for request {}.", response.getRequestId()); + boolean hasError = response.hasError(); + ServerError error; + String message; + if (hasError) { + error = response.getError(); + message = response.getMessage(); + } else { + error = null; + message = null; + } + long requestId = response.getRequestId(); + internalPinnedExecutor.execute(() -> { + OpForVoidCallBack op = (OpForVoidCallBack) pendingRequests.remove(requestId); + if (op == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Add subscription to txn timeout for request {}.", requestId); + } + return; } - return; - } - if (!response.hasError()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Add subscription to txn success for request {}.", response.getRequestId()); + if (!hasError) { + if (LOG.isDebugEnabled()) { + LOG.debug("Add subscription to txn success for request {}.", requestId); + } + op.callback.complete(null); + } else { + LOG.error("Add subscription to txn failed for request {} error {}.", + requestId, error); + if (checkIfNeedRetryByError(error, message, op)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Get a response for {} request {} error TransactionCoordinatorNotFound and try it again", + BaseCommand.Type.ADD_SUBSCRIPTION_TO_TXN.name(), requestId); + } + pendingRequests.put(requestId, op); + timer.newTimeout(timeout -> { + internalPinnedExecutor.execute(() -> { + if (!pendingRequests.containsKey(requestId)) { + if (LOG.isDebugEnabled()) { + LOG.debug("The request {} already timeout", requestId); + } + return; + } + checkStateAndSendRequest(op); + }); + } + , op.backoff.next(), TimeUnit.MILLISECONDS); + return; + } + LOG.error("{} failed for request {} error {}.", BaseCommand.Type.ADD_SUBSCRIPTION_TO_TXN.name(), + requestId, error); + } - op.callback.complete(null); - } else { - LOG.error("Add subscription to txn failed for request {} error {}.", - response.getRequestId(), response.getError()); - handleTransactionFailOp(response.getError(), response.getMessage(), op); - } - onResponse(op); + onResponse(op); + }); } public CompletableFuture endTxnAsync(TxnID txnID, TxnAction action) { @@ -304,68 +427,115 @@ public CompletableFuture endTxnAsync(TxnID txnID, TxnAction action) { LOG.debug("End txn {}, action {}", txnID, action); } CompletableFuture callback = new CompletableFuture<>(); - if (!canSendRequest(callback)) { return callback; } long requestId = client.newRequestId(); BaseCommand cmd = Commands.newEndTxn(requestId, txnID.getLeastSigBits(), txnID.getMostSigBits(), action); ByteBuf buf = Commands.serializeWithSize(cmd); - OpForVoidCallBack op = OpForVoidCallBack.create(buf, callback); - pendingRequests.put(requestId, op); - timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - buf.retain(); - cnx().ctx().writeAndFlush(buf, cnx().ctx().voidPromise()); + OpForVoidCallBack op = OpForVoidCallBack.create(buf, callback, client); + internalPinnedExecutor.execute(() -> { + pendingRequests.put(requestId, op); + timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); + checkStateAndSendRequest(op); + }); return callback; } void handleEndTxnResponse(CommandEndTxnResponse response) { - OpForVoidCallBack op = (OpForVoidCallBack) pendingRequests.remove(response.getRequestId()); - if (op == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Got end txn response for timeout {} - {}", response.getTxnidMostBits(), - response.getTxnidLeastBits()); + boolean hasError = response.hasError(); + ServerError error; + String message; + if (hasError) { + error = response.getError(); + message = response.getMessage(); + } else { + error = null; + message = null; + } + TxnID txnID = new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits()); + long requestId = response.getRequestId(); + internalPinnedExecutor.execute(() -> { + OpForVoidCallBack op = (OpForVoidCallBack) pendingRequests.remove(requestId); + if (op == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Got end txn response for timeout {} - {}", txnID.getMostSigBits(), + txnID.getLeastSigBits()); + } + return; } - return; - } - if (!response.hasError()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Got end txn response success for request {}", response.getRequestId()); - } - op.callback.complete(null); - } else { - LOG.error("Got end txn response for request {} error {}", response.getRequestId(), response.getError()); - handleTransactionFailOp(response.getError(), response.getMessage(), op); - } + if (!hasError) { + if (LOG.isDebugEnabled()) { + LOG.debug("Got end txn response success for request {}", requestId); + } + op.callback.complete(null); + } else { + if (checkIfNeedRetryByError(error, message, op)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Get a response for the {} request {} error " + + "TransactionCoordinatorNotFound and try it again", + BaseCommand.Type.END_TXN.name(), requestId); + } + pendingRequests.put(requestId, op); + timer.newTimeout(timeout -> { + internalPinnedExecutor.execute(() -> { + if (!pendingRequests.containsKey(requestId)) { + if (LOG.isDebugEnabled()) { + LOG.debug("The request {} already timeout", requestId); + } + return; + } + checkStateAndSendRequest(op); + }); + } + , op.backoff.next(), TimeUnit.MILLISECONDS); + return; + } + LOG.error("Got {} response for request {} error {}", BaseCommand.Type.END_TXN.name(), + requestId, error); - onResponse(op); + } + onResponse(op); + }); } - private void handleTransactionFailOp(ServerError error, String message, OpBase op) { - if (error == ServerError.TransactionCoordinatorNotFound && getState() != State.Connecting) { - connectionHandler.reconnectLater(new TransactionCoordinatorClientException - .CoordinatorNotFoundException(message)); + + private boolean checkIfNeedRetryByError(ServerError error, String message, OpBase op) { + if (error == ServerError.TransactionCoordinatorNotFound) { + if (getState() != State.Connecting) { + connectionHandler.reconnectLater(new TransactionCoordinatorClientException + .CoordinatorNotFoundException(message)); + } + return true; } if (op != null) { op.callback.completeExceptionally(getExceptionByServerError(error, message)); } + return false; } private static abstract class OpBase { protected ByteBuf cmd; protected CompletableFuture callback; + protected Backoff backoff; abstract void recycle(); } private static class OpForTxnIdCallBack extends OpBase { - static OpForTxnIdCallBack create(ByteBuf cmd, CompletableFuture callback) { + static OpForTxnIdCallBack create(ByteBuf cmd, CompletableFuture callback, PulsarClientImpl client) { OpForTxnIdCallBack op = RECYCLER.get(); op.callback = callback; op.cmd = cmd; + op.backoff = new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), + TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos() / 10, TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(); return op; } @@ -375,6 +545,9 @@ private OpForTxnIdCallBack(Recycler.Handle recyclerHandle) { @Override void recycle() { + this.backoff = null; + this.cmd = null; + this.callback = null; recyclerHandle.recycle(this); } @@ -389,18 +562,29 @@ protected OpForTxnIdCallBack newObject(Handle handle) { private static class OpForVoidCallBack extends OpBase { - static OpForVoidCallBack create(ByteBuf cmd, CompletableFuture callback) { + + static OpForVoidCallBack create(ByteBuf cmd, CompletableFuture callback, PulsarClientImpl client) { OpForVoidCallBack op = RECYCLER.get(); op.callback = callback; op.cmd = cmd; + op.backoff = new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), + TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos() / 10, TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(); return op; } + private OpForVoidCallBack(Recycler.Handle recyclerHandle) { this.recyclerHandle = recyclerHandle; } @Override void recycle() { + this.backoff = null; + this.cmd = null; + this.callback = null; recyclerHandle.recycle(this); } @@ -433,9 +617,6 @@ private void onResponse(OpBase op) { } private boolean canSendRequest(CompletableFuture callback) { - if (!isValidHandlerState(callback)) { - return false; - } try { if (blockIfReachMaxPendingOps) { semaphore.acquire(); @@ -453,81 +634,89 @@ private boolean canSendRequest(CompletableFuture callback) { return true; } - private boolean isValidHandlerState(CompletableFuture callback) { + private void checkStateAndSendRequest(OpBase op) { switch (getState()) { case Ready: - return true; + ClientCnx cnx = cnx(); + if (cnx != null) { + op.cmd.retain(); + cnx.ctx().writeAndFlush(op.cmd, cnx().ctx().voidPromise()); + } else { + LOG.error("The cnx was null when the TC handler was ready", new NullPointerException()); + } + break; case Connecting: - callback.completeExceptionally( - new TransactionCoordinatorClientException.MetaStoreHandlerNotReadyException( - "Transaction meta store handler for tcId " - + transactionCoordinatorId - + " is connecting now, please try later.")); - return false; + break; case Closing: case Closed: - callback.completeExceptionally( + op.callback.completeExceptionally( new TransactionCoordinatorClientException.MetaStoreHandlerNotReadyException( "Transaction meta store handler for tcId " + transactionCoordinatorId + " is closing or closed.")); - return false; + onResponse(op); + break; case Failed: case Uninitialized: - callback.completeExceptionally( + op.callback.completeExceptionally( new TransactionCoordinatorClientException.MetaStoreHandlerNotReadyException( "Transaction meta store handler for tcId " + transactionCoordinatorId + " not connected.")); - return false; + onResponse(op); + break; default: - callback.completeExceptionally( + op.callback.completeExceptionally( new TransactionCoordinatorClientException.MetaStoreHandlerNotReadyException( transactionCoordinatorId)); - return false; + onResponse(op); + break; } } @Override public void run(Timeout timeout) throws Exception { - if (timeout.isCancelled()) { - return; - } - long timeToWaitMs; - if (getState() == State.Closing || getState() == State.Closed) { - return; - } - RequestTime peeked = timeoutQueue.peek(); - while (peeked != null && peeked.creationTimeMs + client.getConfiguration().getOperationTimeoutMs() - - System.currentTimeMillis() <= 0) { - RequestTime lastPolled = timeoutQueue.poll(); - if (lastPolled != null) { - OpBase op = pendingRequests.remove(lastPolled.requestId); - if (op != null && !op.callback.isDone()) { - op.callback.completeExceptionally(new PulsarClientException.TimeoutException( - "Could not get response from transaction meta store within given timeout.")); - if (LOG.isDebugEnabled()) { - LOG.debug("Transaction coordinator request {} is timeout.", lastPolled.requestId); + internalPinnedExecutor.execute(() -> { + if (timeout.isCancelled()) { + return; + } + long timeToWaitMs; + if (getState() == State.Closing || getState() == State.Closed) { + return; + } + RequestTime peeked = timeoutQueue.peek(); + while (peeked != null && peeked.creationTimeMs + client.getConfiguration().getOperationTimeoutMs() + - System.currentTimeMillis() <= 0) { + RequestTime lastPolled = timeoutQueue.poll(); + if (lastPolled != null) { + OpBase op = pendingRequests.remove(lastPolled.requestId); + if (op != null && !op.callback.isDone()) { + op.callback.completeExceptionally(new PulsarClientException.TimeoutException( + "Could not get response from transaction meta store within given timeout.")); + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction coordinator request {} is timeout.", lastPolled.requestId); + } + onResponse(op); } - onResponse(op); + } else { + break; } - } else { - break; + peeked = timeoutQueue.peek(); } - peeked = timeoutQueue.peek(); - } - if (peeked == null) { - timeToWaitMs = client.getConfiguration().getOperationTimeoutMs(); - } else { - long diff = (peeked.creationTimeMs + client.getConfiguration().getOperationTimeoutMs()) - System.currentTimeMillis(); - if (diff <= 0) { + if (peeked == null) { timeToWaitMs = client.getConfiguration().getOperationTimeoutMs(); } else { - timeToWaitMs = diff; + long diff = (peeked.creationTimeMs + client.getConfiguration().getOperationTimeoutMs()) + - System.currentTimeMillis(); + if (diff <= 0) { + timeToWaitMs = client.getConfiguration().getOperationTimeoutMs(); + } else { + timeToWaitMs = diff; + } } - } - requestTimeout = client.timer().newTimeout(this, timeToWaitMs, TimeUnit.MILLISECONDS); + requestTimeout = client.timer().newTimeout(this, timeToWaitMs, TimeUnit.MILLISECONDS); + }); } private ClientCnx cnx() { From 3e076bbf8804ab1a92386c22d03b0d6e31aa65ba Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Wed, 15 Dec 2021 15:54:41 +0800 Subject: [PATCH 201/823] [Broker] Modify return result of NamespacesBase#internalGetPublishRate (#13237) ### Motivation It should return `null` instead of `RestException` in method `NamespacesBase#internalGetPublishRate`, because `null` means that the `publish-rate` is not configured. It is the same as `internalGetSubscriptionDispatchRate` as below: https://github.com/apache/pulsar/blob/6d9d24d50db5418ddbb845d2c7a2be2b9ac72893/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java#L1303-L1308 (cherry picked from commit 3e55b4f2c40ecd73bb38962cebc4a822bfe5f1ef) --- .../pulsar/broker/admin/impl/NamespacesBase.java | 8 +------- .../apache/pulsar/broker/admin/v1/Namespaces.java | 15 ++++++++++----- .../apache/pulsar/broker/admin/v2/Namespaces.java | 15 ++++++++++----- .../api/AuthorizationProducerConsumerTest.java | 2 ++ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 87c091ae923d9..5f1231fec1305 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1197,13 +1197,7 @@ protected PublishRate internalGetPublishRate() { validateNamespacePolicyOperation(namespaceName, PolicyName.RATE, PolicyOperation.READ); Policies policies = getNamespacePolicies(namespaceName); - PublishRate publishRate = policies.publishMaxMessageRate.get(pulsar().getConfiguration().getClusterName()); - if (publishRate != null) { - return publishRate; - } else { - throw new RestException(Status.NOT_FOUND, - "Publish-rate is not configured for cluster " + pulsar().getConfiguration().getClusterName()); - } + return policies.publishMaxMessageRate.get(pulsar().getConfiguration().getClusterName()); } @SuppressWarnings("deprecation") diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index 51728d63e1b41..afb6174c5508e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -675,7 +675,8 @@ public void setPublishRate(@PathParam("property") String property, @PathParam("c @GET @Path("/{property}/{cluster}/{namespace}/publishRate") @ApiOperation(hidden = true, - value = "Get publish-rate configured for the namespace, -1 represents not configured yet") + value = "Get publish-rate configured for the namespace, null means publish-rate not configured, " + + "-1 means msg-publish-rate or byte-publish-rate not configured in publish-rate yet") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public PublishRate getPublishRate(@PathParam("property") String property, @PathParam("cluster") String cluster, @@ -697,7 +698,8 @@ public void setDispatchRate(@PathParam("property") String property, @PathParam(" @GET @Path("/{property}/{cluster}/{namespace}/dispatchRate") @ApiOperation(hidden = true, - value = "Get dispatch-rate configured for the namespace, -1 represents not configured yet") + value = "Get dispatch-rate configured for the namespace, null means dispatch-rate not configured, " + + "-1 means msg-dispatch-rate or byte-dispatch-rate not configured in dispatch-rate yet") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public DispatchRate getDispatchRate(@PathParam("property") String property, @PathParam("cluster") String cluster, @@ -720,8 +722,9 @@ public void setSubscriptionDispatchRate(@PathParam("property") String property, @GET @Path("/{property}/{cluster}/{namespace}/subscriptionDispatchRate") - @ApiOperation(value = - "Get Subscription dispatch-rate configured for the namespace, -1 represents not configured yet") + @ApiOperation(value = "Get subscription dispatch-rate configured for the namespace, null means subscription " + + "dispatch-rate not configured, -1 means msg-dispatch-rate or byte-dispatch-rate not configured " + + "in dispatch-rate yet") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public DispatchRate getSubscriptionDispatchRate(@PathParam("property") String property, @@ -746,7 +749,9 @@ public void setReplicatorDispatchRate( @GET @Path("/{tenant}/{cluster}/{namespace}/replicatorDispatchRate") - @ApiOperation(value = "Get replicator dispatch-rate configured for the namespace, -1 represents not configured yet") + @ApiOperation(value = "Get replicator dispatch-rate configured for the namespace, null means replicator " + + "dispatch-rate not configured, -1 means msg-dispatch-rate or byte-dispatch-rate not configured " + + "in dispatch-rate yet") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public DispatchRate getReplicatorDispatchRate(@PathParam("tenant") String tenant, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index e1e892c88a235..37b735464c341 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -602,7 +602,8 @@ public void removePublishRate(@PathParam("property") String property, @PathParam @GET @Path("/{property}/{namespace}/publishRate") @ApiOperation(hidden = true, - value = "Get publish-rate configured for the namespace, -1 represents not configured yet") + value = "Get publish-rate configured for the namespace, null means publish-rate not configured, " + + "-1 means msg-publish-rate or byte-publish-rate not configured in publish-rate yet") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public PublishRate getPublishRate( @@ -634,7 +635,8 @@ public void deleteDispatchRate(@PathParam("tenant") String tenant, @PathParam("n @GET @Path("/{tenant}/{namespace}/dispatchRate") - @ApiOperation(value = "Get dispatch-rate configured for the namespace, -1 represents not configured yet") + @ApiOperation(value = "Get dispatch-rate configured for the namespace, null means dispatch-rate not configured, " + + "-1 means msg-dispatch-rate or byte-dispatch-rate not configured in dispatch-rate yet") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public DispatchRate getDispatchRate(@PathParam("tenant") String tenant, @@ -657,8 +659,9 @@ public void setSubscriptionDispatchRate(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/subscriptionDispatchRate") - @ApiOperation( - value = "Get Subscription dispatch-rate configured for the namespace, -1 represents not configured yet") + @ApiOperation(value = "Get subscription dispatch-rate configured for the namespace, null means subscription " + + "dispatch-rate not configured, -1 means msg-dispatch-rate or byte-dispatch-rate not configured " + + "in dispatch-rate yet") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public DispatchRate getSubscriptionDispatchRate(@PathParam("tenant") String tenant, @@ -732,7 +735,9 @@ public void setReplicatorDispatchRate(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/replicatorDispatchRate") - @ApiOperation(value = "Get replicator dispatch-rate configured for the namespace, -1 represents not configured yet") + @ApiOperation(value = "Get replicator dispatch-rate configured for the namespace, null means replicator " + + "dispatch-rate not configured, -1 means msg-dispatch-rate or byte-dispatch-rate not configured " + + "in dispatch-rate yet") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public DispatchRate getReplicatorDispatchRate(@PathParam("tenant") String tenant, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index c9d76b025376b..62aa429436d80 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Lists; @@ -209,6 +210,7 @@ public void testSubscriberPermission() throws Exception { superAdmin.tenants().createTenant("my-property", new TenantInfoImpl(Sets.newHashSet(tenantRole), Sets.newHashSet("test"))); superAdmin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); + assertNull(superAdmin.namespaces().getPublishRate(namespace)); // subscriptionRole doesn't have topic-level authorization, so it will fail to get topic stats-internal info try { From b09fd872aa61138fc979e05868096c21fdcf0509 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 15 Dec 2021 12:48:41 +0800 Subject: [PATCH 202/823] [Transaction] Remove request if can not send (#13308) (cherry picked from commit eb42df7126ac4015c67f6989ec083ee173dce3f4) --- .../impl/TransactionMetaStoreHandler.java | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index b2b756a1ee516..3c6286b0fb26f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -200,7 +200,9 @@ public CompletableFuture newTransactionAsync(long timeout, TimeUnit unit) internalPinnedExecutor.execute(() -> { pendingRequests.put(requestId, op); timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); return callback; } @@ -249,7 +251,9 @@ void handleNewTxnResponse(CommandNewTxnResponse response) { } return; } - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); } , op.backoff.next(), TimeUnit.MILLISECONDS); @@ -279,7 +283,9 @@ public CompletableFuture addPublishPartitionToTxnAsync(TxnID txnID, List { pendingRequests.put(requestId, op); timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); return callback; @@ -329,7 +335,9 @@ void handleAddPublishPartitionToTxnResponse(CommandAddPartitionToTxnResponse res } return; } - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); } , op.backoff.next(), TimeUnit.MILLISECONDS); @@ -360,7 +368,9 @@ public CompletableFuture addSubscriptionToTxn(TxnID txnID, List { pendingRequests.put(requestId, op); timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); return callback; } @@ -408,7 +418,9 @@ public void handleAddSubscriptionToTxnResponse(CommandAddSubscriptionToTxnRespon } return; } - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); } , op.backoff.next(), TimeUnit.MILLISECONDS); @@ -437,7 +449,9 @@ public CompletableFuture endTxnAsync(TxnID txnID, TxnAction action) { internalPinnedExecutor.execute(() -> { pendingRequests.put(requestId, op); timeoutQueue.add(new RequestTime(System.currentTimeMillis(), requestId)); - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); return callback; } @@ -486,7 +500,9 @@ void handleEndTxnResponse(CommandEndTxnResponse response) { } return; } - checkStateAndSendRequest(op); + if (!checkStateAndSendRequest(op)) { + pendingRequests.remove(requestId); + } }); } , op.backoff.next(), TimeUnit.MILLISECONDS); @@ -634,7 +650,7 @@ private boolean canSendRequest(CompletableFuture callback) { return true; } - private void checkStateAndSendRequest(OpBase op) { + private boolean checkStateAndSendRequest(OpBase op) { switch (getState()) { case Ready: ClientCnx cnx = cnx(); @@ -644,9 +660,9 @@ private void checkStateAndSendRequest(OpBase op) { } else { LOG.error("The cnx was null when the TC handler was ready", new NullPointerException()); } - break; + return true; case Connecting: - break; + return true; case Closing: case Closed: op.callback.completeExceptionally( @@ -655,7 +671,7 @@ private void checkStateAndSendRequest(OpBase op) { + transactionCoordinatorId + " is closing or closed.")); onResponse(op); - break; + return false; case Failed: case Uninitialized: op.callback.completeExceptionally( @@ -664,13 +680,13 @@ private void checkStateAndSendRequest(OpBase op) { + transactionCoordinatorId + " not connected.")); onResponse(op); - break; + return false; default: op.callback.completeExceptionally( new TransactionCoordinatorClientException.MetaStoreHandlerNotReadyException( transactionCoordinatorId)); onResponse(op); - break; + return false; } } From 22a24d62e8d4a0da43f6b28ff911696a515b6e0b Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 15 Dec 2021 13:41:02 +0800 Subject: [PATCH 203/823] [Transaction] Allow transaction be commit or abort in the state of aborting or committing. (#13323) Due to operations such as concurrency, retry, and timeout abort. We should allow `abort` or `commit` in the state of aborting or committing Allow `abort` or `commit` in the state of aborting or committing (cherry picked from commit d220c21a601a9a2c97ee3434b3b44b4bd9b8adc1) --- .../namespace/NamespaceServiceTest.java | 1 - .../broker/transaction/TransactionTest.java | 26 ++++++++++++++++ .../impl/transaction/TransactionImpl.java | 31 ++++++++++++++++--- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index d45dcc2d660a8..bb33ac837bbd6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -66,7 +66,6 @@ import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.LocalPolicies; import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.impl.BundlesDataImpl.BundlesDataImplBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.GetResult; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 3d0c406f07c3b..90ddc4b692bed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -76,6 +76,7 @@ import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; @@ -560,4 +561,29 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ Awaitility.await().untilAsserted(() -> assertEquals(metadataStore2.getCoordinatorStats().state, "Ready")); } + + @Test + public void testEndTxnWhenCommittingOrAborting() throws Exception { + Transaction commitTxn = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + Transaction abortTxn = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + + Class transactionClass = TransactionImpl.class; + Field field = transactionClass.getDeclaredField("state"); + field.setAccessible(true); + + field.set(commitTxn, TransactionImpl.State.COMMITTING); + field.set(abortTxn, TransactionImpl.State.ABORTING); + + abortTxn.abort(); + commitTxn.commit(); + } + } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index 60c7829b11f41..458976ae1426a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -144,7 +144,7 @@ public synchronized void registerCumulativeAckConsumer(ConsumerImpl consumer) @Override public CompletableFuture commit() { - return checkIfOpen().thenCompose((value) -> { + return checkIfOpenOrCommitting().thenCompose((value) -> { CompletableFuture commitFuture = new CompletableFuture<>(); this.state = State.COMMITTING; allOpComplete().whenComplete((v, e) -> { @@ -172,7 +172,7 @@ public CompletableFuture commit() { @Override public CompletableFuture abort() { - return checkIfOpen().thenCompose(value -> { + return checkIfOpenOrAborting().thenCompose(value -> { CompletableFuture abortFuture = new CompletableFuture<>(); this.state = State.ABORTING; allOpComplete().whenComplete((v, e) -> { @@ -217,12 +217,33 @@ private CompletableFuture checkIfOpen() { if (state == State.OPEN) { return CompletableFuture.completedFuture(null); } else { - return FutureUtil.failedFuture(new InvalidTxnStatusException("[" + txnIdMostBits + ":" - + txnIdLeastBits + "] with unexpected state : " - + state.name() + ", expect " + State.OPEN + " state!")); + return invalidTxnStatusFuture(); + } + } + + private CompletableFuture checkIfOpenOrCommitting() { + if (state == State.OPEN || state == State.COMMITTING) { + return CompletableFuture.completedFuture(null); + } else { + return invalidTxnStatusFuture(); + } + } + + private CompletableFuture checkIfOpenOrAborting() { + if (state == State.OPEN || state == State.ABORTING) { + return CompletableFuture.completedFuture(null); + } else { + return invalidTxnStatusFuture(); } } + private CompletableFuture invalidTxnStatusFuture() { + return FutureUtil.failedFuture(new InvalidTxnStatusException("[" + txnIdMostBits + ":" + + txnIdLeastBits + "] with unexpected state : " + + state.name() + ", expect " + State.OPEN + " state!")); + } + + private CompletableFuture allOpComplete() { List> futureList = new ArrayList<>(); futureList.addAll(sendFutureList); From ebaf2b7a573e03950e6db52e02c0dae1ed012458 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 15 Dec 2021 17:08:09 +0800 Subject: [PATCH 204/823] Fix Version.h not found when CMake binary directory is customized (#13324) ### Motivation When I build C++ tests on my local env, the following error happened. ``` tests/VersionTest.cc:19:10: fatal error: 'pulsar/Version.h' file not found #include ``` It's because I specified another directory as CMake directory. ```bash mkdir _builds && cd _builds && cmake .. ``` After https://github.com/apache/pulsar/pull/12769, the `Version.h` is generated under `${CMAKE_BINARY_DIR}/include/pulsar` directory but it's not included in `CMakeLists.txt`. CI works well because it's built in the default CMake directory so that `CMAKE_BINARY_DIR` is the same with `CMAKE_SOURCE_DIR`, which is included. ### Modifications Add the `${CMAKE_BINARY_DIR}/include` to `included_directories`. (cherry picked from commit ca37e67211feda4f7e0984e6414e707f1c1dfd07) --- pulsar-client-cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 8521dbf0e6ec3..8ba9569d7f690 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -334,6 +334,7 @@ file(MAKE_DIRECTORY ${AUTOGEN_DIR}) include_directories( ${CMAKE_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include ${AUTOGEN_DIR} ${Boost_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR} From 7ca4552ac4aaeb81b409764da34cccee0e81a97b Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 17 Dec 2021 22:28:29 +0800 Subject: [PATCH 205/823] [Transaction]stop TP replaying with Exception (#12700) When MLPendingAckStore replaying, if any ledger was deleted from bookkeeper, or ManagerLedger was fenced, MLPendingAckStore will not stop recovering and continue to report the exception. End replaying when there is no ledger to read or the managerLedger is fenced. Add a unit test. (cherry picked from commit a962137f530cd2d6c2315749270a1d2cae8b1cc2) --- .../bookkeeper/mledger/ManagedCursor.java | 6 ++ .../mledger/impl/ManagedCursorImpl.java | 1 + .../impl/ManagedCursorContainerTest.java | 5 ++ .../pendingack/impl/MLPendingAckStore.java | 25 +++++--- .../broker/transaction/TransactionTest.java | 61 +++++++++++++++++++ .../impl/MLTransactionLogImpl.java | 2 +- 6 files changed, 89 insertions(+), 11 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 72ee1a1f9c3ea..d1fb90a3ca60e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -710,4 +710,10 @@ Set asyncReplayEntries( * @return if read position changed */ boolean checkAndUpdateReadPositionChanged(); + + /** + * Checks if the cursor is closed. + * @return whether this cursor is closed. + */ + public boolean isClosed(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 521ec7ac18f84..3d382ad9698db 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -837,6 +837,7 @@ private void checkForNewEntries(OpReadEntry op, ReadEntriesCallback callback, Ob } } + @Override public boolean isClosed() { return state == State.Closed || state == State.Closing; } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index 57e1964a2c332..af30c9c372136 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -385,6 +385,11 @@ public List readEntriesOrWait(int maxEntries, long maxSizeBytes) public boolean checkAndUpdateReadPositionChanged() { return false; } + + @Override + public boolean isClosed() { + return false; + } } @Test diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index fb88878d8c826..1592318cf616a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -34,7 +34,6 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; @@ -303,13 +302,12 @@ class PendingAckReplay implements Runnable { @Override public void run() { try { - while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0) { - if (((ManagedCursorImpl) cursor).isClosed()) { - log.warn("[{}] MLPendingAckStore cursor have been closed, close replay thread.", - cursor.getManagedLedger().getName()); - return; - } - fillEntryQueueCallback.fillQueue(); + if (cursor.isClosed()) { + log.warn("[{}] MLPendingAckStore cursor have been closed, close replay thread.", + cursor.getManagedLedger().getName()); + return; + } + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 && fillEntryQueueCallback.fillQueue()) { Entry entry = entryQueue.poll(); if (entry != null) { ByteBuf buffer = entry.getDataBuffer(); @@ -361,15 +359,17 @@ public void run() { class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { + private volatile boolean isReadable = true; private final AtomicLong outstandingReadsRequests = new AtomicLong(0); - void fillQueue() { + boolean fillQueue() { if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); readAsync(100, this); } } + return isReadable; } @Override @@ -389,7 +389,12 @@ public Entry get() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - log.error("MLPendingAckStore stat reply fail!", exception); + if (managedLedger.getConfig().isAutoSkipNonRecoverableData() + && exception instanceof ManagedLedgerException.NonRecoverableLedgerException + || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { + isReadable = false; + } + log.error("MLPendingAckStore of topic [{}] stat reply fail!", managedLedger.getName(), exception); outstandingReadsRequests.decrementAndGet(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 90ddc4b692bed..9cad6fd28d6e0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -49,6 +49,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -60,8 +61,10 @@ import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.buffer.matadata.TransactionBufferSnapshot; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -76,6 +79,7 @@ import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; @@ -502,6 +506,63 @@ public void testMaxReadPositionForNormalPublish() throws Exception { } + @Test + public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ + String topic = NAMESPACE1 + "/testEndTPRecoveringWhenManagerLedgerDisReadable"; + admin.topics().createNonPartitionedTopic(topic); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .producerName("test") + .enableBatching(false) + .sendTimeout(0, TimeUnit.SECONDS) + .topic(topic) + .create(); + producer.newMessage().send(); + + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(topic, false).get().get(); + persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true); + PersistentSubscription persistentSubscription = (PersistentSubscription) persistentTopic + .createSubscription("test", + CommandSubscribe.InitialPosition.Earliest, false).get(); + + ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class); + doReturn(true).when(managedCursor).hasMoreEntries(); + doReturn(false).when(managedCursor).isClosed(); + doReturn(new PositionImpl(-1, -1)).when(managedCursor).getMarkDeletedPosition(); + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), + null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class); + doReturn(CompletableFuture.completedFuture( + new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null))) + .when(pendingAckStoreProvider).newPendingAckStore(any()); + doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any()); + + Class pulsarServiceClass = PulsarService.class; + Field field = pulsarServiceClass.getDeclaredField("transactionPendingAckStoreProvider"); + field.setAccessible(true); + field.set(getPulsarServiceList().get(0), pendingAckStoreProvider); + + PendingAckHandleImpl pendingAckHandle1 = new PendingAckHandleImpl(persistentSubscription); + Awaitility.await().untilAsserted(() -> + assertEquals(pendingAckHandle1.getStats().state, "Ready")); + + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + PendingAckHandleImpl pendingAckHandle2 = new PendingAckHandleImpl(persistentSubscription); + Awaitility.await().untilAsserted(() -> + assertEquals(pendingAckHandle2.getStats().state, "Ready")); + } + @Test public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ String topic = NAMESPACE1 + "/testEndTBRecoveringWhenManagerLedgerDisReadable"; diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index e154bb840d7c6..8bf2ebf4b911a 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -240,7 +240,7 @@ public void start() { class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private final AtomicLong outstandingReadsRequests = new AtomicLong(0); - private boolean isReadable = true; + private volatile boolean isReadable = true; boolean fillQueue() { if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { From 914f172592c6d205cc380ea3edb76e0f03904e0b Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 17 Dec 2021 14:47:12 +0800 Subject: [PATCH 206/823] [Transaction] Delete the redundant code (#13327) The problem was resolved, so there is no need to add a wait and retry method again. 1. Delete the redundant code 2. Optimize some code form (cherry picked from commit fbe010323076ba2339e2339e3031a78e20b09061) --- .../transaction/TransactionTestBase.java | 14 ----- .../testclient/PerformanceConsumer.java | 10 +++- .../testclient/PerformanceProducer.java | 12 ++-- .../testclient/PerformanceTransaction.java | 20 ++++--- .../testclient/utils/PerformanceUtils.java | 59 ------------------- .../PerformanceTransactionTest.java | 23 ++++---- 6 files changed, 38 insertions(+), 100 deletions(-) delete mode 100644 pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index e13365d15f971..fe7a813567efa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -61,8 +61,6 @@ import org.apache.zookeeper.MockZooKeeper; import org.apache.zookeeper.MockZooKeeperSession; import org.apache.zookeeper.ZooKeeper; -import org.awaitility.Awaitility; -import org.testng.Assert; @Slf4j public abstract class TransactionTestBase extends TestRetrySupport { @@ -144,8 +142,6 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int .statsInterval(0, TimeUnit.SECONDS) .enableTransaction(true) .build(); - // wait tc init success to ready state - waitForCoordinatorToBeAvailable(numPartitionsOfTC); } protected void startBroker() throws Exception { @@ -332,14 +328,4 @@ protected final void internalCleanup() { log.warn("Failed to clean up mocked pulsar service:", e); } } - public void waitForCoordinatorToBeAvailable(int numOfTCPerBroker){ - // wait tc init success to ready state - Awaitility.await() - .untilAsserted(() -> { - int transactionMetaStoreCount = pulsarServiceList.stream() - .mapToInt(pulsarService -> pulsarService.getTransactionMetadataStoreService().getStores().size()) - .sum(); - Assert.assertEquals(transactionMetaStoreCount, numOfTCPerBroker); - }); - } } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 9c15a0ec0e845..3c7bed9b743c7 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -20,7 +20,6 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.pulsar.testclient.utils.PerformanceUtils.buildTransaction; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -362,8 +361,13 @@ public static void main(String[] args) throws Exception { } PulsarClient pulsarClient = clientBuilder.build(); - AtomicReference atomicReference = buildTransaction(pulsarClient, arguments.isEnableTransaction, - arguments.transactionTimeout); + AtomicReference atomicReference; + if (arguments.isEnableTransaction) { + atomicReference = new AtomicReference<>(pulsarClient.newTransaction() + .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS).build().get()); + } else { + atomicReference = new AtomicReference<>(null); + } AtomicLong messageAckedCount = new AtomicLong(); Semaphore messageReceiveLimiter = new Semaphore(arguments.numMessagesPerTransaction); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 20eb8f34233c8..0e3d5503734ce 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -74,7 +74,6 @@ import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES; -import static org.apache.pulsar.testclient.utils.PerformanceUtils.buildTransaction; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -422,7 +421,7 @@ public static void main(String[] args) throws Exception { clientBuilder.allowTlsInsecureConnection(arguments.tlsAllowInsecureConnection); } - try (PulsarAdmin client = clientBuilder.build();) { + try (PulsarAdmin client = clientBuilder.build()) { for (String topic : arguments.topics) { log.info("Creating partitioned topic {} with {} partitions", topic, arguments.partitions); try { @@ -592,8 +591,15 @@ private static void runProducer(int producerId, // enable round robin message routing if it is a partitioned topic .messageRoutingMode(MessageRoutingMode.RoundRobinPartition); + AtomicReference transactionAtomicReference; if (arguments.isEnableTransaction) { producerBuilder.sendTimeout(0, TimeUnit.SECONDS); + transactionAtomicReference = new AtomicReference<>(client.newTransaction() + .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .build() + .get()); + } else { + transactionAtomicReference = new AtomicReference<>(null); } if (arguments.producerName != null) { String producerName = String.format("%s%s%d", arguments.producerName, arguments.separator, producerId); @@ -659,8 +665,6 @@ private static void runProducer(int producerId, } // Send messages on all topics/producers long totalSent = 0; - AtomicReference transactionAtomicReference = buildTransaction(client, - arguments.isEnableTransaction, arguments.transactionTimeout); AtomicLong numMessageSend = new AtomicLong(0); Semaphore numMsgPerTxnLimit = new Semaphore(arguments.numMessagesPerTransaction); while (true) { diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java index 5127f85496685..eee284b6bbf2f 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java @@ -19,7 +19,6 @@ package org.apache.pulsar.testclient; import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static org.apache.pulsar.testclient.utils.PerformanceUtils.buildTransaction; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -284,7 +283,7 @@ public static void main(String[] args) ExecutorService executorService = new ThreadPoolExecutor(arguments.numTestThreads, arguments.numTestThreads, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue()); + new LinkedBlockingQueue<>()); long startTime = System.nanoTime(); @@ -311,16 +310,23 @@ public static void main(String[] args) //A thread may perform tasks of multiple transactions in a traversing manner. List> producers = null; List>> consumers = null; + AtomicReference atomicReference = null; try { producers = buildProducers(client, arguments); consumers = buildConsumer(client, arguments); + if (!arguments.isDisableTransaction) { + atomicReference = new AtomicReference<>(client.newTransaction() + .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .build() + .get()); + } else { + atomicReference = new AtomicReference<>(null); + } } catch (Exception e) { log.error("Failed to build Producer/Consumer with exception : ", e); executorService.shutdownNow(); PerfClientUtils.exit(-1); } - AtomicReference atomicReference = buildTransaction(client, - !arguments.isDisableTransaction, arguments.transactionTimeout); //The while loop has no break, and finally ends the execution through the shutdownNow of //the executorService while (true) { @@ -351,7 +357,7 @@ public static void main(String[] args) for (List> subscriptions : consumers) { for (Consumer consumer : subscriptions) { for (int j = 0; j < arguments.numMessagesReceivedPerTransaction; j++) { - Message message = null; + Message message = null; try { message = consumer.receive(); } catch (PulsarClientException e) { @@ -690,9 +696,7 @@ private static List> buildProducers(PulsarClient client, Argume .sendTimeout(0, TimeUnit.SECONDS); final List>> producerFutures = Lists.newArrayList(); - Iterator produceTopicsIterator = arguments.producerTopic.iterator(); - while(produceTopicsIterator.hasNext()){ - String topic = produceTopicsIterator.next(); + for (String topic : arguments.producerTopic) { log.info("Create producer for topic {}", topic); producerFutures.add(producerBuilder.clone().topic(topic).createAsync()); } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java deleted file mode 100644 index ded11315070ed..0000000000000 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/utils/PerformanceUtils.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.testclient.utils; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.transaction.Transaction; -import org.apache.pulsar.testclient.PerformanceProducer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PerformanceUtils { - - private static final Logger log = LoggerFactory.getLogger(PerformanceProducer.class); - - public static AtomicReference buildTransaction(PulsarClient pulsarClient, boolean isEnableTransaction, - long transactionTimeout) { - - AtomicLong numBuildTxnFailed = new AtomicLong(); - if (isEnableTransaction) { - while(true) { - AtomicReference atomicReference = null; - try { - atomicReference = new AtomicReference(pulsarClient.newTransaction() - .withTransactionTimeout(transactionTimeout, TimeUnit.SECONDS).build().get()); - } catch (Exception e) { - numBuildTxnFailed.incrementAndGet(); - if (numBuildTxnFailed.get()%10 == 0) { - log.error("Failed to new a transaction with {} times", numBuildTxnFailed.get(), e); - } - } - if (atomicReference != null && atomicReference.get() != null) { - log.info("After {} failures, the transaction was created successfully for the first time", - numBuildTxnFailed.get()); - return atomicReference; - } - } - } - return new AtomicReference<>(null); - } -} diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java index c5e62f74cc6e3..a08fbe09f62e0 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java @@ -91,8 +91,8 @@ protected void cleanup() throws Exception { @Test public void testTxnPerf() throws Exception { String argString = "--topics-c %s --topics-p %s -threads 1 -ntxn 50 -u %s -ss %s -np 1 -au %s"; - String testConsumeTopic = testTopic + UUID.randomUUID().toString(); - String testProduceTopic = testTopic + UUID.randomUUID().toString(); + String testConsumeTopic = testTopic + UUID.randomUUID(); + String testProduceTopic = testTopic + UUID.randomUUID(); String testSub = "testSub"; admin.topics().createPartitionedTopic(testConsumeTopic, 1); String args = String.format(argString, testConsumeTopic, testProduceTopic, @@ -119,9 +119,8 @@ public void testTxnPerf() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(50); for (int i = 0; i < 50 ; i++) { - produceToConsumeTopic.newMessage().value(("testConsume " + i).getBytes()).sendAsync().thenRun(() -> { - countDownLatch.countDown(); - }); + produceToConsumeTopic.newMessage().value(("testConsume " + i).getBytes()).sendAsync().thenRun( + countDownLatch::countDown); } countDownLatch.await(); @@ -149,11 +148,11 @@ public void testTxnPerf() throws Exception { .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); for (int i = 0; i < 50; i++) { - Message message = consumeFromProduceTopic.receive(2, TimeUnit.SECONDS); + Message message = consumeFromProduceTopic.receive(2, TimeUnit.SECONDS); Assert.assertNotNull(message); consumeFromProduceTopic.acknowledge(message); } - Message message = consumeFromConsumeTopic.receive(2, TimeUnit.SECONDS); + Message message = consumeFromConsumeTopic.receive(2, TimeUnit.SECONDS); Assert.assertNull(message); message = consumeFromProduceTopic.receive(2, TimeUnit.SECONDS); Assert.assertNull(message); @@ -187,16 +186,16 @@ public void testProduceTxnMessage() throws InterruptedException, PulsarClientExc .enableBatchIndexAcknowledgment(false) .subscribe(); for (int i = 0; i < totalMessage; i++) { - Message message = consumer.receive(2, TimeUnit.SECONDS); + Message message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNotNull(message); consumer.acknowledge(message); } - Message message = consumer.receive(2, TimeUnit.SECONDS); + Message message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNull(message); } @Test - public void testConsumeTxnMessage() throws InterruptedException, PulsarClientException, ExecutionException { + public void testConsumeTxnMessage() throws InterruptedException, PulsarClientException { String argString = "%s -r 10 -u %s -txn -ss %s -st %s -sp %s -ntxn %d"; String subName = "sub"; String topic = testTopic + UUID.randomUUID(); @@ -230,10 +229,10 @@ public void testConsumeTxnMessage() throws InterruptedException, PulsarClientExc .enableBatchIndexAcknowledgment(false) .subscribe(); for (int i = 0; i < 5; i++) { - Message message = consumer.receive(2, TimeUnit.SECONDS); + Message message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNotNull(message); } - Message message = consumer.receive(2, TimeUnit.SECONDS); + Message message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNull(message); } From b244d0c3bc625bb4d4d3108735fd2a219d328a50 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 18 Dec 2021 08:35:18 +0800 Subject: [PATCH 207/823] [Transaction]Txn client check timeout (#12521) ### Motivation Optimize the logic on the Transaction Client side. Avoid sending and acking messages with timeout transactions. ### Modifications * TransactionImp * Add a tool field for CAS to replace State : STATE_UPDATE. **When committing and aborted, only the successful cas operation will make subsequent judgments, otherwise it will return a failure future** * Implement TimerTasker to execute tasks that replace the state of the transaction as Aborted. * TransactionBuildImpl * In the callback of the build method, call the timer of PulsarClient to start a Timeout. Pass in the corresponding transactionImpl (TimeTasker has been implemented) (cherry picked from commit c5d7a84c8e5c27e48022df8c7082496840cd3be9) --- .../client/impl/ConsumerAckResponseTest.java | 2 + .../client/impl/TransactionEndToEndTest.java | 67 ++++++++++++++++++- ...TransactionCoordinatorClientException.java | 20 ++++++ .../pulsar/client/impl/ConsumerBase.java | 4 ++ .../client/impl/PartitionedProducerImpl.java | 5 ++ .../pulsar/client/impl/ProducerImpl.java | 4 ++ .../pulsar/client/impl/PulsarClientImpl.java | 1 + .../transaction/TransactionBuilderImpl.java | 9 ++- .../impl/transaction/TransactionImpl.java | 38 ++++++++--- 9 files changed, 137 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckResponseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckResponseTest.java index 0378c53d05be6..698186539b3c6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckResponseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckResponseTest.java @@ -50,8 +50,10 @@ public void setup() throws Exception { super.producerBaseSetup(); doReturn(1L).when(transaction).getTxnIdLeastBits(); doReturn(1L).when(transaction).getTxnIdMostBits(); + doReturn(TransactionImpl.State.OPEN).when(transaction).getState(); CompletableFuture completableFuture = CompletableFuture.completedFuture(null); doNothing().when(transaction).registerAckOp(any()); + doReturn(true).when(transaction).checkIfOpen(any()); doReturn(completableFuture).when(transaction).registerAckedTopic(any(), any()); Thread.sleep(1000 * 3); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 463044969ac10..f52d3193050cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -23,7 +23,9 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.util.Collection; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.ArrayList; @@ -770,18 +772,45 @@ public void produceAndConsumeCloseStateTxnTest() throws Exception { } }); + Class transactionClass = TransactionImpl.class; + Constructor constructor = transactionClass + .getDeclaredConstructor(PulsarClientImpl.class, long.class, long.class, long.class); + constructor.setAccessible(true); + + TransactionImpl timeoutTxnSkipClientTimeout = constructor.newInstance(pulsarClient, 5, + timeoutTxn.getTxnID().getLeastSigBits(), timeoutTxn.getTxnID().getMostSigBits()); + try { - timeoutTxn.commit().get(); + timeoutTxnSkipClientTimeout.commit().get(); fail(); } catch (Exception e) { assertTrue(e.getCause() instanceof TransactionNotFoundException); } Field field = TransactionImpl.class.getDeclaredField("state"); field.setAccessible(true); - TransactionImpl.State state = (TransactionImpl.State) field.get(timeoutTxn); + TransactionImpl.State state = (TransactionImpl.State) field.get(timeoutTxnSkipClientTimeout); assertEquals(state, TransactionImpl.State.ERROR); } + @Test + public void testTxnTimeoutAtTransactionMetadataStore() throws Exception{ + TxnID txnID = pulsarServiceList.get(0).getTransactionMetadataStoreService() + .newTransaction(new TransactionCoordinatorID(0), 1).get(); + Awaitility.await().until(() -> { + try { + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txnID).get(); + return false; + } catch (Exception e) { + return true; + } + }); + Collection transactionMetadataStores = + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores().values(); + long timeoutCount = transactionMetadataStores.stream() + .mapToLong(store -> store.getMetadataStoreStats().timeoutCount).sum(); + Assert.assertEquals(timeoutCount, 1); + } + @Test public void transactionTimeoutTest() throws Exception { String topic = NAMESPACE1 + "/txn-timeout"; @@ -943,4 +972,38 @@ public void oneTransactionOneTopicWithMultiSubTest() throws Exception { } assertTrue(flag); } + + @Test + public void testTxnTimeOutInClient() throws Exception{ + String topic = NAMESPACE1 + "/testTxnTimeOutInClient"; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).producerName("testTxnTimeOut_producer") + .topic(topic).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).consumerName("testTxnTimeOut_consumer") + .topic(topic).subscriptionName("testTxnTimeOut_sub").subscribe(); + + Transaction transaction = pulsarClient.newTransaction().withTransactionTimeout(1, TimeUnit.SECONDS) + .build().get(); + producer.newMessage().send(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(((TransactionImpl)transaction).getState(), TransactionImpl.State.TIMEOUT); + }); + + try { + producer.newMessage(transaction).send(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getCause() instanceof TransactionCoordinatorClientException + .InvalidTxnStatusException); + } + try { + Message message = consumer.receive(); + consumer.acknowledgeAsync(message.getMessageId(), transaction).get(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getCause() instanceof TransactionCoordinatorClientException + .InvalidTxnStatusException); + } + } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionCoordinatorClientException.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionCoordinatorClientException.java index 0e1f6c79cf536..d7df4e3c0754b 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionCoordinatorClientException.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionCoordinatorClientException.java @@ -68,6 +68,11 @@ public static class InvalidTxnStatusException extends TransactionCoordinatorClie public InvalidTxnStatusException(String message) { super(message); } + + public InvalidTxnStatusException(String txnId, String actualState, String expectState) { + super("["+ txnId +"] with unexpected state : " + + actualState + ", expect " + expectState + " state!"); + } } /** @@ -93,6 +98,21 @@ public MetaStoreHandlerNotExistsException(String message) { } } + + /** + * Thrown when transaction meta was timeout. + */ + public static class TransactionTimeotException extends TransactionCoordinatorClientException { + + public TransactionTimeotException(Throwable t) { + super(t); + } + + public TransactionTimeotException(String transactionId) { + super("The transaction " + transactionId + " is timeout."); + } + } + /** * Thrown when send request to transaction meta store but the transaction meta store handler not ready. */ diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 1251593d959c3..0f47208ea1efc 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -487,6 +487,10 @@ public CompletableFuture acknowledgeAsync(MessageId messageId, if (null != txn) { checkArgument(txn instanceof TransactionImpl); txnImpl = (TransactionImpl) txn; + CompletableFuture completableFuture = new CompletableFuture<>(); + if (!txnImpl.checkIfOpen(completableFuture)) { + return completableFuture; + } } return doAcknowledgeWithTxn(messageId, AckType.Individual, Collections.emptyMap(), txnImpl); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index 33110503612e2..216d7755425e9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -48,6 +48,7 @@ import org.apache.pulsar.client.api.TopicMetadata; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; @@ -191,6 +192,10 @@ CompletableFuture internalSendAsync(Message message) { @Override CompletableFuture internalSendWithTxnAsync(Message message, Transaction txn) { + CompletableFuture completableFuture = new CompletableFuture<>(); + if (txn != null && !((TransactionImpl)txn).checkIfOpen(completableFuture)) { + return completableFuture; + } int partition = routerPolicy.choosePartition(message, topicMetadata); checkArgument(partition >= 0 && partition < topicMetadata.numPartitions(), "Illegal partition index chosen by the message routing policy: " + partition); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index a3ca38689a9a1..5944c8f077541 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -371,6 +371,10 @@ CompletableFuture internalSendWithTxnAsync(Message message, Transa if (txn == null) { return internalSendAsync(message); } else { + CompletableFuture completableFuture = new CompletableFuture<>(); + if (!((TransactionImpl)txn).checkIfOpen(completableFuture)) { + return completableFuture; + } return ((TransactionImpl) txn).registerProducedTopic(topic) .thenCompose(ignored -> internalSendAsync(message)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 7703afccb96e1..c5195c9054d75 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -95,6 +95,7 @@ public class PulsarClientImpl implements PulsarClient { protected final ClientConfigurationData conf; private LookupService lookup; private final ConnectionPool cnxPool; + @Getter private final Timer timer; private boolean needStopTimer; private final ExecutorProvider externalExecutorProvider; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java index 84be46fb7b7e2..3ac8676283a1b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.client.impl.transaction; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -67,8 +69,11 @@ public CompletableFuture build() { future.completeExceptionally(throwable); return; } - future.complete(new TransactionImpl(client, txnTimeout, - txnID.getLeastSigBits(), txnID.getMostSigBits())); + TransactionImpl transaction = new TransactionImpl(client, txnTimeout, + txnID.getLeastSigBits(), txnID.getMostSigBits()); + client.getTimer().newTimeout(transaction, + txnTimeout, timeUnit); + future.complete(transaction); }); return future; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index 458976ae1426a..4128a6f223b72 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.client.impl.transaction; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,6 +27,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import com.google.common.collect.Lists; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; @@ -48,7 +51,7 @@ */ @Slf4j @Getter -public class TransactionImpl implements Transaction { +public class TransactionImpl implements Transaction , TimerTask { private final PulsarClientImpl client; private final long transactionTimeoutMs; @@ -63,6 +66,13 @@ public class TransactionImpl implements Transaction { private final ArrayList> sendFutureList; private final ArrayList> ackFutureList; private volatile State state; + private final AtomicReferenceFieldUpdater STATE_UPDATE = + AtomicReferenceFieldUpdater.newUpdater(TransactionImpl.class, State.class, "state"); + + @Override + public void run(Timeout timeout) throws Exception { + STATE_UPDATE.compareAndSet(this, State.OPEN, State.TIMEOUT); + } public enum State { OPEN, @@ -70,7 +80,8 @@ public enum State { ABORTING, COMMITTED, ABORTED, - ERROR + ERROR, + TIMEOUT } TransactionImpl(PulsarClientImpl client, @@ -93,7 +104,8 @@ public enum State { // register the topics that will be modified by this transaction public CompletableFuture registerProducedTopic(String topic) { - return checkIfOpen().thenCompose(value -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + if (checkIfOpen(completableFuture)) { synchronized (TransactionImpl.this) { // we need to issue the request to TC to register the produced topic return registerPartitionMap.compute(topic, (key, future) -> { @@ -106,7 +118,9 @@ public CompletableFuture registerProducedTopic(String topic) { } }); } - }); + } else { + return completableFuture; + } } public synchronized void registerSendOp(CompletableFuture sendFuture) { @@ -115,7 +129,8 @@ public synchronized void registerSendOp(CompletableFuture sendFuture) // register the topics that will be modified by this transaction public CompletableFuture registerAckedTopic(String topic, String subscription) { - return checkIfOpen().thenCompose(value -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + if (checkIfOpen(completableFuture)) { synchronized (TransactionImpl.this) { // we need to issue the request to TC to register the acked topic return registerSubscriptionMap.compute(Pair.of(topic, subscription), (key, future) -> { @@ -128,7 +143,9 @@ public CompletableFuture registerAckedTopic(String topic, String subscript } }); } - }); + } else { + return completableFuture; + } } public synchronized void registerAckOp(CompletableFuture ackFuture) { @@ -213,11 +230,14 @@ public TxnID getTxnID() { return new TxnID(txnIdMostBits, txnIdLeastBits); } - private CompletableFuture checkIfOpen() { + public boolean checkIfOpen(CompletableFuture completableFuture) { if (state == State.OPEN) { - return CompletableFuture.completedFuture(null); + return true; } else { - return invalidTxnStatusFuture(); + completableFuture + .completeExceptionally(new InvalidTxnStatusException( + new TxnID(txnIdMostBits, txnIdLeastBits).toString(), state.name(), State.OPEN.name())); + return false; } } From 1d6324605f348b4bb49bd81ca151c88b4c15bbc8 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 20 Dec 2021 14:49:29 +0800 Subject: [PATCH 208/823] Optimize transaction FieldUpdater to static final (#13396) (cherry picked from commit b261d6abfc3a6a0707bf8ff6712a58cf7f0b6a1b) --- .../apache/pulsar/client/impl/transaction/TransactionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index 4128a6f223b72..ebcb20e8c91ad 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -66,7 +66,7 @@ public class TransactionImpl implements Transaction , TimerTask { private final ArrayList> sendFutureList; private final ArrayList> ackFutureList; private volatile State state; - private final AtomicReferenceFieldUpdater STATE_UPDATE = + private static final AtomicReferenceFieldUpdater STATE_UPDATE = AtomicReferenceFieldUpdater.newUpdater(TransactionImpl.class, State.class, "state"); @Override From f6be11ac90e7f1025b8d289439a4a080f21ad380 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 21 Dec 2021 15:31:29 +0800 Subject: [PATCH 209/823] [Branch-2.9] Trigger CI for new Pull Request (#13423) --- .github/workflows/ci-build-macos.yaml | 1 + .github/workflows/ci-cpp-build-centos7.yaml | 1 + .github/workflows/ci-cpp-build-windows.yaml | 1 + .github/workflows/ci-cpp.yaml | 1 + .github/workflows/ci-go-functions-style.yaml | 1 + .github/workflows/ci-go-functions-test.yaml | 1 + .github/workflows/ci-integration-backwards-compatibility.yaml | 1 + .github/workflows/ci-integration-cli.yaml | 1 + .github/workflows/ci-integration-function.yaml | 1 + .github/workflows/ci-integration-messaging.yaml | 1 + .github/workflows/ci-integration-process.yaml | 1 + .github/workflows/ci-integration-pulsar-io-ora.yaml | 1 + .github/workflows/ci-integration-pulsar-io.yaml | 1 + .github/workflows/ci-integration-schema.yaml | 1 + .github/workflows/ci-integration-sql.yaml | 1 + .github/workflows/ci-integration-standalone.yaml | 1 + .github/workflows/ci-integration-thread.yaml | 1 + .github/workflows/ci-integration-tiered-filesystem.yaml | 1 + .github/workflows/ci-integration-tiered-jcloud.yaml | 1 + .github/workflows/ci-integration-transaction.yaml | 1 + .github/workflows/ci-license.yaml | 1 + .github/workflows/ci-python-build-3.9-client.yaml | 1 + .github/workflows/ci-shade-test.yaml | 1 + .github/workflows/ci-unit-broker-broker-gp1.yaml | 1 + .github/workflows/ci-unit-broker-broker-gp2.yaml | 1 + .github/workflows/ci-unit-broker-client-api.yaml | 1 + .github/workflows/ci-unit-broker-client-impl.yaml | 1 + .github/workflows/ci-unit-broker-jdk8.yaml | 1 + .github/workflows/ci-unit-broker-other.yaml | 1 + .github/workflows/ci-unit-proxy.yaml | 1 + .github/workflows/ci-unit.yaml | 1 + 31 files changed, 31 insertions(+) diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml index 0d128c4422c03..f23c4c0295997 100644 --- a/.github/workflows/ci-build-macos.yaml +++ b/.github/workflows/ci-build-macos.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-cpp-build-centos7.yaml b/.github/workflows/ci-cpp-build-centos7.yaml index 55151ba1cdd5c..bc480d48162f1 100644 --- a/.github/workflows/ci-cpp-build-centos7.yaml +++ b/.github/workflows/ci-cpp-build-centos7.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - '.github/workflows/**' - 'pulsar-client-cpp/**' diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml index a287a4c3560e1..cc99204c31076 100644 --- a/.github/workflows/ci-cpp-build-windows.yaml +++ b/.github/workflows/ci-cpp-build-windows.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - '.github/workflows/**' - 'pulsar-client-cpp/**' diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml index ed823792ded8c..845ae213444d9 100644 --- a/.github/workflows/ci-cpp.yaml +++ b/.github/workflows/ci-cpp.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions-style.yaml index 4c77cf2bf47b8..d033b1a45fb9a 100644 --- a/.github/workflows/ci-go-functions-style.yaml +++ b/.github/workflows/ci-go-functions-style.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - '.github/workflows/**' - 'pulsar-function-go/**' diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml index 37e8b81729efa..c19bc0ed7c442 100644 --- a/.github/workflows/ci-go-functions-test.yaml +++ b/.github/workflows/ci-go-functions-test.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - '.github/workflows/**' - 'pulsar-function-go/**' diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml index 59efbabe45a80..b8e6340b7e655 100644 --- a/.github/workflows/ci-integration-backwards-compatibility.yaml +++ b/.github/workflows/ci-integration-backwards-compatibility.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml index 7fb9b950c8208..68c52cb99ad97 100644 --- a/.github/workflows/ci-integration-cli.yaml +++ b/.github/workflows/ci-integration-cli.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-function.yaml b/.github/workflows/ci-integration-function.yaml index 1e7e5005f996e..ce02e01da838b 100644 --- a/.github/workflows/ci-integration-function.yaml +++ b/.github/workflows/ci-integration-function.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml index 68eeda8aed350..d39368d8de3ee 100644 --- a/.github/workflows/ci-integration-messaging.yaml +++ b/.github/workflows/ci-integration-messaging.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml index 10d3b5ebd2def..46d7a495a55ab 100644 --- a/.github/workflows/ci-integration-process.yaml +++ b/.github/workflows/ci-integration-process.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml index 33b5d11af8329..9a980b93d7beb 100644 --- a/.github/workflows/ci-integration-pulsar-io-ora.yaml +++ b/.github/workflows/ci-integration-pulsar-io-ora.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-pulsar-io.yaml b/.github/workflows/ci-integration-pulsar-io.yaml index 9b811f2b8c540..bd793087d00f8 100644 --- a/.github/workflows/ci-integration-pulsar-io.yaml +++ b/.github/workflows/ci-integration-pulsar-io.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml index 84c02c39471a7..8e22fed107901 100644 --- a/.github/workflows/ci-integration-schema.yaml +++ b/.github/workflows/ci-integration-schema.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml index 6b5d7ba0077bc..482b7c5f84703 100644 --- a/.github/workflows/ci-integration-sql.yaml +++ b/.github/workflows/ci-integration-sql.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml index 17499c4741ff3..9c783812085a7 100644 --- a/.github/workflows/ci-integration-standalone.yaml +++ b/.github/workflows/ci-integration-standalone.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml index 2133a53727521..accbbbac3996b 100644 --- a/.github/workflows/ci-integration-thread.yaml +++ b/.github/workflows/ci-integration-thread.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml index 413838faecc6e..107c53d6482b3 100644 --- a/.github/workflows/ci-integration-tiered-filesystem.yaml +++ b/.github/workflows/ci-integration-tiered-filesystem.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml index 5ba8d357dd154..ba089ad794621 100644 --- a/.github/workflows/ci-integration-tiered-jcloud.yaml +++ b/.github/workflows/ci-integration-tiered-jcloud.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml index cd2624865856f..e1b230edf91b2 100644 --- a/.github/workflows/ci-integration-transaction.yaml +++ b/.github/workflows/ci-integration-transaction.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml index ae60426d482ad..941023f678394 100644 --- a/.github/workflows/ci-license.yaml +++ b/.github/workflows/ci-license.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-python-build-3.9-client.yaml b/.github/workflows/ci-python-build-3.9-client.yaml index f8b04a77f6b2c..1b20a327d9e8f 100644 --- a/.github/workflows/ci-python-build-3.9-client.yaml +++ b/.github/workflows/ci-python-build-3.9-client.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - '.github/workflows/**' - 'pulsar-client-cpp/**' diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml index 1aecf70b17c99..646f8c0f4f72d 100644 --- a/.github/workflows/ci-shade-test.yaml +++ b/.github/workflows/ci-shade-test.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml index 71204e5ff8a6e..946e20dc7fb1b 100644 --- a/.github/workflows/ci-unit-broker-broker-gp1.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp1.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-broker-gp2.yaml b/.github/workflows/ci-unit-broker-broker-gp2.yaml index 6433dd10f0b2b..3763f14be96a5 100644 --- a/.github/workflows/ci-unit-broker-broker-gp2.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp2.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml index 699caf0779d43..9616efd941b3c 100644 --- a/.github/workflows/ci-unit-broker-client-api.yaml +++ b/.github/workflows/ci-unit-broker-client-api.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml index 6841b5782acbb..243ac4cbc46b8 100644 --- a/.github/workflows/ci-unit-broker-client-impl.yaml +++ b/.github/workflows/ci-unit-broker-client-impl.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-jdk8.yaml b/.github/workflows/ci-unit-broker-jdk8.yaml index e0460854b4375..4e728aada3e7a 100644 --- a/.github/workflows/ci-unit-broker-jdk8.yaml +++ b/.github/workflows/ci-unit-broker-jdk8.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml index 5826244751e28..80b244827ffe9 100644 --- a/.github/workflows/ci-unit-broker-other.yaml +++ b/.github/workflows/ci-unit-broker-other.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml index 2ffed49586cde..943abde38ffa5 100644 --- a/.github/workflows/ci-unit-proxy.yaml +++ b/.github/workflows/ci-unit-proxy.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml index fd553b0700632..f89d16b4545a1 100644 --- a/.github/workflows/ci-unit.yaml +++ b/.github/workflows/ci-unit.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* From 44dbe7017f12c98b196afc26b5f5a830f499c522 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Tue, 21 Dec 2021 19:23:28 +0800 Subject: [PATCH 210/823] [Branch-2.9] [Transaction] Stop TB recovering with exception (#13425) Cherry-pick from https://github.com/apache/pulsar/pull/12636 --- .../buffer/impl/TopicTransactionBuffer.java | 14 ++++- .../broker/transaction/TransactionTest.java | 55 ++++++++++++++++++- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 84b209c6cf3a4..9978f6f8ec20a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -582,8 +582,8 @@ public void run() { FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, managedCursor, TopicTransactionBufferRecover.this); if (lastConfirmedEntry.getEntryId() != -1) { - while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0) { - fillEntryQueueCallback.fillQueue(); + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 + && fillEntryQueueCallback.fillQueue()) { Entry entry = entryQueue.poll(); if (entry != null) { try { @@ -641,19 +641,22 @@ static class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallbac private final TopicTransactionBufferRecover recover; + private volatile boolean isReadable = true; + private FillEntryQueueCallback(SpscArrayQueue entryQueue, ManagedCursor cursor, TopicTransactionBufferRecover recover) { this.entryQueue = entryQueue; this.cursor = cursor; this.recover = recover; } - void fillQueue() { + boolean fillQueue() { if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); cursor.asyncReadEntries(100, this, System.nanoTime(), PositionImpl.latest); } } + return isReadable; } @Override @@ -673,6 +676,11 @@ public Entry get() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + if (recover.topic.getManagedLedger().getConfig().isAutoSkipNonRecoverableData() + && exception instanceof ManagedLedgerException.NonRecoverableLedgerException + || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { + isReadable = false; + } recover.callBackException(exception); outstandingReadsRequests.decrementAndGet(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 9cad6fd28d6e0..59d2839f76f53 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -49,6 +49,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; @@ -57,6 +58,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; @@ -363,6 +365,7 @@ public void testTakeSnapshotBeforeBuildTxnProducer() throws Exception { }); } + @Test public void testAppendBufferWithNotManageLedgerExceptionCanCastToMLE() throws Exception { @@ -503,7 +506,57 @@ public void testMaxReadPositionForNormalPublish() throws Exception { PositionImpl position5 = (PositionImpl) maxReadPositionField.get(topicTransactionBuffer); Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); + } + + @Test + public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ + String topic = NAMESPACE1 + "/testEndTBRecoveringWhenManagerLedgerDisReadable"; + admin.topics().createNonPartitionedTopic(topic); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .producerName("test") + .enableBatching(false) + .sendTimeout(0, TimeUnit.SECONDS) + .topic(topic) + .create(); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); + + producer.newMessage(txn).value("test").send(); + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic("persistent://" + topic, false).get().get(); + persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true); + + ManagedCursor managedCursor = mock(ManagedCursor.class); + doReturn("transaction-buffer-sub").when(managedCursor).getName(); + doReturn(true).when(managedCursor).hasMoreEntries(); + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), + null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + Class managedLedgerClass = ManagedLedgerImpl.class; + Field field = managedLedgerClass.getDeclaredField("cursors"); + field.setAccessible(true); + ManagedCursorContainer managedCursors = (ManagedCursorContainer) field.get(persistentTopic.getManagedLedger()); + managedCursors.removeCursor("transaction-buffer-sub"); + managedCursors.add(managedCursor); + + TransactionBuffer buffer1 = new TopicTransactionBuffer(persistentTopic); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(buffer1.getStats().state, "Ready")); + + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + TransactionBuffer buffer2 = new TopicTransactionBuffer(persistentTopic); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(buffer2.getStats().state, "Ready")); } @Test @@ -565,7 +618,7 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ @Test public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ - String topic = NAMESPACE1 + "/testEndTBRecoveringWhenManagerLedgerDisReadable"; + String topic = NAMESPACE1 + "/testEndTCRecoveringWhenManagerLedgerDisReadable"; admin.topics().createNonPartitionedTopic(topic); PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() From ab6f1c4e1906d680ecbfe9611cea8aafdffa0aaa Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 21 Dec 2021 19:23:56 +0800 Subject: [PATCH 211/823] [broker] Bug Fix: topic policy is not properly init if namespace is loaded first. (#12833) (#13422) Cherry-pick #12833 to branch-2.9 Fix some conflicts in test case: testTopicPolicyInitialValueWithNamespaceAlreadyLoaded --- .../SystemTopicBasedTopicPoliciesService.java | 7 ++- .../broker/service/TopicPoliciesService.java | 12 +++++ .../service/persistent/PersistentTopic.java | 16 ++++++- .../broker/admin/TopicPoliciesTest.java | 45 +++++++++++++++++++ 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 12ba88e99bb7a..666220204508b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -172,6 +172,11 @@ public TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesC return policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); } + @Override + public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { + return policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); + } + @Override public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicName topicName) { CompletableFuture result = new CompletableFuture<>(); @@ -469,7 +474,7 @@ boolean checkReaderIsCached(NamespaceName namespaceName) { } @VisibleForTesting - Boolean getPoliciesCacheInit(NamespaceName namespaceName) { + public Boolean getPoliciesCacheInit(NamespaceName namespaceName) { return policyCacheInitMap.get(namespaceName); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index 48d2f1e449673..53de0878100c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -61,6 +61,13 @@ public interface TopicPoliciesService { */ TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesCacheNotInitException; + /** + * Get policies from current cache. + * @param topicName topic name + * @return the topic policies + */ + TopicPolicies getTopicPoliciesIfExists(TopicName topicName); + /** * When getting TopicPolicies, if the initialization has not been completed, * we will go back off and try again until time out. @@ -145,6 +152,11 @@ public TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesC return null; } + @Override + public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { + return null; + } + @Override public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicName topicName) { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 053d72d6d5d0b..1b2e11f4ba0fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -300,6 +300,7 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS @Override public CompletableFuture initialize() { List> futures = new ArrayList<>(); + futures.add(initTopicPolicy()); for (ManagedCursor cursor : ledger.getCursors()) { if (cursor.getName().startsWith(replicatorPrefix)) { String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); @@ -3083,7 +3084,9 @@ public void onUpdate(TopicPolicies policies) { subscriptions.forEach((subName, sub) -> { sub.getConsumers().forEach(Consumer::checkPermissions); Dispatcher dispatcher = sub.getDispatcher(); - dispatcher.updateRateLimiter(policies.getSubscriptionDispatchRate()); + if (dispatcher != null) { + dispatcher.updateRateLimiter(policies.getSubscriptionDispatchRate()); + } }); if (policies.getPublishRate() != null) { @@ -3150,6 +3153,17 @@ private void initializeTopicSubscribeRateLimiterIfNeeded(Optional } } + protected CompletableFuture initTopicPolicy() { + if (brokerService.pulsar().getConfig().isSystemTopicEnabled() + && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { + return CompletableFuture.completedFuture(null).thenRunAsync(() -> onUpdate( + brokerService.getPulsar().getTopicPoliciesService() + .getTopicPoliciesIfExists(TopicName.getPartitionedTopicName(topic))), + brokerService.getTopicOrderedExecutor()); + } + return CompletableFuture.completedFuture(null); + } + private void registerTopicPolicyListener() { if (brokerService.pulsar().getConfig().isSystemTopicEnabled() && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 91de20332e14f..5047c0cad390f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -44,7 +44,9 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BacklogQuotaManager; +import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.PublishRateLimiterImpl; +import org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -63,6 +65,8 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.events.EventsTopicNames; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ClusterData; @@ -125,6 +129,47 @@ public void cleanup() throws Exception { this.resetConfig(); } + @Test + public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception{ + TopicName topicName = TopicName.get( + TopicDomain.persistent.value(), + NamespaceName.get(myNamespace), + "test-" + UUID.randomUUID() + ); + String topic = topicName.toString(); + + SystemTopicBasedTopicPoliciesService policyService = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + + //set up topic with inactiveTopicPolicies.maxInactiveDurationSeconds = 100 + InactiveTopicPolicies inactiveTopicPolicies = + new InactiveTopicPolicies(InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, 100, true); + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setInactiveTopicPolicies(topic, inactiveTopicPolicies); + + //wait until topic loaded with right policy value. + Awaitility.await().untilAsserted(()-> { + AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get(); + assertEquals(topic1.getInactiveTopicPolicies().getMaxInactiveDurationSeconds(), 100); + }); + //unload the topic + pulsar.getNamespaceService().unloadNamespaceBundle(pulsar.getNamespaceService().getBundle(topicName)).get(); + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); + + //load the nameserver, but topic is not init. + log.info("lookup:{}",admin.lookups().lookupTopic(topic)); + assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBroker(topicName)); + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); + //make sure namespace policy reader is fully started. + Awaitility.await().untilAsserted(()-> { + assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject())); + }); + + //load the topic. + AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get(); + assertEquals(topic1.getInactiveTopicPolicies().getMaxInactiveDurationSeconds(), 100); + } + @Test public void testSetSizeBasedBacklogQuota() throws Exception { From c5a4639e87e8c9dd00cbd8bcb040072c9fc0c852 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 21 Dec 2021 19:24:37 +0800 Subject: [PATCH 212/823] [broker] Fix deadlock in metadata-store callback thread for branch 2.9 (#13426) Fixes #12726 for branch 2.9 ### Motivation See #12726 #12753 fixed this in master, but can not merged into branch 2.9. --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 30ce9c11073af..a5d55bf77e6d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1185,7 +1185,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { log.info("[{}][{}] Creating producer. producerId={}", remoteAddress, topicName, producerId); - service.getOrCreateTopic(topicName.toString()).thenAccept((Topic topic) -> { + service.getOrCreateTopic(topicName.toString()).thenAcceptAsync((Topic topic) -> { // Before creating producer, check if backlog quota exceeded // on topic for size based limit and time based limit for (BacklogQuota.BacklogQuotaType backlogQuotaType : @@ -1250,7 +1250,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { return null; }); }); - }).exceptionally(exception -> { + }, getBrokerService().getPulsar().getExecutor()).exceptionally(exception -> { Throwable cause = exception.getCause(); if (cause instanceof NoSuchElementException) { cause = new TopicNotFoundException("Topic Not Found."); From c1f809f9491d6cf172c06b8fd43b4a3ffacaedd7 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 11 Nov 2021 19:49:26 -0800 Subject: [PATCH 213/823] Upgrade to BookKeeper 4.14.3 (#12760) (cherry picked from commit 9e30e26110c077c8e0621914a15ed7af3c7e53ec) --- .../server/src/assemble/LICENSE.bin.txt | 50 +++++++++---------- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 24 ++++----- .../client/BookKeeperTestClient.java | 3 +- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 27d146688088d..77ba2838b07d7 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -392,31 +392,31 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-1.2-api-2.17.0.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.14.2.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.2.jar - - org.apache.bookkeeper-bookkeeper-proto-4.14.2.jar - - org.apache.bookkeeper-bookkeeper-server-4.14.2.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.2.jar - - org.apache.bookkeeper-circe-checksum-4.14.2.jar - - org.apache.bookkeeper-cpu-affinity-4.14.2.jar - - org.apache.bookkeeper-statelib-4.14.2.jar - - org.apache.bookkeeper-stream-storage-api-4.14.2.jar - - org.apache.bookkeeper-stream-storage-common-4.14.2.jar - - org.apache.bookkeeper-stream-storage-java-client-4.14.2.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.14.2.jar - - org.apache.bookkeeper-stream-storage-proto-4.14.2.jar - - org.apache.bookkeeper-stream-storage-server-4.14.2.jar - - org.apache.bookkeeper-stream-storage-service-api-4.14.2.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.14.2.jar - - org.apache.bookkeeper.http-http-server-4.14.2.jar - - org.apache.bookkeeper.http-vertx-http-server-4.14.2.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.2.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.2.jar - - org.apache.distributedlog-distributedlog-common-4.14.2.jar - - org.apache.distributedlog-distributedlog-core-4.14.2-tests.jar - - org.apache.distributedlog-distributedlog-core-4.14.2.jar - - org.apache.distributedlog-distributedlog-protocol-4.14.2.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.2.jar + - org.apache.bookkeeper-bookkeeper-common-4.14.3.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.3.jar + - org.apache.bookkeeper-bookkeeper-proto-4.14.3.jar + - org.apache.bookkeeper-bookkeeper-server-4.14.3.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.3.jar + - org.apache.bookkeeper-circe-checksum-4.14.3.jar + - org.apache.bookkeeper-cpu-affinity-4.14.3.jar + - org.apache.bookkeeper-statelib-4.14.3.jar + - org.apache.bookkeeper-stream-storage-api-4.14.3.jar + - org.apache.bookkeeper-stream-storage-common-4.14.3.jar + - org.apache.bookkeeper-stream-storage-java-client-4.14.3.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.14.3.jar + - org.apache.bookkeeper-stream-storage-proto-4.14.3.jar + - org.apache.bookkeeper-stream-storage-server-4.14.3.jar + - org.apache.bookkeeper-stream-storage-service-api-4.14.3.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.14.3.jar + - org.apache.bookkeeper.http-http-server-4.14.3.jar + - org.apache.bookkeeper.http-vertx-http-server-4.14.3.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.3.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.3.jar + - org.apache.distributedlog-distributedlog-common-4.14.3.jar + - org.apache.distributedlog-distributedlog-core-4.14.3-tests.jar + - org.apache.distributedlog-distributedlog-core-4.14.3.jar + - org.apache.distributedlog-distributedlog-protocol-4.14.3.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.3.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.13.jar diff --git a/pom.xml b/pom.xml index f142eb93b9394..362f348f3d0e1 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.14.2 + 4.14.3 3.6.3 1.1.7 3.2.5 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 94419176d58d1..41f0f938b3baa 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -416,18 +416,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.14.2.jar - - bookkeeper-common-allocator-4.14.2.jar - - bookkeeper-proto-4.14.2.jar - - bookkeeper-server-4.14.2.jar - - bookkeeper-stats-api-4.14.2.jar - - bookkeeper-tools-framework-4.14.2.jar - - circe-checksum-4.14.2.jar - - codahale-metrics-provider-4.14.2.jar - - cpu-affinity-4.14.2.jar - - http-server-4.14.2.jar - - prometheus-metrics-provider-4.14.2.jar - - codahale-metrics-provider-4.14.2.jar + - bookkeeper-common-4.14.3.jar + - bookkeeper-common-allocator-4.14.3.jar + - bookkeeper-proto-4.14.3.jar + - bookkeeper-server-4.14.3.jar + - bookkeeper-stats-api-4.14.3.jar + - bookkeeper-tools-framework-4.14.3.jar + - circe-checksum-4.14.3.jar + - codahale-metrics-provider-4.14.3.jar + - cpu-affinity-4.14.3.jar + - http-server-4.14.3.jar + - prometheus-metrics-provider-4.14.3.jar + - codahale-metrics-provider-4.14.3.jar * Apache Commons - commons-cli-1.2.jar - commons-codec-1.15.jar diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java index 184fc7f4c025d..910c829761be4 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.zk.ZKMetadataClientDriver; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; @@ -32,7 +33,7 @@ public BookKeeperTestClient(ClientConfiguration conf) throws IOException, Interr } public ZooKeeper getZkHandle() { - return super.getZkHandle(); + return ((ZKMetadataClientDriver) metadataDriver).getZk(); } public ClientConfiguration getConf() { From 4a406e0179bed911d44f530c3bf808170abd8f78 Mon Sep 17 00:00:00 2001 From: billowqiu Date: Thu, 9 Dec 2021 10:44:10 +0800 Subject: [PATCH 214/823] [C++]Fix libcurl miss auth header when broker return 307 (#13112) * [C++]Fix libcurl miss auth header when broker return 307 Motivation when broker restart, it will return 307 to client for lookup request. but libcurl miss auth heander when follow new url which will issue the follow lookup req fail with 401. Modifications don't use libcurl CURLOPT_FOLLOWLOCATION to follow new url, instead use retry request with auth header. (cherry picked from commit dd8b47387f1b33de513d02b0778e0622c071518b) --- pulsar-client-cpp/lib/HTTPLookupService.cc | 229 +++++++++++---------- pulsar-client-cpp/lib/HTTPLookupService.h | 2 +- 2 files changed, 123 insertions(+), 108 deletions(-) diff --git a/pulsar-client-cpp/lib/HTTPLookupService.cc b/pulsar-client-cpp/lib/HTTPLookupService.cc index d377171b83c52..340f67c050e53 100644 --- a/pulsar-client-cpp/lib/HTTPLookupService.cc +++ b/pulsar-client-cpp/lib/HTTPLookupService.cc @@ -38,6 +38,8 @@ const static int MAX_HTTP_REDIRECTS = 20; const static std::string PARTITION_METHOD_NAME = "partitions"; const static int NUMBER_OF_LOOKUP_THREADS = 1; +static inline bool needRedirection(long code) { return (code == 307 || code == 302 || code == 301); } + HTTPLookupService::CurlInitializer::CurlInitializer() { // Once per application - https://curl.haxx.se/mail/lib-2015-11/0052.html curl_global_init(CURL_GLOBAL_ALL); @@ -148,132 +150,145 @@ void HTTPLookupService::handleNamespaceTopicsHTTPRequest(NamespaceTopicsPromise } } -Result HTTPLookupService::sendHTTPRequest(const std::string completeUrl, std::string &responseData) { - CURL *handle; - CURLcode res; - std::string version = std::string("Pulsar-CPP-v") + _PULSAR_VERSION_INTERNAL_; - handle = curl_easy_init(); - - if (!handle) { - LOG_ERROR("Unable to curl_easy_init for url " << completeUrl); - // No curl_easy_cleanup required since handle not initialized - return ResultLookupError; - } - // set URL - curl_easy_setopt(handle, CURLOPT_URL, completeUrl.c_str()); - - // Write callback - curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlWriteCallback); - curl_easy_setopt(handle, CURLOPT_WRITEDATA, &responseData); - - // New connection is made for each call - curl_easy_setopt(handle, CURLOPT_FRESH_CONNECT, 1L); - curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); +Result HTTPLookupService::sendHTTPRequest(std::string completeUrl, std::string &responseData) { + uint16_t reqCount = 0; + Result retResult = ResultOk; + while (++reqCount <= MAX_HTTP_REDIRECTS) { + CURL *handle; + CURLcode res; + std::string version = std::string("Pulsar-CPP-v") + _PULSAR_VERSION_INTERNAL_; + handle = curl_easy_init(); + + if (!handle) { + LOG_ERROR("Unable to curl_easy_init for url " << completeUrl); + // No curl_easy_cleanup required since handle not initialized + return ResultLookupError; + } + // set URL + curl_easy_setopt(handle, CURLOPT_URL, completeUrl.c_str()); - // Skipping signal handling - results in timeouts not honored during the DNS lookup - curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); + // Write callback + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlWriteCallback); + curl_easy_setopt(handle, CURLOPT_WRITEDATA, &responseData); - // Timer - curl_easy_setopt(handle, CURLOPT_TIMEOUT, lookupTimeoutInSeconds_); + // New connection is made for each call + curl_easy_setopt(handle, CURLOPT_FRESH_CONNECT, 1L); + curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); - // Set User Agent - curl_easy_setopt(handle, CURLOPT_USERAGENT, version.c_str()); + // Skipping signal handling - results in timeouts not honored during the DNS lookup + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); - // Redirects - curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(handle, CURLOPT_MAXREDIRS, MAX_HTTP_REDIRECTS); + // Timer + curl_easy_setopt(handle, CURLOPT_TIMEOUT, lookupTimeoutInSeconds_); - // Fail if HTTP return code >=400 - curl_easy_setopt(handle, CURLOPT_FAILONERROR, 1L); + // Set User Agent + curl_easy_setopt(handle, CURLOPT_USERAGENT, version.c_str()); - // Authorization data - AuthenticationDataPtr authDataContent; - Result authResult = authenticationPtr_->getAuthData(authDataContent); - if (authResult != ResultOk) { - LOG_ERROR("Failed to getAuthData: " << authResult); - curl_easy_cleanup(handle); - return authResult; - } - struct curl_slist *list = NULL; - if (authDataContent->hasDataForHttp()) { - list = curl_slist_append(list, authDataContent->getHttpHeaders().c_str()); - } - curl_easy_setopt(handle, CURLOPT_HTTPHEADER, list); + // Fail if HTTP return code >=400 + curl_easy_setopt(handle, CURLOPT_FAILONERROR, 1L); - // TLS - if (isUseTls_) { - if (curl_easy_setopt(handle, CURLOPT_SSLENGINE, NULL) != CURLE_OK) { - LOG_ERROR("Unable to load SSL engine for url " << completeUrl); + // Authorization data + AuthenticationDataPtr authDataContent; + Result authResult = authenticationPtr_->getAuthData(authDataContent); + if (authResult != ResultOk) { + LOG_ERROR("Failed to getAuthData: " << authResult); curl_easy_cleanup(handle); - return ResultConnectError; + return authResult; } - if (curl_easy_setopt(handle, CURLOPT_SSLENGINE_DEFAULT, 1L) != CURLE_OK) { - LOG_ERROR("Unable to load SSL engine as default, for url " << completeUrl); - curl_easy_cleanup(handle); - return ResultConnectError; + struct curl_slist *list = NULL; + if (authDataContent->hasDataForHttp()) { + list = curl_slist_append(list, authDataContent->getHttpHeaders().c_str()); } - curl_easy_setopt(handle, CURLOPT_SSLCERTTYPE, "PEM"); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, list); + + // TLS + if (isUseTls_) { + if (curl_easy_setopt(handle, CURLOPT_SSLENGINE, NULL) != CURLE_OK) { + LOG_ERROR("Unable to load SSL engine for url " << completeUrl); + curl_easy_cleanup(handle); + return ResultConnectError; + } + if (curl_easy_setopt(handle, CURLOPT_SSLENGINE_DEFAULT, 1L) != CURLE_OK) { + LOG_ERROR("Unable to load SSL engine as default, for url " << completeUrl); + curl_easy_cleanup(handle); + return ResultConnectError; + } + curl_easy_setopt(handle, CURLOPT_SSLCERTTYPE, "PEM"); - if (tlsAllowInsecure_) { - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); - } else { - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 1L); - } + if (tlsAllowInsecure_) { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + } else { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 1L); + } - if (!tlsTrustCertsFilePath_.empty()) { - curl_easy_setopt(handle, CURLOPT_CAINFO, tlsTrustCertsFilePath_.c_str()); - } + if (!tlsTrustCertsFilePath_.empty()) { + curl_easy_setopt(handle, CURLOPT_CAINFO, tlsTrustCertsFilePath_.c_str()); + } - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, tlsValidateHostname_ ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, tlsValidateHostname_ ? 1L : 0L); - if (authDataContent->hasDataForTls()) { - curl_easy_setopt(handle, CURLOPT_SSLCERT, authDataContent->getTlsCertificates().c_str()); - curl_easy_setopt(handle, CURLOPT_SSLKEY, authDataContent->getTlsPrivateKey().c_str()); + if (authDataContent->hasDataForTls()) { + curl_easy_setopt(handle, CURLOPT_SSLCERT, authDataContent->getTlsCertificates().c_str()); + curl_easy_setopt(handle, CURLOPT_SSLKEY, authDataContent->getTlsPrivateKey().c_str()); + } } - } - - LOG_INFO("Curl Lookup Request sent for " << completeUrl); - - // Make get call to server - res = curl_easy_perform(handle); - - // Free header list - curl_slist_free_all(list); - Result retResult = ResultOk; - - switch (res) { - case CURLE_OK: - long response_code; - curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); - LOG_INFO("Response received for url " << completeUrl << " code " << response_code); - if (response_code == 200) { - retResult = ResultOk; - } else { + LOG_INFO("Curl [" << reqCount << "] Lookup Request sent for " << completeUrl); + + // Make get call to server + res = curl_easy_perform(handle); + + long response_code = -1; + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); + LOG_INFO("Response received for url " << completeUrl << " response_code " << response_code + << " curl res " << res); + + // Free header list + curl_slist_free_all(list); + + switch (res) { + case CURLE_OK: + long response_code; + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); + LOG_INFO("Response received for url " << completeUrl << " code " << response_code); + if (response_code == 200) { + retResult = ResultOk; + } else if (needRedirection(response_code)) { + char *url = NULL; + curl_easy_getinfo(handle, CURLINFO_REDIRECT_URL, &url); + LOG_INFO("Response from url " << completeUrl << " to new url " << url); + completeUrl = url; + retResult = ResultLookupError; + } else { + retResult = ResultLookupError; + } + break; + case CURLE_COULDNT_CONNECT: + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_HTTP_RETURNED_ERROR: + LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); + retResult = ResultConnectError; + break; + case CURLE_READ_ERROR: + LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); + retResult = ResultReadError; + break; + case CURLE_OPERATION_TIMEDOUT: + LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); + retResult = ResultTimeout; + break; + default: + LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); retResult = ResultLookupError; - } - break; - case CURLE_COULDNT_CONNECT: - case CURLE_COULDNT_RESOLVE_PROXY: - case CURLE_COULDNT_RESOLVE_HOST: - case CURLE_HTTP_RETURNED_ERROR: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultConnectError; - break; - case CURLE_READ_ERROR: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultReadError; - break; - case CURLE_OPERATION_TIMEDOUT: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultTimeout; - break; - default: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultLookupError; + break; + } + curl_easy_cleanup(handle); + if (!needRedirection(response_code)) { break; + } } - curl_easy_cleanup(handle); + return retResult; } diff --git a/pulsar-client-cpp/lib/HTTPLookupService.h b/pulsar-client-cpp/lib/HTTPLookupService.h index eb7365447afad..3d0d39ee90a97 100644 --- a/pulsar-client-cpp/lib/HTTPLookupService.h +++ b/pulsar-client-cpp/lib/HTTPLookupService.h @@ -57,7 +57,7 @@ class HTTPLookupService : public LookupService, public std::enable_shared_from_t void handleLookupHTTPRequest(LookupPromise, const std::string, RequestType); void handleNamespaceTopicsHTTPRequest(NamespaceTopicsPromise promise, const std::string completeUrl); - Result sendHTTPRequest(const std::string completeUrl, std::string& responseData); + Result sendHTTPRequest(std::string completeUrl, std::string& responseData); public: HTTPLookupService(const std::string&, const ClientConfiguration&, const AuthenticationPtr&); From ac89c7e71b496204285a1e9c4f6cbc7c8225c79e Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Wed, 15 Dec 2021 15:51:22 +0800 Subject: [PATCH 215/823] Cipher params not work in KeyStoreSSLContext (#13322) ### Motivation The Cipher params not work in KeyStoreSSLContext. ### Modifications - if cipher params is null, use `sslEngine.getSupportedCipherSuites()`, else use the cipher params - make fields final - remove unused throw exception (cherry picked from commit 8625ffe0cce476539faf38c7d8a97adbfbe95855) --- .../util/keystoretls/KeyStoreSSLContext.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java index d35fbc37e605b..3825d80165504 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java @@ -22,7 +22,6 @@ import com.google.common.base.Strings; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.security.GeneralSecurityException; import java.security.KeyStore; @@ -35,7 +34,6 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; -import javax.net.ssl.SSLException; import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -66,22 +64,22 @@ public enum Mode { @Getter private final Mode mode; - private String sslProviderString; - private String keyStoreTypeString; - private String keyStorePath; - private String keyStorePassword; - private boolean allowInsecureConnection; - private String trustStoreTypeString; - private String trustStorePath; - private String trustStorePassword; - private boolean needClientAuth; - private Set ciphers; - private Set protocols; + private final String sslProviderString; + private final String keyStoreTypeString; + private final String keyStorePath; + private final String keyStorePassword; + private final boolean allowInsecureConnection; + private final String trustStoreTypeString; + private final String trustStorePath; + private final String trustStorePassword; + private final boolean needClientAuth; + private final Set ciphers; + private final Set protocols; private SSLContext sslContext; - private String protocol = DEFAULT_SSL_PROTOCOL; - private String kmfAlgorithm = DEFAULT_SSL_KEYMANGER_ALGORITHM; - private String tmfAlgorithm = DEFAULT_SSL_TRUSTMANAGER_ALGORITHM; + private final String protocol = DEFAULT_SSL_PROTOCOL; + private final String kmfAlgorithm = DEFAULT_SSL_KEYMANGER_ALGORITHM; + private final String tmfAlgorithm = DEFAULT_SSL_TRUSTMANAGER_ALGORITHM; // only init vars, before using it, need to call createSSLContext to create ssl context. public KeyStoreSSLContext(Mode mode, @@ -109,8 +107,6 @@ public KeyStoreSSLContext(Mode mode, this.trustStorePath = trustStorePath; this.trustStorePassword = trustStorePassword; this.needClientAuth = requireTrustedClientCertOnConnect; - this.ciphers = ciphers; - this.protocols = protocols; if (protocols != null && protocols.size() > 0) { this.protocols = protocols; @@ -189,7 +185,11 @@ public SSLEngine createSSLEngine(String peerHost, int peerPort) { private SSLEngine configureSSLEngine(SSLEngine sslEngine) { sslEngine.setEnabledProtocols(protocols.toArray(new String[0])); - sslEngine.setEnabledCipherSuites(sslEngine.getSupportedCipherSuites()); + if (this.ciphers == null) { + sslEngine.setEnabledCipherSuites(sslEngine.getSupportedCipherSuites()); + } else { + sslEngine.setEnabledCipherSuites(this.ciphers.toArray(new String[0])); + } if (this.mode == Mode.SERVER) { sslEngine.setNeedClientAuth(this.needClientAuth); @@ -210,7 +210,7 @@ public static KeyStoreSSLContext createClientKeyStoreSslContext(String sslProvid String trustStorePassword, Set ciphers, Set protocols) - throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { + throws GeneralSecurityException, IOException { KeyStoreSSLContext keyStoreSSLContext = new KeyStoreSSLContext(Mode.CLIENT, sslProviderString, keyStoreTypeString, @@ -240,7 +240,7 @@ public static KeyStoreSSLContext createServerKeyStoreSslContext(String sslProvid boolean requireTrustedClientCertOnConnect, Set ciphers, Set protocols) - throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { + throws GeneralSecurityException, IOException { KeyStoreSSLContext keyStoreSSLContext = new KeyStoreSSLContext(Mode.SERVER, sslProviderString, keyStoreTypeString, @@ -268,7 +268,7 @@ public static SSLContext createServerSslContext(String sslProviderString, String trustStorePath, String trustStorePassword, boolean requireTrustedClientCertOnConnect) - throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { + throws GeneralSecurityException, IOException { return createServerKeyStoreSslContext( sslProviderString, @@ -295,7 +295,7 @@ public static SSLContext createClientSslContext(String sslProviderString, String trustStorePassword, Set ciphers, Set protocol) - throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { + throws GeneralSecurityException, IOException { KeyStoreSSLContext keyStoreSSLContext = new KeyStoreSSLContext(Mode.CLIENT, sslProviderString, keyStoreTypeString, @@ -319,7 +319,7 @@ public static SSLContext createClientSslContext(String keyStoreTypeString, String trustStoreTypeString, String trustStorePath, String trustStorePassword) - throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { + throws GeneralSecurityException, IOException { KeyStoreSSLContext keyStoreSSLContext = new KeyStoreSSLContext(Mode.CLIENT, null, keyStoreTypeString, @@ -347,7 +347,7 @@ public static SslContextFactory createSslContextFactory(String sslProviderString String trustStorePassword, boolean requireTrustedClientCertOnConnect, long certRefreshInSec) - throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { + throws GeneralSecurityException, IOException { SslContextFactory sslCtxFactory; if (sslProviderString == null) { From f1aeb06c04daf87d2e5b4dcaa4fa1fa6f5d67b7f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 21 Dec 2021 23:51:16 +0800 Subject: [PATCH 216/823] Apply clang-format check for python wrapper (#13418) (cherry picked from commit 636d5b4c2133bcdf8c1988bbe82eb1064565d5dc) --- pulsar-client-cpp/CMakeLists.txt | 6 +- .../python/src/authentication.cc | 44 +-- pulsar-client-cpp/python/src/client.cc | 63 ++--- pulsar-client-cpp/python/src/config.cc | 256 +++++++++--------- pulsar-client-cpp/python/src/consumer.cc | 81 +++--- .../python/src/cryptoKeyReader.cc | 2 +- pulsar-client-cpp/python/src/enums.cc | 162 ++++++----- pulsar-client-cpp/python/src/exceptions.cc | 49 ++-- pulsar-client-cpp/python/src/message.cc | 147 +++++----- pulsar-client-cpp/python/src/producer.cc | 71 +++-- pulsar-client-cpp/python/src/pulsar.cc | 4 +- pulsar-client-cpp/python/src/reader.cc | 48 ++-- pulsar-client-cpp/python/src/schema.cc | 10 +- pulsar-client-cpp/python/src/utils.h | 3 +- 14 files changed, 441 insertions(+), 505 deletions(-) diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 8ba9569d7f690..3fadb05a092a7 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -446,7 +446,8 @@ add_custom_target(format python ${BUILD_SUPPORT_DIR}/run_clang_format.py ${CMAKE_SOURCE_DIR}/perf ${CMAKE_SOURCE_DIR}/examples ${CMAKE_SOURCE_DIR}/tests - ${CMAKE_SOURCE_DIR}/include) + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/python/src) # `make check-format` option (for CI test) add_custom_target(check-format python ${BUILD_SUPPORT_DIR}/run_clang_format.py @@ -457,4 +458,5 @@ add_custom_target(check-format python ${BUILD_SUPPORT_DIR}/run_clang_format.py ${CMAKE_SOURCE_DIR}/perf ${CMAKE_SOURCE_DIR}/examples ${CMAKE_SOURCE_DIR}/tests - ${CMAKE_SOURCE_DIR}/include) + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/python/src) diff --git a/pulsar-client-cpp/python/src/authentication.cc b/pulsar-client-cpp/python/src/authentication.cc index e236c7e0843c5..920a7174b47bb 100644 --- a/pulsar-client-cpp/python/src/authentication.cc +++ b/pulsar-client-cpp/python/src/authentication.cc @@ -26,8 +26,8 @@ AuthenticationWrapper::AuthenticationWrapper(const std::string& dynamicLibPath, } struct AuthenticationTlsWrapper : public AuthenticationWrapper { - AuthenticationTlsWrapper(const std::string& certificatePath, const std::string& privateKeyPath) : - AuthenticationWrapper() { + AuthenticationTlsWrapper(const std::string& certificatePath, const std::string& privateKeyPath) + : AuthenticationWrapper() { this->auth = AuthTls::create(certificatePath, privateKeyPath); } }; @@ -35,13 +35,10 @@ struct AuthenticationTlsWrapper : public AuthenticationWrapper { struct TokenSupplierWrapper { PyObject* _pySupplier; - TokenSupplierWrapper(py::object pySupplier) : - _pySupplier(pySupplier.ptr()) { - Py_XINCREF(_pySupplier); - } + TokenSupplierWrapper(py::object pySupplier) : _pySupplier(pySupplier.ptr()) { Py_XINCREF(_pySupplier); } TokenSupplierWrapper(const TokenSupplierWrapper& other) { - _pySupplier= other._pySupplier; + _pySupplier = other._pySupplier; Py_XINCREF(_pySupplier); } @@ -51,9 +48,7 @@ struct TokenSupplierWrapper { return *this; } - virtual ~TokenSupplierWrapper() { - Py_XDECREF(_pySupplier); - } + virtual ~TokenSupplierWrapper() { Py_XDECREF(_pySupplier); } std::string operator()() { PyGILState_STATE state = PyGILState_Ensure(); @@ -61,7 +56,7 @@ struct TokenSupplierWrapper { std::string token; try { token = py::call(_pySupplier); - } catch(const py::error_already_set& e) { + } catch (const py::error_already_set& e) { PyErr_Print(); } @@ -70,10 +65,8 @@ struct TokenSupplierWrapper { } }; - struct AuthenticationTokenWrapper : public AuthenticationWrapper { - AuthenticationTokenWrapper(py::object token) : - AuthenticationWrapper() { + AuthenticationTokenWrapper(py::object token) : AuthenticationWrapper() { if (py::extract(token).check()) { // It's a string std::string tokenStr = py::extract(token); @@ -86,15 +79,13 @@ struct AuthenticationTokenWrapper : public AuthenticationWrapper { }; struct AuthenticationAthenzWrapper : public AuthenticationWrapper { - AuthenticationAthenzWrapper(const std::string& authParamsString) : - AuthenticationWrapper() { + AuthenticationAthenzWrapper(const std::string& authParamsString) : AuthenticationWrapper() { this->auth = AuthAthenz::create(authParamsString); } }; struct AuthenticationOauth2Wrapper : public AuthenticationWrapper { - AuthenticationOauth2Wrapper(const std::string& authParamsString) : - AuthenticationWrapper() { + AuthenticationOauth2Wrapper(const std::string& authParamsString) : AuthenticationWrapper() { this->auth = AuthOauth2::create(authParamsString); } }; @@ -102,22 +93,17 @@ struct AuthenticationOauth2Wrapper : public AuthenticationWrapper { void export_authentication() { using namespace boost::python; - class_("Authentication", init()) - ; + class_("Authentication", init()); - class_ >("AuthenticationTLS", - init()) - ; + class_ >( + "AuthenticationTLS", init()); class_ >("AuthenticationToken", - init()) - ; + init()); class_ >("AuthenticationAthenz", - init()) - ; + init()); class_ >("AuthenticationOauth2", - init()) - ; + init()); } diff --git a/pulsar-client-cpp/python/src/client.cc b/pulsar-client-cpp/python/src/client.cc index 3dcbf7fc831a0..445a6dcfa0c37 100644 --- a/pulsar-client-cpp/python/src/client.cc +++ b/pulsar-client-cpp/python/src/client.cc @@ -22,11 +22,10 @@ Producer Client_createProducer(Client& client, const std::string& topic, const P Producer producer; Result res; - Py_BEGIN_ALLOW_THREADS - res = client.createProducer(topic, conf, producer); + Py_BEGIN_ALLOW_THREADS res = client.createProducer(topic, conf, producer); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return producer; } @@ -35,11 +34,10 @@ Consumer Client_subscribe(Client& client, const std::string& topic, const std::s Consumer consumer; Result res; - Py_BEGIN_ALLOW_THREADS - res = client.subscribe(topic, subscriptionName, conf, consumer); + Py_BEGIN_ALLOW_THREADS res = client.subscribe(topic, subscriptionName, conf, consumer); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return consumer; } @@ -50,43 +48,39 @@ Consumer Client_subscribe_topics(Client& client, boost::python::list& topics, std::vector topics_vector; - for (int i = 0; i < len(topics); i ++) { + for (int i = 0; i < len(topics); i++) { std::string content = boost::python::extract(topics[i]); topics_vector.push_back(content); } - Py_BEGIN_ALLOW_THREADS - res = client.subscribe(topics_vector, subscriptionName, conf, consumer); + Py_BEGIN_ALLOW_THREADS res = client.subscribe(topics_vector, subscriptionName, conf, consumer); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return consumer; } -Consumer Client_subscribe_pattern(Client& client, const std::string& topic_pattern, const std::string& subscriptionName, - const ConsumerConfiguration& conf) { +Consumer Client_subscribe_pattern(Client& client, const std::string& topic_pattern, + const std::string& subscriptionName, const ConsumerConfiguration& conf) { Consumer consumer; Result res; - Py_BEGIN_ALLOW_THREADS - res = client.subscribeWithRegex(topic_pattern, subscriptionName, conf, consumer); + Py_BEGIN_ALLOW_THREADS res = client.subscribeWithRegex(topic_pattern, subscriptionName, conf, consumer); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return consumer; } -Reader Client_createReader(Client& client, const std::string& topic, - const MessageId& startMessageId, +Reader Client_createReader(Client& client, const std::string& topic, const MessageId& startMessageId, const ReaderConfiguration& conf) { Reader reader; Result res; - Py_BEGIN_ALLOW_THREADS - res = client.createReader(topic, startMessageId, conf, reader); + Py_BEGIN_ALLOW_THREADS res = client.createReader(topic, startMessageId, conf, reader); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return reader; } @@ -94,11 +88,10 @@ boost::python::list Client_getTopicPartitions(Client& client, const std::string& std::vector partitions; Result res; - Py_BEGIN_ALLOW_THREADS - res = client.getPartitionsForTopic(topic, partitions); + Py_BEGIN_ALLOW_THREADS res = client.getPartitionsForTopic(topic, partitions); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); boost::python::list pyList; for (int i = 0; i < partitions.size(); i++) { @@ -111,24 +104,22 @@ boost::python::list Client_getTopicPartitions(Client& client, const std::string& void Client_close(Client& client) { Result res; - Py_BEGIN_ALLOW_THREADS - res = client.close(); + Py_BEGIN_ALLOW_THREADS res = client.close(); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void export_client() { using namespace boost::python; - class_("Client", init()) - .def("create_producer", &Client_createProducer) - .def("subscribe", &Client_subscribe) - .def("subscribe_topics", &Client_subscribe_topics) - .def("subscribe_pattern", &Client_subscribe_pattern) - .def("create_reader", &Client_createReader) - .def("get_topic_partitions", &Client_getTopicPartitions) - .def("close", &Client_close) - .def("shutdown", &Client::shutdown) - ; + class_("Client", init()) + .def("create_producer", &Client_createProducer) + .def("subscribe", &Client_subscribe) + .def("subscribe_topics", &Client_subscribe_topics) + .def("subscribe_pattern", &Client_subscribe_pattern) + .def("create_reader", &Client_createReader) + .def("get_topic_partitions", &Client_getTopicPartitions) + .def("close", &Client_close) + .def("shutdown", &Client::shutdown); } diff --git a/pulsar-client-cpp/python/src/config.cc b/pulsar-client-cpp/python/src/config.cc index a2872481b10da..2dee1a1183d81 100644 --- a/pulsar-client-cpp/python/src/config.cc +++ b/pulsar-client-cpp/python/src/config.cc @@ -21,14 +21,11 @@ #include "lib/Utils.h" #include -template +template struct ListenerWrapper { PyObject* _pyListener; - ListenerWrapper(py::object pyListener) : - _pyListener(pyListener.ptr()) { - Py_XINCREF(_pyListener); - } + ListenerWrapper(py::object pyListener) : _pyListener(pyListener.ptr()) { Py_XINCREF(_pyListener); } ListenerWrapper(const ListenerWrapper& other) { _pyListener = other._pyListener; @@ -41,9 +38,7 @@ struct ListenerWrapper { return *this; } - virtual ~ListenerWrapper() { - Py_XDECREF(_pyListener); - } + virtual ~ListenerWrapper() { Py_XDECREF(_pyListener); } void operator()(T consumer, const Message& msg) { PyGILState_STATE state = PyGILState_Ensure(); @@ -65,7 +60,7 @@ static ConsumerConfiguration& ConsumerConfiguration_setMessageListener(ConsumerC } static ReaderConfiguration& ReaderConfiguration_setReaderListener(ReaderConfiguration& conf, - py::object pyListener) { + py::object pyListener) { conf.setReaderListener(ListenerWrapper(pyListener)); return conf; } @@ -78,41 +73,36 @@ static ClientConfiguration& ClientConfiguration_setAuthentication(ClientConfigur } static ConsumerConfiguration& ConsumerConfiguration_setCryptoKeyReader(ConsumerConfiguration& conf, - py::object cryptoKeyReader) { + py::object cryptoKeyReader) { CryptoKeyReaderWrapper cryptoKeyReaderWrapper = py::extract(cryptoKeyReader); conf.setCryptoKeyReader(cryptoKeyReaderWrapper.cryptoKeyReader); return conf; } static ProducerConfiguration& ProducerConfiguration_setCryptoKeyReader(ProducerConfiguration& conf, - py::object cryptoKeyReader) { + py::object cryptoKeyReader) { CryptoKeyReaderWrapper cryptoKeyReaderWrapper = py::extract(cryptoKeyReader); conf.setCryptoKeyReader(cryptoKeyReaderWrapper.cryptoKeyReader); return conf; } static ReaderConfiguration& ReaderConfiguration_setCryptoKeyReader(ReaderConfiguration& conf, - py::object cryptoKeyReader) { + py::object cryptoKeyReader) { CryptoKeyReaderWrapper cryptoKeyReaderWrapper = py::extract(cryptoKeyReader); conf.setCryptoKeyReader(cryptoKeyReaderWrapper.cryptoKeyReader); return conf; } -class LoggerWrapper: public Logger { +class LoggerWrapper : public Logger { PyObject* const _pyLogger; const int _pythonLogLevel; const std::unique_ptr _fallbackLogger; - static constexpr int _getLogLevelValue(Level level) { - return 10 + (level * 10); - } + static constexpr int _getLogLevelValue(Level level) { return 10 + (level * 10); } public: - LoggerWrapper(PyObject* pyLogger, int pythonLogLevel, Logger* fallbackLogger) - : _pyLogger(pyLogger), - _pythonLogLevel(pythonLogLevel), - _fallbackLogger(fallbackLogger) { + : _pyLogger(pyLogger), _pythonLogLevel(pythonLogLevel), _fallbackLogger(fallbackLogger) { Py_XINCREF(_pyLogger); } @@ -121,13 +111,9 @@ class LoggerWrapper: public Logger { LoggerWrapper& operator=(const LoggerWrapper&) = delete; LoggerWrapper& operator=(LoggerWrapper&&) = delete; - virtual ~LoggerWrapper() { - Py_XDECREF(_pyLogger); - } + virtual ~LoggerWrapper() { Py_XDECREF(_pyLogger); } - bool isEnabled(Level level) { - return _getLogLevelValue(level) >= _pythonLogLevel; - } + bool isEnabled(Level level) { return _getLogLevelValue(level) >= _pythonLogLevel; } void log(Level level, int line, const std::string& message) { if (!Py_IsInitialized()) { @@ -187,11 +173,9 @@ class LoggerWrapperFactory : public LoggerFactory { initializePythonLogLevel(); } - virtual ~LoggerWrapperFactory() { - Py_XDECREF(_pyLogger); - } + virtual ~LoggerWrapperFactory() { Py_XDECREF(_pyLogger); } - Logger* getLogger(const std::string &fileName) { + Logger* getLogger(const std::string& fileName) { const auto fallbackLogger = _fallbackLoggerFactory->getLogger(fileName); if (_pythonLogLevel.is_present()) { return new LoggerWrapper(_pyLogger, _pythonLogLevel.value(), fallbackLogger); @@ -206,112 +190,128 @@ static ClientConfiguration& ClientConfiguration_setLogger(ClientConfiguration& c return conf; } - void export_config() { using namespace boost::python; class_("ClientConfiguration") - .def("authentication", &ClientConfiguration_setAuthentication, return_self<>()) - .def("operation_timeout_seconds", &ClientConfiguration::getOperationTimeoutSeconds) - .def("operation_timeout_seconds", &ClientConfiguration::setOperationTimeoutSeconds, return_self<>()) - .def("connection_timeout", &ClientConfiguration::getConnectionTimeout) - .def("connection_timeout", &ClientConfiguration::setConnectionTimeout, return_self<>()) - .def("io_threads", &ClientConfiguration::getIOThreads) - .def("io_threads", &ClientConfiguration::setIOThreads, return_self<>()) - .def("message_listener_threads", &ClientConfiguration::getMessageListenerThreads) - .def("message_listener_threads", &ClientConfiguration::setMessageListenerThreads, return_self<>()) - .def("concurrent_lookup_requests", &ClientConfiguration::getConcurrentLookupRequest) - .def("concurrent_lookup_requests", &ClientConfiguration::setConcurrentLookupRequest, return_self<>()) - .def("log_conf_file_path", &ClientConfiguration::getLogConfFilePath, return_value_policy()) - .def("log_conf_file_path", &ClientConfiguration::setLogConfFilePath, return_self<>()) - .def("use_tls", &ClientConfiguration::isUseTls) - .def("use_tls", &ClientConfiguration::setUseTls, return_self<>()) - .def("tls_trust_certs_file_path", &ClientConfiguration::getTlsTrustCertsFilePath, return_value_policy()) - .def("tls_trust_certs_file_path", &ClientConfiguration::setTlsTrustCertsFilePath, return_self<>()) - .def("tls_allow_insecure_connection", &ClientConfiguration::isTlsAllowInsecureConnection) - .def("tls_allow_insecure_connection", &ClientConfiguration::setTlsAllowInsecureConnection, return_self<>()) - .def("tls_validate_hostname", &ClientConfiguration::setValidateHostName, return_self<>()) - .def("set_logger", &ClientConfiguration_setLogger, return_self<>()) - ; + .def("authentication", &ClientConfiguration_setAuthentication, return_self<>()) + .def("operation_timeout_seconds", &ClientConfiguration::getOperationTimeoutSeconds) + .def("operation_timeout_seconds", &ClientConfiguration::setOperationTimeoutSeconds, return_self<>()) + .def("connection_timeout", &ClientConfiguration::getConnectionTimeout) + .def("connection_timeout", &ClientConfiguration::setConnectionTimeout, return_self<>()) + .def("io_threads", &ClientConfiguration::getIOThreads) + .def("io_threads", &ClientConfiguration::setIOThreads, return_self<>()) + .def("message_listener_threads", &ClientConfiguration::getMessageListenerThreads) + .def("message_listener_threads", &ClientConfiguration::setMessageListenerThreads, return_self<>()) + .def("concurrent_lookup_requests", &ClientConfiguration::getConcurrentLookupRequest) + .def("concurrent_lookup_requests", &ClientConfiguration::setConcurrentLookupRequest, return_self<>()) + .def("log_conf_file_path", &ClientConfiguration::getLogConfFilePath, + return_value_policy()) + .def("log_conf_file_path", &ClientConfiguration::setLogConfFilePath, return_self<>()) + .def("use_tls", &ClientConfiguration::isUseTls) + .def("use_tls", &ClientConfiguration::setUseTls, return_self<>()) + .def("tls_trust_certs_file_path", &ClientConfiguration::getTlsTrustCertsFilePath, + return_value_policy()) + .def("tls_trust_certs_file_path", &ClientConfiguration::setTlsTrustCertsFilePath, return_self<>()) + .def("tls_allow_insecure_connection", &ClientConfiguration::isTlsAllowInsecureConnection) + .def("tls_allow_insecure_connection", &ClientConfiguration::setTlsAllowInsecureConnection, + return_self<>()) + .def("tls_validate_hostname", &ClientConfiguration::setValidateHostName, return_self<>()) + .def("set_logger", &ClientConfiguration_setLogger, return_self<>()); class_("ProducerConfiguration") - .def("producer_name", &ProducerConfiguration::getProducerName, return_value_policy()) - .def("producer_name", &ProducerConfiguration::setProducerName, return_self<>()) - .def("schema", &ProducerConfiguration::getSchema, return_value_policy()) - .def("schema", &ProducerConfiguration::setSchema, return_self<>()) - .def("send_timeout_millis", &ProducerConfiguration::getSendTimeout) - .def("send_timeout_millis", &ProducerConfiguration::setSendTimeout, return_self<>()) - .def("initial_sequence_id", &ProducerConfiguration::getInitialSequenceId) - .def("initial_sequence_id", &ProducerConfiguration::setInitialSequenceId, return_self<>()) - .def("compression_type", &ProducerConfiguration::getCompressionType) - .def("compression_type", &ProducerConfiguration::setCompressionType, return_self<>()) - .def("max_pending_messages", &ProducerConfiguration::getMaxPendingMessages) - .def("max_pending_messages", &ProducerConfiguration::setMaxPendingMessages, return_self<>()) - .def("max_pending_messages_across_partitions", &ProducerConfiguration::getMaxPendingMessagesAcrossPartitions) - .def("max_pending_messages_across_partitions", &ProducerConfiguration::setMaxPendingMessagesAcrossPartitions, return_self<>()) - .def("block_if_queue_full", &ProducerConfiguration::getBlockIfQueueFull) - .def("block_if_queue_full", &ProducerConfiguration::setBlockIfQueueFull, return_self<>()) - .def("partitions_routing_mode", &ProducerConfiguration::getPartitionsRoutingMode) - .def("partitions_routing_mode", &ProducerConfiguration::setPartitionsRoutingMode, return_self<>()) - .def("lazy_start_partitioned_producers", &ProducerConfiguration::getLazyStartPartitionedProducers) - .def("lazy_start_partitioned_producers", &ProducerConfiguration::setLazyStartPartitionedProducers, return_self<>()) - .def("batching_enabled", &ProducerConfiguration::getBatchingEnabled, return_value_policy()) - .def("batching_enabled", &ProducerConfiguration::setBatchingEnabled, return_self<>()) - .def("batching_max_messages", &ProducerConfiguration::getBatchingMaxMessages, return_value_policy()) - .def("batching_max_messages", &ProducerConfiguration::setBatchingMaxMessages, return_self<>()) - .def("batching_max_allowed_size_in_bytes", &ProducerConfiguration::getBatchingMaxAllowedSizeInBytes, return_value_policy()) - .def("batching_max_allowed_size_in_bytes", &ProducerConfiguration::setBatchingMaxAllowedSizeInBytes, return_self<>()) - .def("batching_max_publish_delay_ms", &ProducerConfiguration::getBatchingMaxPublishDelayMs, return_value_policy()) - .def("batching_max_publish_delay_ms", &ProducerConfiguration::setBatchingMaxPublishDelayMs, return_self<>()) - .def("property", &ProducerConfiguration::setProperty, return_self<>()) - .def("batching_type", &ProducerConfiguration::setBatchingType, return_self<>()) - .def("batching_type", &ProducerConfiguration::getBatchingType) - .def("encryption_key", &ProducerConfiguration::addEncryptionKey, return_self<>()) - .def("crypto_key_reader", &ProducerConfiguration_setCryptoKeyReader, return_self<>()) - ; + .def("producer_name", &ProducerConfiguration::getProducerName, + return_value_policy()) + .def("producer_name", &ProducerConfiguration::setProducerName, return_self<>()) + .def("schema", &ProducerConfiguration::getSchema, return_value_policy()) + .def("schema", &ProducerConfiguration::setSchema, return_self<>()) + .def("send_timeout_millis", &ProducerConfiguration::getSendTimeout) + .def("send_timeout_millis", &ProducerConfiguration::setSendTimeout, return_self<>()) + .def("initial_sequence_id", &ProducerConfiguration::getInitialSequenceId) + .def("initial_sequence_id", &ProducerConfiguration::setInitialSequenceId, return_self<>()) + .def("compression_type", &ProducerConfiguration::getCompressionType) + .def("compression_type", &ProducerConfiguration::setCompressionType, return_self<>()) + .def("max_pending_messages", &ProducerConfiguration::getMaxPendingMessages) + .def("max_pending_messages", &ProducerConfiguration::setMaxPendingMessages, return_self<>()) + .def("max_pending_messages_across_partitions", + &ProducerConfiguration::getMaxPendingMessagesAcrossPartitions) + .def("max_pending_messages_across_partitions", + &ProducerConfiguration::setMaxPendingMessagesAcrossPartitions, return_self<>()) + .def("block_if_queue_full", &ProducerConfiguration::getBlockIfQueueFull) + .def("block_if_queue_full", &ProducerConfiguration::setBlockIfQueueFull, return_self<>()) + .def("partitions_routing_mode", &ProducerConfiguration::getPartitionsRoutingMode) + .def("partitions_routing_mode", &ProducerConfiguration::setPartitionsRoutingMode, return_self<>()) + .def("lazy_start_partitioned_producers", &ProducerConfiguration::getLazyStartPartitionedProducers) + .def("lazy_start_partitioned_producers", &ProducerConfiguration::setLazyStartPartitionedProducers, + return_self<>()) + .def("batching_enabled", &ProducerConfiguration::getBatchingEnabled, + return_value_policy()) + .def("batching_enabled", &ProducerConfiguration::setBatchingEnabled, return_self<>()) + .def("batching_max_messages", &ProducerConfiguration::getBatchingMaxMessages, + return_value_policy()) + .def("batching_max_messages", &ProducerConfiguration::setBatchingMaxMessages, return_self<>()) + .def("batching_max_allowed_size_in_bytes", &ProducerConfiguration::getBatchingMaxAllowedSizeInBytes, + return_value_policy()) + .def("batching_max_allowed_size_in_bytes", &ProducerConfiguration::setBatchingMaxAllowedSizeInBytes, + return_self<>()) + .def("batching_max_publish_delay_ms", &ProducerConfiguration::getBatchingMaxPublishDelayMs, + return_value_policy()) + .def("batching_max_publish_delay_ms", &ProducerConfiguration::setBatchingMaxPublishDelayMs, + return_self<>()) + .def("property", &ProducerConfiguration::setProperty, return_self<>()) + .def("batching_type", &ProducerConfiguration::setBatchingType, return_self<>()) + .def("batching_type", &ProducerConfiguration::getBatchingType) + .def("encryption_key", &ProducerConfiguration::addEncryptionKey, return_self<>()) + .def("crypto_key_reader", &ProducerConfiguration_setCryptoKeyReader, return_self<>()); class_("ConsumerConfiguration") - .def("consumer_type", &ConsumerConfiguration::getConsumerType) - .def("consumer_type", &ConsumerConfiguration::setConsumerType, return_self<>()) - .def("schema", &ConsumerConfiguration::getSchema, return_value_policy()) - .def("schema", &ConsumerConfiguration::setSchema, return_self<>()) - .def("message_listener", &ConsumerConfiguration_setMessageListener, return_self<>()) - .def("receiver_queue_size", &ConsumerConfiguration::getReceiverQueueSize) - .def("receiver_queue_size", &ConsumerConfiguration::setReceiverQueueSize) - .def("max_total_receiver_queue_size_across_partitions", &ConsumerConfiguration::getMaxTotalReceiverQueueSizeAcrossPartitions) - .def("max_total_receiver_queue_size_across_partitions", &ConsumerConfiguration::setMaxTotalReceiverQueueSizeAcrossPartitions) - .def("consumer_name", &ConsumerConfiguration::getConsumerName, return_value_policy()) - .def("consumer_name", &ConsumerConfiguration::setConsumerName) - .def("unacked_messages_timeout_ms", &ConsumerConfiguration::getUnAckedMessagesTimeoutMs) - .def("unacked_messages_timeout_ms", &ConsumerConfiguration::setUnAckedMessagesTimeoutMs) - .def("negative_ack_redelivery_delay_ms", &ConsumerConfiguration::getNegativeAckRedeliveryDelayMs) - .def("negative_ack_redelivery_delay_ms", &ConsumerConfiguration::setNegativeAckRedeliveryDelayMs) - .def("broker_consumer_stats_cache_time_ms", &ConsumerConfiguration::getBrokerConsumerStatsCacheTimeInMs) - .def("broker_consumer_stats_cache_time_ms", &ConsumerConfiguration::setBrokerConsumerStatsCacheTimeInMs) - .def("pattern_auto_discovery_period", &ConsumerConfiguration::getPatternAutoDiscoveryPeriod) - .def("pattern_auto_discovery_period", &ConsumerConfiguration::setPatternAutoDiscoveryPeriod) - .def("read_compacted", &ConsumerConfiguration::isReadCompacted) - .def("read_compacted", &ConsumerConfiguration::setReadCompacted) - .def("property", &ConsumerConfiguration::setProperty, return_self<>()) - .def("subscription_initial_position", &ConsumerConfiguration::getSubscriptionInitialPosition) - .def("subscription_initial_position", &ConsumerConfiguration::setSubscriptionInitialPosition) - .def("crypto_key_reader", &ConsumerConfiguration_setCryptoKeyReader, return_self<>()) - .def("replicate_subscription_state_enabled", &ConsumerConfiguration::setReplicateSubscriptionStateEnabled) - .def("replicate_subscription_state_enabled", &ConsumerConfiguration::isReplicateSubscriptionStateEnabled) - ; + .def("consumer_type", &ConsumerConfiguration::getConsumerType) + .def("consumer_type", &ConsumerConfiguration::setConsumerType, return_self<>()) + .def("schema", &ConsumerConfiguration::getSchema, return_value_policy()) + .def("schema", &ConsumerConfiguration::setSchema, return_self<>()) + .def("message_listener", &ConsumerConfiguration_setMessageListener, return_self<>()) + .def("receiver_queue_size", &ConsumerConfiguration::getReceiverQueueSize) + .def("receiver_queue_size", &ConsumerConfiguration::setReceiverQueueSize) + .def("max_total_receiver_queue_size_across_partitions", + &ConsumerConfiguration::getMaxTotalReceiverQueueSizeAcrossPartitions) + .def("max_total_receiver_queue_size_across_partitions", + &ConsumerConfiguration::setMaxTotalReceiverQueueSizeAcrossPartitions) + .def("consumer_name", &ConsumerConfiguration::getConsumerName, + return_value_policy()) + .def("consumer_name", &ConsumerConfiguration::setConsumerName) + .def("unacked_messages_timeout_ms", &ConsumerConfiguration::getUnAckedMessagesTimeoutMs) + .def("unacked_messages_timeout_ms", &ConsumerConfiguration::setUnAckedMessagesTimeoutMs) + .def("negative_ack_redelivery_delay_ms", &ConsumerConfiguration::getNegativeAckRedeliveryDelayMs) + .def("negative_ack_redelivery_delay_ms", &ConsumerConfiguration::setNegativeAckRedeliveryDelayMs) + .def("broker_consumer_stats_cache_time_ms", + &ConsumerConfiguration::getBrokerConsumerStatsCacheTimeInMs) + .def("broker_consumer_stats_cache_time_ms", + &ConsumerConfiguration::setBrokerConsumerStatsCacheTimeInMs) + .def("pattern_auto_discovery_period", &ConsumerConfiguration::getPatternAutoDiscoveryPeriod) + .def("pattern_auto_discovery_period", &ConsumerConfiguration::setPatternAutoDiscoveryPeriod) + .def("read_compacted", &ConsumerConfiguration::isReadCompacted) + .def("read_compacted", &ConsumerConfiguration::setReadCompacted) + .def("property", &ConsumerConfiguration::setProperty, return_self<>()) + .def("subscription_initial_position", &ConsumerConfiguration::getSubscriptionInitialPosition) + .def("subscription_initial_position", &ConsumerConfiguration::setSubscriptionInitialPosition) + .def("crypto_key_reader", &ConsumerConfiguration_setCryptoKeyReader, return_self<>()) + .def("replicate_subscription_state_enabled", + &ConsumerConfiguration::setReplicateSubscriptionStateEnabled) + .def("replicate_subscription_state_enabled", + &ConsumerConfiguration::isReplicateSubscriptionStateEnabled); class_("ReaderConfiguration") - .def("reader_listener", &ReaderConfiguration_setReaderListener, return_self<>()) - .def("schema", &ReaderConfiguration::getSchema, return_value_policy()) - .def("schema", &ReaderConfiguration::setSchema, return_self<>()) - .def("receiver_queue_size", &ReaderConfiguration::getReceiverQueueSize) - .def("receiver_queue_size", &ReaderConfiguration::setReceiverQueueSize) - .def("reader_name", &ReaderConfiguration::getReaderName, return_value_policy()) - .def("reader_name", &ReaderConfiguration::setReaderName) - .def("subscription_role_prefix", &ReaderConfiguration::getSubscriptionRolePrefix, return_value_policy()) - .def("subscription_role_prefix", &ReaderConfiguration::setSubscriptionRolePrefix) - .def("read_compacted", &ReaderConfiguration::isReadCompacted) - .def("read_compacted", &ReaderConfiguration::setReadCompacted) - .def("crypto_key_reader", &ReaderConfiguration_setCryptoKeyReader, return_self<>()) - ; + .def("reader_listener", &ReaderConfiguration_setReaderListener, return_self<>()) + .def("schema", &ReaderConfiguration::getSchema, return_value_policy()) + .def("schema", &ReaderConfiguration::setSchema, return_self<>()) + .def("receiver_queue_size", &ReaderConfiguration::getReceiverQueueSize) + .def("receiver_queue_size", &ReaderConfiguration::setReceiverQueueSize) + .def("reader_name", &ReaderConfiguration::getReaderName, return_value_policy()) + .def("reader_name", &ReaderConfiguration::setReaderName) + .def("subscription_role_prefix", &ReaderConfiguration::getSubscriptionRolePrefix, + return_value_policy()) + .def("subscription_role_prefix", &ReaderConfiguration::setSubscriptionRolePrefix) + .def("read_compacted", &ReaderConfiguration::isReadCompacted) + .def("read_compacted", &ReaderConfiguration::setReadCompacted) + .def("crypto_key_reader", &ReaderConfiguration_setCryptoKeyReader, return_self<>()); } diff --git a/pulsar-client-cpp/python/src/consumer.cc b/pulsar-client-cpp/python/src/consumer.cc index 815282d8876ca..28bedad99482a 100644 --- a/pulsar-client-cpp/python/src/consumer.cc +++ b/pulsar-client-cpp/python/src/consumer.cc @@ -20,11 +20,10 @@ void Consumer_unsubscribe(Consumer& consumer) { Result res; - Py_BEGIN_ALLOW_THREADS - res = consumer.unsubscribe(); + Py_BEGIN_ALLOW_THREADS res = consumer.unsubscribe(); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } Message Consumer_receive(Consumer& consumer) { @@ -32,11 +31,10 @@ Message Consumer_receive(Consumer& consumer) { Result res; while (true) { - Py_BEGIN_ALLOW_THREADS - res = consumer.receive(msg); + Py_BEGIN_ALLOW_THREADS res = consumer.receive(msg); Py_END_ALLOW_THREADS - if (res != ResultTimeout) { + if (res != ResultTimeout) { // In case of timeout we keep calling receive() to simulate a // blocking call until a message is available, while breaking // every once in a while to check the Python signal status @@ -56,17 +54,14 @@ Message Consumer_receive(Consumer& consumer) { Message Consumer_receive_timeout(Consumer& consumer, int timeoutMs) { Message msg; Result res; - Py_BEGIN_ALLOW_THREADS - res = consumer.receive(msg, timeoutMs); + Py_BEGIN_ALLOW_THREADS res = consumer.receive(msg, timeoutMs); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return msg; } -void Consumer_acknowledge(Consumer& consumer, const Message& msg) { - consumer.acknowledgeAsync(msg, nullptr); -} +void Consumer_acknowledge(Consumer& consumer, const Message& msg) { consumer.acknowledgeAsync(msg, nullptr); } void Consumer_acknowledge_message_id(Consumer& consumer, const MessageId& msgId) { consumer.acknowledgeAsync(msgId, nullptr); @@ -77,7 +72,7 @@ void Consumer_negative_acknowledge(Consumer& consumer, const Message& msg) { } void Consumer_negative_acknowledge_message_id(Consumer& consumer, const MessageId& msgId) { - consumer.negativeAcknowledge(msgId); + consumer.negativeAcknowledge(msgId); } void Consumer_acknowledge_cumulative(Consumer& consumer, const Message& msg) { @@ -90,60 +85,52 @@ void Consumer_acknowledge_cumulative_message_id(Consumer& consumer, const Messag void Consumer_close(Consumer& consumer) { Result res; - Py_BEGIN_ALLOW_THREADS - res = consumer.close(); + Py_BEGIN_ALLOW_THREADS res = consumer.close(); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } -void Consumer_pauseMessageListener(Consumer& consumer) { - CHECK_RESULT(consumer.pauseMessageListener()); -} +void Consumer_pauseMessageListener(Consumer& consumer) { CHECK_RESULT(consumer.pauseMessageListener()); } -void Consumer_resumeMessageListener(Consumer& consumer) { - CHECK_RESULT(consumer.resumeMessageListener()); -} +void Consumer_resumeMessageListener(Consumer& consumer) { CHECK_RESULT(consumer.resumeMessageListener()); } void Consumer_seek(Consumer& consumer, const MessageId& msgId) { Result res; - Py_BEGIN_ALLOW_THREADS - res = consumer.seek(msgId); + Py_BEGIN_ALLOW_THREADS res = consumer.seek(msgId); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void Consumer_seek_timestamp(Consumer& consumer, uint64_t timestamp) { Result res; - Py_BEGIN_ALLOW_THREADS - res = consumer.seek(timestamp); + Py_BEGIN_ALLOW_THREADS res = consumer.seek(timestamp); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void export_consumer() { using namespace boost::python; class_("Consumer", no_init) - .def("topic", &Consumer::getTopic, "return the topic this consumer is subscribed to", - return_value_policy()) - .def("subscription_name", &Consumer::getSubscriptionName, return_value_policy()) - .def("unsubscribe", &Consumer_unsubscribe) - .def("receive", &Consumer_receive) - .def("receive", &Consumer_receive_timeout) - .def("acknowledge", &Consumer_acknowledge) - .def("acknowledge", &Consumer_acknowledge_message_id) - .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative) - .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative_message_id) - .def("negative_acknowledge", &Consumer_negative_acknowledge) - .def("negative_acknowledge", &Consumer_negative_acknowledge_message_id) - .def("close", &Consumer_close) - .def("pause_message_listener", &Consumer_pauseMessageListener) - .def("resume_message_listener", &Consumer_resumeMessageListener) - .def("redeliver_unacknowledged_messages", &Consumer::redeliverUnacknowledgedMessages) - .def("seek", &Consumer_seek) - .def("seek", &Consumer_seek_timestamp) - ; + .def("topic", &Consumer::getTopic, "return the topic this consumer is subscribed to", + return_value_policy()) + .def("subscription_name", &Consumer::getSubscriptionName, return_value_policy()) + .def("unsubscribe", &Consumer_unsubscribe) + .def("receive", &Consumer_receive) + .def("receive", &Consumer_receive_timeout) + .def("acknowledge", &Consumer_acknowledge) + .def("acknowledge", &Consumer_acknowledge_message_id) + .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative) + .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative_message_id) + .def("negative_acknowledge", &Consumer_negative_acknowledge) + .def("negative_acknowledge", &Consumer_negative_acknowledge_message_id) + .def("close", &Consumer_close) + .def("pause_message_listener", &Consumer_pauseMessageListener) + .def("resume_message_listener", &Consumer_resumeMessageListener) + .def("redeliver_unacknowledged_messages", &Consumer::redeliverUnacknowledgedMessages) + .def("seek", &Consumer_seek) + .def("seek", &Consumer_seek_timestamp); } diff --git a/pulsar-client-cpp/python/src/cryptoKeyReader.cc b/pulsar-client-cpp/python/src/cryptoKeyReader.cc index ccefe6f18b970..2c46b6fb5af19 100644 --- a/pulsar-client-cpp/python/src/cryptoKeyReader.cc +++ b/pulsar-client-cpp/python/src/cryptoKeyReader.cc @@ -21,7 +21,7 @@ CryptoKeyReaderWrapper::CryptoKeyReaderWrapper() {} CryptoKeyReaderWrapper::CryptoKeyReaderWrapper(const std::string& publicKeyPath, - const std::string& privateKeyPath) { + const std::string& privateKeyPath) { this->cryptoKeyReader = DefaultCryptoKeyReader::create(publicKeyPath, privateKeyPath); } diff --git a/pulsar-client-cpp/python/src/enums.cc b/pulsar-client-cpp/python/src/enums.cc index c23b211ffdc15..1b21af585ed54 100644 --- a/pulsar-client-cpp/python/src/enums.cc +++ b/pulsar-client-cpp/python/src/enums.cc @@ -18,104 +18,96 @@ */ #include "utils.h" - void export_enums() { using namespace boost::python; enum_("PartitionsRoutingMode") - .value("UseSinglePartition", ProducerConfiguration::UseSinglePartition) - .value("RoundRobinDistribution", ProducerConfiguration::RoundRobinDistribution) - .value("CustomPartition", ProducerConfiguration::CustomPartition) - ; + .value("UseSinglePartition", ProducerConfiguration::UseSinglePartition) + .value("RoundRobinDistribution", ProducerConfiguration::RoundRobinDistribution) + .value("CustomPartition", ProducerConfiguration::CustomPartition); enum_("CompressionType") - .value("NONE", CompressionNone) // Don't use 'None' since it's a keyword in py3 - .value("LZ4", CompressionLZ4) - .value("ZLib", CompressionZLib) - .value("ZSTD", CompressionZSTD) - .value("SNAPPY", CompressionSNAPPY) - ; + .value("NONE", CompressionNone) // Don't use 'None' since it's a keyword in py3 + .value("LZ4", CompressionLZ4) + .value("ZLib", CompressionZLib) + .value("ZSTD", CompressionZSTD) + .value("SNAPPY", CompressionSNAPPY); enum_("ConsumerType") - .value("Exclusive", ConsumerExclusive) - .value("Shared", ConsumerShared) - .value("Failover", ConsumerFailover) - .value("KeyShared", ConsumerKeyShared) - ; + .value("Exclusive", ConsumerExclusive) + .value("Shared", ConsumerShared) + .value("Failover", ConsumerFailover) + .value("KeyShared", ConsumerKeyShared); - enum_("Result", "Collection of return codes") - .value("Ok", ResultOk) - .value("UnknownError", ResultUnknownError) - .value("InvalidConfiguration", ResultInvalidConfiguration) - .value("Timeout", ResultTimeout) - .value("LookupError", ResultLookupError) - .value("ConnectError", ResultConnectError) - .value("ReadError", ResultReadError) - .value("AuthenticationError", ResultAuthenticationError) - .value("AuthorizationError", ResultAuthorizationError) - .value("ErrorGettingAuthenticationData", ResultErrorGettingAuthenticationData) - .value("BrokerMetadataError", ResultBrokerMetadataError) - .value("BrokerPersistenceError", ResultBrokerPersistenceError) - .value("ChecksumError", ResultChecksumError) - .value("ConsumerBusy", ResultConsumerBusy) - .value("NotConnected", ResultNotConnected) - .value("AlreadyClosed", ResultAlreadyClosed) - .value("InvalidMessage", ResultInvalidMessage) - .value("ConsumerNotInitialized", ResultConsumerNotInitialized) - .value("ProducerNotInitialized", ResultProducerNotInitialized) - .value("ProducerBusy", ResultProducerBusy) - .value("TooManyLookupRequestException", ResultTooManyLookupRequestException) - .value("InvalidTopicName", ResultInvalidTopicName) - .value("InvalidUrl", ResultInvalidUrl) - .value("ServiceUnitNotReady", ResultServiceUnitNotReady) - .value("OperationNotSupported", ResultOperationNotSupported) - .value("ProducerBlockedQuotaExceededError", ResultProducerBlockedQuotaExceededError) - .value("ProducerBlockedQuotaExceededException", ResultProducerBlockedQuotaExceededException) - .value("ProducerQueueIsFull", ResultProducerQueueIsFull) - .value("MessageTooBig", ResultMessageTooBig) - .value("TopicNotFound", ResultTopicNotFound) - .value("SubscriptionNotFound", ResultSubscriptionNotFound) - .value("ConsumerNotFound", ResultConsumerNotFound) - .value("UnsupportedVersionError", ResultUnsupportedVersionError) - .value("TopicTerminated", ResultTopicTerminated) - .value("CryptoError", ResultCryptoError) - .value("IncompatibleSchema", ResultIncompatibleSchema) - .value("ConsumerAssignError", ResultConsumerAssignError) - .value("CumulativeAcknowledgementNotAllowedError", ResultCumulativeAcknowledgementNotAllowedError) - .value("TransactionCoordinatorNotFoundError", ResultTransactionCoordinatorNotFoundError) - .value("InvalidTxnStatusError", ResultInvalidTxnStatusError) - .value("NotAllowedError", ResultNotAllowedError) - .value("TransactionConflict", ResultTransactionConflict) - .value("TransactionNotFound", ResultTransactionNotFound) - .value("ProducerFenced", ResultProducerFenced) - .value("MemoryBufferIsFull", ResultMemoryBufferIsFull) - ; + enum_("Result", "Collection of return codes") + .value("Ok", ResultOk) + .value("UnknownError", ResultUnknownError) + .value("InvalidConfiguration", ResultInvalidConfiguration) + .value("Timeout", ResultTimeout) + .value("LookupError", ResultLookupError) + .value("ConnectError", ResultConnectError) + .value("ReadError", ResultReadError) + .value("AuthenticationError", ResultAuthenticationError) + .value("AuthorizationError", ResultAuthorizationError) + .value("ErrorGettingAuthenticationData", ResultErrorGettingAuthenticationData) + .value("BrokerMetadataError", ResultBrokerMetadataError) + .value("BrokerPersistenceError", ResultBrokerPersistenceError) + .value("ChecksumError", ResultChecksumError) + .value("ConsumerBusy", ResultConsumerBusy) + .value("NotConnected", ResultNotConnected) + .value("AlreadyClosed", ResultAlreadyClosed) + .value("InvalidMessage", ResultInvalidMessage) + .value("ConsumerNotInitialized", ResultConsumerNotInitialized) + .value("ProducerNotInitialized", ResultProducerNotInitialized) + .value("ProducerBusy", ResultProducerBusy) + .value("TooManyLookupRequestException", ResultTooManyLookupRequestException) + .value("InvalidTopicName", ResultInvalidTopicName) + .value("InvalidUrl", ResultInvalidUrl) + .value("ServiceUnitNotReady", ResultServiceUnitNotReady) + .value("OperationNotSupported", ResultOperationNotSupported) + .value("ProducerBlockedQuotaExceededError", ResultProducerBlockedQuotaExceededError) + .value("ProducerBlockedQuotaExceededException", ResultProducerBlockedQuotaExceededException) + .value("ProducerQueueIsFull", ResultProducerQueueIsFull) + .value("MessageTooBig", ResultMessageTooBig) + .value("TopicNotFound", ResultTopicNotFound) + .value("SubscriptionNotFound", ResultSubscriptionNotFound) + .value("ConsumerNotFound", ResultConsumerNotFound) + .value("UnsupportedVersionError", ResultUnsupportedVersionError) + .value("TopicTerminated", ResultTopicTerminated) + .value("CryptoError", ResultCryptoError) + .value("IncompatibleSchema", ResultIncompatibleSchema) + .value("ConsumerAssignError", ResultConsumerAssignError) + .value("CumulativeAcknowledgementNotAllowedError", ResultCumulativeAcknowledgementNotAllowedError) + .value("TransactionCoordinatorNotFoundError", ResultTransactionCoordinatorNotFoundError) + .value("InvalidTxnStatusError", ResultInvalidTxnStatusError) + .value("NotAllowedError", ResultNotAllowedError) + .value("TransactionConflict", ResultTransactionConflict) + .value("TransactionNotFound", ResultTransactionNotFound) + .value("ProducerFenced", ResultProducerFenced) + .value("MemoryBufferIsFull", ResultMemoryBufferIsFull); enum_("SchemaType", "Supported schema types") - .value("NONE", pulsar::NONE) - .value("STRING", pulsar::STRING) - .value("INT8", pulsar::INT8) - .value("INT16", pulsar::INT16) - .value("INT32", pulsar::INT32) - .value("INT64", pulsar::INT64) - .value("FLOAT", pulsar::FLOAT) - .value("DOUBLE", pulsar::DOUBLE) - .value("BYTES", pulsar::BYTES) - .value("JSON", pulsar::JSON) - .value("PROTOBUF", pulsar::PROTOBUF) - .value("AVRO", pulsar::AVRO) - .value("AUTO_CONSUME", pulsar::AUTO_CONSUME) - .value("AUTO_PUBLISH", pulsar::AUTO_PUBLISH) - .value("KEY_VALUE", pulsar::KEY_VALUE) - ; + .value("NONE", pulsar::NONE) + .value("STRING", pulsar::STRING) + .value("INT8", pulsar::INT8) + .value("INT16", pulsar::INT16) + .value("INT32", pulsar::INT32) + .value("INT64", pulsar::INT64) + .value("FLOAT", pulsar::FLOAT) + .value("DOUBLE", pulsar::DOUBLE) + .value("BYTES", pulsar::BYTES) + .value("JSON", pulsar::JSON) + .value("PROTOBUF", pulsar::PROTOBUF) + .value("AVRO", pulsar::AVRO) + .value("AUTO_CONSUME", pulsar::AUTO_CONSUME) + .value("AUTO_PUBLISH", pulsar::AUTO_PUBLISH) + .value("KEY_VALUE", pulsar::KEY_VALUE); enum_("InitialPosition", "Supported initial position") - .value("Latest", InitialPositionLatest) - .value("Earliest", InitialPositionEarliest) - ; + .value("Latest", InitialPositionLatest) + .value("Earliest", InitialPositionEarliest); enum_("BatchingType", "Supported batching types") - .value("Default", ProducerConfiguration::DefaultBatching) - .value("KeyBased", ProducerConfiguration::KeyBasedBatching) - ; + .value("Default", ProducerConfiguration::DefaultBatching) + .value("KeyBased", ProducerConfiguration::KeyBasedBatching); } diff --git a/pulsar-client-cpp/python/src/exceptions.cc b/pulsar-client-cpp/python/src/exceptions.cc index c39b52d737198..25b3bd07e1cb1 100644 --- a/pulsar-client-cpp/python/src/exceptions.cc +++ b/pulsar-client-cpp/python/src/exceptions.cc @@ -29,16 +29,13 @@ PyObject* createExceptionClass(const char* name, PyObject* baseTypeObj = PyExc_E std::string fullName = "_pulsar."; fullName += name; - PyObject* typeObj = PyErr_NewException(const_cast(fullName.c_str()), - baseTypeObj, nullptr); + PyObject* typeObj = PyErr_NewException(const_cast(fullName.c_str()), baseTypeObj, nullptr); if (!typeObj) throw_error_already_set(); scope().attr(name) = handle<>(borrowed(typeObj)); return typeObj; } -PyObject* get_exception_class(Result result) { - return exceptions[result]; -} +PyObject* get_exception_class(Result result) { return exceptions[result]; } void export_exceptions() { using namespace boost::python; @@ -46,44 +43,58 @@ void export_exceptions() { basePulsarException = createExceptionClass("PulsarException"); exceptions[ResultUnknownError] = createExceptionClass("UnknownError", basePulsarException); - exceptions[ResultInvalidConfiguration] = createExceptionClass("InvalidConfiguration", basePulsarException); + exceptions[ResultInvalidConfiguration] = + createExceptionClass("InvalidConfiguration", basePulsarException); exceptions[ResultTimeout] = createExceptionClass("Timeout", basePulsarException); exceptions[ResultLookupError] = createExceptionClass("LookupError", basePulsarException); exceptions[ResultConnectError] = createExceptionClass("ConnectError", basePulsarException); exceptions[ResultReadError] = createExceptionClass("ReadError", basePulsarException); exceptions[ResultAuthenticationError] = createExceptionClass("AuthenticationError", basePulsarException); exceptions[ResultAuthorizationError] = createExceptionClass("AuthorizationError", basePulsarException); - exceptions[ResultErrorGettingAuthenticationData] = createExceptionClass("ErrorGettingAuthenticationData", basePulsarException); + exceptions[ResultErrorGettingAuthenticationData] = + createExceptionClass("ErrorGettingAuthenticationData", basePulsarException); exceptions[ResultBrokerMetadataError] = createExceptionClass("BrokerMetadataError", basePulsarException); - exceptions[ResultBrokerPersistenceError] = createExceptionClass("BrokerPersistenceError", basePulsarException); + exceptions[ResultBrokerPersistenceError] = + createExceptionClass("BrokerPersistenceError", basePulsarException); exceptions[ResultChecksumError] = createExceptionClass("ChecksumError", basePulsarException); exceptions[ResultConsumerBusy] = createExceptionClass("ConsumerBusy", basePulsarException); exceptions[ResultNotConnected] = createExceptionClass("NotConnected", basePulsarException); exceptions[ResultAlreadyClosed] = createExceptionClass("AlreadyClosed", basePulsarException); exceptions[ResultInvalidMessage] = createExceptionClass("InvalidMessage", basePulsarException); - exceptions[ResultConsumerNotInitialized] = createExceptionClass("ConsumerNotInitialized", basePulsarException); - exceptions[ResultProducerNotInitialized] = createExceptionClass("ProducerNotInitialized", basePulsarException); + exceptions[ResultConsumerNotInitialized] = + createExceptionClass("ConsumerNotInitialized", basePulsarException); + exceptions[ResultProducerNotInitialized] = + createExceptionClass("ProducerNotInitialized", basePulsarException); exceptions[ResultProducerBusy] = createExceptionClass("ProducerBusy", basePulsarException); - exceptions[ResultTooManyLookupRequestException] = createExceptionClass("TooManyLookupRequestException", basePulsarException); + exceptions[ResultTooManyLookupRequestException] = + createExceptionClass("TooManyLookupRequestException", basePulsarException); exceptions[ResultInvalidTopicName] = createExceptionClass("InvalidTopicName", basePulsarException); exceptions[ResultInvalidUrl] = createExceptionClass("InvalidUrl", basePulsarException); exceptions[ResultServiceUnitNotReady] = createExceptionClass("ServiceUnitNotReady", basePulsarException); - exceptions[ResultOperationNotSupported] = createExceptionClass("OperationNotSupported", basePulsarException); - exceptions[ResultProducerBlockedQuotaExceededError] = createExceptionClass("ProducerBlockedQuotaExceededError", basePulsarException); - exceptions[ResultProducerBlockedQuotaExceededException] = createExceptionClass("ProducerBlockedQuotaExceededException", basePulsarException); + exceptions[ResultOperationNotSupported] = + createExceptionClass("OperationNotSupported", basePulsarException); + exceptions[ResultProducerBlockedQuotaExceededError] = + createExceptionClass("ProducerBlockedQuotaExceededError", basePulsarException); + exceptions[ResultProducerBlockedQuotaExceededException] = + createExceptionClass("ProducerBlockedQuotaExceededException", basePulsarException); exceptions[ResultProducerQueueIsFull] = createExceptionClass("ProducerQueueIsFull", basePulsarException); exceptions[ResultMessageTooBig] = createExceptionClass("MessageTooBig", basePulsarException); exceptions[ResultTopicNotFound] = createExceptionClass("TopicNotFound", basePulsarException); - exceptions[ResultSubscriptionNotFound] = createExceptionClass("SubscriptionNotFound", basePulsarException); + exceptions[ResultSubscriptionNotFound] = + createExceptionClass("SubscriptionNotFound", basePulsarException); exceptions[ResultConsumerNotFound] = createExceptionClass("ConsumerNotFound", basePulsarException); - exceptions[ResultUnsupportedVersionError] = createExceptionClass("UnsupportedVersionError", basePulsarException); + exceptions[ResultUnsupportedVersionError] = + createExceptionClass("UnsupportedVersionError", basePulsarException); exceptions[ResultTopicTerminated] = createExceptionClass("TopicTerminated", basePulsarException); exceptions[ResultCryptoError] = createExceptionClass("CryptoError", basePulsarException); exceptions[ResultIncompatibleSchema] = createExceptionClass("IncompatibleSchema", basePulsarException); exceptions[ResultConsumerAssignError] = createExceptionClass("ConsumerAssignError", basePulsarException); - exceptions[ResultCumulativeAcknowledgementNotAllowedError] = createExceptionClass("CumulativeAcknowledgementNotAllowedError", basePulsarException); - exceptions[ResultTransactionCoordinatorNotFoundError] = createExceptionClass("TransactionCoordinatorNotFoundError", basePulsarException); - exceptions[ResultInvalidTxnStatusError] = createExceptionClass("InvalidTxnStatusError", basePulsarException); + exceptions[ResultCumulativeAcknowledgementNotAllowedError] = + createExceptionClass("CumulativeAcknowledgementNotAllowedError", basePulsarException); + exceptions[ResultTransactionCoordinatorNotFoundError] = + createExceptionClass("TransactionCoordinatorNotFoundError", basePulsarException); + exceptions[ResultInvalidTxnStatusError] = + createExceptionClass("InvalidTxnStatusError", basePulsarException); exceptions[ResultNotAllowedError] = createExceptionClass("NotAllowedError", basePulsarException); exceptions[ResultTransactionConflict] = createExceptionClass("TransactionConflict", basePulsarException); exceptions[ResultTransactionNotFound] = createExceptionClass("TransactionNotFound", basePulsarException); diff --git a/pulsar-client-cpp/python/src/message.cc b/pulsar-client-cpp/python/src/message.cc index 8532966648d04..b93380bc7afb9 100644 --- a/pulsar-client-cpp/python/src/message.cc +++ b/pulsar-client-cpp/python/src/message.cc @@ -28,34 +28,23 @@ std::string MessageId_str(const MessageId& msgId) { return ss.str(); } -bool MessageId_eq(const MessageId& a, const MessageId& b) { - return a == b; -} +bool MessageId_eq(const MessageId& a, const MessageId& b) { return a == b; } -bool MessageId_ne(const MessageId& a, const MessageId& b) { - return a != b; -} +bool MessageId_ne(const MessageId& a, const MessageId& b) { return a != b; } -bool MessageId_lt(const MessageId& a, const MessageId& b) { - return a < b; -} +bool MessageId_lt(const MessageId& a, const MessageId& b) { return a < b; } -bool MessageId_le(const MessageId& a, const MessageId& b) { - return a <= b; -} +bool MessageId_le(const MessageId& a, const MessageId& b) { return a <= b; } -bool MessageId_gt(const MessageId& a, const MessageId& b) { - return a > b; -} +bool MessageId_gt(const MessageId& a, const MessageId& b) { return a > b; } -bool MessageId_ge(const MessageId& a, const MessageId& b) { - return a >= b; -} +bool MessageId_ge(const MessageId& a, const MessageId& b) { return a >= b; } boost::python::object MessageId_serialize(const MessageId& msgId) { std::string serialized; msgId.serialize(serialized); - return boost::python::object(boost::python::handle<>(PyBytes_FromStringAndSize(serialized.c_str(), serialized.length()))); + return boost::python::object( + boost::python::handle<>(PyBytes_FromStringAndSize(serialized.c_str(), serialized.length()))); } std::string Message_str(const Message& msg) { @@ -65,7 +54,8 @@ std::string Message_str(const Message& msg) { } boost::python::object Message_data(const Message& msg) { - return boost::python::object(boost::python::handle<>(PyBytes_FromStringAndSize((const char*)msg.getData(), msg.getLength()))); + return boost::python::object( + boost::python::handle<>(PyBytes_FromStringAndSize((const char*)msg.getData(), msg.getLength()))); } boost::python::object Message_properties(const Message& msg) { @@ -88,9 +78,7 @@ std::string schema_version_str(const Message& msg) { return ss.str(); } -const MessageId& Message_getMessageId(const Message& msg) { - return msg.getMessageId(); -} +const MessageId& Message_getMessageId(const Message& msg) { return msg.getMessageId(); } void deliverAfter(MessageBuilder* const builder, PyObject* obj_delta) { PyDateTime_Delta const* pydelta = reinterpret_cast(obj_delta); @@ -102,12 +90,9 @@ void deliverAfter(MessageBuilder* const builder, PyObject* obj_delta) { } // Create chrono duration object - std::chrono::milliseconds - duration = std::chrono::duration_cast( - std::chrono::hours(24)*days - + std::chrono::seconds(pydelta->seconds) - + std::chrono::microseconds(pydelta->microseconds) - ); + std::chrono::milliseconds duration = std::chrono::duration_cast( + std::chrono::hours(24) * days + std::chrono::seconds(pydelta->seconds) + + std::chrono::microseconds(pydelta->microseconds)); if (is_negative) { duration = duration * -1; @@ -121,70 +106,66 @@ void export_message() { PyDateTime_IMPORT; - MessageBuilder& (MessageBuilder::*MessageBuilderSetContentString)(const std::string&) = &MessageBuilder::setContent; + MessageBuilder& (MessageBuilder::*MessageBuilderSetContentString)(const std::string&) = + &MessageBuilder::setContent; class_("MessageBuilder") - .def("content", MessageBuilderSetContentString, return_self<>()) - .def("property", &MessageBuilder::setProperty, return_self<>()) - .def("properties", &MessageBuilder::setProperties, return_self<>()) - .def("sequence_id", &MessageBuilder::setSequenceId, return_self<>()) - .def("deliver_after", &deliverAfter, return_self<>()) - .def("deliver_at", &MessageBuilder::setDeliverAt, return_self<>()) - .def("partition_key", &MessageBuilder::setPartitionKey, return_self<>()) - .def("event_timestamp", &MessageBuilder::setEventTimestamp, return_self<>()) - .def("replication_clusters", &MessageBuilder::setReplicationClusters, return_self<>()) - .def("disable_replication", &MessageBuilder::disableReplication, return_self<>()) - .def("build", &MessageBuilder::build) - ; - - class_("MessageStringMap") - .def(map_indexing_suite()) - ; + .def("content", MessageBuilderSetContentString, return_self<>()) + .def("property", &MessageBuilder::setProperty, return_self<>()) + .def("properties", &MessageBuilder::setProperties, return_self<>()) + .def("sequence_id", &MessageBuilder::setSequenceId, return_self<>()) + .def("deliver_after", &deliverAfter, return_self<>()) + .def("deliver_at", &MessageBuilder::setDeliverAt, return_self<>()) + .def("partition_key", &MessageBuilder::setPartitionKey, return_self<>()) + .def("event_timestamp", &MessageBuilder::setEventTimestamp, return_self<>()) + .def("replication_clusters", &MessageBuilder::setReplicationClusters, return_self<>()) + .def("disable_replication", &MessageBuilder::disableReplication, return_self<>()) + .def("build", &MessageBuilder::build); + + class_("MessageStringMap").def(map_indexing_suite()); static const MessageId& _MessageId_earliest = MessageId::earliest(); static const MessageId& _MessageId_latest = MessageId::latest(); class_("MessageId") - .def(init()) - .def("__str__", &MessageId_str) - .def("__eq__", &MessageId_eq) - .def("__ne__", &MessageId_ne) - .def("__le__", &MessageId_le) - .def("__lt__", &MessageId_lt) - .def("__ge__", &MessageId_ge) - .def("__gt__", &MessageId_gt) - .def("ledger_id", &MessageId::ledgerId) - .def("entry_id", &MessageId::entryId) - .def("batch_index", &MessageId::batchIndex) - .def("partition", &MessageId::partition) - .add_static_property("earliest", make_getter(&_MessageId_earliest)) - .add_static_property("latest", make_getter(&_MessageId_latest)) - .def("serialize", &MessageId_serialize) - .def("deserialize", &MessageId::deserialize).staticmethod("deserialize") - ; + .def(init()) + .def("__str__", &MessageId_str) + .def("__eq__", &MessageId_eq) + .def("__ne__", &MessageId_ne) + .def("__le__", &MessageId_le) + .def("__lt__", &MessageId_lt) + .def("__ge__", &MessageId_ge) + .def("__gt__", &MessageId_gt) + .def("ledger_id", &MessageId::ledgerId) + .def("entry_id", &MessageId::entryId) + .def("batch_index", &MessageId::batchIndex) + .def("partition", &MessageId::partition) + .add_static_property("earliest", make_getter(&_MessageId_earliest)) + .add_static_property("latest", make_getter(&_MessageId_latest)) + .def("serialize", &MessageId_serialize) + .def("deserialize", &MessageId::deserialize) + .staticmethod("deserialize"); class_("Message") - .def("properties", &Message_properties) - .def("data", &Message_data) - .def("length", &Message::getLength) - .def("partition_key", &Message::getPartitionKey, return_value_policy()) - .def("publish_timestamp", &Message::getPublishTimestamp) - .def("event_timestamp", &Message::getEventTimestamp) - .def("message_id", &Message_getMessageId, return_value_policy()) - .def("__str__", &Message_str) - .def("topic_name", &Topic_name_str) - .def("redelivery_count", &Message::getRedeliveryCount) - .def("schema_version", &schema_version_str) - ; - - MessageBatch& (MessageBatch::*MessageBatchParseFromString)(const std::string& payload, uint32_t batchSize) = &MessageBatch::parseFrom; + .def("properties", &Message_properties) + .def("data", &Message_data) + .def("length", &Message::getLength) + .def("partition_key", &Message::getPartitionKey, return_value_policy()) + .def("publish_timestamp", &Message::getPublishTimestamp) + .def("event_timestamp", &Message::getEventTimestamp) + .def("message_id", &Message_getMessageId, return_value_policy()) + .def("__str__", &Message_str) + .def("topic_name", &Topic_name_str) + .def("redelivery_count", &Message::getRedeliveryCount) + .def("schema_version", &schema_version_str); + + MessageBatch& (MessageBatch::*MessageBatchParseFromString)(const std::string& payload, + uint32_t batchSize) = &MessageBatch::parseFrom; class_("MessageBatch") - .def("with_message_id", &MessageBatch::withMessageId, return_self<>()) - .def("parse_from", MessageBatchParseFromString, return_self<>()) - .def("messages", &MessageBatch::messages, return_value_policy()) - ; + .def("with_message_id", &MessageBatch::withMessageId, return_self<>()) + .def("parse_from", MessageBatchParseFromString, return_self<>()) + .def("messages", &MessageBatch::messages, return_value_policy()); - class_ >("Messages") - .def(vector_indexing_suite >() ); + class_ >("Messages").def(vector_indexing_suite >()); } diff --git a/pulsar-client-cpp/python/src/producer.cc b/pulsar-client-cpp/python/src/producer.cc index 343650f4b240f..345639e2d951a 100644 --- a/pulsar-client-cpp/python/src/producer.cc +++ b/pulsar-client-cpp/python/src/producer.cc @@ -25,11 +25,10 @@ extern boost::python::object MessageId_serialize(const MessageId& msgId); boost::python::object Producer_send(Producer& producer, const Message& message) { Result res; MessageId messageId; - Py_BEGIN_ALLOW_THREADS - res = producer.send(message, messageId); + Py_BEGIN_ALLOW_THREADS res = producer.send(message, messageId); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return MessageId_serialize(messageId); } @@ -54,57 +53,55 @@ void Producer_sendAsync(Producer& producer, const Message& message, py::object c PyObject* pyCallback = callback.ptr(); Py_XINCREF(pyCallback); - Py_BEGIN_ALLOW_THREADS - producer.sendAsync(message, std::bind(Producer_sendAsyncCallback, pyCallback, - std::placeholders::_1, std::placeholders::_2)); + Py_BEGIN_ALLOW_THREADS producer.sendAsync( + message, + std::bind(Producer_sendAsyncCallback, pyCallback, std::placeholders::_1, std::placeholders::_2)); Py_END_ALLOW_THREADS } void Producer_flush(Producer& producer) { Result res; - Py_BEGIN_ALLOW_THREADS - res = producer.flush(); + Py_BEGIN_ALLOW_THREADS res = producer.flush(); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void Producer_close(Producer& producer) { Result res; - Py_BEGIN_ALLOW_THREADS - res = producer.close(); + Py_BEGIN_ALLOW_THREADS res = producer.close(); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void export_producer() { using namespace boost::python; class_("Producer", no_init) - .def("topic", &Producer::getTopic, "return the topic to which producer is publishing to", - return_value_policy()) - .def("producer_name", &Producer::getProducerName, - "return the producer name which could have been assigned by the system or specified by the client", - return_value_policy()) - .def("last_sequence_id", &Producer::getLastSequenceId) - .def("send", &Producer_send, - "Publish a message on the topic associated with this Producer.\n" - "\n" - "This method will block until the message will be accepted and persisted\n" - "by the broker. In case of errors, the client library will try to\n" - "automatically recover and use a different broker.\n" - "\n" - "If it wasn't possible to successfully publish the message within the sendTimeout,\n" - "an error will be returned.\n" - "\n" - "This method is equivalent to asyncSend() and wait until the callback is triggered.\n" - "\n" - "@param msg message to publish\n") - .def("send_async", &Producer_sendAsync) - .def("flush", &Producer_flush, - "Flush all the messages buffered in the client and wait until all messages have been\n" - "successfully persisted\n") - .def("close", &Producer_close) - ; + .def("topic", &Producer::getTopic, "return the topic to which producer is publishing to", + return_value_policy()) + .def("producer_name", &Producer::getProducerName, + "return the producer name which could have been assigned by the system or specified by the " + "client", + return_value_policy()) + .def("last_sequence_id", &Producer::getLastSequenceId) + .def("send", &Producer_send, + "Publish a message on the topic associated with this Producer.\n" + "\n" + "This method will block until the message will be accepted and persisted\n" + "by the broker. In case of errors, the client library will try to\n" + "automatically recover and use a different broker.\n" + "\n" + "If it wasn't possible to successfully publish the message within the sendTimeout,\n" + "an error will be returned.\n" + "\n" + "This method is equivalent to asyncSend() and wait until the callback is triggered.\n" + "\n" + "@param msg message to publish\n") + .def("send_async", &Producer_sendAsync) + .def("flush", &Producer_flush, + "Flush all the messages buffered in the client and wait until all messages have been\n" + "successfully persisted\n") + .def("close", &Producer_close); } diff --git a/pulsar-client-cpp/python/src/pulsar.cc b/pulsar-client-cpp/python/src/pulsar.cc index a46ce53692d07..e591b738728c5 100644 --- a/pulsar-client-cpp/python/src/pulsar.cc +++ b/pulsar-client-cpp/python/src/pulsar.cc @@ -32,7 +32,6 @@ void export_exceptions(); PyObject* get_exception_class(Result result); - static void translateException(const PulsarException& ex) { std::string err = "Pulsar error: "; err += strResult(ex._result); @@ -40,8 +39,7 @@ static void translateException(const PulsarException& ex) { PyErr_SetString(get_exception_class(ex._result), err.c_str()); } -BOOST_PYTHON_MODULE(_pulsar) -{ +BOOST_PYTHON_MODULE(_pulsar) { py::register_exception_translator(translateException); // Initialize thread support so that we can grab the GIL mutex diff --git a/pulsar-client-cpp/python/src/reader.cc b/pulsar-client-cpp/python/src/reader.cc index fec65da270dd6..668fb9499ed7f 100644 --- a/pulsar-client-cpp/python/src/reader.cc +++ b/pulsar-client-cpp/python/src/reader.cc @@ -24,12 +24,12 @@ Message Reader_readNext(Reader& reader) { while (true) { Py_BEGIN_ALLOW_THREADS - // Use 100ms timeout to periodically check whether the - // interpreter was interrupted - res = reader.readNext(msg, 100); + // Use 100ms timeout to periodically check whether the + // interpreter was interrupted + res = reader.readNext(msg, 100); Py_END_ALLOW_THREADS - if (res != ResultTimeout) { + if (res != ResultTimeout) { // In case of timeout we keep calling receive() to simulate a // blocking call until a message is available, while breaking // every once in a while to check the Python signal status @@ -49,62 +49,56 @@ Message Reader_readNext(Reader& reader) { Message Reader_readNextTimeout(Reader& reader, int timeoutMs) { Message msg; Result res; - Py_BEGIN_ALLOW_THREADS - res = reader.readNext(msg, timeoutMs); + Py_BEGIN_ALLOW_THREADS res = reader.readNext(msg, timeoutMs); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return msg; } bool Reader_hasMessageAvailable(Reader& reader) { bool available = false; Result res; - Py_BEGIN_ALLOW_THREADS - res = reader.hasMessageAvailable(available); + Py_BEGIN_ALLOW_THREADS res = reader.hasMessageAvailable(available); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); return available; } void Reader_close(Reader& reader) { Result res; - Py_BEGIN_ALLOW_THREADS - res = reader.close(); + Py_BEGIN_ALLOW_THREADS res = reader.close(); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void Reader_seek(Reader& reader, const MessageId& msgId) { Result res; - Py_BEGIN_ALLOW_THREADS - res = reader.seek(msgId); + Py_BEGIN_ALLOW_THREADS res = reader.seek(msgId); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void Reader_seek_timestamp(Reader& reader, uint64_t timestamp) { Result res; - Py_BEGIN_ALLOW_THREADS - res = reader.seek(timestamp); + Py_BEGIN_ALLOW_THREADS res = reader.seek(timestamp); Py_END_ALLOW_THREADS - CHECK_RESULT(res); + CHECK_RESULT(res); } void export_reader() { using namespace boost::python; class_("Reader", no_init) - .def("topic", &Reader::getTopic, return_value_policy()) - .def("read_next", &Reader_readNext) - .def("read_next", &Reader_readNextTimeout) - .def("has_message_available", &Reader_hasMessageAvailable) - .def("close", &Reader_close) - .def("seek", &Reader_seek) - .def("seek", &Reader_seek_timestamp) - ; + .def("topic", &Reader::getTopic, return_value_policy()) + .def("read_next", &Reader_readNext) + .def("read_next", &Reader_readNextTimeout) + .def("has_message_available", &Reader_hasMessageAvailable) + .def("close", &Reader_close) + .def("seek", &Reader_seek) + .def("seek", &Reader_seek_timestamp); } diff --git a/pulsar-client-cpp/python/src/schema.cc b/pulsar-client-cpp/python/src/schema.cc index 397ec658d23da..cdfcda6aff14b 100644 --- a/pulsar-client-cpp/python/src/schema.cc +++ b/pulsar-client-cpp/python/src/schema.cc @@ -21,10 +21,8 @@ void export_schema() { using namespace boost::python; - class_("SchemaInfo", - init()) - .def("schema_type", &SchemaInfo::getSchemaType) - .def("name", &SchemaInfo::getName, return_value_policy()) - .def("schema", &SchemaInfo::getSchema, return_value_policy()) - ; + class_("SchemaInfo", init()) + .def("schema_type", &SchemaInfo::getSchemaType) + .def("name", &SchemaInfo::getName, return_value_policy()) + .def("schema", &SchemaInfo::getSchema, return_value_policy()); } diff --git a/pulsar-client-cpp/python/src/utils.h b/pulsar-client-cpp/python/src/utils.h index 457d1f85382b0..5be44732fb704 100644 --- a/pulsar-client-cpp/python/src/utils.h +++ b/pulsar-client-cpp/python/src/utils.h @@ -27,8 +27,7 @@ namespace py = boost::python; struct PulsarException { Result _result; - PulsarException(Result res) : - _result(res) {} + PulsarException(Result res) : _result(res) {} }; inline void CHECK_RESULT(Result res) { From e26afd99c9f91887eb63d9d05463f8e41c9a2548 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 21 Dec 2021 15:17:21 -0600 Subject: [PATCH 217/823] [Java Client] Improve consumer listener logic (#13273) * [Java Client] Improve consumer listener logic * Move isListenerHandlingMessage update to before submitting to executor (cherry picked from commit 9f46c4af5e0dad7f9223add57842591822c3dea3) --- .../apache/pulsar/client/impl/ConsumerBase.java | 17 +++++++++-------- .../apache/pulsar/client/impl/ConsumerImpl.java | 3 +-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 0f47208ea1efc..20baf47d9def8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -34,7 +34,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -85,7 +84,7 @@ public abstract class ConsumerBase extends HandlerState implements Consumer conf, int receiverQueueSize, ExecutorProvider executorProvider, @@ -915,15 +914,17 @@ private void doPendingBatchReceiveTask(Timeout timeout) { } protected void triggerListener() { - // Trigger the notification on the message listener in a separate thread to avoid blocking the networking - // thread while the message processing happens + // Use internalPinnedExecutor to maintain message ordering internalPinnedExecutor.execute(() -> { try { - // Control executor to call MessageListener one by one. - if (executorQueueSize.get() < 1) { + // Listener should only have one pending/running executable to process a message + // See https://github.com/apache/pulsar/issues/11008 for context. + if (!isListenerHandlingMessage) { final Message msg = internalReceive(0, TimeUnit.MILLISECONDS); if (msg != null) { - executorQueueSize.incrementAndGet(); + isListenerHandlingMessage = true; + // Trigger the notification on the message listener in a separate thread to avoid blocking the + // internal pinned executor thread while the message processing happens if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) { executorProvider.getExecutor(peekMessageKey(msg)).execute(() -> callMessageListener(msg)); @@ -956,7 +957,7 @@ protected void callMessageListener(Message msg) { log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, msg.getMessageId(), t); } finally { - executorQueueSize.decrementAndGet(); + isListenerHandlingMessage = false; triggerListener(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index ff0b826f8d48b..ac0465153c817 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1127,8 +1127,7 @@ private void processPayloadByProcessor(final BrokerEntryMetadata brokerEntryMeta increaseAvailablePermits(cnx(), skippedMessages.get()); } - internalPinnedExecutor.execute(() - -> tryTriggerListener()); + tryTriggerListener(); } void messageReceived(MessageIdData messageId, int redeliveryCount, List ackSet, ByteBuf headersAndPayload, ClientCnx cnx) { From 655fe3f339c2830769103f1c3fcc3ba09cf1f1a0 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 21 Dec 2021 23:20:23 +0200 Subject: [PATCH 218/823] Fix invalid setting of enabled ciphers to fix warning from BoringSSL (#13435) (cherry picked from commit 601e141ac1cedbea236b6c9d2d46550454eee669) --- .../pulsar/common/util/keystoretls/KeyStoreSSLContext.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java index 3825d80165504..987a32b216cbc 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java @@ -185,9 +185,7 @@ public SSLEngine createSSLEngine(String peerHost, int peerPort) { private SSLEngine configureSSLEngine(SSLEngine sslEngine) { sslEngine.setEnabledProtocols(protocols.toArray(new String[0])); - if (this.ciphers == null) { - sslEngine.setEnabledCipherSuites(sslEngine.getSupportedCipherSuites()); - } else { + if (this.ciphers != null) { sslEngine.setEnabledCipherSuites(this.ciphers.toArray(new String[0])); } From da61b915d9f648754b40dc928430746d89d00cbb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Dec 2021 07:05:48 +0200 Subject: [PATCH 219/823] [Broker] Fix race conditions in closing producers and consumers (#13428) - closing ServerCnx while producers or consumers are created can lead to a producer or consumer never getting removed from the topic's list of producers (cherry picked from commit 3316db5a52cdeee49bc90fe18baac28d5688bfe8) --- .../pulsar/broker/service/ServerCnx.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index a5d55bf77e6d8..dbab10dce48e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -292,6 +292,11 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { // Connection is gone, close the producers immediately producers.forEach((__, producerFuture) -> { + // prevent race conditions in completing producers + if (!producerFuture.isDone() + && producerFuture.completeExceptionally(new IllegalStateException("Connection closed."))) { + return; + } if (producerFuture.isDone() && !producerFuture.isCompletedExceptionally()) { Producer producer = producerFuture.getNow(null); producer.closeNow(true); @@ -299,17 +304,18 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { }); consumers.forEach((__, consumerFuture) -> { - Consumer consumer; - if (consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally()) { - consumer = consumerFuture.getNow(null); - } else { + // prevent race conditions in completing consumers + if (!consumerFuture.isDone() + && consumerFuture.completeExceptionally(new IllegalStateException("Connection closed."))) { return; } - - try { - consumer.close(); - } catch (BrokerServiceException e) { - log.warn("Consumer {} was already closed: {}", consumer, e); + if (consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally()) { + Consumer consumer = consumerFuture.getNow(null); + try { + consumer.close(); + } catch (BrokerServiceException e) { + log.warn("Consumer {} was already closed: {}", consumer, e); + } } }); this.service.getPulsarStats().recordConnectionClose(); From 519ff4310e68a99232a2d2290792dbd818184fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 17 Dec 2021 16:43:48 +0100 Subject: [PATCH 220/823] [owasp] suppress false positive netty-tc-native (#13364) (cherry picked from commit f657a3f64f690d0066629e3e088f85c6d2e9538f) (cherry picked from commit dbb508162032a78f5d76c73113ababb8b0df7fe4) --- src/owasp-dependency-check-false-positives.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 7336feedec244..28e53b5d0b876 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -42,4 +42,11 @@ org\.apache\.avro:.* CVE-2019-17195 + + + ^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$ + cpe:/a:netty:netty + \ No newline at end of file From 8435cfe95ee03aa115b065d2b132e6d2a9a8f49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 19 Nov 2021 17:13:10 +0100 Subject: [PATCH 221/823] Flaky tests: ElasticSearchClientTests tests time out (#12694) (cherry picked from commit dc48d29dc5d37793b0b2e6da28e7ba9b79bd4e49) --- .../ElasticSearchClientTests.java | 6 +- .../testcontainers/ChaosContainer.java | 61 +++++++++++++++---- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java index aeacaf85ff1f6..fb927c5723e22 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java @@ -201,7 +201,7 @@ public void testBulkRetry() throws Exception { assertEquals(mockRecord.failed, 0); assertEquals(client.totalHits(index), 2); - ChaosContainer chaosContainer = new ChaosContainer<>(container.getContainerName(), "15s"); + ChaosContainer chaosContainer = ChaosContainer.pauseContainerForSeconds(container.getContainerName(), 15); chaosContainer.start(); client.bulkIndex(mockRecord, Pair.of("3", "{\"a\":3}")); @@ -248,12 +248,12 @@ public void testBulkBlocking() throws Exception { }); client.flush(); Awaitility.await().untilAsserted(() -> { - assertEquals(mockRecord.acked, 5); assertEquals(mockRecord.failed, 0); + assertEquals(mockRecord.acked, 5); assertEquals(client.totalHits(index), 5); }); - ChaosContainer chaosContainer = new ChaosContainer<>(container.getContainerName(), "30s"); + ChaosContainer chaosContainer = ChaosContainer.pauseContainerForSeconds(container.getContainerName(), 30); chaosContainer.start(); Thread.sleep(1000L); diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/testcontainers/ChaosContainer.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/testcontainers/ChaosContainer.java index 7e7734f020c3e..4b296bb10137d 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/testcontainers/ChaosContainer.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/testcontainers/ChaosContainer.java @@ -19,26 +19,63 @@ package org.apache.pulsar.io.elasticsearch.testcontainers; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitStrategy; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Predicate; // see https://github.com/alexei-led/pumba @Slf4j public class ChaosContainer> extends GenericContainer { - public static final String PUMBA_IMAGE = Optional.ofNullable(System.getenv("PUMBA_IMAGE")) - .orElse("gaiaadm/pumba:latest"); - - public ChaosContainer(String targetContainer, String pause) { - super(PUMBA_IMAGE); - setCommand("--log-level info --interval 60s pause --duration " + pause + " " + targetContainer); - addFileSystemBind("/var/run/docker.sock", "/var/run/docker.sock", BindMode.READ_WRITE); - setWaitStrategy(Wait.forLogMessage(".*pausing container.*", 1)); - withLogConsumer(o -> { - log.info("pumba> {}", o.getUtf8String()); - }); - } + public static final String PUMBA_IMAGE = Optional.ofNullable(System.getenv("PUMBA_IMAGE")) + .orElse("gaiaadm/pumba:0.8.0"); + + private final List logs = new ArrayList<>(); + private Consumer beforeStop; + + public static ChaosContainer pauseContainerForSeconds(String targetContainer, int seconds) { + return new ChaosContainer(targetContainer, "pause --duration " + seconds + "s", Wait.forLogMessage(".*pausing container.*", 1), + (Consumer) chaosContainer -> Awaitility + .await() + .atMost(seconds + 5, TimeUnit.SECONDS) + .until(() -> { + boolean found = chaosContainer.logs.stream().anyMatch((Predicate) line -> line.contains("stop pausing container")); + if (!found) { + log.debug("ChaosContainer stop requested. waiting for \"stop pausing container\" log"); + log.debug(String.join("\n", chaosContainer.logs)); + } + return found; + } + )); + } + + private ChaosContainer(String targetContainer, String command, WaitStrategy waitStrategy, Consumer beforeStop) { + super(PUMBA_IMAGE); + setCommand("--log-level info " + command + " " + targetContainer); + addFileSystemBind("/var/run/docker.sock", "/var/run/docker.sock", BindMode.READ_WRITE); + setWaitStrategy(waitStrategy); + withLogConsumer(o -> { + final String string = o.getUtf8String(); + log.info("pumba> {}", string); + logs.add(string); + }); + this.beforeStop = beforeStop; + } + + @Override + public void stop() { + if (getContainerId() != null && beforeStop != null) { + beforeStop.accept(this); + } + super.stop(); + } } \ No newline at end of file From ae1b29adc430dcd91c11254f5a523e3462f07f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Oct 2021 18:06:49 +0200 Subject: [PATCH 222/823] Fix ProxyServiceStarterTest flaky tests (#12344) * Fix ProxyServiceStarterTest flaky tests * revert formatting * use ephimeral port (cherry picked from commit 7ad46c8c18bb8365c9a2d1233a6cd58ecd6f541f) --- .../pulsar/proxy/server/ProxyServiceStarter.java | 5 +++++ .../proxy/server/ProxyServiceStarterTest.java | 16 +++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index c16844c1effa0..315607e9d3457 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -313,6 +313,11 @@ public ProxyConfiguration getConfig() { return config; } + @VisibleForTesting + public WebServer getServer() { + return server; + } + private static final Logger log = LoggerFactory.getLogger(ProxyServiceStarter.class); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java index 3377ec266a713..bdba8d35c5515 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java @@ -59,7 +59,8 @@ protected void setup() throws Exception { serviceStarter = new ProxyServiceStarter(ARGS); serviceStarter.getConfig().setBrokerServiceURL(pulsar.getBrokerServiceUrl()); serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress()); - serviceStarter.getConfig().setServicePort(Optional.of(11000)); + serviceStarter.getConfig().setWebServicePort(Optional.of(0)); + serviceStarter.getConfig().setServicePort(Optional.of(0)); serviceStarter.getConfig().setWebSocketServiceEnabled(true); serviceStarter.start(); } @@ -71,14 +72,19 @@ protected void cleanup() throws Exception { serviceStarter.close(); } + private String computeWsBasePath() { + return String.format("ws://localhost:%d/ws", serviceStarter.getServer().getListenPortHTTP().get()); + } + @Test public void testEnableWebSocketServer() throws Exception { HttpClient httpClient = new HttpClient(); WebSocketClient webSocketClient = new WebSocketClient(httpClient); webSocketClient.start(); MyWebSocket myWebSocket = new MyWebSocket(); - String webSocketUri = "ws://localhost:8080/ws/pingpong"; + String webSocketUri = computeWsBasePath() + "/pingpong"; Future sessionFuture = webSocketClient.connect(myWebSocket, URI.create(webSocketUri)); + System.out.println("uri" + webSocketUri); sessionFuture.get().getRemote().sendPing(ByteBuffer.wrap("ping".getBytes())); assertTrue(myWebSocket.getResponse().contains("ping")); } @@ -86,7 +92,7 @@ public void testEnableWebSocketServer() throws Exception { @Test public void testProducer() throws Exception { @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:11000") + PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:" + this.pulsar.getBrokerService().getListenPort().get()) .build(); @Cleanup @@ -105,7 +111,7 @@ public void testProduceAndConsumeMessageWithWebsocket() throws Exception { WebSocketClient producerWebSocketClient = new WebSocketClient(producerClient); producerWebSocketClient.start(); MyWebSocket producerSocket = new MyWebSocket(); - String produceUri = "ws://localhost:8080/ws/producer/persistent/sample/test/local/websocket-topic"; + String produceUri = computeWsBasePath() + "/producer/persistent/sample/test/local/websocket-topic"; Future producerSession = producerWebSocketClient.connect(producerSocket, URI.create(produceUri)); ProducerMessage produceRequest = new ProducerMessage(); @@ -116,7 +122,7 @@ public void testProduceAndConsumeMessageWithWebsocket() throws Exception { WebSocketClient consumerWebSocketClient = new WebSocketClient(consumerClient); consumerWebSocketClient.start(); MyWebSocket consumerSocket = new MyWebSocket(); - String consumeUri = "ws://localhost:8080/ws/consumer/persistent/sample/test/local/websocket-topic/my-sub"; + String consumeUri = computeWsBasePath() + "/consumer/persistent/sample/test/local/websocket-topic/my-sub"; Future consumerSession = consumerWebSocketClient.connect(consumerSocket, URI.create(consumeUri)); consumerSession.get().getRemote().sendPing(ByteBuffer.wrap("ping".getBytes())); producerSession.get().getRemote().sendString(ObjectMapperFactory.getThreadLocal().writeValueAsString(produceRequest)); From 107161f7ae2e0317f752142b169259b2f61b4d84 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Dec 2021 17:57:58 +0200 Subject: [PATCH 223/823] Bump version to 2.9.2-SNAPSHOT --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 126 files changed, 126 insertions(+), 126 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 257f396902094..6128d5be5283e 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index a58e46afed585..31a706eab97eb 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 726420995c7de..0a3f88abe6471 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 57cfabb0284d4..d5aed2f76fe48 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 4b6f4e4b0a54f..b43161a5d8f7f 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.1 + 2.9.2-SNAPSHOT jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 84e8ee7ff5970..ce672387c061b 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 6bfc10dab10b1..a09bdfd322c82 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index ff3b0c63e70f3..33f4c3c0ab61e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 35e9ea3be0562..990455bd577db 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index f771f22396f9f..988705f44be2d 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 2a56ba7e8bd08..0166e0bbe05f5 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 4a120af5bfea8..f909fa855e4ed 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 86dc7600605c2..a5a0cd067ec26 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 989bcfe051f31..2900155df6d06 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 05504478b863d..96a0b567594c1 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 27ef66377f4a1..e7e9d98a5b25a 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 362f348f3d0e1..088e9cb442c8f 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index cc5e217d41204..bf0b444140cfc 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index af45d2bb3d371..426e89b13b49e 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 16866564f0cad..d78bf0db6b443 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 7e074afc22e68..569b910491d0b 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 86bc5621b04ff..68efb88f3eb02 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 45c4a8c858014..304eb8ab3d8e8 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 4f9b9dcda4a6f..56afd8aa7cbe8 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index f291d1c58a5a3..4063145668122 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index ce5114459c28b..e1261ac0b3f8d 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index c37d74b8fcb72..f567b378adc49 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index e4337335f9cec..7f49866e227a3 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index c5296c3627d89..fa98349bb4559 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 4cdd80b71533d..e7a890d652e60 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index b6256f53b931b..ae23cf1e1ddc1 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index bb7d4d61ca2f0..ce82078d557e9 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 58e1d3d733eb9..73cc277ccd326 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 6f3487ddf5d00..8e533d6b25331 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 922b5ebec7a5c..cae4afe2927dd 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 60e0f7a0f8309..c6d261bfa4214 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 0b5564ce3a0c7..ce192866f6645 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index ba2502e99893e..31706b7225b9a 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 2319a17180b27..c10d161d5ada7 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 9c786d99690ad..14f574489aac8 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 2c16c1fdb5d44..180936d3e9d78 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index e75a9fcf33df3..c5f84bc17973a 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 092a71474e167..05f57fde7a960 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index f5bf0ea04428e..f71052bdf62bd 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index fcc43a8868e34..ef69d3ac92a3a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 846e519f26626..45dcdf2f842ee 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index c1a058f84a596..f486f526b487e 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 316c3009bac9a..4a1a7a4f679b6 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 4c1f434cbcbaa..6526a1d8bc5fb 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 2f29075a692aa..accf6a4d51ca8 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 992415c24bd5b..100e582fa7d7f 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 0a9961b7940e0..9ccb68be14d42 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index fcd079c48cdf2..9bc8460e41a56 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 742ed49cfa1da..8141194fec88b 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 056e13ec4e348..0f9b8a966da0f 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 7e32670dbd3e6..84491460cd140 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 1ead477c20039..103c3c96efb1c 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index c643d2040f3bc..a93eea0682bcb 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index bba433e3990bd..1cb0867dd71b3 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 7906e4453cef7..ed00d32bbcc86 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 29aea0e8e92c5..9e2bbe3af5ef7 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 3e651d4fcb2c8..e44b0763b9976 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 2410c8e1e2470..14744436d374e 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 99ecc53aa3291..122c4f5d71f01 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 9248e2e882e9a..08203dca6c303 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 245b1684e7583..25e5248234e62 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index a3d38fdf3552e..0408076b0b54f 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 07c9654f3dbb4..836345e3935ca 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 3562cdc321707..9074b9144e10b 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index b01ac3f0ce46a..779c4ffe7d3b6 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 3f610661905d5..338bdfc678503 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 8600a85e2b8fe..051902af15764 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 17257fa3073ca..388bcd662dfc5 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index f3ffa7163bb02..bc4f19c2225fb 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index e80e3662dc612..2d4f3320bb7eb 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index a929ab3c914db..8c7c6d06475e3 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 2f93507780ce0..c368fba85138a 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 26df287659d9f..1e2c66e7b6615 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index e357409c90cb0..807baaab0b616 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 774dc3044d691..b81637ee81b15 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 06ad0cc5b50b4..5166bad50e331 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 8a61e14ab9b09..4500bf9a5e92c 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 27eb40211bbdd..7518cf7794ba5 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index f0af99ca394d7..24baed819b918 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 9be6b78cad54c..dfb80b7a47c9a 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index b1fe4c9ee0438..fdfa4cb613a62 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 011609b6ba7c6..8cfc461fbab60 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 8a55c34f03857..957b26b4b3729 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index a69f436986f8a..cd83ffb411311 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 77b329c796403..e3a96afb1eaf8 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 3dc46c0b6a422..9a011ed133d42 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index b68dc8da0047e..0fb713cc0edb7 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 04b3e88ab2f87..db3cafc5813b6 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index f6e45a4a8a992..01e5a0aef1499 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.1 + 2.9.2-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 4aaba09bb4c81..56b3789056f99 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 598c5971a2945..0566dc0ded57c 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index ba6b957cacdde..79b0c8790ec2c 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index dfab92a8e5d02..6d954a3560032 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index ffb9cd714d752..fa8bd92a1870c 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 0336b16cfabae..279b6a1e08708 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 211d4575c0b57..39096520c6c4d 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.1 + 2.9.2-SNAPSHOT pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index cc91cb58c5b18..c7a81a2030100 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.1 + 2.9.2-SNAPSHOT pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 0afa345945868..7e81449d3c634 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.1 + 2.9.2-SNAPSHOT pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 2d21778990a54..15bc57d1f0c50 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 704191882006d..9a86ac77503f1 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.1 + 2.9.2-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index cbe82dbc29e57..be2b9201669bf 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.1 + 2.9.2-SNAPSHOT pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 93982b614f0df..0d0d9a0cfc227 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 373ce329762f0..03be75fe4846a 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index fbb37083eaaa8..f1b5c8f3f6823 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 859a9f686c3d8..805f7f93f56d0 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 1babdf1f68349..267ba52fa2262 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.1 + 2.9.2-SNAPSHOT testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index c6a80c04bf5c1..3ed771e4d54f9 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 4364c18e8b599..fe187e0e822c9 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 40b4ea322e89f..a0f682a916159 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index f02d10e3c720e..d632fa4ff6df6 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 626eb5fa11100..be94489d99240 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 5bee911934e0d..9ec7712e5470b 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.1 + 2.9.2-SNAPSHOT 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 2ac23ba096be1..02fad581b2a30 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index dcbaa8ff2b753..48646fa6c82a8 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT integration diff --git a/tests/pom.xml b/tests/pom.xml index 5eef28b6970c5..cfe68cd54ce61 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 2a4d3c71f8c38..08fd384d01d7f 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 7fc147fa7ff1b..573c6cc2eadea 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 9ed80c851388d..92560ecaf93cf 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.1 + 2.9.2-SNAPSHOT pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index b0c394c2f94ca..9dc296310cf8f 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index f221b7d7cf487..4913bd894506f 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.1 + 2.9.2-SNAPSHOT .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index e4e935b2ba3b7..2abd509b8c8fc 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.1 + 2.9.2-SNAPSHOT .. From 048154c7ec47c427f139fc71b8afaaef26b9f5bd Mon Sep 17 00:00:00 2001 From: litao Date: Thu, 23 Dec 2021 22:08:20 +0800 Subject: [PATCH 224/823] [Producer] Change the time units from ns to ms (#13057) ### Motivation The time unit in this exception message is ns, which is not very readable. We can change it from ns to ms. ``` org.apache.pulsar.client.api.PulsarClientException$TimeoutException: The producer xxx can not send message to the topic xxx within given timeout : createdAt 461913074 ns ago, firstSentAt 29545553038276935 ns ago, lastSentAt 29545553038276935 ns ago, retryCount 0 at org.apache.pulsar.client.api.PulsarClientException.unwrap(PulsarClientException.java:916) at org.apache.pulsar.client.impl.TypedMessageBuilderImpl.send(TypedMessageBuilderImpl.java:93) at org.apache.pulsar.client.impl.ProducerBase.send(ProducerBase.java:63) at com.yum.boh.oh.service.impl.StoreOrderPostServiceImpl.generalProcessing(StoreOrderPostServiceImpl.java:272) at com.yum.boh.oh.service.impl.StoreOrderPostServiceImpl.saveThirdOrder(StoreOrderPostServiceImpl.java:72) at com.yum.boh.oh.controller.StoreOrderController.postOrderInfo$original$T8425mfx(StoreOrderController.java:39) at com.yum.boh.oh.controller.StoreOrderController.postOrderInfo$original$T8425mfx$accessor$vJljNzML(StoreOrderController.java) at com.yum.boh.oh.controller.StoreOrderController$auxiliary$nysalhgy.call(Unknown Source) at org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstMethodsInter.intercept(InstMethodsInter.java:86) ``` ### Modifications Change the time units from ns to ms for ProducerImpl#OpSendMsg. (cherry picked from commit 891660e396ad2831ec2bc94b535fc5d0c9543ec7) --- .../org/apache/pulsar/client/impl/ProducerImpl.java | 8 +++++--- .../apache/pulsar/common/util/RelativeTimeUtil.java | 12 ++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 5944c8f077541..abc771c068a89 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -89,6 +89,7 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.RelativeTimeUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1240,9 +1241,9 @@ void sendComplete(final Exception e) { String errMsg = String.format( "%s : createdAt %s ns ago, firstSentAt %s ns ago, lastSentAt %s ns ago, retryCount %s", te.getMessage(), - ns - this.createdAt, - this.firstSentAt <= 0 ? ns - this.lastSentAt : ns - this.firstSentAt, - ns - this.lastSentAt, + RelativeTimeUtil.nsToSeconds(ns - this.createdAt), + RelativeTimeUtil.nsToSeconds(this.firstSentAt <= 0 ? ns - this.lastSentAt : ns - this.firstSentAt), + RelativeTimeUtil.nsToSeconds(ns - this.lastSentAt), retryCount ); @@ -1305,6 +1306,7 @@ protected OpSendMsg newObject(Handle handle) { }; } + /** * Queue implementation that is used as the pending messages queue. * diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RelativeTimeUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RelativeTimeUtil.java index 36e6adfd32303..454cfda2c20db 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RelativeTimeUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RelativeTimeUtil.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.common.util; +import java.math.BigDecimal; import java.util.concurrent.TimeUnit; import lombok.experimental.UtilityClass; @@ -63,4 +64,15 @@ public static long parseRelativeTimeInSeconds(String relativeTime) { throw new IllegalArgumentException("Invalid time unit '" + lastChar + "'"); } } + + /** + * Convert nanoseconds to seconds and keep three decimal places. + * @param ns + * @return seconds + */ + public static double nsToSeconds(long ns) { + double seconds = (double) ns / 1_000_000_000; + BigDecimal bd = new BigDecimal(seconds); + return bd.setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue(); + } } From 04455e30c0201b75e2a4cb723cffa4bde269a2d1 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 23 Dec 2021 16:36:27 +0800 Subject: [PATCH 225/823] [pulsar-client] Fix multi topic reader has message available behavior (#13332) ### Motivation When we use a multi-topic reader, the `hasMessageAvailable` method might have the wrong behavior, since the multi-topics consumer receives all messages from the single-topic consumer, the single-topic consumer `hasMessageAvailable` might always be `false` (The lastDequeuedMessageId reach to the end of the queue, all message enqueue to multi-topic consumer's `incomingMessages` queue). We should check the multi-topics consumer `incomingMessages` size > 0 when calling `hasMessageAvailable `. ### Modifications 1. Add a check of `incomingMessages` size > 0 2. Add units test `testHasMessageAvailableAsync` to verify the behavior. (cherry picked from commit 6c7dcc0cf877cfcb8bcea18cde7662ebacb01d4c) --- .../client/impl/MultiTopicsReaderTest.java | 67 +++++++++++++++++++ .../client/impl/MultiTopicsConsumerImpl.java | 3 + .../client/impl/MultiTopicsReaderImpl.java | 2 +- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java index a8a6ced5a325e..f6230e2ae5485 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java @@ -26,15 +26,19 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.StickyKeyConsumerSelector; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -43,6 +47,7 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; @@ -61,6 +66,7 @@ import org.testng.annotations.Test; @Test(groups = "flaky") +@Slf4j public class MultiTopicsReaderTest extends MockedPulsarServiceBaseTest { private static final String subscription = "reader-multi-topics-sub"; @@ -121,6 +127,67 @@ public void testReadMessageWithBatching() throws Exception { testReadMessages(topic, true); } + @Test(timeOut = 10000) + public void testHasMessageAvailableAsync() throws Exception { + String topic = "persistent://my-property/my-ns/testHasMessageAvailableAsync"; + String content = "my-message-"; + int msgNum = 10; + admin.topics().createPartitionedTopic(topic, 2); + // stop retention from cleaning up + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close(); + + try (Reader reader = pulsarClient.newReader().topic(topic).readCompacted(true) + .startMessageId(MessageId.earliest).create()) { + Assert.assertFalse(reader.hasMessageAvailable()); + Assert.assertFalse(reader.hasMessageAvailableAsync().get(10, TimeUnit.SECONDS)); + } + + try (Reader reader = pulsarClient.newReader() + .topic(topic).startMessageId(MessageId.earliest).create()) { + try (Producer producer = pulsarClient.newProducer().topic(topic).create()) { + for (int i = 0; i < msgNum; i++) { + producer.newMessage().key(content + i) + .value((content + i).getBytes(StandardCharsets.UTF_8)).send(); + } + } + // Should have message available + Assert.assertTrue(reader.hasMessageAvailableAsync().get()); + try { + // Should have message available too + Assert.assertTrue(reader.hasMessageAvailable()); + } catch (PulsarClientException e) { + fail("Expect success but failed.", e); + } + List> msgs = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch latch = new CountDownLatch(1); + readMessageUseAsync(reader, msgs, latch); + latch.await(); + Assert.assertEquals(msgs.size(), msgNum); + } + } + + private static void readMessageUseAsync(Reader reader, List> msgs, CountDownLatch latch) { + reader.hasMessageAvailableAsync().thenAccept(hasMessageAvailable -> { + if (hasMessageAvailable) { + try { + Message msg = reader.readNext(); + msgs.add(msg); + } catch (PulsarClientException e) { + log.error("Read message failed.", e); + latch.countDown(); + return; + } + readMessageUseAsync(reader, msgs, latch); + } else { + latch.countDown(); + } + }).exceptionally(throwable -> { + log.error("Read message failed.", throwable); + latch.countDown(); + return null; + }); + } + @Test(timeOut = 10000) public void testReadMessageWithBatchingWithMessageInclusive() throws Exception { String topic = "persistent://my-property/my-ns/my-reader-topic-with-batching-inclusive" + UUID.randomUUID(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 09dc5643698d3..d2646a8cd0643 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -754,6 +754,9 @@ public boolean hasMessageAvailable() throws PulsarClientException { } public CompletableFuture hasMessageAvailableAsync() { + if (numMessagesInQueue() > 0) { + return CompletableFuture.completedFuture(true); + } List> futureList = new ArrayList<>(); final AtomicBoolean hasMessageAvailable = new AtomicBoolean(false); for (ConsumerImpl consumer : consumers.values()) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java index fab61b2b51130..b656c005db9a5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java @@ -144,7 +144,7 @@ public boolean hasReachedEndOfTopic() { @Override public boolean hasMessageAvailable() throws PulsarClientException { - return multiTopicsConsumer.hasMessageAvailable() || multiTopicsConsumer.numMessagesInQueue() > 0; + return multiTopicsConsumer.hasMessageAvailable(); } @Override From 3bef733e5bc7b24036a2b8d4ed3489b0387eeb47 Mon Sep 17 00:00:00 2001 From: feynmanlin Date: Thu, 23 Dec 2021 17:25:49 +0800 Subject: [PATCH 226/823] Fix NPE in cmdTopics (#13450) (cherry picked from commit 76f35666deb5a956b7eef9732a3028b246e5294c) --- .../src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index e37954de7419b..c541cc1a5752a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -597,6 +597,10 @@ private class GetInternalInfo extends CliCommand { void run() throws PulsarAdminException { String topic = validateTopicName(params); String internalInfo = getTopics().getInternalInfo(topic); + if (internalInfo == null) { + System.out.println("Did not find any internal metadata info"); + return; + } JsonObject result = JsonParser.parseString(internalInfo).getAsJsonObject(); Gson gson = new GsonBuilder().setPrettyPrinting().create(); System.out.println(gson.toJson(result)); From 4a8d6ef06af4619f41049e2184283c77c5c0bb3c Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Thu, 23 Dec 2021 19:27:23 +0800 Subject: [PATCH 227/823] Fixes the NPE in system topics policies service (#13469) --- *Motivation* The `namespaceEventsSystemTopicFactory` is created when you will use it. But the `createSystemTopicFactoryIfNeeded()` may failed which will cause the `namespaceEventsSystemTopicFactory` is null and throw a NPE error from the method. *Modifications* - throw the error and failed the method when there has exceptions in `createSystemTopicFactoryIfNeeded()` (cherry picked from commit 4022b2884f46bb5e1593da419bb226ad1e0fc768) --- .../SystemTopicBasedTopicPoliciesService.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 666220204508b..bf27736e97e47 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -88,9 +88,13 @@ public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, Top private CompletableFuture sendTopicPolicyEvent(TopicName topicName, ActionType actionType, TopicPolicies policies) { - createSystemTopicFactoryIfNeeded(); - CompletableFuture result = new CompletableFuture<>(); + try { + createSystemTopicFactoryIfNeeded(); + } catch (PulsarServerException e) { + result.completeExceptionally(e); + return result; + } SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory.createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); @@ -180,8 +184,9 @@ public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { @Override public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicName topicName) { CompletableFuture result = new CompletableFuture<>(); - createSystemTopicFactoryIfNeeded(); - if (namespaceEventsSystemTopicFactory == null) { + try { + createSystemTopicFactoryIfNeeded(); + } catch (PulsarServerException e) { result.complete(null); return result; } @@ -201,7 +206,6 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name result.complete(null); return result; } - createSystemTopicFactoryIfNeeded(); synchronized (this) { if (readerCaches.get(namespace) != null) { ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); @@ -235,9 +239,15 @@ private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture protected CompletableFuture> creatSystemTopicClientWithRetry( NamespaceName namespace) { + CompletableFuture> result = new CompletableFuture<>(); + try { + createSystemTopicFactoryIfNeeded(); + } catch (PulsarServerException e) { + result.completeExceptionally(e); + return result; + } SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory .createTopicPoliciesSystemTopicClient(namespace); - CompletableFuture> result = new CompletableFuture<>(); Backoff backoff = new Backoff(1, TimeUnit.SECONDS, 3, TimeUnit.SECONDS, 10, TimeUnit.SECONDS); RetryUtil.retryAsynchronously(() -> { try { @@ -386,6 +396,12 @@ private void refreshTopicPoliciesCache(Message msg) { // However, due to compatibility, it is temporarily retained here // and can be deleted in the future. policiesCache.remove(topicName); + try { + createSystemTopicFactoryIfNeeded(); + } catch (PulsarServerException e) { + log.error("Failed to create system topic factory"); + break; + } SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); systemTopicClient.newWriterAsync().thenAccept(writer @@ -405,7 +421,7 @@ private void refreshTopicPoliciesCache(Message msg) { } } - private void createSystemTopicFactoryIfNeeded() { + private void createSystemTopicFactoryIfNeeded() throws PulsarServerException { if (namespaceEventsSystemTopicFactory == null) { synchronized (this) { if (namespaceEventsSystemTopicFactory == null) { @@ -414,6 +430,7 @@ private void createSystemTopicFactoryIfNeeded() { new NamespaceEventsSystemTopicFactory(pulsarService.getClient()); } catch (PulsarServerException e) { log.error("Create namespace event system topic factory error.", e); + throw e; } } } From e4291454da5c291ea70f380008cddf4a3561b817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Fri, 24 Dec 2021 00:46:31 +0800 Subject: [PATCH 228/823] Fix dead loop in BacklogQuotaManager.dropBacklogForTimeLimit (#13194) (#13249) Fixes #13194 ### Motivation https://github.com/apache/pulsar/blob/38fb839154462fc5c6b0b4293f02762ed4021cd9/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java#L200-L219 BacklogQuotaManager.dropBacklogForTimeLimit may fall into dead loop in some conditions, e.g. `backlogQuotaDefaultLimitSecond` is enabled 1. producer stop produce after produced some messages, current ledger is A 2. times up, triggered ledger rollover, a new ledger B created which is empty (no entries) 3. now lastConfirmedEntry is `A:last-entry-id` 4. after `backlogQuotaDefaultLimitSecond` times up, it'll reset cursor to position `A:last-entry-id+1` which is only valid, so loop begin until the producer resume produce ### Modifications Record the previous slowestReaderPosition, if it is same with newer slowestReaderPosition after `resetCursor`, then exit loop. (cherry picked from commit 021409b6e1d3c910afe8e05d51a536cee647cb90) --- .../broker/service/BacklogQuotaManager.java | 28 ++++++----- .../service/BacklogQuotaManagerTest.java | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index 042f9ff0b3ffa..6efc1a4ed9b9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -26,9 +26,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -238,17 +239,22 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo Long currentMillis = ((ManagedLedgerImpl) persistentTopic.getManagedLedger()).getClock().millis(); ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); try { - Long ledgerId = mLedger.getCursors().getSlowestReaderPosition().getLedgerId(); - MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = mLedger.getLedgerInfo(ledgerId).get(); - // Timestamp only > 0 if ledger has been closed - while (ledgerInfo.getTimestamp() > 0 - && currentMillis - ledgerInfo.getTimestamp() > quota.getLimitTime()) { + for (;;) { ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); - // skip whole ledger for the slowest cursor - slowestConsumer.resetCursor(mLedger.getNextValidPosition( - PositionImpl.get(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1))); - ledgerId = mLedger.getCursors().getSlowestReaderPosition().getLedgerId(); - ledgerInfo = mLedger.getLedgerInfo(ledgerId).get(); + Position oldestPosition = slowestConsumer.getMarkDeletedPosition(); + ManagedLedgerInfo.LedgerInfo ledgerInfo = mLedger.getLedgerInfo(oldestPosition.getLedgerId()).get(); + // Timestamp only > 0 if ledger has been closed + if (ledgerInfo.getTimestamp() > 0 + && currentMillis - ledgerInfo.getTimestamp() > quota.getLimitTime()) { + // skip whole ledger for the slowest cursor + PositionImpl nextPosition = mLedger.getNextValidPosition( + PositionImpl.get(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1)); + if (!nextPosition.equals(oldestPosition)) { + slowestConsumer.resetCursor(nextPosition); + continue; + } + } + break; } } catch (Exception e) { log.error("[{}] Error resetting cursor for slowest consumer [{}]", persistentTopic.getName(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index 0dac0c2468ed5..f706b3f4b125b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -483,6 +483,54 @@ public void testConsumerBacklogEvictionTimeQuota() throws Exception { client.close(); } + @Test + public void testConsumerBacklogEvictionTimeQuotaWithEmptyLedger() throws Exception { + assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), + Maps.newHashMap()); + admin.namespaces().setBacklogQuota("prop/ns-quota", + BacklogQuota.builder() + .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) + .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) + .build(), BacklogQuota.BacklogQuotaType.message_age); + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + final String topic = "persistent://prop/ns-quota/topic4"; + final String subName = "c1"; + + Consumer consumer = client.newConsumer().topic(topic).subscriptionName(subName).subscribe(); + org.apache.pulsar.client.api.Producer producer = createProducer(client, topic); + producer.send(new byte[1024]); + consumer.receive(); + + admin.topics().unload(topic); + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topic); + assertEquals(internalStats.ledgers.size(), 2); + assertEquals(internalStats.ledgers.get(1).entries, 0); + + TopicStats stats = admin.topics().getStats(topic); + assertEquals(stats.getSubscriptions().get(subName).getMsgBacklog(), 1); + + TimeUnit.SECONDS.sleep(TIME_TO_CHECK_BACKLOG_QUOTA); + + Awaitility.await() + .pollInterval(Duration.ofSeconds(1)) + .atMost(Duration.ofSeconds(TIME_TO_CHECK_BACKLOG_QUOTA)) + .untilAsserted(() -> { + rolloverStats(); + + // Cause the last ledger is empty, it is not possible to skip first ledger, + // so the number of ledgers will keep unchanged, and backlog is clear + PersistentTopicInternalStats latestInternalStats = admin.topics().getInternalStats(topic); + assertEquals(latestInternalStats.ledgers.size(), 2); + assertEquals(latestInternalStats.ledgers.get(1).entries, 0); + TopicStats latestStats = admin.topics().getStats(topic); + assertEquals(latestStats.getSubscriptions().get(subName).getMsgBacklog(), 0); + }); + + client.close(); + } + @Test public void testConsumerBacklogEvictionWithAckSizeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), From 5ce12583439f2dc1ee80b815c6fbfae4643f7ccd Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Dec 2021 00:44:29 +0800 Subject: [PATCH 229/823] [Broker] Fix create the dynamic configuration resource if not exist (#13420) ### Motivation When request the `DELETE: /admin/brokers/configuration/dispatcherMinReadBatchSize`, which return the`org.apache.pulsar.metadata.api.MetadataStoreException$NotFoundException`. The Pulsar does not create the dynamic configuration resource path on metadata if it does not exist, this behavior is different with Pulsar 2.8. (cherry picked from commit 3a1e8da9230343a9f16da503e27525263369743b) --- .../DynamicConfigurationResources.java | 4 +- .../pulsar/broker/admin/impl/BrokersBase.java | 3 +- .../pulsar/broker/service/BrokerService.java | 11 ++- .../AdminApiDynamicConfigurationsTest.java | 72 +++++++++++++++++++ 4 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/DynamicConfigurationResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/DynamicConfigurationResources.java index 8137dd837ef96..d918dc8f2c5d3 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/DynamicConfigurationResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/DynamicConfigurationResources.java @@ -40,8 +40,8 @@ public CompletableFuture>> getDynamicConfigurationA return getAsync(BROKER_SERVICE_CONFIGURATION_PATH); } - public Map getDynamicConfiguration() throws MetadataStoreException { - return get(BROKER_SERVICE_CONFIGURATION_PATH).orElse(Collections.emptyMap()); + public Optional> getDynamicConfiguration() throws MetadataStoreException { + return get(BROKER_SERVICE_CONFIGURATION_PATH); } public void setDynamicConfigurationWithCreate( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 17f497c2e8fe3..756d141fab444 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -24,6 +24,7 @@ import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import java.time.Duration; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -181,7 +182,7 @@ public void deleteDynamicConfiguration(@PathParam("configName") String configNam public Map getAllDynamicConfigurations() throws Exception { validateSuperUserAccess(); try { - return dynamicConfigurationResources().getDynamicConfiguration(); + return dynamicConfigurationResources().getDynamicConfiguration().orElseGet(Collections::emptyMap); } catch (RestException e) { LOG.error("[{}] couldn't find any configuration in zk {}", clientAppId(), e.getMessage(), e); throw e; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index b3fca060cd811..cbe02c8c15f82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2236,9 +2236,16 @@ private void validateConfigKey(String key) { */ private void updateDynamicServiceConfiguration() { Optional> configCache = Optional.empty(); + try { - configCache = - Optional.of(pulsar().getPulsarResources().getDynamicConfigResources().getDynamicConfiguration()); + configCache = + pulsar().getPulsarResources().getDynamicConfigResources().getDynamicConfiguration(); + + // create dynamic-config if not exist. + if (!configCache.isPresent()) { + pulsar().getPulsarResources().getDynamicConfigResources() + .setDynamicConfigurationWithCreate(n -> Maps.newHashMap()); + } } catch (Exception e) { log.warn("Failed to read dynamic broker configuration", e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java new file mode 100644 index 0000000000000..d3e4b2a4bd829 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.junit.Assert.fail; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.util.Map; +import javax.ws.rs.core.Response; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class AdminApiDynamicConfigurationsTest extends MockedPulsarServiceBaseTest { + @BeforeMethod + @Override + public void setup() throws Exception { + super.internalSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void TestGetAllDynamicConfigurations() throws Exception { + Map configs = admin.brokers().getAllDynamicConfigurations(); + assertNotNull(configs); + } + + @Test + public void TestDeleteDynamicConfiguration() throws Exception { + admin.brokers().deleteDynamicConfiguration("dispatcherMinReadBatchSize"); + } + + @Test + public void TestDeleteInvalidDynamicConfiguration() { + try { + admin.brokers().deleteDynamicConfiguration("errorName"); + fail("exception should be thrown"); + } catch (Exception e) { + if (e instanceof PulsarAdminException) { + assertEquals(((PulsarAdminException) e).getStatusCode(), Response.Status.PRECONDITION_FAILED.getStatusCode()); + } else { + fail("PulsarAdminException should be thrown"); + } + } + } +} From 43df4dbb8909470228dca53cf9be1c19193f02c7 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 24 Dec 2021 10:56:17 +0800 Subject: [PATCH 230/823] Fix semaphore and memory leak when chunks failed to enqueue (#13454) ### Motivation When a large message is sent by chunks, each chunk needs to reserve a spot of the semaphore. However, when it failed, the already reserved memory from limiter and spots from semaphore are not released. ### Modifications - Release the semaphore and memory when `canEnqueueRequest` returns false for chunks. - Add `testChunksEnqueueFailed` to cover this case. It sends a large message whose number of chunks is greater than the `maxPendingMessages` so that the first time `canEnqueueRequest` returns true while the following `canEnqueueRequest` calls will return false. (cherry picked from commit 2e2cd57f984b601a878dd11e1d27f4c169a84e5b) --- .../client/impl/MessageChunkingTest.java | 43 +++++++++++++++++++ .../pulsar/client/impl/ProducerImpl.java | 2 + 2 files changed, 45 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java index d4eab77a7c21f..40191efe241a8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Lists; @@ -27,17 +28,20 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -47,6 +51,7 @@ import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.impl.MessageImpl.SchemaState; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -369,6 +374,44 @@ public void testExpireIncompleteChunkMessage() throws Exception{ producer = null; // clean reference of mocked producer } + @Test + public void testChunksEnqueueFailed() throws Exception { + final String topicName = "persistent://my-property/my-ns/test-chunks-enqueue-failed"; + log.info("-- Starting {} test --", methodName); + this.conf.setMaxMessageSize(5); + + final MemoryLimitController controller = ((PulsarClientImpl) pulsarClient).getMemoryLimitController(); + assertEquals(controller.currentUsage(), 0); + + final int maxPendingMessages = 10; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .maxPendingMessages(maxPendingMessages) + .enableChunking(true) + .enableBatching(false) + .create(); + assertTrue(producer instanceof ProducerImpl); + Semaphore semaphore = ((ProducerImpl) producer).getSemaphore().orElse(null); + assertNotNull(semaphore); + assertEquals(semaphore.availablePermits(), maxPendingMessages); + producer.send(createMessagePayload(1).getBytes()); + try { + producer.send(createMessagePayload(100).getBytes(StandardCharsets.UTF_8)); + fail("It should fail with ProducerQueueIsFullError"); + } catch (PulsarClientException e) { + assertTrue(e instanceof PulsarClientException.ProducerQueueIsFullError); + assertEquals(controller.currentUsage(), 0); + assertEquals(semaphore.availablePermits(), maxPendingMessages); + } + } + + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + clientBuilder.memoryLimit(10000L, SizeUnit.BYTES); + } + private String createMessagePayload(int size) { StringBuilder str = new StringBuilder(); Random rand = new Random(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index abc771c068a89..5c22f13b408bf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -453,6 +453,8 @@ public void sendAsync(Message message, SendCallback callback) { // chunked message also sent individually so, try to acquire send-permits for (int i = 0; i < (totalChunks - 1); i++) { if (!canEnqueueRequest(callback, message.getSequenceId(), 0 /* The memory was already reserved */)) { + client.getMemoryLimitController().releaseMemory(uncompressedSize); + semaphoreRelease(i + 1); return; } } From 1a9f28b0ef889b02be0ba8e5e93158f9322b88c7 Mon Sep 17 00:00:00 2001 From: liudezhi <33149602+liudezhi2098@users.noreply.github.com> Date: Mon, 27 Dec 2021 11:01:40 +0800 Subject: [PATCH 231/823] Optimize the debug log that affects performance, and unify the style (#13498) (cherry picked from commit fb4e2c8c9075506188c79ecf8fae96883a56d948) --- .../pulsar/broker/service/AbstractReplicator.java | 6 ++++-- .../pulsar/functions/windowing/WindowManager.java | 13 +++++++++---- .../triggers/WatermarkTimeTriggerPolicy.java | 14 ++++++++++---- .../functions/worker/FunctionMetaDataManager.java | 5 +++-- .../pulsar/sql/presto/PulsarRecordCursor.java | 4 +++- 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index a404f10d3dacb..0af749c0b737f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -182,8 +182,10 @@ public synchronized CompletableFuture disconnect(boolean failIfHasBacklog) if (failIfHasBacklog && getNumberOfEntriesInBacklog() > 0) { CompletableFuture disconnectFuture = new CompletableFuture<>(); disconnectFuture.completeExceptionally(new TopicBusyException("Cannot close a replicator with backlog")); - log.debug("[{}][{} -> {}] Replicator disconnect failed since topic has backlog", topicName, localCluster, - remoteCluster); + if (log.isDebugEnabled()) { + log.debug("[{}][{} -> {}] Replicator disconnect failed since topic has backlog", topicName, localCluster + , remoteCluster); + } return disconnectFuture; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WindowManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WindowManager.java index 06e0b88776959..9f7b5bb0a2d5b 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WindowManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/WindowManager.java @@ -105,7 +105,9 @@ public void add(T event, long ts, Record record) { public void add(Event windowEvent) { // watermark events are not added to the queue. if (windowEvent.isWatermark()) { - log.debug(String.format("Got watermark event with ts %d", windowEvent.getTimestamp())); + if (log.isDebugEnabled()) { + log.debug("Got watermark event with ts {}", windowEvent.getTimestamp()); + } } else { queue.add(windowEvent); } @@ -145,8 +147,9 @@ public boolean onTrigger() { prevWindowEvents.clear(); if (!events.isEmpty()) { prevWindowEvents.addAll(windowEvents); - log.debug(String.format("invoking windowLifecycleListener onActivation, [%d] events in " - + "window.", events.size())); + if (log.isDebugEnabled()) { + log.debug("invoking windowLifecycleListener onActivation, [{}] events in window.", events.size()); + } windowLifecycleListener.onActivation(events, newEvents, expired, evictionPolicy.getContext().getReferenceTime()); } else { @@ -216,7 +219,9 @@ private List> scanEvents(boolean fullScan) { lock.unlock(); } eventsSinceLastExpiry.set(0); - log.debug(String.format("[%d] events expired from window.", eventsToExpire.size())); + if (log.isDebugEnabled()) { + log.debug("[{}] events expired from window.", eventsToExpire.size()); + } if (!eventsToExpire.isEmpty()) { log.debug("invoking windowLifecycleListener.onExpiry"); windowLifecycleListener.onExpiry(eventsToExpire); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkTimeTriggerPolicy.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkTimeTriggerPolicy.java index 22722bf887a2e..a2bfca689b0fd 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkTimeTriggerPolicy.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/WatermarkTimeTriggerPolicy.java @@ -80,7 +80,9 @@ public void shutdown() { private void handleWaterMarkEvent(Event event) { long watermarkTs = event.getTimestamp(); long windowEndTs = nextWindowEndTs; - log.debug(String.format("Window end ts %d Watermark ts %d", windowEndTs, watermarkTs)); + if (log.isDebugEnabled()) { + log.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs); + } while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); @@ -93,10 +95,14 @@ private void handleWaterMarkEvent(Event event) { * window intervals based on event ts. */ long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); - log.debug(String.format("Next aligned window end ts %d", ts)); + if (log.isDebugEnabled()) { + log.debug("Next aligned window end ts {}", ts); + } if (ts == Long.MAX_VALUE) { - log.debug(String.format("No events to process between %d and watermark ts %d", - windowEndTs, watermarkTs)); + if (log.isDebugEnabled()) { + log.debug("No events to process between {} and watermark ts {}", + windowEndTs, watermarkTs); + } break; } windowEndTs = ts; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java index eddb469ba9273..3d8c07e927c20 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java @@ -413,8 +413,9 @@ synchronized boolean processDeregister(String tenant, String namespace, String functionName, long version) throws IllegalArgumentException { boolean needsScheduling = false; - - log.debug("Process deregister request: {}/{}/{}/{}", tenant, namespace, functionName, version); + if (log.isDebugEnabled()) { + log.debug("Process deregister request: {}/{}/{}/{}", tenant, namespace, functionName, version); + } // Check if we still have this function. Maybe already deleted by someone else if (this.containsFunctionMetaData(tenant, namespace, functionName)) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 558b87bcd661d..99be9f969d6cd 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -437,7 +437,9 @@ public boolean hasFinished() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - log.debug(exception, "Failed to read entries from topic %s", topicName.toString()); + if (log.isDebugEnabled()) { + log.debug(exception, "Failed to read entries from topic %s", topicName.toString()); + } outstandingReadsRequests.incrementAndGet(); //set read latency stats for failed From 30acd1399f7f5567a4632194dd5e77f20d44bafc Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 28 Dec 2021 19:30:17 +0800 Subject: [PATCH 232/823] [Broker] Remove check resource when delete failure domain (#13421) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation This PR is used to keep same behavior with Pulsar 2.8 and 2.7 version. In Pulsar 2.7 and 2.8, when delete a non-existent failure domain resource, it will return 404 status code. In Pulsar 2.9, when delete a non-existent failure domain resource, it will return 200 status code. Reproduce in Pulsar 2.8: ``` $ docker run -itd \ -p 6650:6650 \ -p 8080:8080 \ apachepulsar/pulsar:2.8.1 \ bin/pulsar standalone $ pulsarctl clusters delete-failure-domain standalone non-existent-failure-domain [✖] code: 404 reason: Domain-name non-existent-failure-domain or cluster standalone does not exist ``` Reproduce in Pulsar 2.9: ``` $ docker run -itd \ -p 6650:6650 \ -p 8080:8080 \ apachepulsar/pulsar:2.9.0 \ bin/pulsar standalone $ pulsarctl clusters delete-failure-domain standalone non-existent-failure-domain Delete failure domain [non-existent-failure-domain] for cluster [standalone] succeed ``` Affected version: 2.9.x This issue was caused by https://github.com/apache/pulsar/pull/11693. ### Modifications - Remove check resource exists when call the `ClusterResources#deleteFailureDomain()` - Add check resource exists when call the `ClusterResources#deleteFailureDomains()` (cherry picked from commit 7f4aac55a0c9d82fc689e6de60afc9c611a534dc) --- .../broker/resources/ClusterResources.java | 12 +-- .../broker/admin/AdminApiClusterTest.java | 98 +++++++++++++++++++ 2 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java index a4ee27af4929a..32586d246926e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java @@ -121,20 +121,20 @@ public Optional getFailureDomain(String clusterName, String d public void deleteFailureDomain(String clusterName, String domainName) throws MetadataStoreException { String path = joinPath(BASE_CLUSTERS_PATH, clusterName, FAILURE_DOMAIN, domainName); - if (exists(path)) { - delete(path); - } + delete(path); } public void deleteFailureDomains(String clusterName) throws MetadataStoreException { String failureDomainPath = joinPath(BASE_CLUSTERS_PATH, clusterName, FAILURE_DOMAIN); + if (!exists(failureDomainPath)) { + return; + } + for (String domain : getChildren(failureDomainPath)) { delete(joinPath(failureDomainPath, domain)); } - if (exists(failureDomainPath)) { - delete(failureDomainPath); - } + delete(failureDomainPath); } public void setFailureDomainWithCreate(String clusterName, String domainName, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java new file mode 100644 index 0000000000000..83ef9af7a2e8f --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.Sets; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.FailureDomain; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +@Slf4j +public class AdminApiClusterTest extends MockedPulsarServiceBaseTest { + private final String CLUSTER = "test"; + + @BeforeMethod + @Override + public void setup() throws Exception { + resetConfig(); + super.internalSetup(); + admin.clusters() + .createCluster(CLUSTER, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + } + + @AfterMethod(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testDeleteNonExistCluster() { + String cluster = "test-non-exist-cluster-" + UUID.randomUUID(); + + assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.clusters().deleteCluster(cluster)); + } + + @Test + public void testDeleteExistCluster() throws PulsarAdminException { + String cluster = "test-exist-cluster-" + UUID.randomUUID(); + + admin.clusters() + .createCluster(cluster, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + Awaitility.await().untilAsserted(() -> assertNotNull(admin.clusters().getCluster(cluster))); + + admin.clusters().deleteCluster(cluster); + } + + @Test + public void testDeleteNonExistentFailureDomain() { + assertThrows(PulsarAdminException.NotFoundException.class, + () -> admin.clusters().deleteFailureDomain(CLUSTER, "non-existent-failure-domain")); + } + + @Test + public void testDeleteNonExistentFailureDomainInNonExistCluster() { + assertThrows(PulsarAdminException.PreconditionFailedException.class, + () -> admin.clusters().deleteFailureDomain(CLUSTER + UUID.randomUUID(), + "non-existent-failure-domain")); + } + + @Test + public void testDeleteExistFailureDomain() throws PulsarAdminException { + String domainName = CLUSTER + "-failure-domain"; + FailureDomain domain = FailureDomain.builder() + .brokers(Sets.newHashSet("b1", "b2", "b3")) + .build(); + admin.clusters().createFailureDomain(CLUSTER, domainName, domain); + Awaitility.await().untilAsserted(() -> admin.clusters().getFailureDomain(CLUSTER, domainName)); + + admin.clusters().deleteFailureDomain(CLUSTER, domainName); + } +} From 5637742489cb775d9c71ab8d7983f843f579ce91 Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Tue, 28 Dec 2021 19:33:01 +0800 Subject: [PATCH 233/823] fix: bug when allAll bucket (#13467) (cherry picked from commit 1235162cbe91fe04d0014d7ccd9672a7c68f4c36) --- .../bookkeeper/mledger/util/StatsBuckets.java | 2 +- .../bookkeeper/mledger/util/TestStatsBuckets.java | 13 +++++++++++++ .../apache/pulsar/websocket/stats/StatsBuckets.java | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java index 6d08bf498f837..dd779885391dd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java @@ -110,7 +110,7 @@ public void addAll(StatsBuckets other) { buckets[i].add(other.values[i]); } - sumCounter.add(other.count); + sumCounter.add(other.sum); } private boolean isSorted(long[] array) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java index 02366632ed069..12efbb054ded6 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java @@ -99,4 +99,17 @@ public void test() { assertEquals(stats.getCount(), 3); assertEquals(stats.getBuckets(), new long[] { 1, 0, 1, 1 }); } + + @Test + public void testAddAll() { + StatsBuckets stats = new StatsBuckets(10, 20, 30); + stats.addValue(1); + stats.addValue(2); + stats.refresh(); + StatsBuckets stats2 = new StatsBuckets(10, 20, 30); + stats2.addAll(stats); + stats2.refresh(); + assertEquals(stats2.getSum(),3); + assertEquals(stats2.getCount(),2); + } } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/StatsBuckets.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/StatsBuckets.java index 82f34d1b9de81..8625146a8659e 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/StatsBuckets.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/StatsBuckets.java @@ -61,7 +61,7 @@ public void addAll(StatsBuckets other) { for (int i = 0; i < buckets.length; i++) { buckets[i].add(other.values[i]); } - sumCounter.add(other.count); + sumCounter.add(other.sum); } From fc3ab2eaca4b7154b61b18aa360efc8d99a06999 Mon Sep 17 00:00:00 2001 From: Aloys Date: Tue, 28 Dec 2021 19:06:43 +0800 Subject: [PATCH 234/823] Fix reousrce leak when create producer failed (#13505) Fixes #13214 ### Motivation When client create producer failed caused by connection failed, topic terminated, or produce fenced. There are some resources that are not released in the client. ### Modifications When creating producer failed. 1. stop the sendTimout task 2. cancel the batchTimerTask 3. cancel the keyGeneratorTask 4. cancel the statTimeout task (cherry picked from commit 57eccf48e418e0243f586a945134785dbe9b5d08) --- .../broker/service/ExclusiveProducerTest.java | 28 +++++++++++++ .../broker/service/TopicTerminationTest.java | 28 +++++++++++++ .../pulsar/client/impl/ProducerImpl.java | 42 +++++++++++-------- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java index 04da55dab2141..4d2b16a6a0e75 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import io.netty.util.HashedWheelTimer; import lombok.Cleanup; import org.apache.pulsar.client.api.Producer; @@ -33,8 +34,11 @@ import org.apache.pulsar.client.api.PulsarClientException.ProducerBusyException; import org.apache.pulsar.client.api.PulsarClientException.ProducerFencedException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.awaitility.Awaitility; import org.powermock.reflect.Whitebox; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -117,6 +121,30 @@ private void simpleTest(String topic) throws Exception { p2.close(); } + @Test(dataProvider = "topics") + public void testProducerTasksCleanupWhenUsingExclusiveProducers(String type, boolean partitioned) throws Exception { + String topic = newTopic(type, partitioned); + Producer p1 = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .accessMode(ProducerAccessMode.Exclusive) + .create(); + + try { + pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .accessMode(ProducerAccessMode.Exclusive) + .create(); + fail("Should have failed"); + } catch (ProducerFencedException e) { + // Expected + } + + p1.close(); + + HashedWheelTimer timer = (HashedWheelTimer) ((PulsarClientImpl) pulsarClient).timer(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(timer.pendingTimeouts(), 0)); + } + @Test(dataProvider = "topics") public void existingSharedProducer(String type, boolean partitioned) throws Exception { String topic = newTopic(type, partitioned); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java index 0cd84f3a84602..dedc9904ce846 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java @@ -31,6 +31,8 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; + +import io.netty.util.HashedWheelTimer; import org.apache.pulsar.client.admin.PulsarAdminException.NotAllowedException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -41,8 +43,10 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderListener; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -108,6 +112,30 @@ public void testCreateProducerOnTerminatedTopic() throws Exception { } } + public void testCreatingProducerTasksCleanupWhenOnTerminatedTopic() throws Exception { + Producer producer = pulsarClient.newProducer().topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + producer.send("msg-1".getBytes()); + producer.send("msg-2".getBytes()); + MessageId msgId3 = producer.send("msg-3".getBytes()); + + MessageId lastMessageId = admin.topics().terminateTopicAsync(topicName).get(); + assertEquals(lastMessageId, msgId3); + producer.close(); + + try { + pulsarClient.newProducer().topic(topicName).create(); + fail("Should have thrown exception"); + } catch (PulsarClientException.TopicTerminatedException e) { + // Expected + } + HashedWheelTimer timer = (HashedWheelTimer) ((PulsarClientImpl) pulsarClient).timer(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(timer.pendingTimeouts(), 0)); + } + @Test(timeOut = 20000) public void testTerminateWhilePublishing() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 5c22f13b408bf..31a32da3b7152 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -872,23 +872,7 @@ public CompletableFuture closeAsync() { return CompletableFuture.completedFuture(null); } - Timeout timeout = sendTimeout; - if (timeout != null) { - timeout.cancel(); - sendTimeout = null; - } - - ScheduledFuture batchTimerTask = this.batchTimerTask; - if (batchTimerTask != null) { - batchTimerTask.cancel(false); - this.batchTimerTask = null; - } - - if (keyGeneratorTask != null && !keyGeneratorTask.isCancelled()) { - keyGeneratorTask.cancel(false); - } - - stats.cancelStatsTimeout(); + closeProducerTasks(); ClientCnx cnx = cnx(); if (cnx == null || currentState != State.Ready) { @@ -1558,6 +1542,7 @@ public void connectionOpened(final ClientCnx cnx) { failPendingMessages(cnx(), (PulsarClientException) cause); } producerCreatedFuture.completeExceptionally(cause); + closeProducerTasks(); client.cleanupProducer(this); } else if (cause instanceof PulsarClientException.ProducerFencedException) { setState(State.ProducerFenced); @@ -1565,6 +1550,7 @@ public void connectionOpened(final ClientCnx cnx) { failPendingMessages(cnx(), (PulsarClientException) cause); } producerCreatedFuture.completeExceptionally(cause); + closeProducerTasks(); client.cleanupProducer(this); } else if (producerCreatedFuture.isDone() || // (cause instanceof PulsarClientException && PulsarClientException.isRetriableError(cause) @@ -1575,6 +1561,7 @@ public void connectionOpened(final ClientCnx cnx) { } else { setState(State.Failed); producerCreatedFuture.completeExceptionally(cause); + closeProducerTasks(); client.cleanupProducer(this); Timeout timeout = sendTimeout; if (timeout != null) { @@ -1599,6 +1586,7 @@ public void connectionFailed(PulsarClientException exception) { } else { log.info("[{}] Producer creation failed for producer {} after producerTimeout", topic, producerId); } + closeProducerTasks(); setState(State.Failed); client.cleanupProducer(this); } @@ -1607,6 +1595,26 @@ public void connectionFailed(PulsarClientException exception) { } } + private void closeProducerTasks() { + Timeout timeout = sendTimeout; + if (timeout != null) { + timeout.cancel(); + sendTimeout = null; + } + + ScheduledFuture batchTimerTask = this.batchTimerTask; + if (batchTimerTask != null) { + batchTimerTask.cancel(false); + this.batchTimerTask = null; + } + + if (keyGeneratorTask != null && !keyGeneratorTask.isCancelled()) { + keyGeneratorTask.cancel(false); + } + + stats.cancelStatsTimeout(); + } + private void resendMessages(ClientCnx cnx, long expectedEpoch) { cnx.ctx().channel().eventLoop().execute(() -> { synchronized (this) { From 37701915edd38acc8356325bff2a6797998561a5 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Tue, 28 Dec 2021 18:51:20 +0800 Subject: [PATCH 235/823] Remove the unused junit depency in managed ledger (#13514) (cherry picked from commit 892948a86f2bed005c0cb6e3f047bcc77c6436ee) --- managed-ledger/pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index e7e9d98a5b25a..bdcb39d5165f3 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -94,15 +94,6 @@ snappy-java test - - - junit - junit - test - org.awaitility From 008452b32db9b3c258944587b8d3ddd86612ea2a Mon Sep 17 00:00:00 2001 From: Ali Ahmed Date: Wed, 20 Oct 2021 13:16:24 -0700 Subject: [PATCH 236/823] Add log error tracking for semaphore count leak (#12410) Co-authored-by: Ali Ahmed (cherry picked from commit 7c219b11966d4eb8cc20111468c3439d23f8777c) --- .../pulsar/client/impl/ProducerImpl.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 31a32da3b7152..f5165d7d37f3d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -151,6 +151,8 @@ public class ProducerImpl extends ProducerBase implements TimerTask, Conne private Optional topicEpoch = Optional.empty(); private final List previousExceptions = new CopyOnWriteArrayList(); + private boolean errorState; + @SuppressWarnings("rawtypes") private static final AtomicLongFieldUpdater msgIdGeneratorUpdater = AtomicLongFieldUpdater .newUpdater(ProducerImpl.class, "msgIdGenerator"); @@ -261,6 +263,21 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration grabCnx(); } + protected void semaphoreRelease(final int releaseCountRequest) { + if (semaphore.isPresent()) { + if (!errorState) { + final int availablePermits = semaphore.get().availablePermits(); + if (availablePermits - releaseCountRequest < 0) { + log.error("Semaphore permit release count request greater then availablePermits" + + " : availablePermits={}, releaseCountRequest={}", + availablePermits, releaseCountRequest); + errorState = true; + } + } + semaphore.get().release(releaseCountRequest); + } + } + protected OpSendMsgQueue createPendingMessagesQueue() { return new OpSendMsgQueue(); } @@ -1022,9 +1039,9 @@ private long getHighestSequenceId(OpSendMsg op) { } private void releaseSemaphoreForSendOp(OpSendMsg op) { - if (semaphore.isPresent()) { - semaphore.get().release(isBatchMessagingEnabled() ? op.numMessagesInBatch : 1); - } + + semaphoreRelease(isBatchMessagingEnabled() ? op.numMessagesInBatch : 1); + client.getMemoryLimitController().releaseMemory(op.uncompressedSize); } @@ -1778,7 +1795,7 @@ private void failPendingMessages(ClientCnx cnx, PulsarClientException ex) { }); pendingMessages.clear(); - semaphore.ifPresent(s -> s.release(releaseCount.get())); + semaphoreRelease(releaseCount.get()); if (batchMessagingEnabled) { failPendingBatchMessages(ex); } @@ -1804,7 +1821,7 @@ private void failPendingBatchMessages(PulsarClientException ex) { } final int numMessagesInBatch = batchMessageContainer.getNumMessagesInBatch(); batchMessageContainer.discard(ex); - semaphore.ifPresent(s -> s.release(numMessagesInBatch)); + semaphoreRelease(numMessagesInBatch); } @Override @@ -1847,9 +1864,9 @@ private void batchMessageAndSend() { processOpSendMsg(opSendMsg); } } catch (PulsarClientException e) { - semaphore.ifPresent(s -> s.release(batchMessageContainer.getNumMessagesInBatch())); + semaphoreRelease(batchMessageContainer.getNumMessagesInBatch()); } catch (Throwable t) { - semaphore.ifPresent(s -> s.release(batchMessageContainer.getNumMessagesInBatch())); + semaphoreRelease(batchMessageContainer.getNumMessagesInBatch()); log.warn("[{}] [{}] error while create opSendMsg by batch message container", topic, producerName, t); } } From 32c6cdcf8e30d3c3297ad8a8f9b9bd2319464b7d Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 28 Dec 2021 20:35:21 +0800 Subject: [PATCH 237/823] Use PulsarByteBufAllocator to allocate buffer for chunks at consumer side (#13536) ### Motivation Currently Pulsar consumer allocates memory from direct memory via `Unpooled.directBuffer` directly, which doesn't make use of the widely used allocator in Pulsar. ### Modifications Use `PulsarByteBufAllocator` as the memory allocator for chunks buffer. (cherry picked from commit a5d347370286cb67ffa4ee179ec6fb5117546a4e) (cherry picked from commit ddd1f5eb3dfb98b462a42121027cd5b51ed6218b) --- .../main/java/org/apache/pulsar/client/impl/ConsumerImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index ac0465153c817..9042788eb2368 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -26,7 +26,6 @@ import com.scurrilous.circe.checksum.Crc32cIntChecksum; import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; @@ -1255,7 +1254,7 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m } if (msgMetadata.getChunkId() == 0) { - ByteBuf chunkedMsgBuffer = Unpooled.directBuffer(msgMetadata.getTotalChunkMsgSize(), + ByteBuf chunkedMsgBuffer = PulsarByteBufAllocator.DEFAULT.buffer(msgMetadata.getTotalChunkMsgSize(), msgMetadata.getTotalChunkMsgSize()); int totalChunks = msgMetadata.getNumChunksFromMsg(); chunkedMessagesMap.computeIfAbsent(msgMetadata.getUuid(), From c846c5f17f8cfb7969568503c5fc9d12af8455c5 Mon Sep 17 00:00:00 2001 From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com> Date: Wed, 29 Dec 2021 09:38:38 +0800 Subject: [PATCH 238/823] [Pulsar SQL] support protobuf/timestamp (#13287) (cherry picked from commit 1ea4ad814f5f30b8c371db2a86626cd568ace553) (cherry picked from commit bdf5802477a8df39a5826548d4b80ad6b3ddbab7) --- .../PulsarProtobufNativeColumnDecoder.java | 15 +- ...PulsarProtobufNativeRowDecoderFactory.java | 18 +- .../decoder/protobufnative/TestMsg.java | 917 ++++++++++++++---- .../decoder/protobufnative/TestMsg.proto | 2 + .../TestProtobufNativeDecoder.java | 14 + 5 files changed, 769 insertions(+), 197 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java index 56e9b77cf679e..c71e7caaae876 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java @@ -70,7 +70,8 @@ public class PulsarProtobufNativeColumnDecoder { BigintType.BIGINT, RealType.REAL, DoubleType.DOUBLE, - VarbinaryType.VARBINARY); + VarbinaryType.VARBINARY, + TimestampType.TIMESTAMP); private final Type columnType; private final String columnMapping; @@ -193,6 +194,15 @@ public long getLong() { return floatToIntBits((Float) value); } + //return millisecond which parsed from protobuf/timestamp + if (columnType instanceof TimestampType && value instanceof DynamicMessage) { + DynamicMessage message = (DynamicMessage) value; + int nanos = (int) message.getField(message.getDescriptorForType().findFieldByName("nanos")); + long seconds = (long) message.getField(message.getDescriptorForType().findFieldByName("seconds")); + //maybe an exception here, but seems will never happen in hundred years. + return seconds * MILLIS_PER_SECOND + nanos / NANOS_PER_MILLISECOND; + } + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); @@ -377,5 +387,6 @@ private static Block serializeRow(BlockBuilder parentBlockBuilder, Object value, protected static final String PROTOBUF_MAP_KEY_NAME = "key"; protected static final String PROTOBUF_MAP_VALUE_NAME = "value"; - + private static final long MILLIS_PER_SECOND = 1000; + private static final long NANOS_PER_MILLISECOND = 1000000; } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java index 3a415b334db75..2d0f9af72dd32 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.Descriptors; +import com.google.protobuf.TimestampProto; import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; @@ -37,11 +38,13 @@ import io.prestosql.spi.type.RealType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.TimestampType; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.TypeManager; import io.prestosql.spi.type.TypeSignature; import io.prestosql.spi.type.TypeSignatureParameter; import io.prestosql.spi.type.VarbinaryType; + import java.util.List; import java.util.Optional; import java.util.Set; @@ -142,11 +145,16 @@ private Type parseProtobufPrestoType(Descriptors.FieldDescriptor field) { ImmutableList.of(TypeSignatureParameter.typeParameter(keyType), TypeSignatureParameter.typeParameter(valueType))); } else { - //row - dataType = RowType.from(msg.getFields().stream() - .map(rowField -> new RowType.Field(Optional.of(rowField.getName()), - parseProtobufPrestoType(rowField))) - .collect(toImmutableList())); + if (TimestampProto.getDescriptor().toProto().getName().equals(msg.getFile().toProto().getName())) { + //if msg type is protobuf/timestamp + dataType = TimestampType.TIMESTAMP; + } else { + //row + dataType = RowType.from(msg.getFields().stream() + .map(rowField -> new RowType.Field(Optional.of(rowField.getName()), + parseProtobufPrestoType(rowField))) + .collect(toImmutableList())); + } } break; default: diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java index 3065caf02968e..401ded5bd85cc 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java @@ -67,6 +67,8 @@ public final int getNumber() { } /** + * @param value The numeric wire value of the corresponding enum entry. + * @return The enum associated with the given numeric wire value. * @deprecated Use {@link #forNumber(int)} instead. */ @java.lang.Deprecated @@ -74,6 +76,10 @@ public static TestEnum valueOf(int value) { return forNumber(value); } + /** + * @param value The numeric wire value of the corresponding enum entry. + * @return The enum associated with the given numeric wire value. + */ public static TestEnum forNumber(int value) { switch (value) { case 0: return SHARED; @@ -96,6 +102,10 @@ public TestEnum findValueByNumber(int number) { public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() { + if (this == UNRECOGNIZED) { + throw new java.lang.IllegalStateException( + "Can't get the descriptor of an unrecognized enum value."); + } return getDescriptor().getValues().get(ordinal()); } public final com.google.protobuf.Descriptors.EnumDescriptor @@ -136,25 +146,30 @@ public interface SubMessageOrBuilder extends /** * string foo = 1; + * @return The foo. */ java.lang.String getFoo(); /** * string foo = 1; + * @return The bytes for foo. */ com.google.protobuf.ByteString getFooBytes(); /** * double bar = 2; + * @return The bar. */ double getBar(); /** * .proto.SubMessage.NestedMessage nestedMessage = 3; + * @return Whether the nestedMessage field is set. */ boolean hasNestedMessage(); /** * .proto.SubMessage.NestedMessage nestedMessage = 3; + * @return The nestedMessage. */ org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getNestedMessage(); /** @@ -165,7 +180,7 @@ public interface SubMessageOrBuilder extends /** * Protobuf type {@code proto.SubMessage} */ - public static final class SubMessage extends + public static final class SubMessage extends com.google.protobuf.GeneratedMessageV3 implements // @@protoc_insertion_point(message_implements:proto.SubMessage) SubMessageOrBuilder { @@ -176,7 +191,13 @@ private SubMessage(com.google.protobuf.GeneratedMessageV3.Builder builder) { } private SubMessage() { foo_ = ""; - bar_ = 0D; + } + + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance( + UnusedPrivateParameter unused) { + return new SubMessage(); } @java.lang.Override @@ -192,7 +213,6 @@ private SubMessage( if (extensionRegistry == null) { throw new java.lang.NullPointerException(); } - int mutable_bitField0_ = 0; com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet.newBuilder(); try { @@ -203,13 +223,6 @@ private SubMessage( case 0: done = true; break; - default: { - if (!parseUnknownFieldProto3( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 10: { java.lang.String s = input.readStringRequireUtf8(); @@ -234,6 +247,13 @@ private SubMessage( break; } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -251,6 +271,7 @@ private SubMessage( return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_descriptor; } + @java.lang.Override protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_fieldAccessorTable @@ -264,29 +285,37 @@ public interface NestedMessageOrBuilder extends /** * string title = 1; + * @return The title. */ java.lang.String getTitle(); /** * string title = 1; + * @return The bytes for title. */ com.google.protobuf.ByteString getTitleBytes(); /** * repeated string urls = 2; + * @return A list containing the urls. */ java.util.List getUrlsList(); /** * repeated string urls = 2; + * @return The count of urls. */ int getUrlsCount(); /** * repeated string urls = 2; + * @param index The index of the element to return. + * @return The urls at the given index. */ java.lang.String getUrls(int index); /** * repeated string urls = 2; + * @param index The index of the value to return. + * @return The bytes of the urls at the given index. */ com.google.protobuf.ByteString getUrlsBytes(int index); @@ -294,7 +323,7 @@ public interface NestedMessageOrBuilder extends /** * Protobuf type {@code proto.SubMessage.NestedMessage} */ - public static final class NestedMessage extends + public static final class NestedMessage extends com.google.protobuf.GeneratedMessageV3 implements // @@protoc_insertion_point(message_implements:proto.SubMessage.NestedMessage) NestedMessageOrBuilder { @@ -308,6 +337,13 @@ private NestedMessage() { urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; } + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance( + UnusedPrivateParameter unused) { + return new NestedMessage(); + } + @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -332,13 +368,6 @@ private NestedMessage( case 0: done = true; break; - default: { - if (!parseUnknownFieldProto3( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 10: { java.lang.String s = input.readStringRequireUtf8(); @@ -347,13 +376,20 @@ private NestedMessage( } case 18: { java.lang.String s = input.readStringRequireUtf8(); - if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + if (!((mutable_bitField0_ & 0x00000001) != 0)) { urls_ = new com.google.protobuf.LazyStringArrayList(); - mutable_bitField0_ |= 0x00000002; + mutable_bitField0_ |= 0x00000001; } urls_.add(s); break; } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -362,7 +398,7 @@ private NestedMessage( throw new com.google.protobuf.InvalidProtocolBufferException( e).setUnfinishedMessage(this); } finally { - if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + if (((mutable_bitField0_ & 0x00000001) != 0)) { urls_ = urls_.getUnmodifiableView(); } this.unknownFields = unknownFields.build(); @@ -374,6 +410,7 @@ private NestedMessage( return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_descriptor; } + @java.lang.Override protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_fieldAccessorTable @@ -381,12 +418,13 @@ private NestedMessage( org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder.class); } - private int bitField0_; public static final int TITLE_FIELD_NUMBER = 1; private volatile java.lang.Object title_; /** * string title = 1; + * @return The title. */ + @java.lang.Override public java.lang.String getTitle() { java.lang.Object ref = title_; if (ref instanceof java.lang.String) { @@ -401,7 +439,9 @@ public java.lang.String getTitle() { } /** * string title = 1; + * @return The bytes for title. */ + @java.lang.Override public com.google.protobuf.ByteString getTitleBytes() { java.lang.Object ref = title_; @@ -420,6 +460,7 @@ public java.lang.String getTitle() { private com.google.protobuf.LazyStringList urls_; /** * repeated string urls = 2; + * @return A list containing the urls. */ public com.google.protobuf.ProtocolStringList getUrlsList() { @@ -427,18 +468,23 @@ public java.lang.String getTitle() { } /** * repeated string urls = 2; + * @return The count of urls. */ public int getUrlsCount() { return urls_.size(); } /** * repeated string urls = 2; + * @param index The index of the element to return. + * @return The urls at the given index. */ public java.lang.String getUrls(int index) { return urls_.get(index); } /** * repeated string urls = 2; + * @param index The index of the value to return. + * @return The bytes of the urls at the given index. */ public com.google.protobuf.ByteString getUrlsBytes(int index) { @@ -446,6 +492,7 @@ public java.lang.String getUrls(int index) { } private byte memoizedIsInitialized = -1; + @java.lang.Override public final boolean isInitialized() { byte isInitialized = memoizedIsInitialized; if (isInitialized == 1) return true; @@ -455,6 +502,7 @@ public final boolean isInitialized() { return true; } + @java.lang.Override public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { if (!getTitleBytes().isEmpty()) { @@ -466,6 +514,7 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) unknownFields.writeTo(output); } + @java.lang.Override public int getSerializedSize() { int size = memoizedSize; if (size != -1) return size; @@ -497,13 +546,12 @@ public boolean equals(final java.lang.Object obj) { } org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage other = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage) obj; - boolean result = true; - result = result && getTitle() - .equals(other.getTitle()); - result = result && getUrlsList() - .equals(other.getUrlsList()); - result = result && unknownFields.equals(other.unknownFields); - return result; + if (!getTitle() + .equals(other.getTitle())) return false; + if (!getUrlsList() + .equals(other.getUrlsList())) return false; + if (!unknownFields.equals(other.unknownFields)) return false; + return true; } @java.lang.Override @@ -594,6 +642,7 @@ public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMes .parseWithIOException(PARSER, input, extensionRegistry); } + @java.lang.Override public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() { return DEFAULT_INSTANCE.toBuilder(); @@ -601,6 +650,7 @@ public static Builder newBuilder() { public static Builder newBuilder(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage prototype) { return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); } + @java.lang.Override public Builder toBuilder() { return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); @@ -624,6 +674,7 @@ public static final class Builder extends return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_descriptor; } + @java.lang.Override protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_fieldAccessorTable @@ -646,24 +697,28 @@ private void maybeForceBuilderInitialization() { .alwaysUseFieldBuilders) { } } + @java.lang.Override public Builder clear() { super.clear(); title_ = ""; urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00000002); + bitField0_ = (bitField0_ & ~0x00000001); return this; } + @java.lang.Override public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_descriptor; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getDefaultInstanceForType() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance(); } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage build() { org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage result = buildPartial(); if (!result.isInitialized()) { @@ -672,47 +727,53 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Ne return result; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage buildPartial() { org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage result = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage(this); int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; result.title_ = title_; - if (((bitField0_ & 0x00000002) == 0x00000002)) { + if (((bitField0_ & 0x00000001) != 0)) { urls_ = urls_.getUnmodifiableView(); - bitField0_ = (bitField0_ & ~0x00000002); + bitField0_ = (bitField0_ & ~0x00000001); } result.urls_ = urls_; - result.bitField0_ = to_bitField0_; onBuilt(); return result; } + @java.lang.Override public Builder clone() { - return (Builder) super.clone(); + return super.clone(); } + @java.lang.Override public Builder setField( com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { - return (Builder) super.setField(field, value); + return super.setField(field, value); } + @java.lang.Override public Builder clearField( com.google.protobuf.Descriptors.FieldDescriptor field) { - return (Builder) super.clearField(field); + return super.clearField(field); } + @java.lang.Override public Builder clearOneof( com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return (Builder) super.clearOneof(oneof); + return super.clearOneof(oneof); } + @java.lang.Override public Builder setRepeatedField( com.google.protobuf.Descriptors.FieldDescriptor field, int index, java.lang.Object value) { - return (Builder) super.setRepeatedField(field, index, value); + return super.setRepeatedField(field, index, value); } + @java.lang.Override public Builder addRepeatedField( com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { - return (Builder) super.addRepeatedField(field, value); + return super.addRepeatedField(field, value); } + @java.lang.Override public Builder mergeFrom(com.google.protobuf.Message other) { if (other instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage) { return mergeFrom((org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage)other); @@ -731,7 +792,7 @@ public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.Tes if (!other.urls_.isEmpty()) { if (urls_.isEmpty()) { urls_ = other.urls_; - bitField0_ = (bitField0_ & ~0x00000002); + bitField0_ = (bitField0_ & ~0x00000001); } else { ensureUrlsIsMutable(); urls_.addAll(other.urls_); @@ -743,10 +804,12 @@ public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.Tes return this; } + @java.lang.Override public final boolean isInitialized() { return true; } + @java.lang.Override public Builder mergeFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -769,6 +832,7 @@ public Builder mergeFrom( private java.lang.Object title_ = ""; /** * string title = 1; + * @return The title. */ public java.lang.String getTitle() { java.lang.Object ref = title_; @@ -784,6 +848,7 @@ public java.lang.String getTitle() { } /** * string title = 1; + * @return The bytes for title. */ public com.google.protobuf.ByteString getTitleBytes() { @@ -800,6 +865,8 @@ public java.lang.String getTitle() { } /** * string title = 1; + * @param value The title to set. + * @return This builder for chaining. */ public Builder setTitle( java.lang.String value) { @@ -813,6 +880,7 @@ public Builder setTitle( } /** * string title = 1; + * @return This builder for chaining. */ public Builder clearTitle() { @@ -822,6 +890,8 @@ public Builder clearTitle() { } /** * string title = 1; + * @param value The bytes for title to set. + * @return This builder for chaining. */ public Builder setTitleBytes( com.google.protobuf.ByteString value) { @@ -837,13 +907,14 @@ public Builder setTitleBytes( private com.google.protobuf.LazyStringList urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; private void ensureUrlsIsMutable() { - if (!((bitField0_ & 0x00000002) == 0x00000002)) { + if (!((bitField0_ & 0x00000001) != 0)) { urls_ = new com.google.protobuf.LazyStringArrayList(urls_); - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000001; } } /** * repeated string urls = 2; + * @return A list containing the urls. */ public com.google.protobuf.ProtocolStringList getUrlsList() { @@ -851,18 +922,23 @@ private void ensureUrlsIsMutable() { } /** * repeated string urls = 2; + * @return The count of urls. */ public int getUrlsCount() { return urls_.size(); } /** * repeated string urls = 2; + * @param index The index of the element to return. + * @return The urls at the given index. */ public java.lang.String getUrls(int index) { return urls_.get(index); } /** * repeated string urls = 2; + * @param index The index of the value to return. + * @return The bytes of the urls at the given index. */ public com.google.protobuf.ByteString getUrlsBytes(int index) { @@ -870,6 +946,9 @@ public java.lang.String getUrls(int index) { } /** * repeated string urls = 2; + * @param index The index to set the value at. + * @param value The urls to set. + * @return This builder for chaining. */ public Builder setUrls( int index, java.lang.String value) { @@ -883,6 +962,8 @@ public Builder setUrls( } /** * repeated string urls = 2; + * @param value The urls to add. + * @return This builder for chaining. */ public Builder addUrls( java.lang.String value) { @@ -896,6 +977,8 @@ public Builder addUrls( } /** * repeated string urls = 2; + * @param values The urls to add. + * @return This builder for chaining. */ public Builder addAllUrls( java.lang.Iterable values) { @@ -907,15 +990,18 @@ public Builder addAllUrls( } /** * repeated string urls = 2; + * @return This builder for chaining. */ public Builder clearUrls() { urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00000002); + bitField0_ = (bitField0_ & ~0x00000001); onChanged(); return this; } /** * repeated string urls = 2; + * @param value The bytes of the urls to add. + * @return This builder for chaining. */ public Builder addUrlsBytes( com.google.protobuf.ByteString value) { @@ -928,11 +1014,13 @@ public Builder addUrlsBytes( onChanged(); return this; } + @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFieldsProto3(unknownFields); + return super.setUnknownFields(unknownFields); } + @java.lang.Override public final Builder mergeUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { return super.mergeUnknownFields(unknownFields); @@ -954,6 +1042,7 @@ public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMes private static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + @java.lang.Override public NestedMessage parsePartialFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -971,6 +1060,7 @@ public com.google.protobuf.Parser getParserForType() { return PARSER; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getDefaultInstanceForType() { return DEFAULT_INSTANCE; } @@ -981,7 +1071,9 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Ne private volatile java.lang.Object foo_; /** * string foo = 1; + * @return The foo. */ + @java.lang.Override public java.lang.String getFoo() { java.lang.Object ref = foo_; if (ref instanceof java.lang.String) { @@ -996,7 +1088,9 @@ public java.lang.String getFoo() { } /** * string foo = 1; + * @return The bytes for foo. */ + @java.lang.Override public com.google.protobuf.ByteString getFooBytes() { java.lang.Object ref = foo_; @@ -1015,7 +1109,9 @@ public java.lang.String getFoo() { private double bar_; /** * double bar = 2; + * @return The bar. */ + @java.lang.Override public double getBar() { return bar_; } @@ -1024,24 +1120,30 @@ public double getBar() { private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage nestedMessage_; /** * .proto.SubMessage.NestedMessage nestedMessage = 3; + * @return Whether the nestedMessage field is set. */ + @java.lang.Override public boolean hasNestedMessage() { return nestedMessage_ != null; } /** * .proto.SubMessage.NestedMessage nestedMessage = 3; + * @return The nestedMessage. */ + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getNestedMessage() { return nestedMessage_ == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance() : nestedMessage_; } /** * .proto.SubMessage.NestedMessage nestedMessage = 3; */ + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder getNestedMessageOrBuilder() { return getNestedMessage(); } private byte memoizedIsInitialized = -1; + @java.lang.Override public final boolean isInitialized() { byte isInitialized = memoizedIsInitialized; if (isInitialized == 1) return true; @@ -1051,6 +1153,7 @@ public final boolean isInitialized() { return true; } + @java.lang.Override public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { if (!getFooBytes().isEmpty()) { @@ -1065,6 +1168,7 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) unknownFields.writeTo(output); } + @java.lang.Override public int getSerializedSize() { int size = memoizedSize; if (size != -1) return size; @@ -1096,20 +1200,18 @@ public boolean equals(final java.lang.Object obj) { } org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage other = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage) obj; - boolean result = true; - result = result && getFoo() - .equals(other.getFoo()); - result = result && ( - java.lang.Double.doubleToLongBits(getBar()) - == java.lang.Double.doubleToLongBits( - other.getBar())); - result = result && (hasNestedMessage() == other.hasNestedMessage()); + if (!getFoo() + .equals(other.getFoo())) return false; + if (java.lang.Double.doubleToLongBits(getBar()) + != java.lang.Double.doubleToLongBits( + other.getBar())) return false; + if (hasNestedMessage() != other.hasNestedMessage()) return false; if (hasNestedMessage()) { - result = result && getNestedMessage() - .equals(other.getNestedMessage()); + if (!getNestedMessage() + .equals(other.getNestedMessage())) return false; } - result = result && unknownFields.equals(other.unknownFields); - return result; + if (!unknownFields.equals(other.unknownFields)) return false; + return true; } @java.lang.Override @@ -1203,6 +1305,7 @@ public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMes .parseWithIOException(PARSER, input, extensionRegistry); } + @java.lang.Override public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() { return DEFAULT_INSTANCE.toBuilder(); @@ -1210,6 +1313,7 @@ public static Builder newBuilder() { public static Builder newBuilder(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage prototype) { return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); } + @java.lang.Override public Builder toBuilder() { return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); @@ -1233,6 +1337,7 @@ public static final class Builder extends return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_descriptor; } + @java.lang.Override protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_fieldAccessorTable @@ -1255,6 +1360,7 @@ private void maybeForceBuilderInitialization() { .alwaysUseFieldBuilders) { } } + @java.lang.Override public Builder clear() { super.clear(); foo_ = ""; @@ -1270,15 +1376,18 @@ public Builder clear() { return this; } + @java.lang.Override public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_descriptor; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getDefaultInstanceForType() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance(); } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage build() { org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage result = buildPartial(); if (!result.isInitialized()) { @@ -1287,6 +1396,7 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage bu return result; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage buildPartial() { org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage result = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage(this); result.foo_ = foo_; @@ -1300,32 +1410,39 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage bu return result; } + @java.lang.Override public Builder clone() { - return (Builder) super.clone(); + return super.clone(); } + @java.lang.Override public Builder setField( com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { - return (Builder) super.setField(field, value); + return super.setField(field, value); } + @java.lang.Override public Builder clearField( com.google.protobuf.Descriptors.FieldDescriptor field) { - return (Builder) super.clearField(field); + return super.clearField(field); } + @java.lang.Override public Builder clearOneof( com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return (Builder) super.clearOneof(oneof); + return super.clearOneof(oneof); } + @java.lang.Override public Builder setRepeatedField( com.google.protobuf.Descriptors.FieldDescriptor field, int index, java.lang.Object value) { - return (Builder) super.setRepeatedField(field, index, value); + return super.setRepeatedField(field, index, value); } + @java.lang.Override public Builder addRepeatedField( com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { - return (Builder) super.addRepeatedField(field, value); + return super.addRepeatedField(field, value); } + @java.lang.Override public Builder mergeFrom(com.google.protobuf.Message other) { if (other instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage) { return mergeFrom((org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage)other); @@ -1352,10 +1469,12 @@ public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.Tes return this; } + @java.lang.Override public final boolean isInitialized() { return true; } + @java.lang.Override public Builder mergeFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -1377,6 +1496,7 @@ public Builder mergeFrom( private java.lang.Object foo_ = ""; /** * string foo = 1; + * @return The foo. */ public java.lang.String getFoo() { java.lang.Object ref = foo_; @@ -1392,6 +1512,7 @@ public java.lang.String getFoo() { } /** * string foo = 1; + * @return The bytes for foo. */ public com.google.protobuf.ByteString getFooBytes() { @@ -1408,6 +1529,8 @@ public java.lang.String getFoo() { } /** * string foo = 1; + * @param value The foo to set. + * @return This builder for chaining. */ public Builder setFoo( java.lang.String value) { @@ -1421,6 +1544,7 @@ public Builder setFoo( } /** * string foo = 1; + * @return This builder for chaining. */ public Builder clearFoo() { @@ -1430,6 +1554,8 @@ public Builder clearFoo() { } /** * string foo = 1; + * @param value The bytes for foo to set. + * @return This builder for chaining. */ public Builder setFooBytes( com.google.protobuf.ByteString value) { @@ -1446,12 +1572,16 @@ public Builder setFooBytes( private double bar_ ; /** * double bar = 2; + * @return The bar. */ + @java.lang.Override public double getBar() { return bar_; } /** * double bar = 2; + * @param value The bar to set. + * @return This builder for chaining. */ public Builder setBar(double value) { @@ -1461,6 +1591,7 @@ public Builder setBar(double value) { } /** * double bar = 2; + * @return This builder for chaining. */ public Builder clearBar() { @@ -1469,17 +1600,19 @@ public Builder clearBar() { return this; } - private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage nestedMessage_ = null; + private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage nestedMessage_; private com.google.protobuf.SingleFieldBuilderV3< org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder> nestedMessageBuilder_; /** * .proto.SubMessage.NestedMessage nestedMessage = 3; + * @return Whether the nestedMessage field is set. */ public boolean hasNestedMessage() { return nestedMessageBuilder_ != null || nestedMessage_ != null; } /** * .proto.SubMessage.NestedMessage nestedMessage = 3; + * @return The nestedMessage. */ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getNestedMessage() { if (nestedMessageBuilder_ == null) { @@ -1585,11 +1718,13 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Ne } return nestedMessageBuilder_; } + @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFieldsProto3(unknownFields); + return super.setUnknownFields(unknownFields); } + @java.lang.Override public final Builder mergeUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { return super.mergeUnknownFields(unknownFields); @@ -1611,6 +1746,7 @@ public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMes private static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + @java.lang.Override public SubMessage parsePartialFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -1628,6 +1764,7 @@ public com.google.protobuf.Parser getParserForType() { return PARSER; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getDefaultInstanceForType() { return DEFAULT_INSTANCE; } @@ -1640,99 +1777,119 @@ public interface TestMessageOrBuilder extends /** * string stringField = 1; + * @return The stringField. */ java.lang.String getStringField(); /** * string stringField = 1; + * @return The bytes for stringField. */ com.google.protobuf.ByteString getStringFieldBytes(); /** * double doubleField = 2; + * @return The doubleField. */ double getDoubleField(); /** * float floatField = 3; + * @return The floatField. */ float getFloatField(); /** * int32 int32Field = 4; + * @return The int32Field. */ int getInt32Field(); /** * int64 int64Field = 5; + * @return The int64Field. */ long getInt64Field(); /** * uint32 uint32Field = 6; + * @return The uint32Field. */ int getUint32Field(); /** * uint64 uint64Field = 7; + * @return The uint64Field. */ long getUint64Field(); /** * sint32 sint32Field = 8; + * @return The sint32Field. */ int getSint32Field(); /** * sint64 sint64Field = 9; + * @return The sint64Field. */ long getSint64Field(); /** * fixed32 fixed32Field = 10; + * @return The fixed32Field. */ int getFixed32Field(); /** * fixed64 fixed64Field = 11; + * @return The fixed64Field. */ long getFixed64Field(); /** * sfixed32 sfixed32Field = 12; + * @return The sfixed32Field. */ int getSfixed32Field(); /** * sfixed64 sfixed64Field = 13; + * @return The sfixed64Field. */ long getSfixed64Field(); /** * bool boolField = 14; + * @return The boolField. */ boolean getBoolField(); /** * bytes bytesField = 15; + * @return The bytesField. */ com.google.protobuf.ByteString getBytesField(); /** * .proto.TestEnum testEnum = 16; + * @return The enum numeric value on the wire for testEnum. */ int getTestEnumValue(); /** * .proto.TestEnum testEnum = 16; + * @return The testEnum. */ org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum(); /** * .proto.SubMessage subMessage = 17; + * @return Whether the subMessage field is set. */ boolean hasSubMessage(); /** * .proto.SubMessage subMessage = 17; + * @return The subMessage. */ org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getSubMessage(); /** @@ -1742,19 +1899,25 @@ public interface TestMessageOrBuilder extends /** * repeated string repeatedField = 18; + * @return A list containing the repeatedField. */ java.util.List getRepeatedFieldList(); /** * repeated string repeatedField = 18; + * @return The count of repeatedField. */ int getRepeatedFieldCount(); /** * repeated string repeatedField = 18; + * @param index The index of the element to return. + * @return The repeatedField at the given index. */ java.lang.String getRepeatedField(int index); /** * repeated string repeatedField = 18; + * @param index The index of the value to return. + * @return The bytes of the repeatedField at the given index. */ com.google.protobuf.ByteString getRepeatedFieldBytes(int index); @@ -1792,11 +1955,26 @@ boolean containsMapField( double getMapFieldOrThrow( java.lang.String key); + + /** + * .google.protobuf.Timestamp timestampField = 20; + * @return Whether the timestampField field is set. + */ + boolean hasTimestampField(); + /** + * .google.protobuf.Timestamp timestampField = 20; + * @return The timestampField. + */ + com.google.protobuf.Timestamp getTimestampField(); + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + com.google.protobuf.TimestampOrBuilder getTimestampFieldOrBuilder(); } /** * Protobuf type {@code proto.TestMessage} */ - public static final class TestMessage extends + public static final class TestMessage extends com.google.protobuf.GeneratedMessageV3 implements // @@protoc_insertion_point(message_implements:proto.TestMessage) TestMessageOrBuilder { @@ -1807,24 +1985,18 @@ private TestMessage(com.google.protobuf.GeneratedMessageV3.Builder builder) { } private TestMessage() { stringField_ = ""; - doubleField_ = 0D; - floatField_ = 0F; - int32Field_ = 0; - int64Field_ = 0L; - uint32Field_ = 0; - uint64Field_ = 0L; - sint32Field_ = 0; - sint64Field_ = 0L; - fixed32Field_ = 0; - fixed64Field_ = 0L; - sfixed32Field_ = 0; - sfixed64Field_ = 0L; - boolField_ = false; bytesField_ = com.google.protobuf.ByteString.EMPTY; testEnum_ = 0; repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; } + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance( + UnusedPrivateParameter unused) { + return new TestMessage(); + } + @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -1849,13 +2021,6 @@ private TestMessage( case 0: done = true; break; - default: { - if (!parseUnknownFieldProto3( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 10: { java.lang.String s = input.readStringRequireUtf8(); @@ -1953,18 +2118,18 @@ private TestMessage( } case 146: { java.lang.String s = input.readStringRequireUtf8(); - if (!((mutable_bitField0_ & 0x00020000) == 0x00020000)) { + if (!((mutable_bitField0_ & 0x00000001) != 0)) { repeatedField_ = new com.google.protobuf.LazyStringArrayList(); - mutable_bitField0_ |= 0x00020000; + mutable_bitField0_ |= 0x00000001; } repeatedField_.add(s); break; } case 154: { - if (!((mutable_bitField0_ & 0x00040000) == 0x00040000)) { + if (!((mutable_bitField0_ & 0x00000002) != 0)) { mapField_ = com.google.protobuf.MapField.newMapField( MapFieldDefaultEntryHolder.defaultEntry); - mutable_bitField0_ |= 0x00040000; + mutable_bitField0_ |= 0x00000002; } com.google.protobuf.MapEntry mapField__ = input.readMessage( @@ -1973,6 +2138,26 @@ private TestMessage( mapField__.getKey(), mapField__.getValue()); break; } + case 162: { + com.google.protobuf.Timestamp.Builder subBuilder = null; + if (timestampField_ != null) { + subBuilder = timestampField_.toBuilder(); + } + timestampField_ = input.readMessage(com.google.protobuf.Timestamp.parser(), extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(timestampField_); + timestampField_ = subBuilder.buildPartial(); + } + + break; + } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -1981,7 +2166,7 @@ private TestMessage( throw new com.google.protobuf.InvalidProtocolBufferException( e).setUnfinishedMessage(this); } finally { - if (((mutable_bitField0_ & 0x00020000) == 0x00020000)) { + if (((mutable_bitField0_ & 0x00000001) != 0)) { repeatedField_ = repeatedField_.getUnmodifiableView(); } this.unknownFields = unknownFields.build(); @@ -1994,6 +2179,7 @@ private TestMessage( } @SuppressWarnings({"rawtypes"}) + @java.lang.Override protected com.google.protobuf.MapField internalGetMapField( int number) { switch (number) { @@ -2004,6 +2190,7 @@ protected com.google.protobuf.MapField internalGetMapField( "Invalid map field number: " + number); } } + @java.lang.Override protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_fieldAccessorTable @@ -2011,12 +2198,13 @@ protected com.google.protobuf.MapField internalGetMapField( org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.Builder.class); } - private int bitField0_; public static final int STRINGFIELD_FIELD_NUMBER = 1; private volatile java.lang.Object stringField_; /** * string stringField = 1; + * @return The stringField. */ + @java.lang.Override public java.lang.String getStringField() { java.lang.Object ref = stringField_; if (ref instanceof java.lang.String) { @@ -2031,7 +2219,9 @@ public java.lang.String getStringField() { } /** * string stringField = 1; + * @return The bytes for stringField. */ + @java.lang.Override public com.google.protobuf.ByteString getStringFieldBytes() { java.lang.Object ref = stringField_; @@ -2050,7 +2240,9 @@ public java.lang.String getStringField() { private double doubleField_; /** * double doubleField = 2; + * @return The doubleField. */ + @java.lang.Override public double getDoubleField() { return doubleField_; } @@ -2059,7 +2251,9 @@ public double getDoubleField() { private float floatField_; /** * float floatField = 3; + * @return The floatField. */ + @java.lang.Override public float getFloatField() { return floatField_; } @@ -2068,7 +2262,9 @@ public float getFloatField() { private int int32Field_; /** * int32 int32Field = 4; + * @return The int32Field. */ + @java.lang.Override public int getInt32Field() { return int32Field_; } @@ -2077,7 +2273,9 @@ public int getInt32Field() { private long int64Field_; /** * int64 int64Field = 5; + * @return The int64Field. */ + @java.lang.Override public long getInt64Field() { return int64Field_; } @@ -2086,7 +2284,9 @@ public long getInt64Field() { private int uint32Field_; /** * uint32 uint32Field = 6; + * @return The uint32Field. */ + @java.lang.Override public int getUint32Field() { return uint32Field_; } @@ -2095,7 +2295,9 @@ public int getUint32Field() { private long uint64Field_; /** * uint64 uint64Field = 7; + * @return The uint64Field. */ + @java.lang.Override public long getUint64Field() { return uint64Field_; } @@ -2104,7 +2306,9 @@ public long getUint64Field() { private int sint32Field_; /** * sint32 sint32Field = 8; + * @return The sint32Field. */ + @java.lang.Override public int getSint32Field() { return sint32Field_; } @@ -2113,7 +2317,9 @@ public int getSint32Field() { private long sint64Field_; /** * sint64 sint64Field = 9; + * @return The sint64Field. */ + @java.lang.Override public long getSint64Field() { return sint64Field_; } @@ -2122,7 +2328,9 @@ public long getSint64Field() { private int fixed32Field_; /** * fixed32 fixed32Field = 10; + * @return The fixed32Field. */ + @java.lang.Override public int getFixed32Field() { return fixed32Field_; } @@ -2131,7 +2339,9 @@ public int getFixed32Field() { private long fixed64Field_; /** * fixed64 fixed64Field = 11; + * @return The fixed64Field. */ + @java.lang.Override public long getFixed64Field() { return fixed64Field_; } @@ -2140,7 +2350,9 @@ public long getFixed64Field() { private int sfixed32Field_; /** * sfixed32 sfixed32Field = 12; + * @return The sfixed32Field. */ + @java.lang.Override public int getSfixed32Field() { return sfixed32Field_; } @@ -2149,7 +2361,9 @@ public int getSfixed32Field() { private long sfixed64Field_; /** * sfixed64 sfixed64Field = 13; + * @return The sfixed64Field. */ + @java.lang.Override public long getSfixed64Field() { return sfixed64Field_; } @@ -2158,7 +2372,9 @@ public long getSfixed64Field() { private boolean boolField_; /** * bool boolField = 14; + * @return The boolField. */ + @java.lang.Override public boolean getBoolField() { return boolField_; } @@ -2167,7 +2383,9 @@ public boolean getBoolField() { private com.google.protobuf.ByteString bytesField_; /** * bytes bytesField = 15; + * @return The bytesField. */ + @java.lang.Override public com.google.protobuf.ByteString getBytesField() { return bytesField_; } @@ -2176,14 +2394,17 @@ public com.google.protobuf.ByteString getBytesField() { private int testEnum_; /** * .proto.TestEnum testEnum = 16; + * @return The enum numeric value on the wire for testEnum. */ - public int getTestEnumValue() { + @java.lang.Override public int getTestEnumValue() { return testEnum_; } /** * .proto.TestEnum testEnum = 16; + * @return The testEnum. */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum() { + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum() { + @SuppressWarnings("deprecation") org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum result = org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.valueOf(testEnum_); return result == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.UNRECOGNIZED : result; } @@ -2192,19 +2413,24 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getT private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage subMessage_; /** * .proto.SubMessage subMessage = 17; + * @return Whether the subMessage field is set. */ + @java.lang.Override public boolean hasSubMessage() { return subMessage_ != null; } /** * .proto.SubMessage subMessage = 17; + * @return The subMessage. */ + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getSubMessage() { return subMessage_ == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance() : subMessage_; } /** * .proto.SubMessage subMessage = 17; */ + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder getSubMessageOrBuilder() { return getSubMessage(); } @@ -2213,6 +2439,7 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrB private com.google.protobuf.LazyStringList repeatedField_; /** * repeated string repeatedField = 18; + * @return A list containing the repeatedField. */ public com.google.protobuf.ProtocolStringList getRepeatedFieldList() { @@ -2220,18 +2447,23 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrB } /** * repeated string repeatedField = 18; + * @return The count of repeatedField. */ public int getRepeatedFieldCount() { return repeatedField_.size(); } /** * repeated string repeatedField = 18; + * @param index The index of the element to return. + * @return The repeatedField at the given index. */ public java.lang.String getRepeatedField(int index) { return repeatedField_.get(index); } /** * repeated string repeatedField = 18; + * @param index The index of the value to return. + * @return The bytes of the repeatedField at the given index. */ public com.google.protobuf.ByteString getRepeatedFieldBytes(int index) { @@ -2268,6 +2500,7 @@ public int getMapFieldCount() { * map<string, double> mapField = 19; */ + @java.lang.Override public boolean containsMapField( java.lang.String key) { if (key == null) { throw new java.lang.NullPointerException(); } @@ -2276,6 +2509,7 @@ public boolean containsMapField( /** * Use {@link #getMapFieldMap()} instead. */ + @java.lang.Override @java.lang.Deprecated public java.util.Map getMapField() { return getMapFieldMap(); @@ -2283,6 +2517,7 @@ public java.util.Map getMapField() { /** * map<string, double> mapField = 19; */ + @java.lang.Override public java.util.Map getMapFieldMap() { return internalGetMapField().getMap(); @@ -2290,6 +2525,7 @@ public java.util.Map getMapFieldMap() { /** * map<string, double> mapField = 19; */ + @java.lang.Override public double getMapFieldOrDefault( java.lang.String key, @@ -2302,6 +2538,7 @@ public double getMapFieldOrDefault( /** * map<string, double> mapField = 19; */ + @java.lang.Override public double getMapFieldOrThrow( java.lang.String key) { @@ -2314,7 +2551,34 @@ public double getMapFieldOrThrow( return map.get(key); } + public static final int TIMESTAMPFIELD_FIELD_NUMBER = 20; + private com.google.protobuf.Timestamp timestampField_; + /** + * .google.protobuf.Timestamp timestampField = 20; + * @return Whether the timestampField field is set. + */ + @java.lang.Override + public boolean hasTimestampField() { + return timestampField_ != null; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + * @return The timestampField. + */ + @java.lang.Override + public com.google.protobuf.Timestamp getTimestampField() { + return timestampField_ == null ? com.google.protobuf.Timestamp.getDefaultInstance() : timestampField_; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + @java.lang.Override + public com.google.protobuf.TimestampOrBuilder getTimestampFieldOrBuilder() { + return getTimestampField(); + } + private byte memoizedIsInitialized = -1; + @java.lang.Override public final boolean isInitialized() { byte isInitialized = memoizedIsInitialized; if (isInitialized == 1) return true; @@ -2324,6 +2588,7 @@ public final boolean isInitialized() { return true; } + @java.lang.Override public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { if (!getStringFieldBytes().isEmpty()) { @@ -2386,9 +2651,13 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) internalGetMapField(), MapFieldDefaultEntryHolder.defaultEntry, 19); + if (timestampField_ != null) { + output.writeMessage(20, getTimestampField()); + } unknownFields.writeTo(output); } + @java.lang.Override public int getSerializedSize() { int size = memoizedSize; if (size != -1) return size; @@ -2479,6 +2748,10 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeMessageSize(19, mapField__); } + if (timestampField_ != null) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(20, getTimestampField()); + } size += unknownFields.getSerializedSize(); memoizedSize = size; return size; @@ -2494,53 +2767,55 @@ public boolean equals(final java.lang.Object obj) { } org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage other = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage) obj; - boolean result = true; - result = result && getStringField() - .equals(other.getStringField()); - result = result && ( - java.lang.Double.doubleToLongBits(getDoubleField()) - == java.lang.Double.doubleToLongBits( - other.getDoubleField())); - result = result && ( - java.lang.Float.floatToIntBits(getFloatField()) - == java.lang.Float.floatToIntBits( - other.getFloatField())); - result = result && (getInt32Field() - == other.getInt32Field()); - result = result && (getInt64Field() - == other.getInt64Field()); - result = result && (getUint32Field() - == other.getUint32Field()); - result = result && (getUint64Field() - == other.getUint64Field()); - result = result && (getSint32Field() - == other.getSint32Field()); - result = result && (getSint64Field() - == other.getSint64Field()); - result = result && (getFixed32Field() - == other.getFixed32Field()); - result = result && (getFixed64Field() - == other.getFixed64Field()); - result = result && (getSfixed32Field() - == other.getSfixed32Field()); - result = result && (getSfixed64Field() - == other.getSfixed64Field()); - result = result && (getBoolField() - == other.getBoolField()); - result = result && getBytesField() - .equals(other.getBytesField()); - result = result && testEnum_ == other.testEnum_; - result = result && (hasSubMessage() == other.hasSubMessage()); + if (!getStringField() + .equals(other.getStringField())) return false; + if (java.lang.Double.doubleToLongBits(getDoubleField()) + != java.lang.Double.doubleToLongBits( + other.getDoubleField())) return false; + if (java.lang.Float.floatToIntBits(getFloatField()) + != java.lang.Float.floatToIntBits( + other.getFloatField())) return false; + if (getInt32Field() + != other.getInt32Field()) return false; + if (getInt64Field() + != other.getInt64Field()) return false; + if (getUint32Field() + != other.getUint32Field()) return false; + if (getUint64Field() + != other.getUint64Field()) return false; + if (getSint32Field() + != other.getSint32Field()) return false; + if (getSint64Field() + != other.getSint64Field()) return false; + if (getFixed32Field() + != other.getFixed32Field()) return false; + if (getFixed64Field() + != other.getFixed64Field()) return false; + if (getSfixed32Field() + != other.getSfixed32Field()) return false; + if (getSfixed64Field() + != other.getSfixed64Field()) return false; + if (getBoolField() + != other.getBoolField()) return false; + if (!getBytesField() + .equals(other.getBytesField())) return false; + if (testEnum_ != other.testEnum_) return false; + if (hasSubMessage() != other.hasSubMessage()) return false; if (hasSubMessage()) { - result = result && getSubMessage() - .equals(other.getSubMessage()); - } - result = result && getRepeatedFieldList() - .equals(other.getRepeatedFieldList()); - result = result && internalGetMapField().equals( - other.internalGetMapField()); - result = result && unknownFields.equals(other.unknownFields); - return result; + if (!getSubMessage() + .equals(other.getSubMessage())) return false; + } + if (!getRepeatedFieldList() + .equals(other.getRepeatedFieldList())) return false; + if (!internalGetMapField().equals( + other.internalGetMapField())) return false; + if (hasTimestampField() != other.hasTimestampField()) return false; + if (hasTimestampField()) { + if (!getTimestampField() + .equals(other.getTimestampField())) return false; + } + if (!unknownFields.equals(other.unknownFields)) return false; + return true; } @java.lang.Override @@ -2602,6 +2877,10 @@ public int hashCode() { hash = (37 * hash) + MAPFIELD_FIELD_NUMBER; hash = (53 * hash) + internalGetMapField().hashCode(); } + if (hasTimestampField()) { + hash = (37 * hash) + TIMESTAMPFIELD_FIELD_NUMBER; + hash = (53 * hash) + getTimestampField().hashCode(); + } hash = (29 * hash) + unknownFields.hashCode(); memoizedHashCode = hash; return hash; @@ -2677,6 +2956,7 @@ public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMe .parseWithIOException(PARSER, input, extensionRegistry); } + @java.lang.Override public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() { return DEFAULT_INSTANCE.toBuilder(); @@ -2684,6 +2964,7 @@ public static Builder newBuilder() { public static Builder newBuilder(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage prototype) { return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); } + @java.lang.Override public Builder toBuilder() { return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); @@ -2729,6 +3010,7 @@ protected com.google.protobuf.MapField internalGetMutableMapField( "Invalid map field number: " + number); } } + @java.lang.Override protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_fieldAccessorTable @@ -2751,6 +3033,7 @@ private void maybeForceBuilderInitialization() { .alwaysUseFieldBuilders) { } } + @java.lang.Override public Builder clear() { super.clear(); stringField_ = ""; @@ -2792,20 +3075,29 @@ public Builder clear() { subMessageBuilder_ = null; } repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00020000); + bitField0_ = (bitField0_ & ~0x00000001); internalGetMutableMapField().clear(); + if (timestampFieldBuilder_ == null) { + timestampField_ = null; + } else { + timestampField_ = null; + timestampFieldBuilder_ = null; + } return this; } + @java.lang.Override public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_descriptor; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage getDefaultInstanceForType() { return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.getDefaultInstance(); } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage build() { org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage result = buildPartial(); if (!result.isInitialized()) { @@ -2814,10 +3106,10 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage b return result; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage buildPartial() { org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage result = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage(this); int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; result.stringField_ = stringField_; result.doubleField_ = doubleField_; result.floatField_ = floatField_; @@ -2839,44 +3131,55 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage b } else { result.subMessage_ = subMessageBuilder_.build(); } - if (((bitField0_ & 0x00020000) == 0x00020000)) { + if (((bitField0_ & 0x00000001) != 0)) { repeatedField_ = repeatedField_.getUnmodifiableView(); - bitField0_ = (bitField0_ & ~0x00020000); + bitField0_ = (bitField0_ & ~0x00000001); } result.repeatedField_ = repeatedField_; result.mapField_ = internalGetMapField(); result.mapField_.makeImmutable(); - result.bitField0_ = to_bitField0_; + if (timestampFieldBuilder_ == null) { + result.timestampField_ = timestampField_; + } else { + result.timestampField_ = timestampFieldBuilder_.build(); + } onBuilt(); return result; } + @java.lang.Override public Builder clone() { - return (Builder) super.clone(); + return super.clone(); } + @java.lang.Override public Builder setField( com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { - return (Builder) super.setField(field, value); + return super.setField(field, value); } + @java.lang.Override public Builder clearField( com.google.protobuf.Descriptors.FieldDescriptor field) { - return (Builder) super.clearField(field); + return super.clearField(field); } + @java.lang.Override public Builder clearOneof( com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return (Builder) super.clearOneof(oneof); + return super.clearOneof(oneof); } + @java.lang.Override public Builder setRepeatedField( com.google.protobuf.Descriptors.FieldDescriptor field, int index, java.lang.Object value) { - return (Builder) super.setRepeatedField(field, index, value); + return super.setRepeatedField(field, index, value); } + @java.lang.Override public Builder addRepeatedField( com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { - return (Builder) super.addRepeatedField(field, value); + return super.addRepeatedField(field, value); } + @java.lang.Override public Builder mergeFrom(com.google.protobuf.Message other) { if (other instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage) { return mergeFrom((org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage)other); @@ -2943,7 +3246,7 @@ public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.Tes if (!other.repeatedField_.isEmpty()) { if (repeatedField_.isEmpty()) { repeatedField_ = other.repeatedField_; - bitField0_ = (bitField0_ & ~0x00020000); + bitField0_ = (bitField0_ & ~0x00000001); } else { ensureRepeatedFieldIsMutable(); repeatedField_.addAll(other.repeatedField_); @@ -2952,15 +3255,20 @@ public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.Tes } internalGetMutableMapField().mergeFrom( other.internalGetMapField()); + if (other.hasTimestampField()) { + mergeTimestampField(other.getTimestampField()); + } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; } + @java.lang.Override public final boolean isInitialized() { return true; } + @java.lang.Override public Builder mergeFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -2983,6 +3291,7 @@ public Builder mergeFrom( private java.lang.Object stringField_ = ""; /** * string stringField = 1; + * @return The stringField. */ public java.lang.String getStringField() { java.lang.Object ref = stringField_; @@ -2998,6 +3307,7 @@ public java.lang.String getStringField() { } /** * string stringField = 1; + * @return The bytes for stringField. */ public com.google.protobuf.ByteString getStringFieldBytes() { @@ -3014,6 +3324,8 @@ public java.lang.String getStringField() { } /** * string stringField = 1; + * @param value The stringField to set. + * @return This builder for chaining. */ public Builder setStringField( java.lang.String value) { @@ -3027,6 +3339,7 @@ public Builder setStringField( } /** * string stringField = 1; + * @return This builder for chaining. */ public Builder clearStringField() { @@ -3036,6 +3349,8 @@ public Builder clearStringField() { } /** * string stringField = 1; + * @param value The bytes for stringField to set. + * @return This builder for chaining. */ public Builder setStringFieldBytes( com.google.protobuf.ByteString value) { @@ -3052,12 +3367,16 @@ public Builder setStringFieldBytes( private double doubleField_ ; /** * double doubleField = 2; + * @return The doubleField. */ + @java.lang.Override public double getDoubleField() { return doubleField_; } /** * double doubleField = 2; + * @param value The doubleField to set. + * @return This builder for chaining. */ public Builder setDoubleField(double value) { @@ -3067,6 +3386,7 @@ public Builder setDoubleField(double value) { } /** * double doubleField = 2; + * @return This builder for chaining. */ public Builder clearDoubleField() { @@ -3078,12 +3398,16 @@ public Builder clearDoubleField() { private float floatField_ ; /** * float floatField = 3; + * @return The floatField. */ + @java.lang.Override public float getFloatField() { return floatField_; } /** * float floatField = 3; + * @param value The floatField to set. + * @return This builder for chaining. */ public Builder setFloatField(float value) { @@ -3093,6 +3417,7 @@ public Builder setFloatField(float value) { } /** * float floatField = 3; + * @return This builder for chaining. */ public Builder clearFloatField() { @@ -3104,12 +3429,16 @@ public Builder clearFloatField() { private int int32Field_ ; /** * int32 int32Field = 4; + * @return The int32Field. */ + @java.lang.Override public int getInt32Field() { return int32Field_; } /** * int32 int32Field = 4; + * @param value The int32Field to set. + * @return This builder for chaining. */ public Builder setInt32Field(int value) { @@ -3119,6 +3448,7 @@ public Builder setInt32Field(int value) { } /** * int32 int32Field = 4; + * @return This builder for chaining. */ public Builder clearInt32Field() { @@ -3130,12 +3460,16 @@ public Builder clearInt32Field() { private long int64Field_ ; /** * int64 int64Field = 5; + * @return The int64Field. */ + @java.lang.Override public long getInt64Field() { return int64Field_; } /** * int64 int64Field = 5; + * @param value The int64Field to set. + * @return This builder for chaining. */ public Builder setInt64Field(long value) { @@ -3145,6 +3479,7 @@ public Builder setInt64Field(long value) { } /** * int64 int64Field = 5; + * @return This builder for chaining. */ public Builder clearInt64Field() { @@ -3156,12 +3491,16 @@ public Builder clearInt64Field() { private int uint32Field_ ; /** * uint32 uint32Field = 6; + * @return The uint32Field. */ + @java.lang.Override public int getUint32Field() { return uint32Field_; } /** * uint32 uint32Field = 6; + * @param value The uint32Field to set. + * @return This builder for chaining. */ public Builder setUint32Field(int value) { @@ -3171,6 +3510,7 @@ public Builder setUint32Field(int value) { } /** * uint32 uint32Field = 6; + * @return This builder for chaining. */ public Builder clearUint32Field() { @@ -3182,12 +3522,16 @@ public Builder clearUint32Field() { private long uint64Field_ ; /** * uint64 uint64Field = 7; + * @return The uint64Field. */ + @java.lang.Override public long getUint64Field() { return uint64Field_; } /** * uint64 uint64Field = 7; + * @param value The uint64Field to set. + * @return This builder for chaining. */ public Builder setUint64Field(long value) { @@ -3197,6 +3541,7 @@ public Builder setUint64Field(long value) { } /** * uint64 uint64Field = 7; + * @return This builder for chaining. */ public Builder clearUint64Field() { @@ -3208,12 +3553,16 @@ public Builder clearUint64Field() { private int sint32Field_ ; /** * sint32 sint32Field = 8; + * @return The sint32Field. */ + @java.lang.Override public int getSint32Field() { return sint32Field_; } /** * sint32 sint32Field = 8; + * @param value The sint32Field to set. + * @return This builder for chaining. */ public Builder setSint32Field(int value) { @@ -3223,6 +3572,7 @@ public Builder setSint32Field(int value) { } /** * sint32 sint32Field = 8; + * @return This builder for chaining. */ public Builder clearSint32Field() { @@ -3234,12 +3584,16 @@ public Builder clearSint32Field() { private long sint64Field_ ; /** * sint64 sint64Field = 9; + * @return The sint64Field. */ + @java.lang.Override public long getSint64Field() { return sint64Field_; } /** * sint64 sint64Field = 9; + * @param value The sint64Field to set. + * @return This builder for chaining. */ public Builder setSint64Field(long value) { @@ -3249,6 +3603,7 @@ public Builder setSint64Field(long value) { } /** * sint64 sint64Field = 9; + * @return This builder for chaining. */ public Builder clearSint64Field() { @@ -3260,12 +3615,16 @@ public Builder clearSint64Field() { private int fixed32Field_ ; /** * fixed32 fixed32Field = 10; + * @return The fixed32Field. */ + @java.lang.Override public int getFixed32Field() { return fixed32Field_; } /** * fixed32 fixed32Field = 10; + * @param value The fixed32Field to set. + * @return This builder for chaining. */ public Builder setFixed32Field(int value) { @@ -3275,6 +3634,7 @@ public Builder setFixed32Field(int value) { } /** * fixed32 fixed32Field = 10; + * @return This builder for chaining. */ public Builder clearFixed32Field() { @@ -3286,12 +3646,16 @@ public Builder clearFixed32Field() { private long fixed64Field_ ; /** * fixed64 fixed64Field = 11; + * @return The fixed64Field. */ + @java.lang.Override public long getFixed64Field() { return fixed64Field_; } /** * fixed64 fixed64Field = 11; + * @param value The fixed64Field to set. + * @return This builder for chaining. */ public Builder setFixed64Field(long value) { @@ -3301,6 +3665,7 @@ public Builder setFixed64Field(long value) { } /** * fixed64 fixed64Field = 11; + * @return This builder for chaining. */ public Builder clearFixed64Field() { @@ -3312,12 +3677,16 @@ public Builder clearFixed64Field() { private int sfixed32Field_ ; /** * sfixed32 sfixed32Field = 12; + * @return The sfixed32Field. */ + @java.lang.Override public int getSfixed32Field() { return sfixed32Field_; } /** * sfixed32 sfixed32Field = 12; + * @param value The sfixed32Field to set. + * @return This builder for chaining. */ public Builder setSfixed32Field(int value) { @@ -3327,6 +3696,7 @@ public Builder setSfixed32Field(int value) { } /** * sfixed32 sfixed32Field = 12; + * @return This builder for chaining. */ public Builder clearSfixed32Field() { @@ -3338,12 +3708,16 @@ public Builder clearSfixed32Field() { private long sfixed64Field_ ; /** * sfixed64 sfixed64Field = 13; + * @return The sfixed64Field. */ + @java.lang.Override public long getSfixed64Field() { return sfixed64Field_; } /** * sfixed64 sfixed64Field = 13; + * @param value The sfixed64Field to set. + * @return This builder for chaining. */ public Builder setSfixed64Field(long value) { @@ -3353,6 +3727,7 @@ public Builder setSfixed64Field(long value) { } /** * sfixed64 sfixed64Field = 13; + * @return This builder for chaining. */ public Builder clearSfixed64Field() { @@ -3364,12 +3739,16 @@ public Builder clearSfixed64Field() { private boolean boolField_ ; /** * bool boolField = 14; + * @return The boolField. */ + @java.lang.Override public boolean getBoolField() { return boolField_; } /** * bool boolField = 14; + * @param value The boolField to set. + * @return This builder for chaining. */ public Builder setBoolField(boolean value) { @@ -3379,6 +3758,7 @@ public Builder setBoolField(boolean value) { } /** * bool boolField = 14; + * @return This builder for chaining. */ public Builder clearBoolField() { @@ -3390,12 +3770,16 @@ public Builder clearBoolField() { private com.google.protobuf.ByteString bytesField_ = com.google.protobuf.ByteString.EMPTY; /** * bytes bytesField = 15; + * @return The bytesField. */ + @java.lang.Override public com.google.protobuf.ByteString getBytesField() { return bytesField_; } /** * bytes bytesField = 15; + * @param value The bytesField to set. + * @return This builder for chaining. */ public Builder setBytesField(com.google.protobuf.ByteString value) { if (value == null) { @@ -3408,6 +3792,7 @@ public Builder setBytesField(com.google.protobuf.ByteString value) { } /** * bytes bytesField = 15; + * @return This builder for chaining. */ public Builder clearBytesField() { @@ -3419,27 +3804,36 @@ public Builder clearBytesField() { private int testEnum_ = 0; /** * .proto.TestEnum testEnum = 16; + * @return The enum numeric value on the wire for testEnum. */ - public int getTestEnumValue() { + @java.lang.Override public int getTestEnumValue() { return testEnum_; } /** * .proto.TestEnum testEnum = 16; + * @param value The enum numeric value on the wire for testEnum to set. + * @return This builder for chaining. */ public Builder setTestEnumValue(int value) { + testEnum_ = value; onChanged(); return this; } /** * .proto.TestEnum testEnum = 16; + * @return The testEnum. */ + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum() { + @SuppressWarnings("deprecation") org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum result = org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.valueOf(testEnum_); return result == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.UNRECOGNIZED : result; } /** * .proto.TestEnum testEnum = 16; + * @param value The testEnum to set. + * @return This builder for chaining. */ public Builder setTestEnum(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum value) { if (value == null) { @@ -3452,6 +3846,7 @@ public Builder setTestEnum(org.apache.pulsar.sql.presto.decoder.protobufnative.T } /** * .proto.TestEnum testEnum = 16; + * @return This builder for chaining. */ public Builder clearTestEnum() { @@ -3460,17 +3855,19 @@ public Builder clearTestEnum() { return this; } - private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage subMessage_ = null; + private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage subMessage_; private com.google.protobuf.SingleFieldBuilderV3< org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder> subMessageBuilder_; /** * .proto.SubMessage subMessage = 17; + * @return Whether the subMessage field is set. */ public boolean hasSubMessage() { return subMessageBuilder_ != null || subMessage_ != null; } /** * .proto.SubMessage subMessage = 17; + * @return The subMessage. */ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getSubMessage() { if (subMessageBuilder_ == null) { @@ -3579,13 +3976,14 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrB private com.google.protobuf.LazyStringList repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; private void ensureRepeatedFieldIsMutable() { - if (!((bitField0_ & 0x00020000) == 0x00020000)) { + if (!((bitField0_ & 0x00000001) != 0)) { repeatedField_ = new com.google.protobuf.LazyStringArrayList(repeatedField_); - bitField0_ |= 0x00020000; + bitField0_ |= 0x00000001; } } /** * repeated string repeatedField = 18; + * @return A list containing the repeatedField. */ public com.google.protobuf.ProtocolStringList getRepeatedFieldList() { @@ -3593,18 +3991,23 @@ private void ensureRepeatedFieldIsMutable() { } /** * repeated string repeatedField = 18; + * @return The count of repeatedField. */ public int getRepeatedFieldCount() { return repeatedField_.size(); } /** * repeated string repeatedField = 18; + * @param index The index of the element to return. + * @return The repeatedField at the given index. */ public java.lang.String getRepeatedField(int index) { return repeatedField_.get(index); } /** * repeated string repeatedField = 18; + * @param index The index of the value to return. + * @return The bytes of the repeatedField at the given index. */ public com.google.protobuf.ByteString getRepeatedFieldBytes(int index) { @@ -3612,6 +4015,9 @@ public java.lang.String getRepeatedField(int index) { } /** * repeated string repeatedField = 18; + * @param index The index to set the value at. + * @param value The repeatedField to set. + * @return This builder for chaining. */ public Builder setRepeatedField( int index, java.lang.String value) { @@ -3625,6 +4031,8 @@ public Builder setRepeatedField( } /** * repeated string repeatedField = 18; + * @param value The repeatedField to add. + * @return This builder for chaining. */ public Builder addRepeatedField( java.lang.String value) { @@ -3638,6 +4046,8 @@ public Builder addRepeatedField( } /** * repeated string repeatedField = 18; + * @param values The repeatedField to add. + * @return This builder for chaining. */ public Builder addAllRepeatedField( java.lang.Iterable values) { @@ -3649,15 +4059,18 @@ public Builder addAllRepeatedField( } /** * repeated string repeatedField = 18; + * @return This builder for chaining. */ public Builder clearRepeatedField() { repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00020000); + bitField0_ = (bitField0_ & ~0x00000001); onChanged(); return this; } /** * repeated string repeatedField = 18; + * @param value The bytes of the repeatedField to add. + * @return This builder for chaining. */ public Builder addRepeatedFieldBytes( com.google.protobuf.ByteString value) { @@ -3701,6 +4114,7 @@ public int getMapFieldCount() { * map<string, double> mapField = 19; */ + @java.lang.Override public boolean containsMapField( java.lang.String key) { if (key == null) { throw new java.lang.NullPointerException(); } @@ -3709,6 +4123,7 @@ public boolean containsMapField( /** * Use {@link #getMapFieldMap()} instead. */ + @java.lang.Override @java.lang.Deprecated public java.util.Map getMapField() { return getMapFieldMap(); @@ -3716,6 +4131,7 @@ public java.util.Map getMapField() { /** * map<string, double> mapField = 19; */ + @java.lang.Override public java.util.Map getMapFieldMap() { return internalGetMapField().getMap(); @@ -3723,6 +4139,7 @@ public java.util.Map getMapFieldMap() { /** * map<string, double> mapField = 19; */ + @java.lang.Override public double getMapFieldOrDefault( java.lang.String key, @@ -3735,6 +4152,7 @@ public double getMapFieldOrDefault( /** * map<string, double> mapField = 19; */ + @java.lang.Override public double getMapFieldOrThrow( java.lang.String key) { @@ -3793,11 +4211,132 @@ public Builder putAllMapField( .putAll(values); return this; } + + private com.google.protobuf.Timestamp timestampField_; + private com.google.protobuf.SingleFieldBuilderV3< + com.google.protobuf.Timestamp, com.google.protobuf.Timestamp.Builder, com.google.protobuf.TimestampOrBuilder> timestampFieldBuilder_; + /** + * .google.protobuf.Timestamp timestampField = 20; + * @return Whether the timestampField field is set. + */ + public boolean hasTimestampField() { + return timestampFieldBuilder_ != null || timestampField_ != null; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + * @return The timestampField. + */ + public com.google.protobuf.Timestamp getTimestampField() { + if (timestampFieldBuilder_ == null) { + return timestampField_ == null ? com.google.protobuf.Timestamp.getDefaultInstance() : timestampField_; + } else { + return timestampFieldBuilder_.getMessage(); + } + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + public Builder setTimestampField(com.google.protobuf.Timestamp value) { + if (timestampFieldBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + timestampField_ = value; + onChanged(); + } else { + timestampFieldBuilder_.setMessage(value); + } + + return this; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + public Builder setTimestampField( + com.google.protobuf.Timestamp.Builder builderForValue) { + if (timestampFieldBuilder_ == null) { + timestampField_ = builderForValue.build(); + onChanged(); + } else { + timestampFieldBuilder_.setMessage(builderForValue.build()); + } + + return this; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + public Builder mergeTimestampField(com.google.protobuf.Timestamp value) { + if (timestampFieldBuilder_ == null) { + if (timestampField_ != null) { + timestampField_ = + com.google.protobuf.Timestamp.newBuilder(timestampField_).mergeFrom(value).buildPartial(); + } else { + timestampField_ = value; + } + onChanged(); + } else { + timestampFieldBuilder_.mergeFrom(value); + } + + return this; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + public Builder clearTimestampField() { + if (timestampFieldBuilder_ == null) { + timestampField_ = null; + onChanged(); + } else { + timestampField_ = null; + timestampFieldBuilder_ = null; + } + + return this; + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + public com.google.protobuf.Timestamp.Builder getTimestampFieldBuilder() { + + onChanged(); + return getTimestampFieldFieldBuilder().getBuilder(); + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + public com.google.protobuf.TimestampOrBuilder getTimestampFieldOrBuilder() { + if (timestampFieldBuilder_ != null) { + return timestampFieldBuilder_.getMessageOrBuilder(); + } else { + return timestampField_ == null ? + com.google.protobuf.Timestamp.getDefaultInstance() : timestampField_; + } + } + /** + * .google.protobuf.Timestamp timestampField = 20; + */ + private com.google.protobuf.SingleFieldBuilderV3< + com.google.protobuf.Timestamp, com.google.protobuf.Timestamp.Builder, com.google.protobuf.TimestampOrBuilder> + getTimestampFieldFieldBuilder() { + if (timestampFieldBuilder_ == null) { + timestampFieldBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< + com.google.protobuf.Timestamp, com.google.protobuf.Timestamp.Builder, com.google.protobuf.TimestampOrBuilder>( + getTimestampField(), + getParentForChildren(), + isClean()); + timestampField_ = null; + } + return timestampFieldBuilder_; + } + @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFieldsProto3(unknownFields); + return super.setUnknownFields(unknownFields); } + @java.lang.Override public final Builder mergeUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { return super.mergeUnknownFields(unknownFields); @@ -3819,6 +4358,7 @@ public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMe private static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + @java.lang.Override public TestMessage parsePartialFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -3836,6 +4376,7 @@ public com.google.protobuf.Parser getParserForType() { return PARSER; } + @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage getDefaultInstanceForType() { return DEFAULT_INSTANCE; } @@ -3871,40 +4412,35 @@ public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage g descriptor; static { java.lang.String[] descriptorData = { - "\n\rTestMsg.proto\022\005proto\"\214\001\n\nSubMessage\022\013\n" + - "\003foo\030\001 \001(\t\022\013\n\003bar\030\002 \001(\001\0226\n\rnestedMessage" + - "\030\003 \001(\0132\037.proto.SubMessage.NestedMessage\032" + - ",\n\rNestedMessage\022\r\n\005title\030\001 \001(\t\022\014\n\004urls\030" + - "\002 \003(\t\"\216\004\n\013TestMessage\022\023\n\013stringField\030\001 \001" + - "(\t\022\023\n\013doubleField\030\002 \001(\001\022\022\n\nfloatField\030\003 " + - "\001(\002\022\022\n\nint32Field\030\004 \001(\005\022\022\n\nint64Field\030\005 " + - "\001(\003\022\023\n\013uint32Field\030\006 \001(\r\022\023\n\013uint64Field\030" + - "\007 \001(\004\022\023\n\013sint32Field\030\010 \001(\021\022\023\n\013sint64Fiel" + - "d\030\t \001(\022\022\024\n\014fixed32Field\030\n \001(\007\022\024\n\014fixed64" + - "Field\030\013 \001(\006\022\025\n\rsfixed32Field\030\014 \001(\017\022\025\n\rsf" + - "ixed64Field\030\r \001(\020\022\021\n\tboolField\030\016 \001(\010\022\022\n\n" + - "bytesField\030\017 \001(\014\022!\n\010testEnum\030\020 \001(\0162\017.pro" + - "to.TestEnum\022%\n\nsubMessage\030\021 \001(\0132\021.proto." + - "SubMessage\022\025\n\rrepeatedField\030\022 \003(\t\0222\n\010map" + - "Field\030\023 \003(\0132 .proto.TestMessage.MapField" + - "Entry\032/\n\rMapFieldEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005v" + - "alue\030\002 \001(\001:\0028\001*$\n\010TestEnum\022\n\n\006SHARED\020\000\022\014" + - "\n\010FAILOVER\020\001B>\n3org.apache.pulsar.sql.pr" + - "esto.decoder.protobufnativeB\007TestMsgb\006pr" + - "oto3" + "\n\rTestMsg.proto\022\005proto\032\037google/protobuf/" + + "timestamp.proto\"\214\001\n\nSubMessage\022\013\n\003foo\030\001 " + + "\001(\t\022\013\n\003bar\030\002 \001(\001\0226\n\rnestedMessage\030\003 \001(\0132" + + "\037.proto.SubMessage.NestedMessage\032,\n\rNest" + + "edMessage\022\r\n\005title\030\001 \001(\t\022\014\n\004urls\030\002 \003(\t\"\302" + + "\004\n\013TestMessage\022\023\n\013stringField\030\001 \001(\t\022\023\n\013d" + + "oubleField\030\002 \001(\001\022\022\n\nfloatField\030\003 \001(\002\022\022\n\n" + + "int32Field\030\004 \001(\005\022\022\n\nint64Field\030\005 \001(\003\022\023\n\013" + + "uint32Field\030\006 \001(\r\022\023\n\013uint64Field\030\007 \001(\004\022\023" + + "\n\013sint32Field\030\010 \001(\021\022\023\n\013sint64Field\030\t \001(\022" + + "\022\024\n\014fixed32Field\030\n \001(\007\022\024\n\014fixed64Field\030\013" + + " \001(\006\022\025\n\rsfixed32Field\030\014 \001(\017\022\025\n\rsfixed64F" + + "ield\030\r \001(\020\022\021\n\tboolField\030\016 \001(\010\022\022\n\nbytesFi" + + "eld\030\017 \001(\014\022!\n\010testEnum\030\020 \001(\0162\017.proto.Test" + + "Enum\022%\n\nsubMessage\030\021 \001(\0132\021.proto.SubMess" + + "age\022\025\n\rrepeatedField\030\022 \003(\t\0222\n\010mapField\030\023" + + " \003(\0132 .proto.TestMessage.MapFieldEntry\0222" + + "\n\016timestampField\030\024 \001(\0132\032.google.protobuf" + + ".Timestamp\032/\n\rMapFieldEntry\022\013\n\003key\030\001 \001(\t" + + "\022\r\n\005value\030\002 \001(\001:\0028\001*$\n\010TestEnum\022\n\n\006SHARE" + + "D\020\000\022\014\n\010FAILOVER\020\001B>\n3org.apache.pulsar.s" + + "ql.presto.decoder.protobufnativeB\007TestMs" + + "gP\000b\006proto3" }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - return null; - } - }; - com.google.protobuf.Descriptors.FileDescriptor + descriptor = com.google.protobuf.Descriptors.FileDescriptor .internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] { - }, assigner); + com.google.protobuf.TimestampProto.getDescriptor(), + }); internal_static_proto_SubMessage_descriptor = getDescriptor().getMessageTypes().get(0); internal_static_proto_SubMessage_fieldAccessorTable = new @@ -3922,13 +4458,14 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_proto_TestMessage_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_proto_TestMessage_descriptor, - new java.lang.String[] { "StringField", "DoubleField", "FloatField", "Int32Field", "Int64Field", "Uint32Field", "Uint64Field", "Sint32Field", "Sint64Field", "Fixed32Field", "Fixed64Field", "Sfixed32Field", "Sfixed64Field", "BoolField", "BytesField", "TestEnum", "SubMessage", "RepeatedField", "MapField", }); + new java.lang.String[] { "StringField", "DoubleField", "FloatField", "Int32Field", "Int64Field", "Uint32Field", "Uint64Field", "Sint32Field", "Sint64Field", "Fixed32Field", "Fixed64Field", "Sfixed32Field", "Sfixed64Field", "BoolField", "BytesField", "TestEnum", "SubMessage", "RepeatedField", "MapField", "TimestampField", }); internal_static_proto_TestMessage_MapFieldEntry_descriptor = internal_static_proto_TestMessage_descriptor.getNestedTypes().get(0); internal_static_proto_TestMessage_MapFieldEntry_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_proto_TestMessage_MapFieldEntry_descriptor, new java.lang.String[] { "Key", "Value", }); + com.google.protobuf.TimestampProto.getDescriptor(); } // @@protoc_insertion_point(outer_class_scope) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto index 0a0f48917ea88..fd522bdd62652 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto @@ -19,6 +19,7 @@ syntax = "proto3"; package proto; +import public "google/protobuf/timestamp.proto"; option java_package = "org.apache.pulsar.sql.presto.decoder.protobufnative"; option java_outer_classname = "TestMsg"; @@ -58,4 +59,5 @@ message TestMessage { SubMessage subMessage = 17; repeated string repeatedField = 18; map mapField = 19; + google.protobuf.Timestamp timestampField = 20; } \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java index fc1834cae887f..c4c6cb2e6f31d 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -43,6 +44,7 @@ import static io.prestosql.spi.type.BooleanType.BOOLEAN; import static io.prestosql.spi.type.DoubleType.DOUBLE; import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; import static io.prestosql.spi.type.VarbinaryType.VARBINARY; import static io.prestosql.spi.type.VarcharType.VARCHAR; import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; @@ -65,6 +67,12 @@ public void init() { @Test public void testPrimitiveType() { + //Time: 2921-1-1 + long mills = 30010669261001L; + Timestamp timestamp = Timestamp.newBuilder() + .setSeconds(mills / 1000) + .setNanos((int) (mills % 1000) * 1000000) + .build(); TestMsg.TestMessage testMessage = TestMsg.TestMessage.newBuilder() .setStringField("aaa") @@ -83,6 +91,7 @@ public void testPrimitiveType() { .setBoolField(true) .setBytesField(ByteString.copyFrom("abc".getBytes())) .setTestEnum(TestMsg.TestEnum.FAILOVER) + .setTimestampField(timestamp) .build(); ByteBuf payload = io.netty.buffer.Unpooled @@ -163,6 +172,11 @@ public void testPrimitiveType() { PulsarColumnHandle.HandleKeyValueType.NONE); checkValue(decodedRow, enumFieldColumnHandle, testMessage.getTestEnum().name()); + PulsarColumnHandle timestampFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "timestampField", TIMESTAMP,false,false,"timestampField",null,null, + PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, timestampFieldColumnHandle, mills); + } @Test From 29533f1c45015c177ec8d441189bf42bdeddd8e0 Mon Sep 17 00:00:00 2001 From: ran Date: Mon, 27 Dec 2021 09:45:38 +0800 Subject: [PATCH 239/823] Fix getting last message id from empty compact ledger (#13476) ### Motivation Currently, if the last confirmed entry is an empty ledger, getting the last message-id operation will get data from the compact ledger, if the compact ledger is also an empty ledger, it will encounter `IncorrectParameterException`. **Broker error message** ``` [pulsar-io-29-9] ERROR org.apache.bookkeeper.client.LedgerHandle - IncorrectParameterException on ledgerId:617 firstEntry:-1 lastEntry:-1 ``` **Client error log** ``` Exception in thread "main" org.apache.pulsar.client.api.PulsarClientException$BrokerMetadataException: The subscription reader-bf9246cfcb of the topic persistent://public/ns-test/t1 gets the last message id was failed {"errorMsg":"Failed to read last entry of the compacted Ledger Incorrect parameter input","reqId":79405902881798690, "remote":"localhost/127.0.0.1:6650", "local":"/127.0.0.1:55207"} at org.apache.pulsar.client.api.PulsarClientException.unwrap(PulsarClientException.java:1034) at org.apache.pulsar.client.impl.ConsumerImpl.hasMessageAvailable(ConsumerImpl.java:2001) at org.apache.pulsar.client.impl.ReaderImpl.hasMessageAvailable(ReaderImpl.java:181) at org.apache.pulsar.compaction.CompactedTopicTest.main(CompactedTopicTest.java:730) ``` ### Modifications Check the compact ledger entry id before reading an entry from the compact ledger, if there is no entry, return a null value. (cherry picked from commit 8136762e743fe1bc4be2adeb08631a8c44719d37) (cherry picked from commit 472f02d1799eb18ee5a32308f22c570f70ec1073) --- .../pulsar/compaction/CompactedTopicImpl.java | 15 +++++--- .../pulsar/compaction/CompactedTopicTest.java | 38 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index 457754070c8dd..a6d6fc9a75232 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -293,11 +293,16 @@ public CompletableFuture readLastEntryOfCompactedLedger() { if (compactionHorizon == null) { return CompletableFuture.completedFuture(null); } - return compactedTopicContext.thenCompose(context -> - readEntries(context.ledger, context.ledger.getLastAddConfirmed(), context.ledger.getLastAddConfirmed()) - .thenCompose(entries -> entries.size() > 0 - ? CompletableFuture.completedFuture(entries.get(0)) - : CompletableFuture.completedFuture(null))); + return compactedTopicContext.thenCompose(context -> { + if (context.ledger.getLastAddConfirmed() == -1) { + return CompletableFuture.completedFuture(null); + } + return readEntries( + context.ledger, context.ledger.getLastAddConfirmed(), context.ledger.getLastAddConfirmed()) + .thenCompose(entries -> entries.size() > 0 + ? CompletableFuture.completedFuture(entries.get(0)) + : CompletableFuture.completedFuture(null)); + }); } private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 69d66d9d0f1b2..d44868a140166 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -650,4 +650,42 @@ public void testCompactionWithTopicUnloading() throws Exception { reader.close(); producer.close(); } + + @Test(timeOut = 1000 * 30) + public void testReader() throws Exception { + final String ns = "my-property/use/my-ns"; + String topic = "persistent://" + ns + "/t1"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + + producer.newMessage().key("k").value(("value").getBytes()).send(); + producer.newMessage().key("k").value(null).send(); + pulsar.getCompactor().compact(topic).get(); + + Awaitility.await() + .pollInterval(3, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + admin.topics().unload(topic); + Thread.sleep(100); + Assert.assertTrue(admin.topics().getInternalStats(topic).lastConfirmedEntry.endsWith("-1")); + }); + // Make sure the last confirm entry is -1, then get last message id from compact ledger + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topic); + Assert.assertTrue(internalStats.lastConfirmedEntry.endsWith("-1")); + // Because the latest value of the key `k` is null, so there is no data in compact ledger. + Assert.assertEquals(internalStats.compactedLedger.size, 0); + + @Cleanup + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageIdInclusive() + .startMessageId(MessageId.earliest) + .readCompacted(true) + .create(); + Assert.assertFalse(reader.hasMessageAvailable()); + } + } From 50626776c6d69b6d9951e67c2034ae60b48bd408 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Wed, 29 Dec 2021 13:26:57 +0800 Subject: [PATCH 240/823] Return message ID from compacted ledger while the compaction cursor reach the end of the topic (#13533) ### Motivation The problem happens when the compaction cursor reaches the end of the topic but the tail messages of the topic have been removed by producer writes null value messages during the topic compaction. For example: - 5 messages in the original topic with key: 0,1,2,3,4 - the corresponding message IDs are: 1:0, 1:1, 1:2, 1:3, 1:4 - producer send null value messages for key 3 and 4 - trigger the topic compaction task After the compaction task complete, - 5 messages in the original topic: 1:0, 1:1, 1:2, 1:3, 1:4 - 3 messages in the compacted ledger: 1:0, 1:1, 1:2 At this moment, if the reader tries to get the last message ID of the topic, we should return `1:2` not `1:4`, because the reader is not able to read the message with keys `3` and `4` from the compacted topic, otherwise, the `reader.readNext()` method will be blocked until a new message written to the topic. ### Modifications The fix is straightforward, when the broker receives a get last message ID request, the broker will check if the compaction cursor reaches the end of the original topic. If yes, respond last message ID from the compacted ledger. ### Verifying this change New test added `testHasMessageAvailableWithNullValueMessage` which ensure the `hasMessageAvailable()` return false no more messages from the compacted topic if the compaction cursor reaches the end of the topic. (cherry picked from commit d49415e5a558ea9a82c93e55e77b2c3542eacb10) (cherry picked from commit c916233918afc9c441d64b563d0aa4cc6ffd4810) --- .../pulsar/broker/service/ServerCnx.java | 5 +- .../pulsar/compaction/CompactedTopic.java | 2 + .../pulsar/compaction/CompactedTopicImpl.java | 6 +-- .../pulsar/compaction/CompactedTopicTest.java | 48 ++++++++++++++++++- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index dbab10dce48e4..c4ab0e26be326 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1716,7 +1716,10 @@ private void getLargestBatchIndexWhenPossible( ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); // If it's not pointing to a valid entry, respond messageId of the current position. - if (lastPosition.getEntryId() == -1) { + // If the compaction cursor reach the end of the topic, respond messageId from compacted ledger + Optional compactionHorizon = persistentTopic.getCompactedTopic().getCompactionHorizon(); + if (lastPosition.getEntryId() == -1 || (compactionHorizon.isPresent() + && lastPosition.compareTo((PositionImpl) compactionHorizon.get()) <= 0)) { handleLastMessageIdFromCompactedLedger(persistentTopic, requestId, partitionIndex, markDeletePosition); return; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java index 31955a5c5c4c8..9e50fc07152f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.compaction; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; import org.apache.bookkeeper.mledger.Entry; @@ -34,4 +35,5 @@ void asyncReadEntriesOrWait(ManagedCursor cursor, ReadEntriesCallback callback, Consumer consumer); CompletableFuture readLastEntryOfCompactedLedger(); + Optional getCompactionHorizon(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index a6d6fc9a75232..aac213fcadcd1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -20,7 +20,6 @@ import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ComparisonChain; import io.netty.buffer.ByteBuf; import java.util.ArrayList; @@ -311,9 +310,8 @@ private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) .compare(p.getEntryId(), m.getEntryId()).result(); } - @VisibleForTesting - PositionImpl getCompactionHorizon() { - return this.compactionHorizon; + public synchronized Optional getCompactionHorizon() { + return Optional.ofNullable(this.compactionHorizon); } private static final Logger log = LoggerFactory.getLogger(CompactedTopicImpl.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index d44868a140166..4d00d2864e6ff 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -254,7 +254,8 @@ public void testCleanupOldCompactedTopicLedger() throws Exception { Assert.assertTrue(compactedTopic.getCompactedTopicContext().isPresent()); Assert.assertEquals(compactedTopic.getCompactedTopicContext().get().getLedger().getId(), newCompactedLedger.getId()); - Assert.assertEquals(compactedTopic.getCompactionHorizon(), newHorizon); + Assert.assertTrue(compactedTopic.getCompactionHorizon().isPresent()); + Assert.assertEquals(compactedTopic.getCompactionHorizon().get(), newHorizon); compactedTopic.deleteCompactedLedger(oldCompactedLedger.getId()).join(); // old ledger should be deleted, new still there @@ -688,4 +689,49 @@ public void testReader() throws Exception { Assert.assertFalse(reader.hasMessageAvailable()); } + @Test + public void testHasMessageAvailableWithNullValueMessage() throws Exception { + String topic = "persistent://my-property/use/my-ns/testHasMessageAvailable-" + + UUID.randomUUID(); + final int numMessages = 10; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .blockIfQueueFull(true) + .enableBatching(false) + .create(); + CompletableFuture lastMessage = null; + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i + "").value(String.format("msg [%d]", i)).sendAsync(); + } + + for (int i = numMessages / 2; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i + "").value(null).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, numMessages / 2); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + Assert.assertEquals(stats.lastConfirmedEntry, stats.cursors.get(COMPACTION_SUBSCRIPTION).markDeletePosition); + }); + + @Cleanup + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageIdInclusive() + .startMessageId(MessageId.earliest) + .readCompacted(true) + .create(); + for (int i = numMessages / 2; i < numMessages; ++i) { + reader.readNext(); + } + Assert.assertFalse(reader.hasMessageAvailable()); + Assert.assertNull(reader.readNext(3, TimeUnit.SECONDS)); + } + } From 0e6bf549f0438b01633232b36b08c886b42b2b3b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 29 Dec 2021 12:35:57 +0200 Subject: [PATCH 241/823] [Security] Upgrade Log4j to 2.17.1 (#13552) - see https://logging.apache.org/log4j/2.x/security.html - mitigates CVE-2021-44832 (cherry picked from commit 978bb7c1998acef749912075ea8f4e1e1c148e2d) (cherry picked from commit d75dc86f1a78d7624d15ed73f5f2e3c717d29683) --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index b43161a5d8f7f..207a7296fd524 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -39,7 +39,7 @@ 1.8 1.8 3.0.0-M3 - 2.17.0 + 2.17.1 1.7.25 7.3.0 3.11 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 77ba2838b07d7..b641110ca434b 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -385,11 +385,11 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.17.0.jar - - org.apache.logging.log4j-log4j-core-2.17.0.jar - - org.apache.logging.log4j-log4j-slf4j-impl-2.17.0.jar - - org.apache.logging.log4j-log4j-web-2.17.0.jar - - org.apache.logging.log4j-log4j-1.2-api-2.17.0.jar + - org.apache.logging.log4j-log4j-api-2.17.1.jar + - org.apache.logging.log4j-log4j-core-2.17.1.jar + - org.apache.logging.log4j-log4j-slf4j-impl-2.17.1.jar + - org.apache.logging.log4j-log4j-web-2.17.1.jar + - org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - org.apache.bookkeeper-bookkeeper-common-4.14.3.jar diff --git a/pom.xml b/pom.xml index 088e9cb442c8f..1d656584b81bf 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ flexible messaging model and an intuitive client API. 6.10.2 1.7.25 3.2.2 - 2.17.0 + 2.17.1 1.69 1.0.2 2.12.3 From d0bdb74c87b03a4b712884a1c05443021e2d0b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 22 Dec 2021 16:46:03 +0100 Subject: [PATCH 242/823] [owasp] add suppression for pulsar-package-bookkeeper-storage (#13451) (cherry picked from commit 5dd60dbd748e446f8da396b448a5bb16a2ae6902) --- src/owasp-dependency-check-false-positives.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 28e53b5d0b876..6cc464e637623 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -32,6 +32,11 @@ org\.apache\.pulsar:.* cpe:/a:apache:zookeeper + + pulsar-package-bookkeeper-storage gets mixed with bookkeeper. + org\.apache\.pulsar:.* + cpe:/a:apache:bookkeeper + kubernetes client doesn't contain CVE-2020-8554 io\.kubernetes:.* From 2547d5c1176c791b92dc95c22ef1808bed5f21e9 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 6 Jan 2022 07:35:15 +0800 Subject: [PATCH 243/823] Upgrade Gson version 2.8.6 to 2.8.9 (#13610) * Upgrade gson version 2.8.6 to 2.8.9 * fix license (cherry picked from commit 937131a7003d94031b720b3c0d3d64a9bd602bc9) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b641110ca434b..4e3b9b1540a25 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -325,7 +325,7 @@ The Apache Software License, Version 2.0 * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.17.0.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.7.6.jar * Gson - - com.google.code.gson-gson-2.8.6.jar + - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar * Guava - com.google.guava-guava-30.1-jre.jar diff --git a/pom.xml b/pom.xml index 1d656584b81bf..cfa1c34bf1527 100644 --- a/pom.xml +++ b/pom.xml @@ -134,7 +134,7 @@ flexible messaging model and an intuitive client API. 1.33.0 0.19.0 ${grpc.version} - 2.8.6 + 2.8.9 0.8.3 2.2.0 3.6.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 41f0f938b3baa..aaddf3a416411 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -437,7 +437,7 @@ The Apache Software License, Version 2.0 - commons-lang-2.6.jar - commons-logging-1.2.jar * GSON - - gson-2.8.6.jar + - gson-2.8.9.jar * Snappy - snappy-java-1.1.7.jar * Jackson From 406aa4e472af612acf5377ad7def4f5e2e64de2f Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 22 Oct 2021 11:03:40 +0800 Subject: [PATCH 244/823] Fix the null point caused by deleting the system topic policy (#12367) Signed-off-by: Zixuan Liu ### Motivation `Message.getValue()` sometimes returns `null` in `SystemTopicBasedTopicPoliciesService.notifyListener()`, so we need to skip the messages that do not belong to the policy type, this problem can cause the policy service to fail to work. ### Modifications - Checks the `Message.getValue()` value. - Uses the `event` instead of `null` as message value when delete policy. ### Verifying this change - [x] Make sure that the change passes the CI checks. ### Does this pull request potentially affect one of the following parts: *If `yes` was chosen, please highlight the changes* - Dependencies (does it add or upgrade a dependency): no - The public API: no - The schema: no - The default values of configurations: no - The wire protocol: no - The rest endpoints: no - The admin cli options: no - Anything that affects deployment: no ### Documentation Check the box below and label this PR (if you have committer privilege). Need to update docs? - [ ] doc-required - [x] no-need-doc - [ ] doc (cherry picked from commit d310e79f360f41ddabc820fd0e45af67a8a4db82) --- .../SystemTopicBasedTopicPoliciesService.java | 27 +++++++++++++------ .../broker/admin/TopicPoliciesTest.java | 23 ++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index bf27736e97e47..a6c786ba2b98b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -126,7 +126,7 @@ private CompletableFuture sendTopicPolicyEvent(TopicName topicName, Action } } }); - }) + }) ); } }); @@ -149,6 +149,17 @@ private PulsarEvent getPulsarEvent(TopicName topicName, ActionType actionType, T } private void notifyListener(Message msg) { + // delete policies + if (msg.getValue() == null) { + TopicName topicName = TopicName.get(TopicName.get(msg.getKey()).getPartitionedTopicName()); + if (listeners.get(topicName) != null) { + for (TopicPolicyListener listener : listeners.get(topicName)) { + listener.onUpdate(null); + } + } + return; + } + if (!EventType.TOPIC_POLICY.equals(msg.getValue().getEventType())) { return; } @@ -296,12 +307,11 @@ public void unLoad(NamespaceBundle bundle) { removeOwnedNamespaceBundleAsync(bundle); } - @Override - public boolean test(NamespaceBundle namespaceBundle) { - return true; - } - - }); + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return true; + } + }); } private void initPolicesCache(SystemTopicClient.Reader reader, CompletableFuture future) { @@ -449,7 +459,8 @@ private void fetchTopicPoliciesAsyncAndCloseReader(SystemTopicClient.Reader { + Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNull(); + }); + + admin.topics().setMaxConsumersPerSubscription(topic, 1); + Awaitility.await().untilAsserted(() -> { + Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNotNull(); + }); + + admin.topics().delete(topic); + Awaitility.await().untilAsserted(() -> { + Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNull(); + }); + } + } } From 395bd73b3684602fbdcca40b8e09972632cefb4a Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Thu, 30 Dec 2021 19:13:41 +0800 Subject: [PATCH 245/823] fix(Auth): Fix multi roles authz cannot handle empty roles case (#13477) Motivation Currently, if the roles in the token are empty, then he `MultiRolesTokenAuthorizationProvider` will have problems processing it. It will keep waiting for an empty list of futures. Eventually causing the operation to time out. Modification * In `MultiRolesTokenAuthorizationProvider.authorize`, return false immediately when the roles are empty. Signed-off-by: Zike Yang (cherry picked from commit 4f942d71263424f12d9be88f39b79c41485d0f32) --- .../MultiRolesTokenAuthorizationProvider.java | 3 ++ ...tiRolesTokenAuthorizationProviderTest.java | 32 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index dcdf779780ef6..8a91d7f697158 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -128,6 +128,9 @@ private List getRoles(AuthenticationDataSource authData) { public CompletableFuture authorize(AuthenticationDataSource authenticationData, Function> authorizeFunc) { List roles = getRoles(authenticationData); + if (roles.isEmpty()) { + return CompletableFuture.completedFuture(false); + } List> futures = new ArrayList<>(roles.size()); roles.forEach(r -> futures.add(authorizeFunc.apply(r))); return CompletableFuture.supplyAsync(() -> { diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java index fdedf864c488c..edd0baa42ae25 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java @@ -56,7 +56,9 @@ public String getHttpHeader(String name) { }; Assert.assertTrue(provider.authorize(ads, role -> { - if (role.equals(userB)) return CompletableFuture.completedFuture(true); // only userB has permission + if (role.equals(userB)) { + return CompletableFuture.completedFuture(true); // only userB has permission + } return CompletableFuture.completedFuture(false); }).get()); @@ -65,7 +67,33 @@ public String getHttpHeader(String name) { }).get()); Assert.assertFalse(provider.authorize(ads, role -> { - return CompletableFuture.completedFuture(false); // only users has no permission + return CompletableFuture.completedFuture(false); // all users has no permission }).get()); } + + @Test + public void testMultiRolesAuthzWithEmptyRoles() throws Exception { + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + String token = Jwts.builder().claim("sub", new String[]{}).signWith(secretKey).compact(); + + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + + AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + Assert.assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); + } } From 7df7896ec5bb52d3890bde161375f68a81d830c5 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 4 Jan 2022 22:34:35 +0800 Subject: [PATCH 246/823] Change ContextClassLoader to NarClassLoader in AdditionalServlet (#13501) ### Motivation It's ``AddtionalServlet`` side change, like #11270 ### Modifications Change context class loader through Thread.currentThread().setContextClassLoader(classLoader) before every protocol handler callback, and change it back to original class loader afterwards. (cherry picked from commit 40ac7934ed567b2b9f16dbd7f937f7973dcddc21) --- .../pulsar/broker/ClassLoaderSwitcher.java | 37 ++++++ .../AdditionalServletWithClassLoader.java | 18 ++- .../ProtocolHandlerWithClassLoader.java | 19 +-- .../AdditionalServletWithClassLoaderTest.java | 114 ++++++++++++++++++ 4 files changed, 166 insertions(+), 22 deletions(-) create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoaderTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java new file mode 100644 index 0000000000000..787182ef012bc --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker; + +/** + * Help to switch the class loader of current thread to the NarClassLoader, and change it back when it's done. + * With the help of try-with-resources statement, the code would be cleaner than using try finally every time. + */ +public class ClassLoaderSwitcher implements AutoCloseable { + private final ClassLoader prevClassLoader; + + public ClassLoaderSwitcher(ClassLoader classLoader) { + prevClassLoader = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(classLoader); + } + + @Override + public void close() { + Thread.currentThread().setContextClassLoader(prevClassLoader); + } +} \ No newline at end of file diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java index 06a51c856c75d..7f1af734431ef 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java @@ -19,9 +19,11 @@ package org.apache.pulsar.broker.web.plugin.servlet; import java.io.IOException; + import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ClassLoaderSwitcher; import org.apache.pulsar.common.configuration.PulsarConfiguration; import org.apache.pulsar.common.nar.NarClassLoader; import org.eclipse.jetty.servlet.ServletHolder; @@ -39,22 +41,30 @@ public class AdditionalServletWithClassLoader implements AdditionalServlet { @Override public void loadConfig(PulsarConfiguration pulsarConfiguration) { - servlet.loadConfig(pulsarConfiguration); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + servlet.loadConfig(pulsarConfiguration); + } } @Override public String getBasePath() { - return servlet.getBasePath(); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + return servlet.getBasePath(); + } } @Override public ServletHolder getServletHolder() { - return servlet.getServletHolder(); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + return servlet.getServletHolder(); + } } @Override public void close() { - servlet.close(); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + servlet.close(); + } try { classLoader.close(); } catch (IOException e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java index 223cf81ccdb56..63aa6696917d9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ClassLoaderSwitcher; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.common.nar.NarClassLoader; @@ -95,22 +96,4 @@ public void close() { log.warn("Failed to close the protocol handler class loader", e); } } - - /** - * Help to switch the class loader of current thread to the NarClassLoader, and change it back when it's done. - * With the help of try-with-resources statement, the code would be cleaner than using try finally every time. - */ - private static class ClassLoaderSwitcher implements AutoCloseable { - private final ClassLoader prevClassLoader; - - ClassLoaderSwitcher(ClassLoader classLoader) { - prevClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(classLoader); - } - - @Override - public void close() { - Thread.currentThread().setContextClassLoader(prevClassLoader); - } - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoaderTest.java new file mode 100644 index 0000000000000..c875e8ac129cf --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoaderTest.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.web.plugin.servlet; + +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.configuration.PulsarConfiguration; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.eclipse.jetty.servlet.ServletHolder; +import org.testng.annotations.Test; + + +/** + * Unit test {@link AdditionalServletWithClassLoader}. + */ +@Test(groups = "broker") +public class AdditionalServletWithClassLoaderTest { + + @Test + public void testWrapper() { + AdditionalServlet servlet = mock(AdditionalServlet.class); + NarClassLoader loader = mock(NarClassLoader.class); + AdditionalServletWithClassLoader wrapper = new AdditionalServletWithClassLoader(servlet, loader); + // test getBasePath + String basePath = "bathPath"; + when(servlet.getBasePath()).thenReturn(basePath); + assertEquals(basePath, wrapper.getBasePath()); + verify(servlet, times(1)).getBasePath(); + // test loadConfig + ServiceConfiguration conf = new ServiceConfiguration(); + wrapper.loadConfig(conf); + verify(servlet, times(1)).loadConfig(same(conf)); + // test getServlet + assertEquals(wrapper.getServlet(),servlet); + // test getServletHolder + ServletHolder servletHolder = new ServletHolder(); + when(servlet.getServletHolder()).thenReturn(servletHolder); + assertEquals(wrapper.getServletHolder(),servletHolder); + verify(servlet, times(1)).getServletHolder(); + } + + @Test + public void testClassLoaderSwitcher() throws Exception { + NarClassLoader narLoader = mock(NarClassLoader.class); + AdditionalServlet servlet = new AdditionalServlet() { + @Override + public void loadConfig(PulsarConfiguration pulsarConfiguration) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + + @Override + public String getBasePath() { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + return "base-path"; + } + + @Override + public ServletHolder getServletHolder() { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + return null; + } + + @Override + public void close() { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + }; + + AdditionalServletWithClassLoader additionalServletWithClassLoader = + new AdditionalServletWithClassLoader(servlet, narLoader); + ClassLoader curClassLoader = Thread.currentThread().getContextClassLoader(); + // test class loader + assertEquals(additionalServletWithClassLoader.getClassLoader(), narLoader); + // test getBasePath + assertEquals(additionalServletWithClassLoader.getBasePath(), "base-path"); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test loadConfig + ServiceConfiguration conf = new ServiceConfiguration(); + additionalServletWithClassLoader.loadConfig(conf); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test getServletHolder + assertNull(additionalServletWithClassLoader.getServletHolder()); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test getServlet + assertEquals(additionalServletWithClassLoader.getServlet(), servlet); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test close + additionalServletWithClassLoader.close(); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + + } +} From 25fece02f3b021f8f50ecf171a63225ed6621195 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Tue, 4 Jan 2022 20:46:33 +0800 Subject: [PATCH 247/823] Bump slf4j to 1.7.32 (cherry picked from commit 05ea215ddc40e9957da7ed492d33a11831df3177) --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 6 +++--- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 207a7296fd524..be6f1fefc0384 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -40,7 +40,7 @@ 1.8 3.0.0-M3 2.17.1 - 1.7.25 + 1.7.32 7.3.0 3.11 3.2.4 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4e3b9b1540a25..1620c9f61754f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -540,9 +540,9 @@ BSD 2-Clause License MIT License * Java SemVer -- com.github.zafarkhaja-java-semver-0.9.0.jar -- licenses/LICENSE-SemVer.txt * SLF4J -- licenses/LICENSE-SLF4J.txt - - org.slf4j-jul-to-slf4j-1.7.25.jar - - org.slf4j-slf4j-api-1.7.25.jar - - org.slf4j-jcl-over-slf4j-1.7.25.jar + - org.slf4j-jul-to-slf4j-1.7.32.jar + - org.slf4j-slf4j-api-1.7.32.jar + - org.slf4j-jcl-over-slf4j-1.7.32.jar * The Checker Framework - org.checkerframework-checker-qual-3.5.0.jar diff --git a/pom.xml b/pom.xml index cfa1c34bf1527..2358fc740994a 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,7 @@ flexible messaging model and an intuitive client API. 0.5.0 3.9.8 6.10.2 - 1.7.25 + 1.7.32 3.2.2 2.17.1 1.69 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index aaddf3a416411..d3ab8a0f6463b 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -485,12 +485,12 @@ MIT License - pcollections-2.1.2.jar * SLF4J - slf4j-jdk14-1.7.29.jar - - slf4j-api-1.7.25.jar + - slf4j-api-1.7.32.jar - slf4j-jdk14-1.7.30.jar * JCL 1.2 Implemented Over SLF4J - - jcl-over-slf4j-1.7.25.jar + - jcl-over-slf4j-1.7.32.jar * JUL to SLF4J Bridge - - jul-to-slf4j-1.7.25.jar + - jul-to-slf4j-1.7.32.jar * Checker Qual - checker-qual-3.5.0.jar From f1f2f3dabe536a3fdeb2f7f51e93dd43c46f827a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 4 Jan 2022 20:26:54 +0200 Subject: [PATCH 248/823] [Presto] Fix issue of mixed SLF4J library versions in presto libs (#13603) (cherry picked from commit 29840b4ecabd0d997c25d52f6c02086b215f2cae) --- pulsar-sql/pom.xml | 12 ++++++++++++ pulsar-sql/presto-distribution/LICENSE | 6 ++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 279b6a1e08708..0261117dafb87 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -133,6 +133,18 @@ okio ${okio.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-jdk14 + ${slf4j.version} + diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d3ab8a0f6463b..a988218649c3a 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -329,8 +329,7 @@ The Apache Software License, Version 2.0 - leveldb-0.10.jar - leveldb-api-0.10.jar * Log4j implemented over SLF4J - - log4j-over-slf4j-1.7.29.jar - - log4j-over-slf4j-1.7.30.jar + - log4j-over-slf4j-1.7.32.jar * Lucene Common Analyzers - lucene-analyzers-common-8.4.1.jar - lucene-core-8.4.1.jar @@ -484,9 +483,8 @@ MIT License * PCollections - pcollections-2.1.2.jar * SLF4J - - slf4j-jdk14-1.7.29.jar - slf4j-api-1.7.32.jar - - slf4j-jdk14-1.7.30.jar + - slf4j-jdk14-1.7.32.jar * JCL 1.2 Implemented Over SLF4J - jcl-over-slf4j-1.7.32.jar * JUL to SLF4J Bridge From 624bf12934e8e0fccb411321c693b2f601a853b7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 4 Jan 2022 19:32:09 +0200 Subject: [PATCH 249/823] [Tests] Fix flakiness issue when spying ServerCnx (#13608) Fixes #13570 (cherry picked from commit de99c2c9c3c54b31096e6d734de8132e96d50c79) --- .../PersistentDispatcherFailoverConsumerTest.java | 10 ++++++++-- .../service/PersistentTopicConcurrentTest.java | 6 +++++- .../pulsar/broker/service/PersistentTopicTest.java | 14 +++++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index bc5e05cbf1855..84cf739ebcf12 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -21,6 +21,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.matches; import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -28,6 +29,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.assertEquals; @@ -165,7 +167,9 @@ public void setup() throws Exception { return null; }).when(channelCtx).writeAndFlush(any(), any()); - serverCnx = spy(new ServerCnx(pulsar)); + serverCnx = mock(ServerCnx.class, withSettings() + .useConstructor(pulsar) + .defaultAnswer(CALLS_REAL_METHODS)); doReturn(true).when(serverCnx).isActive(); doReturn(true).when(serverCnx).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress(); @@ -174,7 +178,9 @@ public void setup() throws Exception { doReturn(new PulsarCommandSenderImpl(null, serverCnx)) .when(serverCnx).getCommandSender(); - serverCnxWithOldVersion = spy(new ServerCnx(pulsar)); + serverCnxWithOldVersion = mock(ServerCnx.class, withSettings() + .useConstructor(pulsar) + .defaultAnswer(CALLS_REAL_METHODS)); doReturn(true).when(serverCnxWithOldVersion).isActive(); doReturn(true).when(serverCnxWithOldVersion).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java index ef81ed2575e66..78c7e255739a8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.broker.service; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.withSettings; import static org.testng.Assert.assertFalse; import java.lang.reflect.Method; @@ -101,7 +103,9 @@ public void setup(Method m) throws Exception { brokerService = spy(new BrokerService(pulsar, eventLoopGroup)); doReturn(brokerService).when(pulsar).getBrokerService(); - serverCnx = spy(new ServerCnx(pulsar)); + serverCnx = mock(ServerCnx.class, withSettings() + .useConstructor(pulsar) + .defaultAnswer(CALLS_REAL_METHODS)); doReturn(true).when(serverCnx).isActive(); NamespaceService nsSvc = mock(NamespaceService.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index f1a11ad4f9c11..d94320ffc96d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; @@ -34,6 +35,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; @@ -207,7 +209,9 @@ public void setup() throws Exception { brokerService = spy(new BrokerService(pulsar, eventLoopGroup)); doReturn(brokerService).when(pulsar).getBrokerService(); - serverCnx = spy(new ServerCnx(pulsar)); + serverCnx = mock(ServerCnx.class, withSettings() + .useConstructor(pulsar) + .defaultAnswer(CALLS_REAL_METHODS)); doReturn(true).when(serverCnx).isActive(); doReturn(true).when(serverCnx).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress(); @@ -584,7 +588,9 @@ private Producer getMockedProducerWithSpecificAddress(Topic topic, long producer final String producerNameBase = "producer"; final String role = "appid1"; - ServerCnx cnx = spy(new ServerCnx(pulsar)); + ServerCnx cnx = mock(ServerCnx.class, withSettings() + .useConstructor(pulsar) + .defaultAnswer(CALLS_REAL_METHODS)); doReturn(true).when(cnx).isActive(); doReturn(true).when(cnx).isWritable(); doReturn(new InetSocketAddress(address, 1234)).when(cnx).clientAddress(); @@ -1022,7 +1028,9 @@ private Consumer getMockedConsumerWithSpecificAddress(Topic topic, Subscription final String consumerNameBase = "consumer"; final String role = "appid1"; - ServerCnx cnx = spy(new ServerCnx(pulsar)); + ServerCnx cnx = mock(ServerCnx.class, withSettings() + .useConstructor(pulsar) + .defaultAnswer(CALLS_REAL_METHODS)); doReturn(true).when(cnx).isActive(); doReturn(true).when(cnx).isWritable(); doReturn(new InetSocketAddress(address, 1234)).when(cnx).clientAddress(); From 9eef658d12bbe4f187765f324a63677a29047e35 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Thu, 6 Jan 2022 10:23:43 +0800 Subject: [PATCH 250/823] update log content (#13540) * update log content * update log content * check style (cherry picked from commit 825d2fe236306242de639c38a0a60d7f3ec5ced0) --- .../pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 7668c116a3954..7bcfcab0e49c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -607,7 +607,8 @@ public synchronized void doLoadShedding() { return; } - log.info("[Overload shedder] Unloading bundle: {} from broker {}", bundle, broker); + log.info("[{}] Unloading bundle: {} from broker {}", + strategy.getClass().getSimpleName(), bundle, broker); try { pulsar.getAdminClient().namespaces().unloadNamespaceBundle(namespaceName, bundleRange); loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis()); From 6fd742139ddfed161ea0c76582b7dfdd4be7004e Mon Sep 17 00:00:00 2001 From: feynmanlin Date: Wed, 5 Jan 2022 14:46:32 +0800 Subject: [PATCH 251/823] Fix NPE when unloading namespace bundle (#13571) * Fix NPE when unloading namespace bundle (cherry picked from commit ec0a44058d249a7510bb3d05685b2ee5e0874eb6) --- .../pulsar/broker/service/BrokerService.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index cbe02c8c15f82..d37dfb6a08105 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -830,15 +830,17 @@ public void unloadNamespaceBundlesGracefully() { // unload all namespace-bundles gracefully long closeTopicsStartTime = System.nanoTime(); Set serviceUnits = pulsar.getNamespaceService().getOwnedServiceUnits(); - serviceUnits.forEach(su -> { - if (su instanceof NamespaceBundle) { - try { - pulsar.getNamespaceService().unloadNamespaceBundle(su, 1, TimeUnit.MINUTES).get(); - } catch (Exception e) { - log.warn("Failed to unload namespace bundle {}", su, e); + if (serviceUnits != null) { + serviceUnits.forEach(su -> { + if (su instanceof NamespaceBundle) { + try { + pulsar.getNamespaceService().unloadNamespaceBundle(su, 1, TimeUnit.MINUTES).get(); + } catch (Exception e) { + log.warn("Failed to unload namespace bundle {}", su, e); + } } - } - }); + }); + } double closeTopicsTimeSeconds = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - closeTopicsStartTime)) / 1000.0; From 598cf4099be6fbba25a00ab645adc9e0b67bc10e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 5 Jan 2022 14:54:27 +0200 Subject: [PATCH 252/823] [Tests] Upgrade Mockito to latest stable 3.x version, 3.12.4 (#13622) (cherry picked from commit 1f102814b96db723592b2fdfb2bbf288d9b75fb9) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2358fc740994a..88d2e5813e532 100644 --- a/pom.xml +++ b/pom.xml @@ -213,7 +213,7 @@ flexible messaging model and an intuitive client API. 1.1.1 7.3.0 4.13.1 - 3.8.0 + 3.12.4 2.0.9 3.25.0-GA 2.3.1 From a27fb57a183649eede83da50af5bc9c859535367 Mon Sep 17 00:00:00 2001 From: liuchangqing Date: Thu, 6 Jan 2022 13:19:24 +0800 Subject: [PATCH 253/823] Fix time field use error (#12249) * Fix time field use error * rebuid Co-authored-by: liu.changqing (cherry picked from commit 793dd910fc17b0e149f4b554b54cad55ca5bc479) --- .../java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 99be9f969d6cd..d839e05def76f 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -580,7 +580,7 @@ public boolean advanceNextPosition() { currentRowValuesMap.put(columnHandle, longValueProvider(this.partition)); } else if (PulsarInternalColumn.EVENT_TIME.getName().equals(columnHandle.getName())) { currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider( - this.currentMessage.getEventTime(), this.currentMessage.getPublishTime() == 0)); + this.currentMessage.getEventTime(), this.currentMessage.getEventTime() == 0)); } else if (PulsarInternalColumn.PUBLISH_TIME.getName().equals(columnHandle.getName())) { currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider( this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); From 62b4c60b4b01805ee3b374170d20333a8e6bac1d Mon Sep 17 00:00:00 2001 From: baomingyu Date: Fri, 7 Jan 2022 12:51:41 +0800 Subject: [PATCH 254/823] Fix issues 11964, deadlock bug when use key_shared mode (#11965) (cherry picked from commit fd3ba55f8e39992ecbc96830af5ee58e3247a25a) --- .../org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 3d382ad9698db..2f9578ced4d37 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1993,7 +1993,6 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb if (individualDeletedMessages.isEmpty()) { // No changes to individually deleted messages, so nothing to do at this point - callback.deleteComplete(ctx); return; } @@ -2025,6 +2024,9 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb return; } finally { lock.writeLock().unlock(); + if (individualDeletedMessages.isEmpty()) { + callback.deleteComplete(ctx); + } } // Apply rate limiting to mark-delete operations From 89c7b6e0d586812bd39bb2b21175dcad4c237316 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Fri, 7 Jan 2022 15:14:45 +0800 Subject: [PATCH 255/823] [issue #13439][backlogQuota] Allow both limit and limitsize be null (#13557) Fixes #13439 ### Motivation Now we only constraint `limit` or `limitSize` be set on pulsar-admin, we don't limit it on rest endpoint. So there are situtations that both `limit` and `limitSize` be null ### Verifying this change This change added tests and can be verified as follows: - *Added test method in `BacklogQuotaCompatibilityTest` (cherry picked from commit 33491bc2edf698723865d2be168e4b1082742d45) --- .../policies/data/impl/BacklogQuotaImpl.java | 4 +--- .../BacklogQuotaCompatibilityTest.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java index 3d97fa06426f9..c8073d8fb4066 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BacklogQuotaImpl.java @@ -35,7 +35,7 @@ public class BacklogQuotaImpl implements BacklogQuota { * @since 2.9.1 */ @Deprecated - private Long limit; + private long limit; /** * backlog quota by size in byte. @@ -57,7 +57,6 @@ public BacklogQuotaImpl(long limitSize, int limitTime, RetentionPolicy policy) { @Deprecated public long getLimit() { if (limitSize == null) { - // the limitSize and limit can't be both null return limit; } return limitSize; @@ -71,7 +70,6 @@ public void setLimit(long limit) { public long getLimitSize() { if (limitSize == null) { - // the limitSize and limit can't be both null return limit; } return limitSize; diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java index 659825c74858a..f621a3a452cbc 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BacklogQuotaCompatibilityTest.java @@ -105,4 +105,24 @@ public void testBackwardCompatibility() throws IOException { BacklogQuota.RetentionPolicy.consumer_backlog_eviction); } + @Test + public void testBackwardCompatibilityNullLimitAndLimitSize() throws IOException { + String oldPolicyStr = "{\"auth_policies\":{\"namespace_auth\":{},\"destination_auth\":{}," + + "\"subscription_auth_roles\":{}},\"replication_clusters\":[],\"backlog_quota_map\":" + + "{\"destination_storage\":{\"policy\":\"consumer_backlog_eviction\"}}," + + "\"clusterDispatchRate\":{},\"topicDispatchRate\":{},\"subscriptionDispatchRate\":{}," + + "\"replicatorDispatchRate\":{},\"clusterSubscribeRate\":{},\"publishMaxMessageRate\":{}," + + "\"latency_stats_sample_rate\":{},\"subscription_expiration_time_minutes\":0,\"deleted\":false," + + "\"encryption_required\":false,\"subscription_auth_mode\":\"None\"," + + "\"max_consumers_per_subscription\":0,\"offload_threshold\":-1," + + "\"schema_auto_update_compatibility_strategy\":\"Full\",\"schema_compatibility_strategy\":" + + "\"UNDEFINED\",\"is_allow_auto_update_schema\":true,\"schema_validation_enforced\":false," + + "\"subscription_types_enabled\":[]}\n"; + Policies policies = simpleType.deserialize(null, oldPolicyStr.getBytes(), null); + assertEquals(policies.backlog_quota_map.get(BacklogQuota.BacklogQuotaType.destination_storage).getLimitSize(), + 0); + assertEquals(policies.backlog_quota_map.get(BacklogQuota.BacklogQuotaType.destination_storage).getLimitTime(), + 0); + } + } From 23700dbe7eaf871a2160a47546dc3b03f51f3e1e Mon Sep 17 00:00:00 2001 From: lipenghui Date: Fri, 7 Jan 2022 13:44:02 +0800 Subject: [PATCH 256/823] Fix reader skipped remaining compacted data during the topic unloading. (#13629) ### Motivation To fix the reader skipping remaining compacted data while the topic has been unloaded. #11287 fixed the data skipped issue while the reader first time to read the messages with the earliest position. But if the reader has consumed some messages from the compacted ledger but not all, the start position will not be `earliest`, the broker will rewind the cursor for the reader to the next valid position of the original topic. So the remaining messages in the compacted ledger will be skipped. Here are the logs from the broker: ``` 10:44:36.035 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] INFO org.apache.pulsar.broker.service.BrokerService - Created topic persistent://xxx/product-full-prod/5126 - dedup is disabled 10:44:36.035 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] INFO org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://xxx/product-full-prod/5126][xxx] Creating non-durable subscription at msg id 181759:14:-1:-1 10:44:36.035 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] INFO org.apache.bookkeeper.mledger.impl.NonDurableCursorImpl - [xxx/product-full-prod/persistent/5126] Created non-durable cursor read-position=221199:0 mark-delete-position=181759:13 10:44:36.035 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] INFO org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [xxx/product-full-prod/persistent/5126] Opened new cursor: NonDurableCursorImpl{ledger=xxx/product-full-prod/persistent/5126, ackPos=181759:13, readPos=221199:0} 10:44:36.035 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] INFO org.apache.bookkeeper.mledger.impl.ManagedCursorImpl - [xxx/product-full-prod/persistent/5126-xxx] Rewind from 221199:0 to 221199:0 ``` There some many compacted messages after `181759:13`, but the broker will not dispatch them to the reader. The issue also can be reproduced by the unit test that was added in this PR. ### Modification If the cursor with `readCompacted = true`, just rewind to the next message of the mark delete position, so that the reader can continue to read the data from the compacted ledger. ### Verification A new test added for testing the reader can get all the compacted messages and non-compacted messages from the topic during the topic unloading. (cherry picked from commit 07f131fec7db36d6f424ce419d26888592b04207) --- .../bookkeeper/mledger/ManagedLedger.java | 3 +- .../mledger/impl/ManagedLedgerImpl.java | 7 ++- .../mledger/impl/NonDurableCursorImpl.java | 23 +++++--- ...bstractDispatcherSingleActiveConsumer.java | 4 -- .../service/persistent/PersistentTopic.java | 8 ++- .../pulsar/compaction/CompactedTopicTest.java | 55 +++++++++++++++++++ .../jcloud/impl/MockManagedLedger.java | 4 +- 7 files changed, 84 insertions(+), 20 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index 74c67e965afc6..0200e25cff132 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -259,7 +259,8 @@ public interface ManagedLedger { */ ManagedCursor newNonDurableCursor(Position startCursorPosition) throws ManagedLedgerException; ManagedCursor newNonDurableCursor(Position startPosition, String subscriptionName) throws ManagedLedgerException; - ManagedCursor newNonDurableCursor(Position startPosition, String subscriptionName, InitialPosition initialPosition) throws ManagedLedgerException; + ManagedCursor newNonDurableCursor(Position startPosition, String subscriptionName, InitialPosition initialPosition, + boolean isReadCompacted) throws ManagedLedgerException; /** * Delete a ManagedCursor asynchronously. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index cec106c7e66a0..457600ccf1630 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1030,11 +1030,12 @@ public ManagedCursor newNonDurableCursor(Position startCursorPosition) throws Ma @Override public ManagedCursor newNonDurableCursor(Position startPosition, String subscriptionName) throws ManagedLedgerException { - return newNonDurableCursor(startPosition, subscriptionName, InitialPosition.Latest); + return newNonDurableCursor(startPosition, subscriptionName, InitialPosition.Latest, false); } @Override - public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cursorName, InitialPosition initialPosition) + public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cursorName, InitialPosition initialPosition, + boolean isReadCompacted) throws ManagedLedgerException { Objects.requireNonNull(cursorName, "cursor name can't be null"); checkManagedLedgerIsOpen(); @@ -1049,7 +1050,7 @@ public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cu } NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, config, this, cursorName, - (PositionImpl) startCursorPosition, initialPosition); + (PositionImpl) startCursorPosition, initialPosition, isReadCompacted); cursor.setActive(); log.info("[{}] Opened new cursor: {}", name, cursor); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java index 0f7ffe41f8a0b..1d545bd6b48c6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java @@ -33,11 +33,12 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { - private volatile boolean readCompacted; + private final boolean readCompacted; NonDurableCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, String cursorName, - PositionImpl startCursorPosition, CommandSubscribe.InitialPosition initialPosition) { + PositionImpl startCursorPosition, CommandSubscribe.InitialPosition initialPosition, boolean isReadCompacted) { super(bookkeeper, config, ledger, cursorName); + this.readCompacted = isReadCompacted; // Compare with "latest" position marker by using only the ledger id. Since the C++ client is using 48bits to // store the entryId, it's not able to pass a Long.max() as entryId. In this case there's no point to require @@ -67,7 +68,7 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { private void recoverCursor(PositionImpl mdPosition) { Pair lastEntryAndCounter = ledger.getLastPositionAndCounter(); - this.readPosition = ledger.getNextValidPosition(mdPosition); + this.readPosition = isReadCompacted() ? mdPosition.getNext() : ledger.getNextValidPosition(mdPosition); markDeletePosition = mdPosition; // Initialize the counter such that the difference between the messages written on the ML and the @@ -118,14 +119,22 @@ public void asyncDeleteCursor(final String consumerName, final DeleteCursorCallb callback.deleteCursorComplete(ctx); } - public void setReadCompacted(boolean readCompacted) { - this.readCompacted = readCompacted; - } - public boolean isReadCompacted() { return readCompacted; } + @Override + public void rewind() { + // For reading the compacted data, + // we couldn't reset the read position to the next valid position of the original topic. + // Otherwise, the remaining data in the compacted ledger will be skipped. + if (!readCompacted) { + super.rewind(); + } else { + readPosition = markDeletePosition.getNext(); + } + } + @Override public synchronized String toString() { return MoreObjects.toStringHelper(this).add("ledger", ledger.getName()).add("ackPos", markDeletePosition) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java index 4c7ea45f1a4bb..8cab06be116af 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java @@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.NonDurableCursorImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; @@ -181,9 +180,6 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce consumer.notifyActiveConsumerChange(currentActiveConsumer); } } - if (cursor != null && !cursor.isDurable() && cursor instanceof NonDurableCursorImpl) { - ((NonDurableCursorImpl) cursor).setReadCompacted(ACTIVE_CONSUMER_UPDATER.get(this).readCompacted()); - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1b2e11f4ba0fc..e2cc79c5f8e51 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -745,7 +745,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs getDurableSubscription(subscriptionName, initialPosition, startMessageRollbackDurationSec, replicatedSubscriptionState) : getNonDurableSubscription(subscriptionName, startMessageId, initialPosition, - startMessageRollbackDurationSec); + startMessageRollbackDurationSec, readCompacted); int maxUnackedMessages = isDurable ? getMaxUnackedMessagesOnConsumer() @@ -891,7 +891,8 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { } private CompletableFuture getNonDurableSubscription(String subscriptionName, - MessageId startMessageId, InitialPosition initialPosition, long startMessageRollbackDurationSec) { + MessageId startMessageId, InitialPosition initialPosition, long startMessageRollbackDurationSec, + boolean isReadCompacted) { log.info("[{}][{}] Creating non-durable subscription at msg id {}", topic, subscriptionName, startMessageId); CompletableFuture subscriptionFuture = new CompletableFuture<>(); @@ -924,7 +925,8 @@ private CompletableFuture getNonDurableSubscription(Stri Position startPosition = new PositionImpl(ledgerId, entryId); ManagedCursor cursor = null; try { - cursor = ledger.newNonDurableCursor(startPosition, subscriptionName, initialPosition); + cursor = ledger.newNonDurableCursor(startPosition, subscriptionName, initialPosition, + isReadCompacted); } catch (ManagedLedgerException e) { return FutureUtil.failedFuture(e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 4d00d2864e6ff..accce4ef6f285 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -734,4 +734,59 @@ public void testHasMessageAvailableWithNullValueMessage() throws Exception { Assert.assertNull(reader.readNext(3, TimeUnit.SECONDS)); } + public void testReadCompleteMessagesDuringTopicUnloading() throws Exception { + String topic = "persistent://my-property/use/my-ns/testReadCompleteMessagesDuringTopicUnloading-" + + UUID.randomUUID(); + final int numMessages = 1000; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .blockIfQueueFull(true) + .enableBatching(false) + .create(); + CompletableFuture lastMessage = null; + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i + "").value(String.format("msg [%d]", i)).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, numMessages); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + Assert.assertEquals(stats.lastConfirmedEntry, stats.cursors.get(COMPACTION_SUBSCRIPTION).markDeletePosition); + }); + // Unload the topic to make sure the original ledger been deleted. + admin.topics().unload(topic); + // Produce more messages to the original topic + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i + numMessages + "").value(String.format("msg [%d]", i + numMessages)).sendAsync(); + } + producer.flush(); + lastMessage.join(); + // For now the topic has 1000 messages in the compacted ledger and 1000 messages in the original topic. + @Cleanup + Reader reader = pulsarClient.newReader(Schema.STRING) + .topic(topic) + .startMessageIdInclusive() + .startMessageId(MessageId.earliest) + .readCompacted(true) + .create(); + + // Unloading the topic during reading the data to make sure the reader will not miss any messages. + for (int i = 0; i < numMessages / 2; ++i) { + Assert.assertEquals(reader.readNext().getValue(), String.format("msg [%d]", i)); + } + admin.topics().unload(topic); + for (int i = 0; i < numMessages / 2; ++i) { + Assert.assertEquals(reader.readNext().getValue(), String.format("msg [%d]", i + numMessages / 2)); + } + admin.topics().unload(topic); + for (int i = 0; i < numMessages; ++i) { + Assert.assertEquals(reader.readNext().getValue(), String.format("msg [%d]", i + numMessages)); + } + } } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java index 767190c8624aa..229cd666acff8 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java @@ -122,8 +122,8 @@ public ManagedCursor newNonDurableCursor(Position startPosition, String subscrip @Override public ManagedCursor newNonDurableCursor(Position startPosition, String subscriptionName, - CommandSubscribe.InitialPosition initialPosition) throws - ManagedLedgerException { + CommandSubscribe.InitialPosition initialPosition, + boolean isReadCompacted) throws ManagedLedgerException { return null; } From d0904136da4654834248c30730fcaef6da59b270 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 8 Jan 2022 03:30:07 +0800 Subject: [PATCH 257/823] Update notice year. (#13653) ### Motivation Update NOTICE from 2021-2022 (cherry picked from commit bacac04f564536be9c30481d2cab88512906967e) --- NOTICE | 2 +- distribution/server/src/assemble/NOTICE.bin.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE b/NOTICE index 2052417487269..bbbe4fab89b56 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Pulsar -Copyright 2017-2021 The Apache Software Foundation +Copyright 2017-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/distribution/server/src/assemble/NOTICE.bin.txt b/distribution/server/src/assemble/NOTICE.bin.txt index 661921cc81684..bc5e2e6d63b0e 100644 --- a/distribution/server/src/assemble/NOTICE.bin.txt +++ b/distribution/server/src/assemble/NOTICE.bin.txt @@ -1,6 +1,6 @@ Apache Pulsar -Copyright 2017-2021 The Apache Software Foundation +Copyright 2017-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). From ad043fca44450fd8b8b13f27fc3460fc2442fa1a Mon Sep 17 00:00:00 2001 From: lipenghui Date: Mon, 10 Jan 2022 09:44:47 +0800 Subject: [PATCH 258/823] Remove txn add partition info log (#13670) During my test, there are many output logs `handle add partition to txn finish` for the broker. ### Documentation Check the box below or label this PR directly (if you have committer privilege). (cherry picked from commit 53bd7a44f252812a0ff4c1bc22d0a03e606a4875) --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index c4ab0e26be326..583ad3c5a30eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2335,7 +2335,6 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) { } ctx.writeAndFlush(Commands.newAddSubscriptionToTxnResponse(requestId, txnID.getLeastSigBits(), txnID.getMostSigBits())); - log.info("handle add partition to txn finish."); } else { ex = handleTxnException(ex, BaseCommand.Type.ADD_SUBSCRIPTION_TO_TXN.name(), requestId); From a08f0f506e5985df50a818e601ee4b3869e1ce02 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Mon, 10 Jan 2022 19:27:26 +0800 Subject: [PATCH 259/823] [Java Client] fixed Producer semaphore permit release issue (#13682) (cherry picked from commit 916b61d3a497c44c8511347d2c4eac95c53f8506) --- .../client/impl/ProducerSemaphoreTest.java | 14 ++++++++++++++ .../apache/pulsar/client/impl/ProducerImpl.java | 16 +++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java index c719cbda6f0be..0c7f4b137e791 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java @@ -75,11 +75,13 @@ public void testProducerSemaphoreAcquireAndRelease() throws PulsarClientExceptio futures.add(producer.newMessage().value(("Semaphore-test-" + i).getBytes()).sendAsync()); } Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize - messages); + Assert.assertFalse(producer.isErrorStat()); } finally { producer.getClientCnx().channel().config().setAutoRead(true); } FutureUtil.waitForAll(futures).get(); Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); futures.clear(); // Simulate replicator, non batching message but `numMessagesInBatch` of message metadata > 1 @@ -92,15 +94,18 @@ public void testProducerSemaphoreAcquireAndRelease() throws PulsarClientExceptio futures.add(producer.sendAsync(msg)); } Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize - messages/2); + Assert.assertFalse(producer.isErrorStat()); } finally { producer.getClientCnx().channel().config().setAutoRead(true); } FutureUtil.waitForAll(futures).get(); Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); futures.clear(); // Here must ensure that the semaphore available permits is 0 Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); // Acquire 5 and not wait the send ack call back producer.getClientCnx().channel().config().setAutoRead(false); @@ -111,12 +116,14 @@ public void testProducerSemaphoreAcquireAndRelease() throws PulsarClientExceptio // Here must ensure that the Semaphore a acquired 5 Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize - messages / 2); + Assert.assertFalse(producer.isErrorStat()); } finally { producer.getClientCnx().channel().config().setAutoRead(true); } FutureUtil.waitForAll(futures).get(); Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); } /** @@ -141,6 +148,7 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { // Test that when we fill the queue with "replicator" messages, we are notified // (replicator itself would block) Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); producer.getClientCnx().channel().config().setAutoRead(false); try { for (int i = 0; i < pendingQueueSize; i++) { @@ -151,6 +159,7 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { futures.add(producer.sendAsync(msg)); } Assert.assertEquals(producer.getSemaphore().get().availablePermits(), 0); + Assert.assertFalse(producer.isErrorStat()); try { MessageMetadata metadata = new MessageMetadata() .setNumMessagesInBatch(10); @@ -162,6 +171,7 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { Assert.assertEquals(ee.getCause().getClass(), PulsarClientException.ProducerQueueIsFullError.class); Assert.assertEquals(producer.getSemaphore().get().availablePermits(), 0); + Assert.assertFalse(producer.isErrorStat()); } } finally { producer.getClientCnx().channel().config().setAutoRead(true); @@ -171,12 +181,14 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { // Test that when we fill the queue with normal messages, we get an error Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); producer.getClientCnx().channel().config().setAutoRead(false); try { for (int i = 0; i < pendingQueueSize; i++) { futures.add(producer.newMessage().value(("Semaphore-test-" + i).getBytes()).sendAsync()); } Assert.assertEquals(producer.getSemaphore().get().availablePermits(), 0); + Assert.assertFalse(producer.isErrorStat()); try { producer.newMessage().value(("Semaphore-test-Q-full").getBytes()).sendAsync().get(); @@ -184,6 +196,7 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { Assert.assertEquals(ee.getCause().getClass(), PulsarClientException.ProducerQueueIsFullError.class); Assert.assertEquals(producer.getSemaphore().get().availablePermits(), 0); + Assert.assertFalse(producer.isErrorStat()); } } finally { @@ -191,5 +204,6 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { } FutureUtil.waitForAll(futures).get(); Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + Assert.assertFalse(producer.isErrorStat()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index f5165d7d37f3d..46649bde8dc6e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -266,11 +266,12 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration protected void semaphoreRelease(final int releaseCountRequest) { if (semaphore.isPresent()) { if (!errorState) { - final int availablePermits = semaphore.get().availablePermits(); - if (availablePermits - releaseCountRequest < 0) { - log.error("Semaphore permit release count request greater then availablePermits" + - " : availablePermits={}, releaseCountRequest={}", - availablePermits, releaseCountRequest); + final int availableReleasePermits = + conf.getMaxPendingMessages() - this.semaphore.get().availablePermits(); + if (availableReleasePermits - releaseCountRequest < 0) { + log.error("Semaphore permit release count request greater then availableReleasePermits" + + " : availableReleasePermits={}, releaseCountRequest={}", + availableReleasePermits, releaseCountRequest); errorState = true; } } @@ -2043,5 +2044,10 @@ Optional getSemaphore() { return semaphore; } + @VisibleForTesting + boolean isErrorStat() { + return errorState; + } + private static final Logger log = LoggerFactory.getLogger(ProducerImpl.class); } From 9015e1795958e53af09a3f1fee0d5cca0d0243e7 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 14 Jan 2022 01:55:59 +0800 Subject: [PATCH 260/823] [Transaction] Duplicate TYPE of Prometheus metrics (#13704) * [Transaction] Fix Duplicate TYPE of Prometheus metrics ### Motivation Fix duplicate TYPE statements in the metrics output which leads to parsing of Prometheus metrics. ### Modification Add a map in TransactionAggregator.java, which used for tracking duplicate TYPE definitions. * Add test * Fix test and checkstyle * Use threadLocal * Use threadLocal * Fix test and details/*8 * Fix details --- .../prometheus/TransactionAggregator.java | 28 ++++++- .../broker/stats/TransactionMetricsTest.java | 75 +++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java index 65399d4effce3..e6ac1535f43c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java @@ -20,6 +20,8 @@ import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; @@ -36,6 +38,17 @@ @Slf4j public class TransactionAggregator { + /** + * Used for tracking duplicate TYPE definitions. + */ + private static final FastThreadLocal> threadLocalMetricWithTypeDefinition = + new FastThreadLocal() { + @Override + protected Map initialValue() { + return new HashMap<>(); + } + }; + private static final FastThreadLocal localTransactionCoordinatorStats = new FastThreadLocal() { @Override @@ -54,6 +67,8 @@ protected ManagedLedgerStats initialValue() throws Exception { public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, boolean includeTopicMetrics) { String cluster = pulsar.getConfiguration().getClusterName(); + Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); + metricWithTypeDefinition.clear(); if (includeTopicMetrics) { pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { @@ -146,10 +161,19 @@ private static void generateManageLedgerStats(ManagedLedger managedLedger, Simpl subscription, managedLedgerStats); } + private static void metricType(SimpleTextOutputStream stream, String name) { + Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); + if (!metricWithTypeDefinition.containsKey(name)) { + metricWithTypeDefinition.put(name, "gauge"); + stream.write("# TYPE ").write(name).write(" gauge\n"); + } + + } + private static void metric(SimpleTextOutputStream stream, String cluster, String name, double value, long coordinatorId) { - stream.write("# TYPE ").write(name).write(" gauge\n") - .write(name) + metricType(stream, name); + stream.write(name) .write("{cluster=\"").write(cluster) .write("\",coordinator_id=\"").write(coordinatorId).write("\"} ") .write(value).write(' ').write(System.currentTimeMillis()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java index 6a4b5c401a1dc..7c48e81449813 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java @@ -18,8 +18,14 @@ */ package org.apache.pulsar.broker.stats; +import com.google.common.base.Splitter; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; @@ -51,10 +57,13 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.broker.stats.PrometheusMetricsTest.parseMetrics; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +@Slf4j public class TransactionMetricsTest extends BrokerTestBase { @BeforeMethod(alwaysRun = true) @@ -339,6 +348,72 @@ public void testManagedLedgerMetricsWhenPendingAckNotInit() throws Exception{ checkManagedLedgerMetrics(subName2, 32, metric); } + @Test + public void testDuplicateMetricTypeDefinitions() throws Exception{ + admin.lookups().lookupTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()); + TransactionCoordinatorID transactionCoordinatorIDOne = TransactionCoordinatorID.get(0); + TransactionCoordinatorID transactionCoordinatorIDTwo = TransactionCoordinatorID.get(1); + pulsar.getTransactionMetadataStoreService().handleTcClientConnect(transactionCoordinatorIDOne); + pulsar.getTransactionMetadataStoreService().handleTcClientConnect(transactionCoordinatorIDTwo); + + Awaitility.await().until(() -> + pulsar.getTransactionMetadataStoreService().getStores().size() == 2); + pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString()).enableTransaction(true).build(); + Producer p1 = pulsarClient + .newProducer() + .topic("persistent://my-property/use/my-ns/my-topic1") + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + Transaction transaction = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + p1.newMessage(transaction) + .value(message.getBytes()) + .send(); + } + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + String metricsStr = statsOut.toString(); + + Map typeDefs = new HashMap<>(); + Map metricNames = new HashMap<>(); + Pattern typePattern = Pattern.compile("^#\\s+TYPE\\s+(\\w+)\\s+(\\w+)"); + + Splitter.on("\n").split(metricsStr).forEach(line -> { + if (line.isEmpty()) { + return; + } + if (line.startsWith("#")) { + // Check for duplicate type definitions + Matcher typeMatcher = typePattern.matcher(line); + checkArgument(typeMatcher.matches()); + String metricName = typeMatcher.group(1); + String type = typeMatcher.group(2); + // From https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md + // "Only one TYPE line may exist for a given metric name." + if (!typeDefs.containsKey(metricName)) { + typeDefs.put(metricName, type); + } else { + log.warn(metricsStr); + fail("Duplicate type definition found for TYPE definition " + metricName); + + } + // From https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md + // "The TYPE line for a metric name must appear before the first sample is reported for that metric name." + if (metricNames.containsKey(metricName)) { + log.info(metricsStr); + fail("TYPE definition for " + metricName + " appears after first sample"); + + } + } + }); + + } + private void checkManagedLedgerMetrics(String tag, double value, Collection metrics) { boolean exist = false; for (PrometheusMetricsTest.Metric metric1 : metrics) { From dfc298f3d58d7ca78573758fafece104b8daf040 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Wed, 12 Jan 2022 21:57:34 +0800 Subject: [PATCH 261/823] Avoid call sync method in async rest API for delete subscription (#13666) --- .../admin/impl/PersistentTopicsBase.java | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 5a084bf7e10b0..fa57199ed600a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -1485,35 +1485,38 @@ protected void internalDeleteSubscription(AsyncResponse asyncResponse, String su private void internalDeleteSubscriptionForNonPartitionedTopic(AsyncResponse asyncResponse, String subName, boolean authoritative) { - try { - validateTopicOwnership(topicName, authoritative); - validateTopicOperation(topicName, TopicOperation.UNSUBSCRIBE); - - Topic topic = getTopicReference(topicName); - Subscription sub = topic.getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, "Subscription not found")); - return; - } - sub.delete().get(); - log.info("[{}][{}] Deleted subscription {}", clientAppId(), topicName, subName); - asyncResponse.resume(Response.noContent().build()); - } catch (Exception e) { - if (e.getCause() instanceof SubscriptionBusyException) { - log.error("[{}] Failed to delete subscription {} from topic {}", clientAppId(), subName, topicName, e); - asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, - "Subscription has active connected consumers")); - } else if (e instanceof WebApplicationException) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to delete subscription from topic {}, redirecting to other brokers.", - clientAppId(), topicName, e); + validateTopicOwnershipAsync(topicName, authoritative) + .thenRun(() -> validateTopicOperation(topicName, TopicOperation.UNSUBSCRIBE)) + .thenCompose(__ -> { + Topic topic = getTopicReference(topicName); + Subscription sub = topic.getSubscription(subName); + if (sub == null) { + throw new RestException(Status.NOT_FOUND, "Subscription not found"); } - asyncResponse.resume(e); - } else { - log.error("[{}] Failed to delete subscription {} {}", clientAppId(), topicName, subName, e); - asyncResponse.resume(new RestException(e)); - } - } + return sub.delete(); + }).thenRun(() -> { + log.info("[{}][{}] Deleted subscription {}", clientAppId(), topicName, subName); + asyncResponse.resume(Response.noContent().build()); + }).exceptionally(e -> { + Throwable cause = e.getCause(); + if (cause instanceof SubscriptionBusyException) { + log.error("[{}] Failed to delete subscription {} from topic {}", clientAppId(), subName, + topicName, cause); + asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, + "Subscription has active connected consumers")); + } else if (cause instanceof WebApplicationException) { + if (log.isDebugEnabled() && ((WebApplicationException) cause).getResponse().getStatus() + == Status.TEMPORARY_REDIRECT.getStatusCode()) { + log.debug("[{}] Failed to delete subscription from topic {}, redirecting to other brokers.", + clientAppId(), topicName, cause); + } + asyncResponse.resume(cause); + } else { + log.error("[{}] Failed to delete subscription {} {}", clientAppId(), topicName, subName, cause); + asyncResponse.resume(new RestException(cause)); + } + return null; + }); } protected void internalDeleteSubscriptionForcefully(AsyncResponse asyncResponse, From 2a0be4b794e6d8a5738bba7073fe09f06b7961ef Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 13 Jan 2022 00:21:29 +0800 Subject: [PATCH 262/823] Avoid call sync method in async rest API for force delete subscription (#13668) --- .../admin/impl/PersistentTopicsBase.java | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index fa57199ed600a..9e8728c13f597 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -1585,33 +1585,34 @@ protected void internalDeleteSubscriptionForcefully(AsyncResponse asyncResponse, private void internalDeleteSubscriptionForNonPartitionedTopicForcefully(AsyncResponse asyncResponse, String subName, boolean authoritative) { - try { - validateTopicOwnership(topicName, authoritative); - validateTopicOperation(topicName, TopicOperation.UNSUBSCRIBE); - - Topic topic = getTopicReference(topicName); - Subscription sub = topic.getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, "Subscription not found")); - return; - } - sub.deleteForcefully().get(); - log.info("[{}][{}] Deleted subscription forcefully {}", clientAppId(), topicName, subName); - asyncResponse.resume(Response.noContent().build()); - } catch (Exception e) { - if (e instanceof WebApplicationException) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to delete subscription forcefully from topic {}," - + " redirecting to other brokers.", - clientAppId(), topicName, e); - } - asyncResponse.resume(e); - } else { - log.error("[{}] Failed to delete subscription forcefully {} {}", - clientAppId(), topicName, subName, e); - asyncResponse.resume(new RestException(e)); - } - } + validateTopicOwnershipAsync(topicName, authoritative) + .thenRun(() -> validateTopicOperation(topicName, TopicOperation.UNSUBSCRIBE)) + .thenCompose(__ -> { + Topic topic = getTopicReference(topicName); + Subscription sub = topic.getSubscription(subName); + if (sub == null) { + throw new RestException(Status.NOT_FOUND, "Subscription not found"); + } + return sub.deleteForcefully(); + }).thenRun(() -> { + log.info("[{}][{}] Deleted subscription forcefully {}", clientAppId(), topicName, subName); + asyncResponse.resume(Response.noContent().build()); + }).exceptionally(e -> { + Throwable cause = e.getCause(); + if (cause instanceof WebApplicationException) { + if (log.isDebugEnabled() && ((WebApplicationException) cause).getResponse().getStatus() + == Status.TEMPORARY_REDIRECT.getStatusCode()) { + log.debug("[{}] Failed to delete subscription from topic {}, redirecting to other brokers.", + clientAppId(), topicName, cause); + } + asyncResponse.resume(cause); + } else { + log.error("[{}] Failed to delete subscription forcefully {} {}", + clientAppId(), topicName, subName, cause); + asyncResponse.resume(new RestException(cause)); + } + return null; + }); } protected void internalSkipAllMessages(AsyncResponse asyncResponse, String subName, boolean authoritative) { From d22798a3d303a27272ffd91ea3d251e76089b018 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 2 Dec 2021 10:32:55 +0800 Subject: [PATCH 263/823] [es-sink] Use topic name as the index name if indexName is not configured (#13064) ### Motivation According to the [document](https://github.com/apache/pulsar/blob/master/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java#L53), the default index name should be the topic name, but the code still uses `indexName` only when generating write requests which makes it a required config in practice. ### Modifications - Use the method `indexName()` for the index name of write requests which will be the topic name if `indexName` is not configured. - Extract methods `makeIndexRequest` and `makeDeleteRequest` for better code reuse and testability. - Add unit tests to check the index name of generated index/delete requests. (cherry picked from commit 14ea189cb96a4acd2c39ac9c049dfd2fbcf93de7) --- .../io/elasticsearch/ElasticSearchClient.java | 40 ++++++++--------- .../ElasticSearchClientTests.java | 43 +++++++++++++++++++ 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java index af66795ef6ab5..b9e756e24d30c 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java @@ -254,16 +254,27 @@ void hasIrrecoverableError(BulkItemResponse bulkItemResponse) throws Exception { } } + IndexRequest makeIndexRequest(Record record, Pair idAndDoc) throws IOException { + IndexRequest indexRequest = Requests.indexRequest(indexName(record.getTopicName())); + if (!Strings.isNullOrEmpty(idAndDoc.getLeft())) + indexRequest.id(idAndDoc.getLeft()); + indexRequest.type(config.getTypeName()); + indexRequest.source(idAndDoc.getRight(), XContentType.JSON); + return indexRequest; + } + + DeleteRequest makeDeleteRequest(Record record, String id) throws IOException { + DeleteRequest deleteRequest = Requests.deleteRequest(indexName(record.getTopicName())); + deleteRequest.id(id); + deleteRequest.type(config.getTypeName()); + return deleteRequest; + } + public void bulkIndex(Record record, Pair idAndDoc) throws Exception { try { checkNotFailed(); checkIndexExists(record.getTopicName()); - IndexRequest indexRequest = Requests.indexRequest(config.getIndexName()); - if (!Strings.isNullOrEmpty(idAndDoc.getLeft())) - indexRequest.id(idAndDoc.getLeft()); - indexRequest.type(config.getTypeName()); - indexRequest.source(idAndDoc.getRight(), XContentType.JSON); - + IndexRequest indexRequest = makeIndexRequest(record, idAndDoc); records.put(indexRequest, record); bulkProcessor.add(indexRequest); } catch(Exception e) { @@ -284,12 +295,7 @@ public boolean indexDocument(Record record, Pair try { checkNotFailed(); checkIndexExists(record.getTopicName()); - IndexRequest indexRequest = Requests.indexRequest(config.getIndexName()); - if (!Strings.isNullOrEmpty(idAndDoc.getLeft())) - indexRequest.id(idAndDoc.getLeft()); - indexRequest.type(config.getTypeName()); - indexRequest.source(idAndDoc.getRight(), XContentType.JSON); - IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); + IndexResponse indexResponse = client.index(makeIndexRequest(record, idAndDoc), RequestOptions.DEFAULT); if (indexResponse.getResult().equals(DocWriteResponse.Result.CREATED) || indexResponse.getResult().equals(DocWriteResponse.Result.UPDATED)) { record.ack(); @@ -309,10 +315,7 @@ public void bulkDelete(Record record, String id) throws Exception try { checkNotFailed(); checkIndexExists(record.getTopicName()); - DeleteRequest deleteRequest = Requests.deleteRequest(config.getIndexName()); - deleteRequest.id(id); - deleteRequest.type(config.getTypeName()); - + DeleteRequest deleteRequest = makeDeleteRequest(record, id); records.put(deleteRequest, record); bulkProcessor.add(deleteRequest); } catch(Exception e) { @@ -333,10 +336,7 @@ public boolean deleteDocument(Record record, String id) throws Ex try { checkNotFailed(); checkIndexExists(record.getTopicName()); - DeleteRequest deleteRequest = Requests.deleteRequest(config.getIndexName()); - deleteRequest.id(id); - deleteRequest.type(config.getTypeName()); - DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); + DeleteResponse deleteResponse = client.delete(makeDeleteRequest(record, id), RequestOptions.DEFAULT); log.debug("delete result=" + deleteResponse.getResult()); if (deleteResponse.getResult().equals(DocWriteResponse.Result.DELETED) || deleteResponse.getResult().equals(DocWriteResponse.Result.NOT_FOUND)) { diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java index fb927c5723e22..a831d5cee9cec 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java @@ -24,7 +24,10 @@ import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.elasticsearch.testcontainers.ChaosContainer; import org.awaitility.Awaitility; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; import org.junit.AfterClass; +import org.mockito.Mockito; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -34,6 +37,8 @@ import java.util.UUID; import static org.junit.Assert.*; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.hamcrest.MatcherAssert.assertThat; @@ -78,6 +83,44 @@ public void fail() { } } + @Test + public void testIndexRequest() throws Exception { + String index = "myindex-" + UUID.randomUUID(); + Record record = Mockito.mock(Record.class); + String topicName = "topic-" + UUID.randomUUID(); + when(record.getTopicName()).thenReturn(Optional.of(topicName)); + try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() + .setElasticSearchUrl("http://" + container.getHttpHostAddress()) + .setIndexName(index))) { + IndexRequest request = client.makeIndexRequest(record, Pair.of("1", "{ \"a\":1}")); + assertEquals(request.index(), index); + } + try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() + .setElasticSearchUrl("http://" + container.getHttpHostAddress()))) { + IndexRequest request = client.makeIndexRequest(record, Pair.of("1", "{ \"a\":1}")); + assertEquals(request.index(), topicName); + } + } + + @Test + public void testDeleteRequest() throws Exception { + String index = "myindex-" + UUID.randomUUID(); + Record record = Mockito.mock(Record.class); + String topicName = "topic-" + UUID.randomUUID(); + when(record.getTopicName()).thenReturn(Optional.of(topicName)); + try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() + .setElasticSearchUrl("http://" + container.getHttpHostAddress()) + .setIndexName(index))) { + DeleteRequest request = client.makeDeleteRequest(record, "1"); + assertEquals(request.index(), index); + } + try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() + .setElasticSearchUrl("http://" + container.getHttpHostAddress()))) { + DeleteRequest request = client.makeDeleteRequest(record, "1"); + assertEquals(request.index(), topicName); + } + } + @Test public void testIndexDelete() throws Exception { String index = "myindex-" + UUID.randomUUID(); From ebec861178e46f268403d2890313bad82deec7a1 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 11 Jan 2022 13:54:23 +0800 Subject: [PATCH 264/823] Change ``ContextClassLoader`` to ``NarClassLoader`` in BrokerInterceptor (#13589) It's ``BrokerInterceptor`` side change, like #13501 Change context class loader through Thread.currentThread().setContextClassLoader(classLoader) before every interceptor handler callback, and change it back to original class loader afterwards. (cherry picked from commit afc241f02af4e8bf949d8d34b338b31c9fce5b9f) --- .../BrokerInterceptorWithClassLoader.java | 31 +++- .../BrokerInterceptorWithClassLoaderTest.java | 147 +++++++++++++++++- 2 files changed, 166 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java index e1e23e414d8f9..6b06920a9bf2a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java @@ -26,6 +26,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; +import org.apache.pulsar.broker.ClassLoaderSwitcher; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Subscription; @@ -50,39 +51,53 @@ public void beforeSendMessage(Subscription subscription, Entry entry, long[] ackSet, MessageMetadata msgMetadata) { - this.interceptor.beforeSendMessage( - subscription, entry, ackSet, msgMetadata); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + this.interceptor.beforeSendMessage( + subscription, entry, ackSet, msgMetadata); + } } @Override public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { - this.interceptor.onPulsarCommand(command, cnx); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + this.interceptor.onPulsarCommand(command, cnx); + } } @Override public void onConnectionClosed(ServerCnx cnx) { - this.interceptor.onConnectionClosed(cnx); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + this.interceptor.onConnectionClosed(cnx); + } } @Override public void onWebserviceRequest(ServletRequest request) throws IOException, ServletException, InterceptException { - this.interceptor.onWebserviceRequest(request); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + this.interceptor.onWebserviceRequest(request); + } } @Override public void onWebserviceResponse(ServletRequest request, ServletResponse response) throws IOException, ServletException { - this.interceptor.onWebserviceResponse(request, response); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + this.interceptor.onWebserviceResponse(request, response); + } } @Override public void initialize(PulsarService pulsarService) throws Exception { - this.interceptor.initialize(pulsarService); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + this.interceptor.initialize(pulsarService); + } } @Override public void close() { - interceptor.close(); + try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + interceptor.close(); + } try { classLoader.close(); } catch (IOException e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java index ae669373ded70..668323b40099a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java @@ -18,14 +18,30 @@ */ package org.apache.pulsar.broker.intercept; -import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.testng.annotations.Test; - import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import com.google.common.collect.Maps; +import io.netty.buffer.ByteBuf; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Producer; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.common.api.proto.BaseCommand; +import org.apache.pulsar.common.api.proto.CommandAck; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.intercept.InterceptException; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.testng.annotations.Test; +import javax.servlet.FilterChain; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import java.util.Map; /** * Unit test {@link BrokerInterceptorWithClassLoader}. @@ -44,4 +60,127 @@ public void testWrapper() throws Exception { verify(h, times(1)).initialize(same(pulsarService)); } + + @Test + public void testClassLoaderSwitcher() throws Exception { + NarClassLoader narLoader = mock(NarClassLoader.class); + BrokerInterceptor interceptor = new BrokerInterceptor() { + @Override + public void beforeSendMessage(Subscription subscription, Entry entry, long[] ackSet, MessageMetadata msgMetadata) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void onConnectionCreated(ServerCnx cnx) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void messageProduced(ServerCnx cnx, Producer producer, long startTimeNs, + long ledgerId, long entryId, Topic.PublishContext publishContext) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void messageDispatched(ServerCnx cnx, Consumer consumer, long ledgerId, + long entryId, ByteBuf headersAndPayload) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void onConnectionClosed(ServerCnx cnx) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void onWebserviceRequest(ServletRequest request) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void onWebserviceResponse(ServletRequest request, ServletResponse response) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain) { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void initialize(PulsarService pulsarService) throws Exception { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + @Override + public void close() { + assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); + } + }; + + BrokerInterceptorWithClassLoader brokerInterceptorWithClassLoader = + new BrokerInterceptorWithClassLoader(interceptor, narLoader); + ClassLoader curClassLoader = Thread.currentThread().getContextClassLoader(); + // test class loader + assertEquals(brokerInterceptorWithClassLoader.getClassLoader(), narLoader); + // test initialize + brokerInterceptorWithClassLoader.initialize(mock(PulsarService.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test onFilter + brokerInterceptorWithClassLoader.onFilter(mock(ServletRequest.class) + , mock(ServletResponse.class), mock(FilterChain.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test onWebserviceResponse + brokerInterceptorWithClassLoader.onWebserviceResponse(mock(ServletRequest.class) + , mock(ServletResponse.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test onWebserviceRequest + brokerInterceptorWithClassLoader.onWebserviceRequest(mock(ServletRequest.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test onConnectionClosed + brokerInterceptorWithClassLoader.onConnectionClosed(mock(ServerCnx.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test onPulsarCommand + brokerInterceptorWithClassLoader.onPulsarCommand(null, mock(ServerCnx.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test messageAcked + brokerInterceptorWithClassLoader + .messageAcked(mock(ServerCnx.class), mock(Consumer.class), null); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test messageDispatched + brokerInterceptorWithClassLoader + .messageDispatched(mock(ServerCnx.class), mock(Consumer.class), 1, 1, null); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test messageProduced + brokerInterceptorWithClassLoader + .messageProduced(mock(ServerCnx.class), mock(Producer.class), 1, 1, 1, null); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test consumerCreated + brokerInterceptorWithClassLoader + .consumerCreated(mock(ServerCnx.class), mock(Consumer.class), Maps.newHashMap()); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test producerCreated + brokerInterceptorWithClassLoader + .producerCreated(mock(ServerCnx.class), mock(Producer.class), Maps.newHashMap()); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test onConnectionCreated + brokerInterceptorWithClassLoader + .onConnectionCreated(mock(ServerCnx.class)); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test beforeSendMessage + brokerInterceptorWithClassLoader + .beforeSendMessage(mock(Subscription.class), mock(Entry.class), null, null); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + // test close + brokerInterceptorWithClassLoader.close(); + assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); + + } } From 69e0499cf07dac1d993d97f43e588ddc324339f7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 10 Jan 2022 22:09:13 +0200 Subject: [PATCH 265/823] [Security] Upgrade Jackson to 2.12.6 (#13694) * [Security] Upgrade Jackson to 2.12.6 * update LICENSE files (cherry picked from commit f8a9159efd14298e7afff82068aeb50eef95e25e) --- .../server/src/assemble/LICENSE.bin.txt | 16 +++++------ pom.xml | 4 +-- pulsar-sql/presto-distribution/LICENSE | 28 +++++++++---------- pulsar-sql/presto-distribution/pom.xml | 4 +-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 1620c9f61754f..b2c1da45aeadd 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -312,14 +312,14 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.78.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.7.3.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.12.3.jar - - com.fasterxml.jackson.core-jackson-core-2.12.3.jar - - com.fasterxml.jackson.core-jackson-databind-2.12.3.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.12.3.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.12.3.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.12.3.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.12.3.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.12.3.jar + - com.fasterxml.jackson.core-jackson-annotations-2.12.6.jar + - com.fasterxml.jackson.core-jackson-core-2.12.6.jar + - com.fasterxml.jackson.core-jackson-databind-2.12.6.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.12.6.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.12.6.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.12.6.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.12.6.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.12.6.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.17.0.jar diff --git a/pom.xml b/pom.xml index 88d2e5813e532..ed0138d064399 100644 --- a/pom.xml +++ b/pom.xml @@ -122,8 +122,8 @@ flexible messaging model and an intuitive client API. 2.17.1 1.69 1.0.2 - 2.12.3 - 2.12.3 + 2.12.6 + 2.12.6 0.9.11 1.6.2 8.37 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index a988218649c3a..69c387bb5c53b 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.12.3.jar - - jackson-core-2.12.3.jar - - jackson-databind-2.12.3.jar - - jackson-dataformat-smile-2.12.3.jar - - jackson-datatype-guava-2.12.3.jar - - jackson-datatype-jdk8-2.12.3.jar - - jackson-datatype-joda-2.12.3.jar - - jackson-datatype-jsr310-2.12.3.jar - - jackson-dataformat-yaml-2.12.3.jar - - jackson-jaxrs-base-2.12.3.jar - - jackson-jaxrs-json-provider-2.12.3.jar - - jackson-module-jaxb-annotations-2.12.3.jar - - jackson-module-jsonSchema-2.12.3.jar + - jackson-annotations-2.12.6.jar + - jackson-core-2.12.6.jar + - jackson-databind-2.12.6.jar + - jackson-dataformat-smile-2.12.6.jar + - jackson-datatype-guava-2.12.6.jar + - jackson-datatype-jdk8-2.12.6.jar + - jackson-datatype-joda-2.12.6.jar + - jackson-datatype-jsr310-2.12.6.jar + - jackson-dataformat-yaml-2.12.6.jar + - jackson-jaxrs-base-2.12.6.jar + - jackson-jaxrs-json-provider-2.12.6.jar + - jackson-module-jaxb-annotations-2.12.6.jar + - jackson-module-jsonSchema-2.12.6.jar * Guava - guava-30.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -440,7 +440,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.7.jar * Jackson - - jackson-module-parameter-names-2.12.3.jar + - jackson-module-parameter-names-2.12.6.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 39096520c6c4d..24cefa1c54a5b 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -39,10 +39,10 @@ 2.6 0.0.12 4.2.0 - 2.12.3 + 2.12.6 - 2.12.3 + 2.12.6 3.0.5 30.1-jre 2.12.1 From b4822565632419e0c43c237f865578afca73f793 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 11 Jan 2022 07:56:15 +0200 Subject: [PATCH 266/823] [Security] Upgrade protobuf to 3.16.1 to address CVE-2021-22569 (#13695) ### Motivation - protobuf < 3.16.1 contains DoS vulnerability CVE-2021-22569, https://nvd.nist.gov/vuln/detail/CVE-2021-22569. ### Modifications - upgrade protobuf from 3.11.4 to 3.16.1 (cherry picked from commit 1a3688c936bb2320db34ccfbef08500f2c522591) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b2c1da45aeadd..07b43e4ab2a52 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -548,8 +548,8 @@ MIT License Protocol Buffers License * Protocol Buffers - - com.google.protobuf-protobuf-java-3.11.4.jar -- licenses/LICENSE-protobuf.txt - - com.google.protobuf-protobuf-java-util-3.11.4.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-3.16.1.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-util-3.16.1.jar -- licenses/LICENSE-protobuf.txt CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt * Java Annotations API diff --git a/pom.xml b/pom.xml index ed0138d064399..0a7356127d285 100644 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ flexible messaging model and an intuitive client API. 8.37 1.4.13 0.5.0 - 3.11.4 + 3.16.1 ${protobuf3.version} 1.33.0 0.19.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 69c387bb5c53b..ed9d018647fff 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -460,7 +460,7 @@ The Apache Software License, Version 2.0 Protocol Buffers License * Protocol Buffers - - protobuf-java-3.11.4.jar + - protobuf-java-3.16.1.jar BSD 3-clause "New" or "Revised" License * RE2J TD -- re2j-td-1.4.jar From d3aa54f3d6774801e3bc783535e6111376f7d214 Mon Sep 17 00:00:00 2001 From: mingyifei <965162704@qq.com> Date: Thu, 13 Jan 2022 01:06:51 +0800 Subject: [PATCH 267/823] [pulsar-broker] The log prints NamespaceService#isServiceUnitActive exception stack information (#13553) (cherry picked from commit 58c281811b01dfc2bcf1906865964badbee4a952) --- .../apache/pulsar/broker/namespace/NamespaceService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 8f6bfb8d651b1..c0eee85ed6aee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -980,9 +980,13 @@ public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) public boolean isServiceUnitActive(TopicName topicName) { try { - return ownershipCache.getOwnedBundle(getBundle(topicName)).isActive(); + OwnedBundle ownedBundle = ownershipCache.getOwnedBundle(getBundle(topicName)); + if (ownedBundle == null) { + return false; + } + return ownedBundle.isActive(); } catch (Exception e) { - LOG.warn("Unable to find OwnedBundle for topic - [{}]", topicName); + LOG.warn("Unable to find OwnedBundle for topic - [{}]", topicName, e); return false; } } From 3492116968a666383d7c3e4f94c758aae2ec3f22 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Wed, 12 Jan 2022 11:10:52 +0800 Subject: [PATCH 268/823] Fix bug :Infinity value for CPU or Bandwidth usage (#13609) * fix bug :Infinity value for CPU or Bandwidth usage * use executorService.schedule replace to executorService.schedule * 1.use scheduleAtFixedDelay instead of scheduleAtFixedRate; 2.returns when elapsedSeconds <= 0 is true, skip this round of calculateBrokerHostUsage; * 1.move the early return conditional block to before getTotalCpuUsage; * 1.use scheduleWithFixedDelay instead of scheduleAtFixedRate in the class GenericBrokerHostUsageImpl; * update log content: add a space in log (cherry picked from commit 2a7515f9593a76b294bfe2835621a0ab8a904957) --- .../loadbalance/impl/GenericBrokerHostUsageImpl.java | 4 ++-- .../loadbalance/impl/LinuxBrokerHostUsageImpl.java | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java index 72d36abdfb004..1405e8e3953ac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/GenericBrokerHostUsageImpl.java @@ -54,9 +54,9 @@ public GenericBrokerHostUsageImpl(int hostUsageCheckIntervalMin, this.totalCpuLimit = getTotalCpuLimit(); // Call now to initialize values before the constructor returns calculateBrokerHostUsage(); - executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::checkCpuLoad), CPU_CHECK_MILLIS, + executorService.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::checkCpuLoad), CPU_CHECK_MILLIS, CPU_CHECK_MILLIS, TimeUnit.MILLISECONDS); - executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::doCalculateBrokerHostUsage), + executorService.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::doCalculateBrokerHostUsage), hostUsageCheckIntervalMin, hostUsageCheckIntervalMin, TimeUnit.MINUTES); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index ccbe8e3e413ec..5bc7bf952b129 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -87,7 +87,7 @@ public LinuxBrokerHostUsageImpl(int hostUsageCheckIntervalMin, // Call now to initialize values before the constructor returns calculateBrokerHostUsage(); - executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::calculateBrokerHostUsage), + executorService.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::calculateBrokerHostUsage), hostUsageCheckIntervalMin, hostUsageCheckIntervalMin, TimeUnit.MINUTES); } @@ -105,9 +105,14 @@ public void calculateBrokerHostUsage() { double totalNicUsageRx = getTotalNicUsageRxKb(nics); double totalCpuLimit = getTotalCpuLimit(); - SystemResourceUsage usage = new SystemResourceUsage(); long now = System.currentTimeMillis(); double elapsedSeconds = (now - lastCollection) / 1000d; + if (elapsedSeconds <= 0) { + log.warn("elapsedSeconds {} is not expected, skip this round of calculateBrokerHostUsage", elapsedSeconds); + return; + } + + SystemResourceUsage usage = new SystemResourceUsage(); double cpuUsage = getTotalCpuUsage(elapsedSeconds); if (lastCollection == 0L) { From 9a25e599eef808cc71b48b999987df55abb0070f Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 13 Jan 2022 00:52:38 +0800 Subject: [PATCH 269/823] [ Issue 13479 ] Fixed internal topic effect by InactiveTopicPolicy. (#13611) (cherry picked from commit 5835191295371b3a5f49ed1019a8ad197554424e) --- .../pulsar/broker/admin/impl/BrokersBase.java | 3 +- .../service/persistent/SystemTopic.java | 5 ++ .../broker/systopic/SystemTopicClient.java | 6 ++- .../service/InactiveTopicDeleteTest.java | 49 +++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 756d141fab444..bb9453e09bfdd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -70,6 +70,7 @@ public class BrokersBase extends PulsarWebResource { private static final Logger LOG = LoggerFactory.getLogger(BrokersBase.class); private static final Duration HEALTHCHECK_READ_TIMEOUT = Duration.ofSeconds(10); + public static final String HEALTH_CHECK_TOPIC_SUFFIX = "healthcheck"; @GET @Path("/{cluster}") @@ -317,7 +318,7 @@ public void healthcheck(@Suspended AsyncResponse asyncResponse, pulsar().getConfiguration()); - topic = String.format("persistent://%s/healthcheck", heartbeatNamespace); + topic = String.format("persistent://%s/%s", heartbeatNamespace, HEALTH_CHECK_TOPIC_SUFFIX); LOG.info("Running healthCheck with topic={}", topic); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java index 6e3173f0c1708..f1d7b0665f357 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java @@ -30,6 +30,11 @@ public SystemTopic(String topic, ManagedLedger ledger, BrokerService brokerServi super(topic, ledger, brokerService); } + @Override + public boolean isDeleteWhileInactive() { + return false; + } + @Override public boolean isSizeBacklogExceeded() { return false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java index e838a694233a2..122bd9f52b392 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.admin.impl.BrokersBase; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -201,7 +202,10 @@ static boolean isSystemTopic(TopicName topicName) { if (StringUtils.endsWith(localName, MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { return true; } - + // health check topic + if (StringUtils.endsWith(localName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX)){ + return true; + } return false; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java index 94ac6e8b4afc8..3b7d9aaec3424 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.admin.impl.BrokersBase; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; @@ -576,4 +578,51 @@ public void testInactiveTopicApplied() throws Exception { Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getInactiveTopicPolicies(topic, true), brokerLevelPolicy)); } + + @Test(timeOut = 30000) + public void testInternalTopicInactiveNotClean() throws Exception { + conf.setSystemTopicEnabled(true); + conf.setBrokerDeleteInactiveTopicsMode(InactiveTopicDeleteMode.delete_when_no_subscriptions); + conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1); + super.baseSetup(); + // init topic + final String healthCheckTopic = "persistent://prop/ns-abc/"+ BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX; + final String topic = "persistent://prop/ns-abc/testDeleteWhenNoSubscriptions"; + + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub") + .subscribe(); + + Producer heathCheckProducer = pulsarClient.newProducer() + .topic(healthCheckTopic) + .create(); + Consumer heathCheckConsumer = pulsarClient.newConsumer() + .topic(healthCheckTopic) + .subscriptionName("healthCheck") + .subscribe(); + + consumer.close(); + producer.close(); + heathCheckConsumer.close(); + heathCheckProducer.close(); + + Awaitility.await().untilAsserted(() -> Assert.assertTrue(admin.topics().getList("prop/ns-abc") + .contains(topic))); + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue(admin.topics().getList("prop/ns-abc").contains(healthCheckTopic)); + }); + + admin.topics().deleteSubscription(topic, "sub"); + admin.topics().deleteSubscription(healthCheckTopic, "healthCheck"); + + Awaitility.await().untilAsserted(() -> Assert.assertFalse(admin.topics().getList("prop/ns-abc") + .contains(topic))); + Awaitility.await().pollDelay(2, TimeUnit.SECONDS) + .untilAsserted(() -> Assert.assertTrue(admin.topics().getList("prop/ns-abc") + .contains(healthCheckTopic))); + } } From 42f6b28d507f31c0785bbe5099a50836d13f5079 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 13 Jan 2022 00:46:45 +0800 Subject: [PATCH 270/823] Fix the wrong multi-topic has message available behavior (#13634) Fixes #13605 ### Motivation Currently, the multiTopicReader `hasMessageAvailable` might get the wrong result, we must check `numMessagesInQueue() > 0` again after finish all consumer `hasMessageAvaliableAsync` future, bacause some message might already in `MultiTopicsConsumerImpl#incomingMessages`. ### Modifications * Fix the wrong multi-topic has message available behavior. * Use `reader.readNextAsync()` instead of block method `reader.readNext()`. * Reduce the units test running time by changing `MultiTopicsReaderTest` to use `@BeforeClass`, `@AfterClass`. (cherry picked from commit e57dc8faa2979b6d03410fa2cef5f1dd84e46785) --- .../client/impl/MultiTopicsReaderTest.java | 26 +++++++++---------- .../client/impl/MultiTopicsConsumerImpl.java | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java index f6230e2ae5485..6b6bf9594836a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java @@ -61,17 +61,17 @@ import org.apache.pulsar.common.util.Murmur3_32Hash; import org.awaitility.Awaitility; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -@Test(groups = "flaky") @Slf4j +@Test(groups = "flaky") public class MultiTopicsReaderTest extends MockedPulsarServiceBaseTest { private static final String subscription = "reader-multi-topics-sub"; - @BeforeMethod(alwaysRun = true) + @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { super.internalSetup(); @@ -87,7 +87,7 @@ protected void setup() throws Exception { admin.namespaces().createNamespace("my-property/my-ns", policies); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); @@ -169,15 +169,15 @@ public void testHasMessageAvailableAsync() throws Exception { private static void readMessageUseAsync(Reader reader, List> msgs, CountDownLatch latch) { reader.hasMessageAvailableAsync().thenAccept(hasMessageAvailable -> { if (hasMessageAvailable) { - try { - Message msg = reader.readNext(); + reader.readNextAsync().whenComplete((msg, ex) -> { + if (ex != null) { + log.error("Read message failed.", ex); + latch.countDown(); + return; + } msgs.add(msg); - } catch (PulsarClientException e) { - log.error("Read message failed.", e); - latch.countDown(); - return; - } - readMessageUseAsync(reader, msgs, latch); + readMessageUseAsync(reader, msgs, latch); + }); } else { latch.countDown(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index d2646a8cd0643..74898a2a62a3e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -771,7 +771,7 @@ public CompletableFuture hasMessageAvailableAsync() { if (exception != null) { completableFuture.completeExceptionally(exception); } else { - completableFuture.complete(hasMessageAvailable.get()); + completableFuture.complete(hasMessageAvailable.get() || numMessagesInQueue() > 0); } }); return completableFuture; From 00694f2ebf18cd7f025070e3a20e8fe7594c3d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8C=85=E5=AD=90?= Date: Thu, 13 Jan 2022 00:27:33 +0800 Subject: [PATCH 271/823] Fix getInternalStats occasional lack of LeaderInfo (#13656) ### Motivation #11704 AdminApiTest is flaky. The testGetPartitionedStatsInternal test method fails sporadically. The root cause it that `stateFuture` is returned, but `availableBookiesFuture` is not accomplished, so we need ensure that `futures` is all completed before `stateFuture.complete`. https://github.com/apache/pulsar/blob/5faae1c4e683f4796ad34c1c612de0bcfe754d2a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java#L2176-L2178 (cherry picked from commit dcf01e910a4f8d30cd86ab95ab800952af92692a) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index e2cc79c5f8e51..fb87b74c3361d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1990,9 +1990,10 @@ public CompletableFuture getInternalStats(boolean stats.state = ml.getState().toString(); stats.ledgers = Lists.newArrayList(); - List> futures = includeLedgerMetadata ? Lists.newArrayList() : null; + List> futures = Lists.newArrayList(); CompletableFuture> availableBookiesFuture = brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync(); + futures.add(availableBookiesFuture.handle((strings, throwable) -> null)); availableBookiesFuture.whenComplete((bookies, e) -> { if (e != null) { log.error("[{}] Failed to fetch available bookies.", topic, e); @@ -2005,7 +2006,7 @@ public CompletableFuture getInternalStats(boolean info.size = li.getSize(); info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); stats.ledgers.add(info); - if (futures != null) { + if (includeLedgerMetadata) { futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> { if (ex == null) { info.metadata = lMetadata; From 2e78d93f1e1e0dd1c61aa59224178783d0321ec7 Mon Sep 17 00:00:00 2001 From: wenbingshen Date: Fri, 14 Jan 2022 10:45:55 +0800 Subject: [PATCH 272/823] fix no response to client when handleSubscribe because PendingAckHandleImpl init fail (#13655) Fixes #13654 ### Modifications When the initialization of `PendingAckHandleImpl` fails, `pendingAckHandleCompletableFuture` will not be exception or complete, then `org.apache.pulsar.broker.service.persistent.PersistentSubscription#addConsumer` will not return any response to the client. ``` public CompletableFuture addConsumer(Consumer consumer) { return pendingAckHandle.pendingAckHandleFuture().thenCompose(future -> ...) } ``` (cherry picked from commit 528c972eff6d9c41adb40a6dad42e9432487d187) --- .../transaction/pendingack/PendingAckReplyCallBack.java | 6 ++++++ .../pendingack/impl/MLPendingAckReplyCallBack.java | 7 +++++++ .../transaction/pendingack/impl/MLPendingAckStore.java | 3 +++ .../transaction/pendingack/impl/PendingAckHandleImpl.java | 7 +++++++ 4 files changed, 23 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java index 3f2cc51f8ecb6..fed9add15657a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java @@ -36,4 +36,10 @@ public interface PendingAckReplyCallBack { * @param pendingAckMetadataEntry {@link PendingAckMetadataEntry} the metadata entry of pending ack */ void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry); + + /** + * Pending ack replay failed callback for pending ack store. + */ + void replayFailed(Throwable t); + } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java index 6bcc344ddf392..b98e64ef9f5c9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -60,6 +60,13 @@ public void replayComplete() { pendingAckHandle.handleCacheRequest(); } + @Override + public void replayFailed(Throwable t) { + synchronized (pendingAckHandle) { + pendingAckHandle.exceptionHandleFuture(t); + } + } + @Override public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) { TxnID txnID = new TxnID(pendingAckMetadataEntry.getTxnidMostBits(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 1592318cf616a..1ed0992b2ff2d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -303,6 +303,8 @@ class PendingAckReplay implements Runnable { public void run() { try { if (cursor.isClosed()) { + pendingAckReplyCallBack.replayFailed(new ManagedLedgerException + .CursorAlreadyClosedException("MLPendingAckStore cursor have been closed.")); log.warn("[{}] MLPendingAckStore cursor have been closed, close replay thread.", cursor.getManagedLedger().getName()); return; @@ -350,6 +352,7 @@ public void run() { } } } catch (Exception e) { + pendingAckReplyCallBack.replayFailed(e); log.error("[{}] Pending ack recover fail!", subManagedCursor.getManagedLedger().getName(), e); return; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index d92793a933e09..ba74d78c8ee9e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -141,6 +141,7 @@ private void initPendingAckStore() { }).exceptionally(e -> { acceptQueue.clear(); changeToErrorState(); + exceptionHandleFuture(e.getCause()); log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); return null; }); @@ -889,6 +890,12 @@ public synchronized void completeHandleFuture() { } } + public synchronized void exceptionHandleFuture(Throwable t) { + if (!this.pendingAckHandleCompletableFuture.isDone()) { + this.pendingAckHandleCompletableFuture.completeExceptionally(t); + } + } + @Override public TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID) { TransactionInPendingAckStats transactionInPendingAckStats = new TransactionInPendingAckStats(); From f3bf499b83a263f5bfc3da77def18c9c55b0d972 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Fri, 14 Jan 2022 11:44:56 +0800 Subject: [PATCH 273/823] Fix Flaky test BacklogQuotaManagerTest.testConsumerBacklogEvictionTimeQuotaWithEmptyLedger (#13724) (cherry picked from commit 1fd44908e76881cd8035f0772a9cdccc7bafb62d) --- .../apache/pulsar/broker/service/BacklogQuotaManagerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index f706b3f4b125b..f531119386808 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -504,6 +504,7 @@ public void testConsumerBacklogEvictionTimeQuotaWithEmptyLedger() throws Excepti consumer.receive(); admin.topics().unload(topic); + Awaitility.await().until(consumer::isConnected); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topic); assertEquals(internalStats.ledgers.size(), 2); assertEquals(internalStats.ledgers.get(1).entries, 0); From c07764f03d2f77197ff72fe8734756637a4079ad Mon Sep 17 00:00:00 2001 From: lipenghui Date: Fri, 14 Jan 2022 17:47:13 +0800 Subject: [PATCH 274/823] Fix the deadlock while using zookeeper thread to create ledger (#13744) ### Motivation Fixes: #13736, the deadlock when using ZK thread to create a ledger. ### Modification Use the executor of the managed ledger to create the ledger to avoid the deadlock. (cherry picked from commit 1d4c374d9a87e5188c8b64677b8c6bea652a2d2a) --- .../apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 457600ccf1630..528816047b709 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1593,7 +1593,10 @@ synchronized void createLedgerAfterClosed() { STATE_UPDATER.set(this, State.CreatingLedger); this.lastLedgerCreationInitiationTimestamp = System.currentTimeMillis(); mbean.startDataLedgerCreateOp(); - asyncCreateLedger(bookKeeper, config, digestType, this, Collections.emptyMap()); + // Use the executor here is to avoid use the Zookeeper thread to create the ledger which will lead + // to deadlock at the zookeeper client, details to see https://github.com/apache/pulsar/issues/13736 + this.executor.execute(() -> + asyncCreateLedger(bookKeeper, config, digestType, this, Collections.emptyMap())); } } From 022d43b265a63e5dc9cb87a84e7f63e100339037 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Sun, 16 Jan 2022 10:58:15 +0800 Subject: [PATCH 275/823] Fix invalid rack name cause bookie join rack failed (#13683) * Fix invalid rackname cause bookie join rack failed (cherry picked from commit af761e2f8d3aee35c2cf081998f556972a92b574) --- .../BrokerInterceptorWithClassLoaderTest.java | 50 ------------------- .../policies/data/impl/BookieInfoImpl.java | 10 ++++ .../pulsar/admin/cli/PulsarAdminToolTest.java | 18 +++++++ .../apache/pulsar/admin/cli/CmdBookies.java | 13 +++++ .../ZkBookieRackAffinityMapping.java | 5 +- .../ZkBookieRackAffinityMappingTest.java | 29 +++++++++++ 6 files changed, 74 insertions(+), 51 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java index 668323b40099a..5288ab9954343 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java @@ -70,32 +70,6 @@ public void beforeSendMessage(Subscription subscription, Entry entry, long[] ack assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); } @Override - public void onConnectionCreated(ServerCnx cnx) { - assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); - } - @Override - public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { - assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); - } - @Override - public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { - assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); - } - @Override - public void messageProduced(ServerCnx cnx, Producer producer, long startTimeNs, - long ledgerId, long entryId, Topic.PublishContext publishContext) { - assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); - } - @Override - public void messageDispatched(ServerCnx cnx, Consumer consumer, long ledgerId, - long entryId, ByteBuf headersAndPayload) { - assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); - } - @Override - public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { - assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); - } - @Override public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { assertEquals(Thread.currentThread().getContextClassLoader(), narLoader); } @@ -150,30 +124,6 @@ public void close() { // test onPulsarCommand brokerInterceptorWithClassLoader.onPulsarCommand(null, mock(ServerCnx.class)); assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); - // test messageAcked - brokerInterceptorWithClassLoader - .messageAcked(mock(ServerCnx.class), mock(Consumer.class), null); - assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); - // test messageDispatched - brokerInterceptorWithClassLoader - .messageDispatched(mock(ServerCnx.class), mock(Consumer.class), 1, 1, null); - assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); - // test messageProduced - brokerInterceptorWithClassLoader - .messageProduced(mock(ServerCnx.class), mock(Producer.class), 1, 1, 1, null); - assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); - // test consumerCreated - brokerInterceptorWithClassLoader - .consumerCreated(mock(ServerCnx.class), mock(Consumer.class), Maps.newHashMap()); - assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); - // test producerCreated - brokerInterceptorWithClassLoader - .producerCreated(mock(ServerCnx.class), mock(Producer.class), Maps.newHashMap()); - assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); - // test onConnectionCreated - brokerInterceptorWithClassLoader - .onConnectionCreated(mock(ServerCnx.class)); - assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); // test beforeSendMessage brokerInterceptorWithClassLoader .beforeSendMessage(mock(Subscription.class), mock(Entry.class), null, null); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BookieInfoImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BookieInfoImpl.java index de316e612f3c1..b58498903ab48 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BookieInfoImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BookieInfoImpl.java @@ -21,6 +21,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.NonNull; import org.apache.pulsar.common.policies.data.BookieInfo; /** @@ -40,6 +41,7 @@ public static BookieInfoImplBuilder builder() { public static class BookieInfoImplBuilder implements BookieInfo.Builder { private String rack; private String hostname; + private static final String PATH_SEPARATOR = "/"; public BookieInfoImplBuilder rack(String rack) { this.rack = rack; @@ -52,7 +54,15 @@ public BookieInfoImplBuilder hostname(String hostname) { } public BookieInfoImpl build() { + checkArgument(rack != null && !rack.isEmpty() && !rack.equals(PATH_SEPARATOR), + "rack name is invalid, it should not be null, empty or '/'"); return new BookieInfoImpl(rack, hostname); } + + public static void checkArgument(boolean expression, @NonNull Object errorMessage) { + if (!expression) { + throw new IllegalArgumentException(String.valueOf(errorMessage)); + } + } } } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 300eefc368c88..5b8b4da52cb6b 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -28,6 +28,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -1460,6 +1461,23 @@ public void bookies() throws Exception { .rack("rack-1") .hostname("host-1") .build()); + + // test invalid rack name "" + try { + BookieInfo.builder().rack("").hostname("host-1").build(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "rack name is invalid, it should not be null, empty or '/'"); + } + + // test invalid rack name "/" + try { + BookieInfo.builder().rack("/").hostname("host-1").build(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "rack name is invalid, it should not be null, empty or '/'"); + } + } @Test diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java index c15e8dbc60d6a..38fd8473f7aba 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java @@ -25,6 +25,8 @@ import com.beust.jcommander.Parameters; import java.util.function.Supplier; +import com.google.common.base.Strings; +import lombok.NonNull; @Parameters(commandDescription = "Operations about bookies rack placement") public class CmdBookies extends CmdBase { @@ -73,6 +75,8 @@ void run() throws Exception { @Parameters(commandDescription = "Updates the rack placement information for a specific bookie in the cluster (note. bookie address format:`address:port`)") private class UpdateBookie extends CliCommand { + private static final String PATH_SEPARATOR = "/"; + @Parameter(names = { "-g", "--group" }, description = "Bookie group name", required = false) private String group = "default"; @@ -87,12 +91,21 @@ private class UpdateBookie extends CliCommand { @Override void run() throws Exception { + checkArgument(!Strings.isNullOrEmpty(bookieRack) && !bookieRack.trim().equals(PATH_SEPARATOR), + "rack name is invalid, it should not be null, empty or '/'"); + getAdmin().bookies().updateBookieRackInfo(bookieAddress, group, BookieInfo.builder() .rack(bookieRack) .hostname(bookieHost) .build()); } + + private void checkArgument(boolean expression, @NonNull Object errorMessage) { + if (!expression) { + throw new IllegalArgumentException(String.valueOf(errorMessage)); + } + } } public CmdBookies(Supplier admin) { diff --git a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java index caeba510fd8f5..86d39884099ab 100644 --- a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java +++ b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.util.Strings; import java.net.InetAddress; import java.net.URI; import java.util.ArrayList; @@ -212,7 +213,9 @@ private String getRack(String bookieAddress) { } } - if (bi != null) { + if (bi != null + && !Strings.isNullOrEmpty(bi.getRack()) + && !bi.getRack().trim().equals("/")) { String rack = bi.getRack(); if (!rack.startsWith("/")) { rack = "/" + rack; diff --git a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMappingTest.java b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMappingTest.java index 2f814062e1d87..fadffa789dc68 100644 --- a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMappingTest.java +++ b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMappingTest.java @@ -27,6 +27,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; + import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.util.ZkUtils; @@ -34,6 +36,7 @@ import org.apache.pulsar.common.policies.data.BookieInfo; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; import org.awaitility.Awaitility; @@ -104,6 +107,32 @@ public void testBasic() throws Exception { assertEquals(racks2.get(2), null); localZkc.delete(ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, -1); + assertNull(racks1.get(2)); + } + + @Test + public void testInvalidRackName() throws InterruptedException, KeeperException { + String data = "{\"group1\": {\"" + BOOKIE1 + + "\": {\"rack\": \"/\", \"hostname\": \"bookie1.example.com\"}, \"" + BOOKIE2 + + "\": {\"rack\": \"\", \"hostname\": \"bookie2.example.com\"}}}"; + + ZkUtils.createFullPathOptimistic(localZkc, ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, data.getBytes(), + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + // Case1: ZKCache is given + ZkBookieRackAffinityMapping mapping1 = new ZkBookieRackAffinityMapping(); + ClientConfiguration bkClientConf1 = new ClientConfiguration(); + bkClientConf1.setProperty(ZooKeeperCache.ZK_CACHE_INSTANCE, new ZooKeeperCache("test", localZkc, 30) { + }); + + mapping1.setBookieAddressResolver(BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + mapping1.setConf(bkClientConf1); + List racks1 = mapping1 + .resolve(Lists.newArrayList(BOOKIE1.getHostName(), BOOKIE2.getHostName(), BOOKIE3.getHostName())); + + assertNull(racks1.get(0)); + assertNull(racks1.get(1)); + assertNull(racks1.get(2)); } @Test From 2c73edd27b93f1146615b178396f37826a8803c6 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 18 Jan 2022 08:46:45 +0800 Subject: [PATCH 276/823] [Transaction] Fix recover of TransactionBuffer (#13739) Fix the concurrency issue caused by TransactionBuffer when recovering. 1. use TransactionReplayExecutor when thenAcceptAsync in recover::run was called. 2. delete repeated changeToInitializingState 3. if cursor.hasMoreEntries() == false && entryQueue.size() == 0, return false and stop recovering. - If the cursor is cleared when transaction is recovering , there will no entries can be read, but currentLoadPosition < lastConfirmedEntry. (cherry picked from commit 7dee63ed707c8784989692b764b058bc8ea7f4ac) --- .../buffer/impl/TopicTransactionBuffer.java | 17 +++++++--- .../broker/transaction/TransactionTest.java | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 9978f6f8ec20a..9a324fbc82b32 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -96,7 +96,6 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; - this.changeToInitializingState(); this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotService().createWriter(TopicName.get(topic.getName())); this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); @@ -531,7 +530,11 @@ private TopicTransactionBufferRecover(TopicTransactionBufferRecoverCallBack call @SneakyThrows @Override public void run() { - this.topicTransactionBuffer.changeToInitializingState(); + if (!this.topicTransactionBuffer.changeToInitializingState()) { + log.warn("TransactionBuffer {} of topic {} can not change state to Initializing", + this, topic.getName()); + return; + } topic.getBrokerService().getPulsar().getTransactionBufferSnapshotService() .createReader(TopicName.get(topic.getName())).thenAcceptAsync(reader -> { try { @@ -579,8 +582,8 @@ public void run() { } PositionImpl lastConfirmedEntry = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; - FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, managedCursor, - TopicTransactionBufferRecover.this); + FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, + managedCursor, TopicTransactionBufferRecover.this); if (lastConfirmedEntry.getEntryId() != -1) { while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 && fillEntryQueueCallback.fillQueue()) { @@ -604,7 +607,7 @@ public void run() { closeCursor(managedCursor); callBack.recoverComplete(); - }).exceptionally(e -> { + }, topic.getBrokerService().getPulsar().getTransactionReplayExecutor()).exceptionally(e -> { callBack.recoverExceptionally(new Exception(e)); log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); return null; @@ -654,6 +657,10 @@ boolean fillQueue() { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); cursor.asyncReadEntries(100, this, System.nanoTime(), PositionImpl.latest); + } else { + if (entryQueue.size() == 0) { + isReadable = false; + } } } return isReadable; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 59d2839f76f53..6fe99a765f306 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -34,6 +35,7 @@ import io.netty.buffer.Unpooled; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.Map; import java.util.List; @@ -700,4 +702,35 @@ public void testEndTxnWhenCommittingOrAborting() throws Exception { commitTxn.commit(); } + @Test + public void testNoEntryCanBeReadWhenRecovery() throws Exception { + String topic = NAMESPACE1 + "/test"; + PersistentTopic persistentTopic = + (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(TopicName.get(topic).toString(), true) + .get() + .get(); + + Class persistentTopicClass = PersistentTopic.class; + Field filed1 = persistentTopicClass.getDeclaredField("ledger"); + Field field2 = persistentTopicClass.getDeclaredField("transactionBuffer"); + filed1.setAccessible(true); + field2.setAccessible(true); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) spy(filed1.get(persistentTopic)); + filed1.set(persistentTopic, managedLedger); + + TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field2.get(persistentTopic); + Method method = TopicTransactionBuffer.class.getDeclaredMethod("takeSnapshot"); + method.setAccessible(true); + CompletableFuture completableFuture = (CompletableFuture) method.invoke(topicTransactionBuffer); + completableFuture.get(); + + doReturn(PositionImpl.latest).when(managedLedger).getLastConfirmedEntry(); + ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class); + doReturn(false).when(managedCursor).hasMoreEntries(); + doReturn(managedCursor).when(managedLedger).newNonDurableCursor(any(), any()); + + TopicTransactionBuffer transactionBuffer = new TopicTransactionBuffer(persistentTopic); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(transactionBuffer.checkIfReady())); + } } From 5d4fe65a7713789346c600fc87c068e41648b377 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 18 Jan 2022 11:18:27 +0800 Subject: [PATCH 277/823] Fix encrption bug with chunked message (#13689) # Motivation Fix issue #13688. Send chunking message failed with `org.apache.pulsar.client.api.PulsarClientException$TimeoutException` when encryption is enabled. ### Modifications The root cause is that all chunked messages share the same msgMetadata object. The `EncryptionKeys` will be repeated added into message metadata. And proto buffer objects do not support serialization with bytes value type. (cherry picked from commit c1ff87c99828e0d4dfb45b0c98e3ce5ed91e2221) --- .../api/SimpleProducerConsumerTest.java | 33 +++++++++++++++++++ .../client/impl/crypto/MessageCryptoBc.java | 1 + 2 files changed, 34 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 2eebbf8a0dc1b..fc238d125c66d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -77,6 +77,7 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.mledger.impl.EntryCacheImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -2620,6 +2621,38 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe log.info("-- Exiting {} test --", methodName); } + @Test + public void testCryptoWithChunking() throws Exception { + final String topic = "persistent://my-property/my-ns/testCryptoWithChunking" + System.currentTimeMillis(); + final String ecdsaPublicKeyFile = "file:./src/test/resources/certificate/public-key.client-ecdsa.pem"; + final String ecdsaPrivateKeyFile = "file:./src/test/resources/certificate/private-key.client-ecdsa.pem"; + + this.conf.setMaxMessageSize(1000); + + @Cleanup + PulsarClient pulsarClient = newPulsarClient(lookupUrl.toString(), 0); + + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .defaultCryptoKeyReader(ecdsaPrivateKeyFile).subscribe(); + @Cleanup + Producer producer1 = pulsarClient.newProducer().topic(topic) + .enableChunking(true) + .enableBatching(false) + .addEncryptionKey("client-ecdsa.pem") + .defaultCryptoKeyReader(ecdsaPublicKeyFile) + .create(); + + byte[] data = RandomUtils.nextBytes(5100); + MessageId id = producer1.send(data); + log.info("Message Id={}", id); + + MessageImpl message; + message = (MessageImpl) consumer1.receive(); + Assert.assertEquals(message.getData(), data); + Assert.assertEquals(message.getEncryptionCtx().get().getKeys().size(), 1); + } + @Test public void testDefaultCryptoKeyReader() throws Exception { final String topic = "persistent://my-property/my-ns/default-crypto-key-reader" + System.currentTimeMillis(); diff --git a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java index e4f3200735edb..4ce457401cd70 100644 --- a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java +++ b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java @@ -389,6 +389,7 @@ public synchronized void encrypt(Set encKeys, CryptoKeyReader keyReader, return; } + msgMetadata.clearEncryptionKeys(); // Update message metadata with encrypted data key for (String keyName : encKeys) { if (encryptedDataKeyMap.get(keyName) == null) { From 280498299b9efbe5b72ac4195920d80bff2068a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Tue, 18 Jan 2022 11:08:46 +0800 Subject: [PATCH 278/823] Fix inefficient forEach loop (#13742) ### Motivation There are some methods implemented with an inefficient forEach loop, so fix it to get better performance. It is similar to #12953 ### Modifications I rewrite it with the `for` loop. (cherry picked from commit 9c94cd7803f3653e5aa170f80f5591fe68cc0366) --- .../pulsar/broker/service/AbstractTopic.java | 14 +++++++------- .../broker/service/persistent/PersistentTopic.java | 13 +++++++------ .../collections/ConcurrentSortedLongPairSet.java | 14 ++++++++------ .../ConcurrentSortedLongPairSetTest.java | 12 ++++++++---- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 30c0faeab100a..ba00a20ee6608 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -30,7 +30,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -262,14 +261,15 @@ public void enableCnxAutoRead() { } protected boolean hasLocalProducers() { - AtomicBoolean foundLocal = new AtomicBoolean(false); - producers.values().forEach(producer -> { + if (producers.isEmpty()) { + return false; + } + for (Producer producer : producers.values()) { if (!producer.isRemote()) { - foundLocal.set(true); + return true; } - }); - - return foundLocal.get(); + } + return false; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index fb87b74c3361d..d13e65e458fa1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -601,14 +601,15 @@ public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) } private boolean hasRemoteProducers() { - AtomicBoolean foundRemote = new AtomicBoolean(false); - producers.values().forEach(producer -> { + if (producers.isEmpty()) { + return false; + } + for (Producer producer : producers.values()) { if (producer.isRemote()) { - foundRemote.set(true); + return true; } - }); - - return foundRemote.get(); + } + return false; } public CompletableFuture startReplProducers() { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java index d3321f9ad35ac..95e2302dcb7b9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java @@ -166,13 +166,15 @@ public String toString() { @Override public boolean isEmpty() { - AtomicBoolean isEmpty = new AtomicBoolean(true); - longPairSets.forEach((item1, longPairSet) -> { - if (isEmpty.get() && !longPairSet.isEmpty()) { - isEmpty.set(false); + if (longPairSets.isEmpty()) { + return true; + } + for (ConcurrentLongPairSet subSet : longPairSets.values()) { + if (!subSet.isEmpty()) { + return false; } - }); - return isEmpty.get(); + } + return true; } @Override diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java index 821bb8819554b..fcb9884a795ad 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java @@ -22,7 +22,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; - +import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -30,13 +30,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - import lombok.Cleanup; import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet.LongPair; import org.testng.annotations.Test; -import com.google.common.collect.Lists; - public class ConcurrentSortedLongPairSetTest { @Test @@ -241,4 +238,11 @@ public void testToString() { assertEquals(set.toString(), toString); } + @Test + public void testIsEmpty() { + LongPairSet set = new ConcurrentSortedLongPairSet(); + assertTrue(set.isEmpty()); + set.add(1, 1); + assertFalse(set.isEmpty()); + } } From 36f409a85ff0697f710ae935864a662436e2d8fd Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:09:21 +0800 Subject: [PATCH 279/823] [Broker] Change create topic return error to Status.BAD_REQUEST (#12919) (cherry picked from commit 2262a5d5dc3b835026ade95c8a5fa953338c6e89) --- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 6 +++--- .../apache/pulsar/broker/transaction/TransactionTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 9e8728c13f597..7f5aafc027c17 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -247,8 +247,8 @@ protected void validateAdminAndClientPermission() { protected void validateCreateTopic(TopicName topicName) { if (isTransactionInternalName(topicName)) { - log.warn("Try to create a topic in the system topic format! {}", topicName); - throw new RestException(Status.CONFLICT, "Cannot create topic in system topic format!"); + log.warn("Forbidden to create transaction internal topic: {}", topicName); + throw new RestException(Status.BAD_REQUEST, "Cannot create topic in system topic format!"); } } @@ -3642,7 +3642,7 @@ private Topic getTopicReference(TopicName topicName) { throw e; } catch (Exception e) { if (e.getCause() instanceof NotAllowedException) { - throw new RestException(Status.CONFLICT, e.getCause()); + throw new RestException(Status.BAD_REQUEST, e.getCause()); } throw new RestException(e.getCause()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 6fe99a765f306..d7d90891624de 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -155,7 +155,7 @@ public void testCreateTransactionSystemTopic() throws Exception { try { admin.topics().getSubscriptions(topicName); fail(); - } catch (PulsarAdminException.ConflictException e) { + } catch (PulsarAdminException e) { assertEquals(e.getMessage(), "Can not create transaction system topic " + topicName); } @@ -163,7 +163,7 @@ public void testCreateTransactionSystemTopic() throws Exception { try { admin.topics().createPartitionedTopic(topicName, 3); fail(); - } catch (PulsarAdminException.ConflictException e) { + } catch (PulsarAdminException e) { assertEquals(e.getMessage(), "Cannot create topic in system topic format!"); } @@ -171,7 +171,7 @@ public void testCreateTransactionSystemTopic() throws Exception { try { admin.topics().createNonPartitionedTopic(topicName); fail(); - } catch (PulsarAdminException.ConflictException e) { + } catch (PulsarAdminException e) { assertEquals(e.getMessage(), "Cannot create topic in system topic format!"); } } From 16aef57eefd15b7bf9036fbca9c3ff821b63ef02 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Tue, 18 Jan 2022 12:10:07 +0800 Subject: [PATCH 280/823] [Issuse 13640][broker] Fix non persistent topic subscription error. (#13685) Fixes #13640 ### Motivation If pulsar broker started with `allowAutoSubscriptionCreation=false`, there are no way to subscribe non persistent topic. ### Modifications Add `isPersistent` check in `ServerCnx`. (cherry picked from commit 3dbe418b70e35d710fef7c8da6689e5ef6b8574a) --- .../pulsar/broker/admin/v1/PersistentTopics.java | 2 +- .../pulsar/broker/admin/v2/PersistentTopics.java | 2 +- .../apache/pulsar/broker/service/ServerCnx.java | 3 ++- .../org/apache/pulsar/broker/service/Topic.java | 2 ++ .../service/nonpersistent/NonPersistentTopic.java | 5 +++++ .../broker/service/persistent/PersistentTopic.java | 5 +++++ .../BrokerServiceAutoSubscriptionCreationTest.java | 14 ++++++++++++++ 7 files changed, 30 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index c0c4b486dd782..fcb2ba8305730 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -638,7 +638,7 @@ public void createSubscription(@Suspended final AsyncResponse asyncResponse, @Pa try { validateTopicName(property, cluster, namespace, topic); if (!topicName.isPersistent()) { - throw new RestException(Response.Status.BAD_REQUEST, "Create subscription on non-persistent topic" + throw new RestException(Response.Status.BAD_REQUEST, "Create subscription on non-persistent topic " + "can only be done through client"); } internalCreateSubscription(asyncResponse, decode(encodedSubName), messageId, authoritative, replicated); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 977d54a99d21b..3b2ba8d670ce6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -1329,7 +1329,7 @@ public void createSubscription( try { validateTopicName(tenant, namespace, topic); if (!topicName.isPersistent()) { - throw new RestException(Response.Status.BAD_REQUEST, "Create subscription on non-persistent topic" + throw new RestException(Response.Status.BAD_REQUEST, "Create subscription on non-persistent topic " + "can only be done through client"); } internalCreateSubscription(asyncResponse, decode(encodedSubName), messageId, authoritative, replicated); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 583ad3c5a30eb..9f317be3280ff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1008,7 +1008,8 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { boolean rejectSubscriptionIfDoesNotExist = isDurable && !service.isAllowAutoSubscriptionCreation(topicName.toString()) - && !topic.getSubscriptions().containsKey(subscriptionName); + && !topic.getSubscriptions().containsKey(subscriptionName) + && topic.isPersistent(); if (rejectSubscriptionIfDoesNotExist) { return FutureUtil diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 4e5c698178112..0e7589ade92ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -258,6 +258,8 @@ default boolean isSystemTopic() { return false; } + boolean isPersistent(); + /* ------ Transaction related ------ */ /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 43245b9e672e6..808b403b655ac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -1076,4 +1076,9 @@ public CompletableFuture truncate() { protected boolean isTerminated() { return false; } + + @Override + public boolean isPersistent() { + return false; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index d13e65e458fa1..3c08bd9ea10f7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2929,6 +2929,11 @@ public boolean isSystemTopic() { return false; } + @Override + public boolean isPersistent() { + return true; + } + private synchronized void fence() { isFenced = true; ScheduledFuture monitoringTask = this.fencedTopicMonitoringTask; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java index d0eb3bfad2db4..dc4b3d9d51d88 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java @@ -138,4 +138,18 @@ public void testAutoSubscriptionCreationNamespaceDisallowOverridesBroker() throw assertFalse(admin.topics().getSubscriptions(topicName.toString()).contains(subscriptionName)); } + @Test + public void testNonPersistentTopicSubscriptionCreationWithAutoCreationDisable() throws Exception { + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(false); + + final String topicName = "non-persistent://prop/ns-abc/test-subtopic-" + testId.getAndIncrement(); + final String subscriptionName = "test-subtopic-sub"; + + admin.topics().createNonPartitionedTopic(topicName); + + // Subscribe operation should be successful + pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe(); + assertTrue(admin.topics().getSubscriptions(topicName).contains(subscriptionName)); + } + } From ea3c064fb2f78ee412b8708b966048ac46f681de Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 18 Jan 2022 11:51:56 +0800 Subject: [PATCH 281/823] fix ordering key with chunking (#13699) Master Issue: #13688 ### Motivation The root cause is the same as #13688. Chunking messages share the same metadata. And the type of ordering key is bytebuf, which can not be serialized twice. ### Modifications Reset the field before sending. (cherry picked from commit 8dc29755b8448d2a94f6e6af60b7d56ef591e05f) --- .../client/impl/MessageChunkingTest.java | 25 +++++++++++++++++++ .../pulsar/client/impl/ProducerImpl.java | 11 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java index 40191efe241a8..fc935a2b7f688 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java @@ -40,6 +40,7 @@ import lombok.Cleanup; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.RandomUtils; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.CompressionType; @@ -62,6 +63,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -167,6 +169,29 @@ public void testLargeMessage(boolean ackReceiptEnabled) throws Exception { } + @Test + public void testChunkingWithOrderingKey() throws Exception { + this.conf.setMaxMessageSize(5); + + final String topicName = "persistent://my-property/my-ns/testChunkingWithOrderingKey"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name") + .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).enableChunking(true) + .enableBatching(false).create(); + + byte[] data = RandomUtils.nextBytes(20); + byte[] ok = RandomUtils.nextBytes(10); + producer.newMessage().value(data).orderingKey(ok).send(); + + Message msg = consumer.receive(); + Assert.assertEquals(msg.getData(), data); + Assert.assertEquals(msg.getOrderingKey(), ok); + } + @Test(dataProvider = "ackReceiptEnabled") public void testLargeMessageAckTimeOut(boolean ackReceiptEnabled) throws Exception { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 46649bde8dc6e..95adbcc6f34b7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -490,13 +490,20 @@ public void sendAsync(Message message, SendCallback callback) { String uuid = totalChunks > 1 ? String.format("%s-%d", producerName, sequenceId) : null; byte[] schemaVersion = totalChunks > 1 && msg.getMessageBuilder().hasSchemaVersion() ? msg.getMessageBuilder().getSchemaVersion() : null; + byte[] orderingKey = totalChunks > 1 && msg.getMessageBuilder().hasOrderingKey() ? + msg.getMessageBuilder().getOrderingKey() : null; for (int chunkId = 0; chunkId < totalChunks; chunkId++) { // Need to reset the schemaVersion, because the schemaVersion is based on a ByteBuf object in // `MessageMetadata`, if we want to re-serialize the `SEND` command using a same `MessageMetadata`, // we need to reset the ByteBuf of the schemaVersion in `MessageMetadata`, I think we need to // reset `ByteBuf` objects in `MessageMetadata` after call the method `MessageMetadata#writeTo()`. - if (chunkId > 0 && schemaVersion != null) { - msg.getMessageBuilder().setSchemaVersion(schemaVersion); + if (chunkId > 0) { + if (schemaVersion != null) { + msg.getMessageBuilder().setSchemaVersion(schemaVersion); + } + if (orderingKey != null) { + msg.getMessageBuilder().setOrderingKey(orderingKey); + } } serializeAndSendMessage(msg, payload, sequenceId, uuid, chunkId, totalChunks, readStartIndex, ClientCnx.getMaxMessageSize(), compressedPayload, compressed, From f36b701dbb22af19ad66089441673a602f70513c Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:07:39 +0800 Subject: [PATCH 282/823] Fix call sync method in async rest API for ``internalGetSubscriptionsForNonPartitionedTopic`` (#13745) (cherry picked from commit 0a046b9f6dc4a1122bbd7d5e8a1da3a74bd2c7b0) --- .../pulsar/broker/admin/AdminResource.java | 5 +- .../admin/impl/PersistentTopicsBase.java | 49 +++++++++---------- .../pulsar/broker/web/PulsarWebResource.java | 38 ++++++++++---- 3 files changed, 55 insertions(+), 37 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 9045462f193ed..6a9311ca4ab5e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -38,6 +38,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; @@ -757,7 +758,9 @@ private CompletableFuture provisionPartitionedTopicPath(AsyncResponse asyn protected void resumeAsyncResponseExceptionally(AsyncResponse asyncResponse, Throwable throwable) { if (throwable instanceof WebApplicationException) { - asyncResponse.resume((WebApplicationException) throwable); + asyncResponse.resume(throwable); + } else if (throwable instanceof BrokerServiceException.NotAllowedException) { + asyncResponse.resume(new RestException(Status.CONFLICT, throwable)); } else { asyncResponse.resume(new RestException(throwable)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 7f5aafc027c17..a1481e0c92b6e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -1124,26 +1124,25 @@ private void resumeAsyncResponse(AsyncResponse asyncResponse, Set subscr } private void internalGetSubscriptionsForNonPartitionedTopic(AsyncResponse asyncResponse, boolean authoritative) { - try { - validateTopicOwnership(topicName, authoritative); - validateTopicOperation(topicName, TopicOperation.GET_SUBSCRIPTIONS); - - Topic topic = getTopicReference(topicName); - final List subscriptions = Lists.newArrayList(); - topic.getSubscriptions().forEach((subName, sub) -> subscriptions.add(subName)); - asyncResponse.resume(subscriptions); - } catch (WebApplicationException wae) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to get subscriptions for non-partitioned topic {}," - + " redirecting to other brokers.", - clientAppId(), topicName, wae); - } - resumeAsyncResponseExceptionally(asyncResponse, wae); - return; - } catch (Exception e) { - log.error("[{}] Failed to get list of subscriptions for {}", clientAppId(), topicName, e); - resumeAsyncResponseExceptionally(asyncResponse, e); - } + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.GET_SUBSCRIPTIONS)) + .thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenAccept(topic -> asyncResponse.resume(Lists.newArrayList(topic.getSubscriptions().keys()))) + .exceptionally(ex -> { + Throwable cause = ex.getCause(); + if (cause instanceof WebApplicationException + && ((WebApplicationException) cause).getResponse().getStatus() + == Status.TEMPORARY_REDIRECT.getStatusCode()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to get subscriptions for non-partitioned topic {}," + + " redirecting to other brokers.", clientAppId(), topicName, cause); + } + } else { + log.error("[{}] Failed to get list of subscriptions for {}", clientAppId(), topicName, cause); + } + resumeAsyncResponseExceptionally(asyncResponse, cause); + return null; + }); } protected TopicStats internalGetStats(boolean authoritative, boolean getPreciseBacklog, @@ -3650,13 +3649,9 @@ private Topic getTopicReference(TopicName topicName) { private CompletableFuture getTopicReferenceAsync(TopicName topicName) { return pulsar().getBrokerService().getTopicIfExists(topicName.toString()) - .thenCompose(optTopic -> { - if (optTopic.isPresent()) { - return CompletableFuture.completedFuture(optTopic.get()); - } else { - return topicNotFoundReasonAsync(topicName); - } - }); + .thenCompose(optTopic -> optTopic + .map(CompletableFuture::completedFuture) + .orElseGet(() -> topicNotFoundReasonAsync(topicName))); } private RestException topicNotFoundReason(TopicName topicName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index d4955de8ba999..696e3d108b45d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -1016,22 +1016,42 @@ public void validateTopicOperation(TopicName topicName, TopicOperation operation } public void validateTopicOperation(TopicName topicName, TopicOperation operation, String subscription) { + try { + validateTopicOperationAsync(topicName, operation, subscription).get(); + } catch (InterruptedException | ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof WebApplicationException){ + throw (WebApplicationException) cause; + } else { + throw new RestException(cause); + } + } + } + + public CompletableFuture validateTopicOperationAsync(TopicName topicName, TopicOperation operation) { + return validateTopicOperationAsync(topicName, operation, null); + } + + public CompletableFuture validateTopicOperationAsync(TopicName topicName, + TopicOperation operation, String subscription) { if (pulsar().getConfiguration().isAuthenticationEnabled() && pulsar().getBrokerService().isAuthorizationEnabled()) { if (!isClientAuthenticated(clientAppId())) { throw new RestException(Status.UNAUTHORIZED, "Need to authenticate to perform the request"); } - AuthenticationDataHttps authData = clientAuthData(); authData.setSubscription(subscription); - - Boolean isAuthorized = pulsar().getBrokerService().getAuthorizationService() - .allowTopicOperation(topicName, operation, originalPrincipal(), clientAppId(), authData); - - if (!isAuthorized) { - throw new RestException(Status.UNAUTHORIZED, String.format("Unauthorized to validateTopicOperation for" - + " operation [%s] on topic [%s]", operation.toString(), topicName)); - } + return pulsar().getBrokerService().getAuthorizationService() + .allowTopicOperationAsync(topicName, operation, originalPrincipal(), clientAppId(), authData) + .thenAccept(isAuthorized -> { + if (!isAuthorized) { + throw new RestException(Status.UNAUTHORIZED, String.format( + "Unauthorized to validateTopicOperation for operation [%s] on topic [%s]", + operation.toString(), topicName)); + } + }); + } else { + return CompletableFuture.completedFuture(null); } } From 0bfd38ee3041b9d6be4b81b431cfe4da08dd39f2 Mon Sep 17 00:00:00 2001 From: Nicklee007 <84127069+Nicklee007@users.noreply.github.com> Date: Tue, 18 Jan 2022 14:55:16 +0800 Subject: [PATCH 283/823] Release old bundle from ownership cache when operator split bundle (#13678) Fixes #13677 Modifications release the old bundle from ownership and temporary znode cache when we split the old bundle; (cherry picked from commit 0cd9462802b51b0026c9c641b82861aaaa8f2d9b) --- .../broker/namespace/NamespaceService.java | 2 + .../namespace/NamespaceServiceTest.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index c0eee85ed6aee..3c31f711d6a25 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -894,6 +894,8 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, // update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); loadManager.get().setLoadReportForceUpdateFlag(); + // release old bundle from ownership cache + pulsar.getNamespaceService().getOwnershipCache().removeOwnership(bundle); completionFuture.complete(null); if (unload) { // Unload new split bundles, in background. This will not diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index bb33ac837bbd6..1420bc6de9cda 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -73,6 +73,7 @@ import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.awaitility.Awaitility; import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -491,6 +492,48 @@ public void testRemoveOwnershipAndSplitBundle() throws Exception { } } + + @Test + public void testSplitBundleAndRemoveOldBundleFromOwnerShipCache() throws Exception { + OwnershipCache ownershipCache = spy(pulsar.getNamespaceService().getOwnershipCache()); + doReturn(CompletableFuture.completedFuture(null)).when(ownershipCache).disableOwnership(any(NamespaceBundle.class)); + + Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); + ownership.setAccessible(true); + ownership.set(pulsar.getNamespaceService(), ownershipCache); + + NamespaceService namespaceService = pulsar.getNamespaceService(); + NamespaceName nsname = NamespaceName.get("pulsar/global/ns1"); + TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1"); + NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname); + + NamespaceBundle splitBundle1 = bundles.findBundle(topicName); + ownershipCache.tryAcquiringOwnership(splitBundle1); + CompletableFuture result1 = namespaceService.splitAndOwnBundle(splitBundle1, false, NamespaceBundleSplitAlgorithm.RANGE_EQUALLY_DIVIDE_ALGO); + try { + result1.get(); + } catch (Exception e) { + fail("split bundle failed", e); + } + Awaitility.await().untilAsserted(() + -> assertNull(namespaceService.getOwnershipCache().getOwnedBundles().get(splitBundle1))); + + //unload split + bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname); + assertNotNull(bundles); + NamespaceBundle splitBundle2 = bundles.findBundle(topicName); + CompletableFuture result2 = namespaceService.splitAndOwnBundle(splitBundle2, true, NamespaceBundleSplitAlgorithm.RANGE_EQUALLY_DIVIDE_ALGO); + try { + result2.get(); + } catch (Exception e) { + // make sure: NPE does not occur + fail("split bundle failed", e); + } + Awaitility.await().untilAsserted(() + -> assertNull(namespaceService.getOwnershipCache().getOwnedBundles().get(splitBundle2))); + } + + @Test public void testSplitLargestBundle() throws Exception { String namespace = "prop/test/ns-abc2"; From 9bfbd08655722304418fe2d1702a1c46c4ddeace Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 11 Jan 2022 14:47:28 -0800 Subject: [PATCH 284/823] Upgrade BK to 4.14.4 and Grpc to 1.42.1 (#13714) * Upgrade BK to 4.14.4 * Upgrade Grpc to 1.42.1 * Fixed license file (cherry picked from commit 104adbe66ae327305227ce8f57ba3bb95ce83492) --- .../server/src/assemble/LICENSE.bin.txt | 93 ++++++++++--------- pom.xml | 4 +- pulsar-sql/presto-distribution/LICENSE | 24 ++--- 3 files changed, 61 insertions(+), 60 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 07b43e4ab2a52..68297da5fc1e2 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -322,7 +322,7 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.12.6.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar - * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.17.0.jar + * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.7.6.jar * Gson - com.google.code.gson-gson-2.8.9.jar @@ -392,31 +392,31 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.14.3.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.3.jar - - org.apache.bookkeeper-bookkeeper-proto-4.14.3.jar - - org.apache.bookkeeper-bookkeeper-server-4.14.3.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.3.jar - - org.apache.bookkeeper-circe-checksum-4.14.3.jar - - org.apache.bookkeeper-cpu-affinity-4.14.3.jar - - org.apache.bookkeeper-statelib-4.14.3.jar - - org.apache.bookkeeper-stream-storage-api-4.14.3.jar - - org.apache.bookkeeper-stream-storage-common-4.14.3.jar - - org.apache.bookkeeper-stream-storage-java-client-4.14.3.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.14.3.jar - - org.apache.bookkeeper-stream-storage-proto-4.14.3.jar - - org.apache.bookkeeper-stream-storage-server-4.14.3.jar - - org.apache.bookkeeper-stream-storage-service-api-4.14.3.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.14.3.jar - - org.apache.bookkeeper.http-http-server-4.14.3.jar - - org.apache.bookkeeper.http-vertx-http-server-4.14.3.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.3.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.3.jar - - org.apache.distributedlog-distributedlog-common-4.14.3.jar - - org.apache.distributedlog-distributedlog-core-4.14.3-tests.jar - - org.apache.distributedlog-distributedlog-core-4.14.3.jar - - org.apache.distributedlog-distributedlog-protocol-4.14.3.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.3.jar + - org.apache.bookkeeper-bookkeeper-common-4.14.4.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.4.jar + - org.apache.bookkeeper-bookkeeper-proto-4.14.4.jar + - org.apache.bookkeeper-bookkeeper-server-4.14.4.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.4.jar + - org.apache.bookkeeper-circe-checksum-4.14.4.jar + - org.apache.bookkeeper-cpu-affinity-4.14.4.jar + - org.apache.bookkeeper-statelib-4.14.4.jar + - org.apache.bookkeeper-stream-storage-api-4.14.4.jar + - org.apache.bookkeeper-stream-storage-common-4.14.4.jar + - org.apache.bookkeeper-stream-storage-java-client-4.14.4.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.14.4.jar + - org.apache.bookkeeper-stream-storage-proto-4.14.4.jar + - org.apache.bookkeeper-stream-storage-server-4.14.4.jar + - org.apache.bookkeeper-stream-storage-service-api-4.14.4.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.14.4.jar + - org.apache.bookkeeper.http-http-server-4.14.4.jar + - org.apache.bookkeeper.http-vertx-http-server-4.14.4.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.4.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.4.jar + - org.apache.distributedlog-distributedlog-common-4.14.4.jar + - org.apache.distributedlog-distributedlog-core-4.14.4-tests.jar + - org.apache.distributedlog-distributedlog-core-4.14.4.jar + - org.apache.distributedlog-distributedlog-protocol-4.14.4.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.4.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.13.jar @@ -461,20 +461,21 @@ The Apache Software License, Version 2.0 - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.4.32.jar - org.jetbrains-annotations-13.0.jar * gRPC - - io.grpc-grpc-all-1.33.0.jar - - io.grpc-grpc-auth-1.33.0.jar - - io.grpc-grpc-context-1.33.0.jar - - io.grpc-grpc-core-1.33.0.jar - - io.grpc-grpc-netty-1.33.0.jar - - io.grpc-grpc-protobuf-1.33.0.jar - - io.grpc-grpc-protobuf-lite-1.33.0.jar - - io.grpc-grpc-stub-1.33.0.jar - - io.grpc-grpc-alts-1.33.0.jar - - io.grpc-grpc-api-1.33.0.jar - - io.grpc-grpc-grpclb-1.33.0.jar - - io.grpc-grpc-netty-shaded-1.33.0.jar - - io.grpc-grpc-services-1.33.0.jar - - io.grpc-grpc-xds-1.33.0.jar + - io.grpc-grpc-all-1.42.1.jar + - io.grpc-grpc-auth-1.42.1.jar + - io.grpc-grpc-context-1.42.1.jar + - io.grpc-grpc-core-1.42.1.jar + - io.grpc-grpc-netty-1.42.1.jar + - io.grpc-grpc-protobuf-1.42.1.jar + - io.grpc-grpc-protobuf-lite-1.42.1.jar + - io.grpc-grpc-stub-1.42.1.jar + - io.grpc-grpc-alts-1.42.1.jar + - io.grpc-grpc-api-1.42.1.jar + - io.grpc-grpc-grpclb-1.42.1.jar + - io.grpc-grpc-netty-shaded-1.42.1.jar + - io.grpc-grpc-services-1.42.1.jar + - io.grpc-grpc-xds-1.42.1.jar + - io.grpc-grpc-rls-1.42.1.jar * Perfmark - io.perfmark-perfmark-api-0.19.0.jar * OpenCensus @@ -521,15 +522,15 @@ The Apache Software License, Version 2.0 * Snappy Java - org.xerial.snappy-snappy-java-1.1.7.jar * Google HTTP Client - - com.google.http-client-google-http-client-jackson2-1.34.0.jar - - com.google.http-client-google-http-client-1.34.0.jar - - com.google.auto.value-auto-value-annotations-1.7.jar - - com.google.re2j-re2j-1.2.jar + - com.google.http-client-google-http-client-jackson2-1.38.0.jar + - com.google.http-client-google-http-client-1.38.0.jar + - com.google.auto.value-auto-value-annotations-1.7.4.jar + - com.google.re2j-re2j-1.5.jar BSD 3-clause "New" or "Revised" License * Google auth library - - com.google.auth-google-auth-library-credentials-0.20.0.jar -- licenses/LICENSE-google-auth-library.txt - - com.google.auth-google-auth-library-oauth2-http-0.20.0.jar -- licenses/LICENSE-google-auth-library.txt + - com.google.auth-google-auth-library-credentials-0.22.2.jar -- licenses/LICENSE-google-auth-library.txt + - com.google.auth-google-auth-library-oauth2-http-0.22.2.jar -- licenses/LICENSE-google-auth-library.txt * LevelDB -- (included in org.rocksdb.*.jar) -- licenses/LICENSE-LevelDB.txt * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- licenses/LICENSE-JSR305.txt * JLine -- jline-jline-2.14.6.jar -- licenses/LICENSE-JLine.txt diff --git a/pom.xml b/pom.xml index 0a7356127d285..91a9a4accadb8 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.14.3 + 4.14.4 3.6.3 1.1.7 3.2.5 @@ -131,7 +131,7 @@ flexible messaging model and an intuitive client API. 0.5.0 3.16.1 ${protobuf3.version} - 1.33.0 + 1.42.1 0.19.0 ${grpc.version} 2.8.9 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index ed9d018647fff..c6840b8346428 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -415,18 +415,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.14.3.jar - - bookkeeper-common-allocator-4.14.3.jar - - bookkeeper-proto-4.14.3.jar - - bookkeeper-server-4.14.3.jar - - bookkeeper-stats-api-4.14.3.jar - - bookkeeper-tools-framework-4.14.3.jar - - circe-checksum-4.14.3.jar - - codahale-metrics-provider-4.14.3.jar - - cpu-affinity-4.14.3.jar - - http-server-4.14.3.jar - - prometheus-metrics-provider-4.14.3.jar - - codahale-metrics-provider-4.14.3.jar + - bookkeeper-common-4.14.4.jar + - bookkeeper-common-allocator-4.14.4.jar + - bookkeeper-proto-4.14.4.jar + - bookkeeper-server-4.14.4.jar + - bookkeeper-stats-api-4.14.4.jar + - bookkeeper-tools-framework-4.14.4.jar + - circe-checksum-4.14.4.jar + - codahale-metrics-provider-4.14.4.jar + - cpu-affinity-4.14.4.jar + - http-server-4.14.4.jar + - prometheus-metrics-provider-4.14.4.jar + - codahale-metrics-provider-4.14.4.jar * Apache Commons - commons-cli-1.2.jar - commons-codec-1.15.jar From 85bd12acbf740db62aad078ec3ccf5f9f803b2ea Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 20 Jan 2022 15:34:56 +0800 Subject: [PATCH 285/823] Fix SystemTopicBasedTopicPoliciesService NPE issue. (#13840) --- .../broker/service/SystemTopicBasedTopicPoliciesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index a6c786ba2b98b..44b866f4fc134 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -222,7 +222,6 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); result.complete(null); } else { - ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); prepareInitPoliciesCache(namespace, result); } } @@ -234,6 +233,7 @@ private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture CompletableFuture> readerCompletableFuture = creatSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); + ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); readerCompletableFuture.whenComplete((reader, ex) -> { if (ex != null) { log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); From d3a2ee444ecb54076b15dd41d69701658b435a4e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Jan 2022 06:41:12 +0200 Subject: [PATCH 286/823] [Proxy] Prevent leak of unreleased lookupRequestSemaphore permits (#13812) * [Proxy] Prevent leak of unreleased lookupRequestSemaphore permits - should release permit in try-finally block * Cleanup code in LookupProxyHandler (cherry picked from commit 85b62e050b01b591a4b5751aab48b418ac9e4e76) --- .../proxy/server/LookupProxyHandler.java | 91 ++++++++----------- 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java index b14bea56f812c..30e11a185a038 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java @@ -18,14 +18,14 @@ */ package org.apache.pulsar.proxy.server; -import static org.apache.commons.lang3.StringUtils.isBlank; - +import io.netty.buffer.ByteBuf; +import io.prometheus.client.Counter; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.Optional; - +import java.util.concurrent.Semaphore; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.protocol.Commands; @@ -42,13 +42,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.netty.buffer.ByteBuf; -import io.prometheus.client.Counter; - public class LookupProxyHandler { private final String throttlingErrorMessage = "Too many concurrent lookup and partitionsMetadata requests"; - private final ProxyService service; private final ProxyConnection proxyConnection; + private final BrokerDiscoveryProvider discoveryProvider; private final boolean connectWithTLS; private SocketAddress clientAddress; @@ -83,9 +80,11 @@ public class LookupProxyHandler { .build("pulsar_proxy_rejected_get_topics_of_namespace_requests", "Counter of getTopicsOfNamespace requests rejected due to throttling") .create().register(); + private final Semaphore lookupRequestSemaphore; public LookupProxyHandler(ProxyService proxy, ProxyConnection proxyConnection) { - this.service = proxy; + this.discoveryProvider = proxy.getDiscoveryProvider(); + this.lookupRequestSemaphore = proxy.getLookupRequestSemaphore(); this.proxyConnection = proxyConnection; this.clientAddress = proxyConnection.clientAddress(); this.connectWithTLS = proxy.getConfiguration().isTlsEnabledWithBroker(); @@ -98,28 +97,16 @@ public void handleLookup(CommandLookupTopic lookup) { log.debug("Received Lookup from {}", clientAddress); } long clientRequestId = lookup.getRequestId(); - if (this.service.getLookupRequestSemaphore().tryAcquire()) { - lookupRequests.inc(); - String topic = lookup.getTopic(); - String serviceUrl; - if (isBlank(brokerServiceURL)) { - ServiceLookupData availableBroker = null; - try { - availableBroker = service.getDiscoveryProvider().nextBroker(); - } catch (Exception e) { - log.warn("[{}] Failed to get next active broker {}", clientAddress, e.getMessage(), e); - proxyConnection.ctx().writeAndFlush(Commands.newLookupErrorResponse(ServerError.ServiceNotReady, - e.getMessage(), clientRequestId)); - return; + if (lookupRequestSemaphore.tryAcquire()) { + try { + lookupRequests.inc(); + String serviceUrl = getBrokerServiceUrl(clientRequestId); + if (serviceUrl != null) { + performLookup(clientRequestId, lookup.getTopic(), serviceUrl, false, 10); } - serviceUrl = this.connectWithTLS ? availableBroker.getPulsarServiceUrlTls() - : availableBroker.getPulsarServiceUrl(); - } else { - serviceUrl = this.connectWithTLS ? service.getConfiguration().getBrokerServiceURLTLS() - : service.getConfiguration().getBrokerServiceURL(); + } finally { + lookupRequestSemaphore.release(); } - performLookup(clientRequestId, topic, serviceUrl, false, 10); - this.service.getLookupRequestSemaphore().release(); } else { rejectedLookupRequests.inc(); if (log.isDebugEnabled()) { @@ -203,9 +190,12 @@ public void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata part log.debug("[{}] Received PartitionMetadataLookup", clientAddress); } final long clientRequestId = partitionMetadata.getRequestId(); - if (this.service.getLookupRequestSemaphore().tryAcquire()) { - handlePartitionMetadataResponse(partitionMetadata, clientRequestId); - this.service.getLookupRequestSemaphore().release(); + if (lookupRequestSemaphore.tryAcquire()) { + try { + handlePartitionMetadataResponse(partitionMetadata, clientRequestId); + } finally { + lookupRequestSemaphore.release(); + } } else { rejectedPartitionsMetadataRequests.inc(); if (log.isDebugEnabled()) { @@ -277,9 +267,12 @@ public void handleGetTopicsOfNamespace(CommandGetTopicsOfNamespace commandGetTop final long requestId = commandGetTopicsOfNamespace.getRequestId(); - if (this.service.getLookupRequestSemaphore().tryAcquire()) { - handleGetTopicsOfNamespace(commandGetTopicsOfNamespace, requestId); - this.service.getLookupRequestSemaphore().release(); + if (lookupRequestSemaphore.tryAcquire()) { + try { + handleGetTopicsOfNamespace(commandGetTopicsOfNamespace, requestId); + } finally { + lookupRequestSemaphore.release(); + } } else { rejectedGetTopicsOfNamespaceRequests.inc(); if (log.isDebugEnabled()) { @@ -408,24 +401,20 @@ public void handleGetSchema(CommandGetSchema commandGetSchema) { * Get default broker service url or discovery an available broker **/ private String getBrokerServiceUrl(long clientRequestId) { - if (isBlank(brokerServiceURL)) { - ServiceLookupData availableBroker; - try { - availableBroker = service.getDiscoveryProvider().nextBroker(); - } catch (Exception e) { - log.warn("[{}] Failed to get next active broker {}", clientAddress, e.getMessage(), e); - proxyConnection.ctx().writeAndFlush(Commands.newError( - clientRequestId, ServerError.ServiceNotReady, e.getMessage() - )); - return null; - } - return this.connectWithTLS ? - availableBroker.getPulsarServiceUrlTls() : availableBroker.getPulsarServiceUrl(); - } else { - return this.connectWithTLS ? - service.getConfiguration().getBrokerServiceURLTLS() : service.getConfiguration().getBrokerServiceURL(); + if (StringUtils.isNotBlank(brokerServiceURL)) { + return brokerServiceURL; } - + ServiceLookupData availableBroker; + try { + availableBroker = discoveryProvider.nextBroker(); + } catch (Exception e) { + log.warn("[{}] Failed to get next active broker {}", clientAddress, e.getMessage(), e); + proxyConnection.ctx().writeAndFlush(Commands.newError( + clientRequestId, ServerError.ServiceNotReady, e.getMessage() + )); + return null; + } + return this.connectWithTLS ? availableBroker.getPulsarServiceUrlTls() : availableBroker.getPulsarServiceUrl(); } private InetSocketAddress getAddr(String brokerServiceUrl, long clientRequestId) { From 2dac261ea55d604a39cc82593818a13639725c5c Mon Sep 17 00:00:00 2001 From: liudezhi <33149602+liudezhi2098@users.noreply.github.com> Date: Thu, 20 Jan 2022 15:35:23 +0800 Subject: [PATCH 287/823] [broker] Fix topic produced through REST not support Authorization (#13771) (cherry picked from commit f551fdc1d34570bc28ae1eccb9c409f02a6e00dd) --- .../apache/pulsar/broker/rest/TopicsBase.java | 3 +- .../pulsar/broker/admin/TopicsAuthTest.java | 193 ++++++++++++++++++ 2 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 1df2c14bc9c94..f89abf9bea30d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -756,7 +756,8 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) { } boolean isAuthorized = pulsar().getBrokerService().getAuthorizationService() - .canProduce(topicName, originalPrincipal(), clientAuthData()); + .canProduce(topicName, originalPrincipal() == null ? clientAppId() : originalPrincipal(), + clientAuthData()); if (!isAuthorized) { throw new RestException(Status.UNAUTHORIZED, String.format("Unauthorized to produce to topic %s" + " with clientAppId [%s] and authdata %s", topicName.toString(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java new file mode 100644 index 0000000000000..185053ed7029b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import java.util.Base64; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.crypto.SecretKey; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.client.impl.schema.StringSchema; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.websocket.data.ProducerMessage; +import org.apache.pulsar.websocket.data.ProducerMessages; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TopicsAuthTest extends MockedPulsarServiceBaseTest { + + private final String testLocalCluster = "test"; + private final String testTenant = "my-tenant"; + private final String testNamespace = "my-namespace"; + private final String testTopicName = "my-topic"; + + private static final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + private static final String ADMIN_TOKEN = Jwts.builder().setSubject("admin").signWith(SECRET_KEY).compact(); + private static final String PRODUCE_TOKEN = Jwts.builder().setSubject("producer").signWith(SECRET_KEY).compact(); + private static final String CONSUME_TOKEN = Jwts.builder().setSubject("consumer").signWith(SECRET_KEY).compact(); + + @Override + @BeforeMethod + protected void setup() throws Exception { + // enable auth&auth and use JWT at broker + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(true); + conf.getProperties().setProperty("tokenSecretKey", "data:;base64," + + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); + Set superUserRoles = new HashSet<>(); + superUserRoles.add("admin"); + conf.setSuperUserRoles(superUserRoles); + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + conf.setAuthenticationProviders(providers); + super.internalSetup(); + PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null + ? brokerUrl.toString() : brokerUrlTls.toString()) + .authentication(AuthenticationToken.class.getName(), + ADMIN_TOKEN); + admin = Mockito.spy(pulsarAdminBuilder.build()); + admin.clusters().createCluster(testLocalCluster, new ClusterDataImpl()); + admin.tenants().createTenant(testTenant, new TenantInfoImpl(Sets.newHashSet("role1", "role2"), + Sets.newHashSet(testLocalCluster))); + admin.namespaces().createNamespace(testTenant + "/" + testNamespace, + Sets.newHashSet(testLocalCluster)); + admin.namespaces().grantPermissionOnNamespace(testTenant + "/" + testNamespace, "producer", + EnumSet.of(AuthAction.produce)); + admin.namespaces().grantPermissionOnNamespace(testTenant + "/" + testNamespace, "consumer", + EnumSet.of(AuthAction.consume)); + } + + @Override + @AfterMethod + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "variations") + public static Object[][] variations() { + return new Object[][]{ + {CONSUME_TOKEN, 401}, + {PRODUCE_TOKEN, 200} + }; + } + + @Test(dataProvider = "variations") + public void testProduceToNonPartitionedTopic(String token, int status) throws Exception { + innerTestProduce(testTopicName, true, false, token, status); + } + + @Test(dataProvider = "variations") + public void testProduceToPartitionedTopic(String token, int status) throws Exception { + innerTestProduce(testTopicName, true, true, token, status); + } + + @Test(dataProvider = "variations") + public void testProduceOnNonPersistentNonPartitionedTopic(String token, int status) throws Exception { + innerTestProduce(testTopicName, false, false, token, status); + } + + @Test(dataProvider = "variations") + public void testProduceOnNonPersistentPartitionedTopic(String token, int status) throws Exception { + innerTestProduce(testTopicName, false, true, token, status); + } + + private void innerTestProduce(String createTopicName, boolean isPersistent, boolean isPartition, + String token, int status) throws Exception { + String topicPrefix = null; + if (isPersistent == true) { + topicPrefix = "persistent"; + } else { + topicPrefix = "non-persistent"; + } + if (isPartition == true) { + admin.topics().createPartitionedTopic(topicPrefix + "://" + testTenant + "/" + + testNamespace + "/" + createTopicName, 5); + } else { + admin.topics().createNonPartitionedTopic(topicPrefix + "://" + testTenant + "/" + + testNamespace + "/" + createTopicName); + } + Schema schema = StringSchema.utf8(); + ProducerMessages producerMessages = new ProducerMessages(); + producerMessages.setKeySchema(ObjectMapperFactory.getThreadLocal(). + writeValueAsString(schema.getSchemaInfo())); + producerMessages.setValueSchema(ObjectMapperFactory.getThreadLocal(). + writeValueAsString(schema.getSchemaInfo())); + String message = "[" + + "{\"key\":\"my-key\",\"payload\":\"RestProducer:1\",\"eventTime\":1603045262772,\"sequenceId\":1}," + + "{\"key\":\"my-key\",\"payload\":\"RestProducer:2\",\"eventTime\":1603045262772,\"sequenceId\":2}]"; + producerMessages.setMessages(ObjectMapperFactory.getThreadLocal().readValue(message, + new TypeReference>() { + })); + + WebTarget root = buildWebClient(); + String requestPath = null; + if (isPartition == true) { + requestPath = "/topics/" + topicPrefix + "/" + testTenant + "/" + testNamespace + "/" + + createTopicName + "/partitions/2"; + } else { + requestPath = "/topics/" + topicPrefix + "/" + testTenant + "/" + testNamespace + "/" + createTopicName; + } + + Response response = root.path(requestPath) + .request(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .post(Entity.json(producerMessages)); + Assert.assertEquals(response.getStatus(), status); + } + + WebTarget buildWebClient() throws Exception { + ClientConfig httpConfig = new ClientConfig(); + httpConfig.property(ClientProperties.FOLLOW_REDIRECTS, true); + httpConfig.property(ClientProperties.ASYNC_THREADPOOL_SIZE, 8); + httpConfig.register(MultiPartFeature.class); + + javax.ws.rs.client.ClientBuilder clientBuilder = ClientBuilder.newBuilder().withConfig(httpConfig); + Client client = clientBuilder.build(); + return client.target(brokerUrl.toString()); + } + +} From bf7af7943895377a465943a19c93ba191920edea Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:51:31 -0800 Subject: [PATCH 288/823] Getting rid of CVes brought in with aerospike (#13819) CVE-2020-26939 CVE-2015-0886 (cherry picked from commit 8a452b733c91d6a60ddf32b403c9169be5ba7978) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91a9a4accadb8..cfa1c2db0e546 100644 --- a/pom.xml +++ b/pom.xml @@ -138,7 +138,7 @@ flexible messaging model and an intuitive client API. 0.8.3 2.2.0 3.6.0 - 4.4.8 + 4.4.20 2.7.0 5.1.1 1.11.774 From ff96162732b6978d8046ddb6caa28d62f5bf101a Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:51:57 -0800 Subject: [PATCH 289/823] Getting rid of CVEs in batch-data-generator (#13820) CVE-2021-41269 CVE-2021-22060 CVE-2021-22096 CVE-2021-22118 (cherry picked from commit 5a33d106dd282fa85008888a687c94203f888df7) --- pom.xml | 4 ++-- pulsar-io/batch-data-generator/pom.xml | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index cfa1c2db0e546..22afc5c9accc9 100644 --- a/pom.xml +++ b/pom.xml @@ -198,8 +198,8 @@ flexible messaging model and an intuitive client API. 1.4.32 1.0 - 9.1.3 - 5.3.1 + 9.1.6 + 5.3.15 4.5.13 diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 8141194fec88b..288583b2c16f1 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -44,6 +44,12 @@ ${project.version} + + org.springframework + spring-context + ${spring-context.version} + + io.codearte.jfairy jfairy From f68892c03c4393f7105878a6ca96268a4a206b7d Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:52:23 -0800 Subject: [PATCH 290/823] Getitng rid of CVEs in InfluxDB connector (#13821) CVE-2019-20933 CVE-2018-1000850 (cherry picked from commit 85746390c4421cce710ec6988d1747fd5a3f7ade) --- pulsar-io/influxdb/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 8c7c6d06475e3..f95f19af912fd 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -51,13 +51,13 @@ com.influxdb influxdb-client-java - 1.6.0 + 4.0.0 org.influxdb influxdb-java - 2.7 + 2.22 com.squareup.okhttp3 From 9909ec859a5668005591a105565b4abde87b3341 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:52:46 -0800 Subject: [PATCH 291/823] Getting rid of CVEs in Solr connector (#13822) CVE-2019-17638 CVE-2020-27216 CVE-2020-27218 CVE-2020-27223 CVE-2021-28165 CVE-2021-28169 CVE-2021-34428 CVE-2021-27905 CVE-2021-29262 CVE-2021-29943 CVE-2021-44548 (cherry picked from commit 76fc2fbc9801da5232cd4ba32d25298b74d460db) --- pulsar-io/solr/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index db3cafc5813b6..615d476296ea3 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -29,7 +29,7 @@ - 8.6.3 + 8.11.1 pulsar-io-solr From 2ca5a0469a466510ff801a8e737430ee47edaf39 Mon Sep 17 00:00:00 2001 From: ran Date: Thu, 20 Jan 2022 20:50:53 +0800 Subject: [PATCH 292/823] [Broker] Avoid thread deadlock problem when creating topic policy reader (#13837) Currently, the topic policy reader creation thread is the `executor` of the `PulsarService`, this thread is also used to search the candidate broker. If they use the same thread in some conditions, the lookup request will be blocked and result in a lookup request timeout. ![image](https://user-images.githubusercontent.com/15029908/150124902-c4318182-56f2-4b31-9149-ae64a9919aa4.png) ![image](https://user-images.githubusercontent.com/15029908/150124952-66a9f095-bab2-40fa-9370-76d3f2158dac.png) We could find out that the lookup request was blocked 1 minute until the lookup request timeout. The thread `pulsar-2-8` was blocked by topic policy reader creation. Change the topic policy reader creation to be asynchronous. Modify the method `RetryUtil.retryAsynchronously` to handle asynchronous execution. Add a new test to verify consumer creations can be successful when enabling the topic policy feature. (cherry picked from commit 760bfec51ed6f043024c3c42bc73639fc2c117bb) --- .../SystemTopicBasedTopicPoliciesService.java | 12 ++----- .../broker/service/TopicPoliciesService.java | 6 ++-- ...temTopicBasedTopicPoliciesServiceTest.java | 9 ++--- .../systopic/PartitionedSystemTopicTest.java | 33 ++++++++++++++++++ .../pulsar/client/impl/RetryUtilTest.java | 13 ++++--- .../apache/pulsar/client/util/RetryUtil.java | 34 +++++++++---------- 6 files changed, 69 insertions(+), 38 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 44b866f4fc134..838edfe7f05c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -231,7 +231,7 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture result) { if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { CompletableFuture> readerCompletableFuture = - creatSystemTopicClientWithRetry(namespace); + createSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); readerCompletableFuture.whenComplete((reader, ex) -> { @@ -248,7 +248,7 @@ private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture } } - protected CompletableFuture> creatSystemTopicClientWithRetry( + protected CompletableFuture> createSystemTopicClientWithRetry( NamespaceName namespace) { CompletableFuture> result = new CompletableFuture<>(); try { @@ -260,13 +260,7 @@ protected CompletableFuture> creatSystemTo SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory .createTopicPoliciesSystemTopicClient(namespace); Backoff backoff = new Backoff(1, TimeUnit.SECONDS, 3, TimeUnit.SECONDS, 10, TimeUnit.SECONDS); - RetryUtil.retryAsynchronously(() -> { - try { - return systemTopicClient.newReader(); - } catch (PulsarClientException e) { - throw new RuntimeException(e); - } - }, backoff, pulsarService.getExecutor(), result); + RetryUtil.retryAsynchronously(systemTopicClient::newReaderAsync, backoff, pulsarService.getExecutor(), result); return result; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index 53de0878100c7..0165dc13cd331 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -85,11 +85,13 @@ default CompletableFuture> getTopicPoliciesAsyncWithRetr .create() : backoff; try { RetryUtil.retryAsynchronously(() -> { + CompletableFuture> future = new CompletableFuture<>(); try { - return Optional.ofNullable(getTopicPolicies(topicName)); + future.complete(Optional.ofNullable(getTopicPolicies(topicName))); } catch (BrokerServiceException.TopicPoliciesCacheNotInitException exception) { - throw new RuntimeException(exception); + future.completeExceptionally(exception); } + return future; }, usedBackoff, scheduledExecutorService, response); } catch (Exception e) { response.completeExceptionally(e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 9a489cf593984..8b5f4203c8447 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -20,7 +20,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -56,6 +55,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -313,7 +313,7 @@ public void testGetPolicyTimeout() throws Exception { try { service.getTopicPoliciesAsyncWithRetry(TOPIC1, backoff, pulsar.getExecutor()).get(); } catch (Exception e) { - assertTrue(e.getCause().getCause() instanceof TopicPoliciesCacheNotInitException); + assertTrue(e.getCause() instanceof TopicPoliciesCacheNotInitException); } long cost = System.currentTimeMillis() - start; assertTrue("actual:" + cost, cost >= 5000 - 1000); @@ -333,9 +333,10 @@ public void testCreatSystemTopicClientWithRetry() throws Exception { SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class); // Throw an exception first, create successfully after retrying - doThrow(new PulsarClientException("test")).doReturn(reader).when(client).newReader(); + doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))) + .doReturn(CompletableFuture.completedFuture(reader)).when(client).newReaderAsync(); - SystemTopicClient.Reader reader1 = service.creatSystemTopicClientWithRetry(null).get(); + SystemTopicClient.Reader reader1 = service.createSystemTopicClientWithRetry(null).get(); assertEquals(reader1, reader); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index bd4ef7870da9f..7dd02bdeae7ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -18,14 +18,23 @@ */ package org.apache.pulsar.broker.systopic; +import com.google.common.collect.Sets; +import org.apache.commons.lang.RandomStringUtils; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + @Test(groups = "broker") public class PartitionedSystemTopicTest extends BrokerTestBase { @@ -66,4 +75,28 @@ public void testAutoCreatedPartitionedSystemTopic() throws Exception { Assert.assertEquals(partitions, PARTITIONS); Assert.assertEquals(admin.topics().getList(ns).size(), PARTITIONS); } + + @Test(timeOut = 1000 * 60) + public void testConsumerCreationWhenEnablingTopicPolicy() throws Exception { + String tenant = "tenant-" + RandomStringUtils.randomAlphabetic(4).toLowerCase(); + admin.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet("test"))); + int namespaceCount = 30; + for (int i = 0; i < namespaceCount; i++) { + String ns = tenant + "/ns-" + i; + admin.namespaces().createNamespace(ns, 4); + String topic = ns + "/t1"; + admin.topics().createPartitionedTopic(topic, 2); + } + + List>> futureList = new ArrayList<>(); + for (int i = 0; i < namespaceCount; i++) { + String topic = tenant + "/ns-" + i + "/t1"; + futureList.add(pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub") + .subscribeAsync()); + } + FutureUtil.waitForAll(futureList).get(); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java index e17f376fcafb8..f5544307c3711 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import org.apache.pulsar.client.util.RetryUtil; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.annotations.Test; import java.util.concurrent.CompletableFuture; @@ -45,11 +46,14 @@ public void testFailAndRetry() throws Exception { .setMandatoryStop(5000, TimeUnit.MILLISECONDS) .create(); RetryUtil.retryAsynchronously(() -> { + CompletableFuture future = new CompletableFuture<>(); atomicInteger.incrementAndGet(); if (atomicInteger.get() < 5) { - throw new RuntimeException("fail"); + future.completeExceptionally(new RuntimeException("fail")); + } else { + future.complete(true); } - return true; + return future; }, backoff, executor, callback); assertTrue(callback.get()); assertEquals(atomicInteger.get(), 5); @@ -66,9 +70,8 @@ public void testFail() throws Exception { .setMandatoryStop(5000, TimeUnit.MILLISECONDS) .create(); long start = System.currentTimeMillis(); - RetryUtil.retryAsynchronously(() -> { - throw new RuntimeException("fail"); - }, backoff, executor, callback); + RetryUtil.retryAsynchronously(() -> + FutureUtil.failedFuture(new RuntimeException("fail")), backoff, executor, callback); try { callback.get(); } catch (Exception e) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java index 084a5839157d5..b3ed2c398d8bf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java @@ -30,7 +30,7 @@ public class RetryUtil { private static final Logger log = LoggerFactory.getLogger(RetryUtil.class); - public static void retryAsynchronously(Supplier supplier, Backoff backoff, + public static void retryAsynchronously(Supplier> supplier, Backoff backoff, ScheduledExecutorService scheduledExecutorService, CompletableFuture callback) { if (backoff.getMax() <= 0) { @@ -43,26 +43,24 @@ public static void retryAsynchronously(Supplier supplier, Backoff backoff executeWithRetry(supplier, backoff, scheduledExecutorService, callback)); } - private static void executeWithRetry(Supplier supplier, Backoff backoff, + private static void executeWithRetry(Supplier> supplier, Backoff backoff, ScheduledExecutorService scheduledExecutorService, CompletableFuture callback) { - try { - T result = supplier.get(); - callback.complete(result); - } catch (Exception e) { - long next = backoff.next(); - boolean isMandatoryStop = backoff.isMandatoryStopMade(); - if (isMandatoryStop) { - callback.completeExceptionally(e); - } else { - if (log.isDebugEnabled()) { - log.debug("execute with retry fail, will retry in {} ms", next, e); + supplier.get().whenComplete((result, e) -> { + if (e != null) { + long next = backoff.next(); + boolean isMandatoryStop = backoff.isMandatoryStopMade(); + if (isMandatoryStop) { + callback.completeExceptionally(e); + } else { + log.warn("Execution with retry fail, because of {}, will retry in {} ms", e.getMessage(), next); + scheduledExecutorService.schedule(() -> + executeWithRetry(supplier, backoff, scheduledExecutorService, callback), + next, TimeUnit.MILLISECONDS); } - log.info("Because of {} , will retry in {} ms", e.getMessage(), next); - scheduledExecutorService.schedule(() -> - executeWithRetry(supplier, backoff, scheduledExecutorService, callback), - next, TimeUnit.MILLISECONDS); + return; } - } + callback.complete(result); + }); } } From e9b585caf484bb5b2b1d1adb221cb42527d17cc9 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Fri, 21 Jan 2022 20:28:56 +0800 Subject: [PATCH 293/823] NPE when get isAllowAutoUploadSchema (#13831) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation there were some potential NPE bug when get null value by pulsar-admin ``` Caused by: java.lang.NullPointerException at org.apache.pulsar.broker.admin.impl.NamespacesBase.internalGetIsAllowAutoUpdateSchema(NamespacesBase.java:2423) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1] at org.apache.pulsar.broker.admin.v2.Namespaces.getIsAllowAutoUpdateSchema(Namespaces.java:1631) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] ``` ### Modifications 1、set default value true to is_allow_auto_update_schema in Policies as same as ServiceConfiguration 2、changes from boolean to Boolean, long to Long, prevent potential NPE. (cherry picked from commit 0a8794b768b414f8e5bc9b3abb76e57de9455add) --- .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 5f1231fec1305..7f4b288bfc22f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2399,6 +2399,9 @@ protected void internalSetSchemaValidationEnforced(boolean schemaValidationEnfor protected boolean internalGetIsAllowAutoUpdateSchema() { validateNamespacePolicyOperation(namespaceName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); + if (getNamespacePolicies(namespaceName).is_allow_auto_update_schema == null) { + return pulsar().getConfig().isAllowAutoUpdateSchemaEnabled(); + } return getNamespacePolicies(namespaceName).is_allow_auto_update_schema; } From 8a5d2253b888b3b865a2aedf635d6727777821c7 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Sat, 22 Jan 2022 00:35:48 +0800 Subject: [PATCH 294/823] Release 2.9.2 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- deployment/terraform-ansible/deploy-pulsar.yaml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 127 files changed, 127 insertions(+), 127 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 6128d5be5283e..ddca4e6494ada 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 31a706eab97eb..fb84e6fe63d32 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 0a3f88abe6471..4fa3ead0eadd4 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index d5aed2f76fe48..979803c96f58c 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index be6f1fefc0384..2a6002297c1cb 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.2-SNAPSHOT + 2.9.2 jar Pulsar Build Tools diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml index c016d548e6af1..cdde04d8aab5c 100644 --- a/deployment/terraform-ansible/deploy-pulsar.yaml +++ b/deployment/terraform-ansible/deploy-pulsar.yaml @@ -39,7 +39,7 @@ zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" service_url: "{{ pulsar_service_url }}" http_url: "{{ pulsar_web_url }}" - pulsar_version: "2.8.1" + pulsar_version: "2.9.2" - name: Download Pulsar binary package unarchive: src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index ce672387c061b..155b7da8a080d 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index a09bdfd322c82..ff4055cafd909 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 33f4c3c0ab61e..72afff04ad6ab 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 990455bd577db..5fa966d476187 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index 988705f44be2d..df9617f314ec9 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 0166e0bbe05f5..94fabef4d82be 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index f909fa855e4ed..a897b00b648cb 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index a5a0cd067ec26..9ee0334651e1c 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 2900155df6d06..3e51419304178 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 96a0b567594c1..85d189f87b5b5 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index bdcb39d5165f3..63525923b0003 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pom.xml b/pom.xml index 22afc5c9accc9..2a14a858dfe2f 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index bf0b444140cfc..c6ff0570e1bde 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 426e89b13b49e..660097361abad 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index d78bf0db6b443..77757b42fa6d8 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 569b910491d0b..289105f78f2f8 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 68efb88f3eb02..60b175c3d4ecd 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 304eb8ab3d8e8..6078062dc1e69 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 56afd8aa7cbe8..59cf04834338d 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 4063145668122..1f3346fd0da53 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index e1261ac0b3f8d..60840c9f37f44 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index f567b378adc49..faebb94d05622 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 7f49866e227a3..f6db9d39ee693 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index fa98349bb4559..2722b42911976 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index e7a890d652e60..c9ef5d87e4106 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index ae23cf1e1ddc1..40f5f8407b8d5 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index ce82078d557e9..020af46f5f855 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 73cc277ccd326..9f9528b58e80e 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 8e533d6b25331..97002332870b7 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index cae4afe2927dd..2397289149c1e 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index c6d261bfa4214..9252d603f32b4 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index ce192866f6645..089ac49c5d579 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 31706b7225b9a..42c823a7f8615 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index c10d161d5ada7..afbc14ec0eec1 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 14f574489aac8..6e42f63c66acc 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 180936d3e9d78..efbb8fb6229c4 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index c5f84bc17973a..ad37c3588e0fe 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 05f57fde7a960..254ed1437d129 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index f71052bdf62bd..b5c22b5cff2af 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index ef69d3ac92a3a..36591e6f2f194 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 45dcdf2f842ee..871a07a9658dc 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index f486f526b487e..bf01c32367513 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 4a1a7a4f679b6..2037b5d3f6194 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 6526a1d8bc5fb..30c8375a0c7b9 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index accf6a4d51ca8..676a983f20dc2 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 100e582fa7d7f..8bc57600a8759 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 9ccb68be14d42..8bc73937e2029 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 9bc8460e41a56..176d920631e2f 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 288583b2c16f1..6ce7e8c415c03 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 0f9b8a966da0f..b66f5bddd275b 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 84491460cd140..4c5b57d67fbbd 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 103c3c96efb1c..311197c9f2756 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index a93eea0682bcb..d4a711ed01186 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 1cb0867dd71b3..917981f55658c 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index ed00d32bbcc86..1a49c3eb32b6e 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 9e2bbe3af5ef7..5401ace104fc4 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index e44b0763b9976..c6c8c58f95e27 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 14744436d374e..0a0f831bef961 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 122c4f5d71f01..c963ae626c7e4 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 08203dca6c303..cde53aeb6359b 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 25e5248234e62..79b92b89999c2 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 0408076b0b54f..ad331e30847d1 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 836345e3935ca..38c8b53a6bd0b 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 9074b9144e10b..0a173aa8650e8 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 779c4ffe7d3b6..180e338099f7f 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 338bdfc678503..4a7a2a335e4b7 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 051902af15764..b259cbb1de580 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 388bcd662dfc5..cde101487ff3e 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index bc4f19c2225fb..2f3e178a9de09 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 2d4f3320bb7eb..d6dabf60355ba 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index f95f19af912fd..3fa60cd1807d7 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index c368fba85138a..b1fcbc010f996 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 1e2c66e7b6615..2db6fb2ef7d99 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 807baaab0b616..483a855523e08 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index b81637ee81b15..5f95ff1e50211 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 5166bad50e331..127c6f72ef6fa 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 4500bf9a5e92c..e1d65e87302bf 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 7518cf7794ba5..af7f025330897 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 24baed819b918..4df689dfff5d2 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index dfb80b7a47c9a..f3060d71166bc 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index fdfa4cb613a62..5d6166e13ffad 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 8cfc461fbab60..1678a9294dc89 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 957b26b4b3729..2bf4d0baf3452 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index cd83ffb411311..bdf1081106964 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index e3a96afb1eaf8..5974289444586 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 9a011ed133d42..e573833baba2d 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 0fb713cc0edb7..7291f3e88f29e 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 615d476296ea3..76fcf52c380fa 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 01e5a0aef1499..780e216c96484 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2-SNAPSHOT + 2.9.2 pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 56b3789056f99..809def86ed1dc 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 0566dc0ded57c..e688a104a16f2 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 79b0c8790ec2c..168397dacb44d 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 6d954a3560032..f6f776ce015a3 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index fa8bd92a1870c..39a783c27ac7e 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 0261117dafb87..31ec8436754ff 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 24cefa1c54a5b..e3f0ae3b51475 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2-SNAPSHOT + 2.9.2 pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index c7a81a2030100..b1f9486286769 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2-SNAPSHOT + 2.9.2 pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 7e81449d3c634..48376baa59270 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2-SNAPSHOT + 2.9.2 pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 15bc57d1f0c50..c31872944da0d 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 9a86ac77503f1..ddc76bebb24e9 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.2-SNAPSHOT + 2.9.2 pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index be2b9201669bf..b4ddf32ae19de 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.2-SNAPSHOT + 2.9.2 pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 0d0d9a0cfc227..e0b566a0c3434 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 03be75fe4846a..2fff29d1c0165 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index f1b5c8f3f6823..e2c29fe00ae0b 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 805f7f93f56d0..e52e5b7350f2c 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 267ba52fa2262..b18cd2eb75c9b 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.2-SNAPSHOT + 2.9.2 testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 3ed771e4d54f9..80f166bb0c575 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index fe187e0e822c9..9a61cb41d4f6e 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index a0f682a916159..559bc5fe6f658 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index d632fa4ff6df6..20e71e217ce44 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index be94489d99240..46558ca492aa2 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 9ec7712e5470b..c9e16e115f056 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2-SNAPSHOT + 2.9.2 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 02fad581b2a30..f2681751f3eda 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 48646fa6c82a8..1314c15dcb45c 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 integration diff --git a/tests/pom.xml b/tests/pom.xml index cfe68cd54ce61..512c1529b195f 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 08fd384d01d7f..a84660e326864 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 573c6cc2eadea..9bdd8fea40493 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 92560ecaf93cf..281cf4cd64c02 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2-SNAPSHOT + 2.9.2 pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 9dc296310cf8f..f839ad30810ee 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 4913bd894506f..716aed18b98f0 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.2-SNAPSHOT + 2.9.2 .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 2abd509b8c8fc..5b8ae94a8e323 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2-SNAPSHOT + 2.9.2 .. From 9d42eacdb88ffdaa034d13b5daf6434aa4024b46 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 21 Jan 2022 17:08:23 -0600 Subject: [PATCH 295/823] [Authorization] Return if namespace policies are read only (#12514) * [Authorization Provider] Return early when namespace policies are read only * Remove typo fix to simplify cherry-picking (cherry picked from commit f1e72d6154332ca69e278005243ae6bde4e8c554) --- .../pulsar/broker/authorization/PulsarAuthorizationProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index 61d77a7a8a0a9..cfb05ab1e2c39 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -286,6 +286,7 @@ public CompletableFuture grantPermissionAsync(NamespaceName namespaceName, validatePoliciesReadOnlyAccess(); } catch (Exception e) { result.completeExceptionally(e); + return result; } try { From 20cfa08580b4d31a9dbda50fb7fa0c398fea0bb4 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 24 Jan 2022 12:06:00 +0800 Subject: [PATCH 296/823] [C++] Fix hasMessageAvailable returns wrong value for last message (#13883) In C++ client, there is a corner case that when a reader's start message ID is the last message of a topic, `hasMessageAvailable` returns true. However, it should return false because the start message ID is exclusive and in this case `readNext` would never return a message unless new messages arrived. The current C++ implementation of `hasMessageAvailable` is from long days ago and has many problems. So this PR migrates the Java implementation of `hasMessageAvailable` to C++ client. Since after the modifications we need to access `startMessageId` in `hasMessageAvailable`, which is called in a different thread from `connectionOpened` that might modify `startMessageId`. We use a common mutex `mutexForMessageIds` to protect the access to `lastDequedMessageId_` and `lastMessageIdInBroker_`. To fix the original tests when `startMessageId` is latest, this PR adds a `GetLastMessageIdResponse` as the response of `GetLastMessageId` request. The `GetLastMessageIdResponse` contains the `consumer_mark_delete_position` introduced from https://github.com/apache/pulsar/pull/9652 to compare with `last_message_id` when `startMessageId` is latest. This change added tests `ReaderTest#testHasMessageAvailableWhenCreated` and `MessageIdTest# testCompareLedgerAndEntryId`. (cherry picked from commit e50493ea17dd5f2f9d4527d74cc4f40e12439df2) Fix the conflicts by - Remove new fields introduced from https://github.com/apache/pulsar/pull/13627 --- pulsar-client-cpp/lib/ClientConnection.cc | 23 ++-- pulsar-client-cpp/lib/ClientConnection.h | 5 +- pulsar-client-cpp/lib/ConsumerImpl.cc | 112 +++++++++++------- pulsar-client-cpp/lib/ConsumerImpl.h | 23 ++-- .../lib/GetLastMessageIdResponse.h | 56 +++++++++ pulsar-client-cpp/lib/MessageIdUtil.h | 44 +++++++ pulsar-client-cpp/lib/ReaderImpl.cc | 4 +- pulsar-client-cpp/tests/MessageIdTest.cc | 22 ++++ pulsar-client-cpp/tests/ReaderTest.cc | 45 +++++++ 9 files changed, 263 insertions(+), 71 deletions(-) create mode 100644 pulsar-client-cpp/lib/GetLastMessageIdResponse.h create mode 100644 pulsar-client-cpp/lib/MessageIdUtil.h diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 3ad6f4062f4b9..24cf11c120548 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -37,6 +37,7 @@ #include "ProducerImpl.h" #include "ConsumerImpl.h" #include "checksum/ChecksumProvider.h" +#include "MessageIdUtil.h" DECLARE_LOG_OBJECT() @@ -1072,7 +1073,7 @@ void ClientConnection::handleIncomingCommand() { PendingGetLastMessageIdRequestsMap::iterator it = pendingGetLastMessageIdRequests_.find(error.request_id()); if (it != pendingGetLastMessageIdRequests_.end()) { - Promise getLastMessageIdPromise = it->second; + auto getLastMessageIdPromise = it->second; pendingGetLastMessageIdRequests_.erase(it); lock.unlock(); @@ -1191,15 +1192,18 @@ void ClientConnection::handleIncomingCommand() { pendingGetLastMessageIdRequests_.find(getLastMessageIdResponse.request_id()); if (it != pendingGetLastMessageIdRequests_.end()) { - Promise getLastMessageIdPromise = it->second; + auto getLastMessageIdPromise = it->second; pendingGetLastMessageIdRequests_.erase(it); lock.unlock(); - MessageIdData messageIdData = getLastMessageIdResponse.last_message_id(); - MessageId messageId = MessageId(messageIdData.partition(), messageIdData.ledgerid(), - messageIdData.entryid(), messageIdData.batch_index()); - - getLastMessageIdPromise.setValue(messageId); + if (getLastMessageIdResponse.has_consumer_mark_delete_position()) { + getLastMessageIdPromise.setValue( + {toMessageId(getLastMessageIdResponse.last_message_id()), + toMessageId(getLastMessageIdResponse.consumer_mark_delete_position())}); + } else { + getLastMessageIdPromise.setValue( + {toMessageId(getLastMessageIdResponse.last_message_id())}); + } } else { lock.unlock(); LOG_WARN( @@ -1608,9 +1612,10 @@ Commands::ChecksumType ClientConnection::getChecksumType() const { return getServerProtocolVersion() >= proto::v6 ? Commands::Crc32c : Commands::None; } -Future ClientConnection::newGetLastMessageId(uint64_t consumerId, uint64_t requestId) { +Future ClientConnection::newGetLastMessageId(uint64_t consumerId, + uint64_t requestId) { Lock lock(mutex_); - Promise promise; + Promise promise; if (isClosed()) { lock.unlock(); LOG_ERROR(cnxString_ << " Client is not connected to the broker"); diff --git a/pulsar-client-cpp/lib/ClientConnection.h b/pulsar-client-cpp/lib/ClientConnection.h index 48e6d57a0a23c..7ca5f378f30e5 100644 --- a/pulsar-client-cpp/lib/ClientConnection.h +++ b/pulsar-client-cpp/lib/ClientConnection.h @@ -46,6 +46,7 @@ #include #include #include "lib/PeriodicTask.h" +#include "lib/GetLastMessageIdResponse.h" using namespace pulsar; @@ -156,7 +157,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this newConsumerStats(uint64_t consumerId, uint64_t requestId); - Future newGetLastMessageId(uint64_t consumerId, uint64_t requestId); + Future newGetLastMessageId(uint64_t consumerId, uint64_t requestId); Future newGetTopicsOfNamespace(const std::string& nsName, uint64_t requestId); @@ -306,7 +307,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this> PendingConsumerStatsMap; PendingConsumerStatsMap pendingConsumerStatsMap_; - typedef std::map> PendingGetLastMessageIdRequestsMap; + typedef std::map> PendingGetLastMessageIdRequestsMap; PendingGetLastMessageIdRequestsMap pendingGetLastMessageIdRequests_; typedef std::map> PendingGetNamespaceTopicsMap; diff --git a/pulsar-client-cpp/lib/ConsumerImpl.cc b/pulsar-client-cpp/lib/ConsumerImpl.cc index 77c0fa9d52a94..84583d521cc3f 100644 --- a/pulsar-client-cpp/lib/ConsumerImpl.cc +++ b/pulsar-client-cpp/lib/ConsumerImpl.cc @@ -25,6 +25,7 @@ #include "pulsar/Result.h" #include "pulsar/MessageId.h" #include "Utils.h" +#include "MessageIdUtil.h" #include "AckGroupingTracker.h" #include "AckGroupingTrackerEnabled.h" #include "AckGroupingTrackerDisabled.h" @@ -51,7 +52,6 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, hasParent_(hasParent), consumerTopicType_(consumerTopicType), subscriptionMode_(subscriptionMode), - startMessageId_(startMessageId), // This is the initial capacity of the queue incomingMessages_(std::max(config_.getReceiverQueueSize(), 1)), availablePermits_(0), @@ -63,7 +63,7 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, negativeAcksTracker_(client, *this, conf), ackGroupingTrackerPtr_(std::make_shared()), readCompacted_(conf.isReadCompacted()), - lastMessageInBroker_(Optional::of(MessageId())) { + startMessageId_(startMessageId) { std::stringstream consumerStrStream; consumerStrStream << "[" << topic_ << ", " << subscription_ << ", " << consumerId_ << "] "; consumerStr_ = consumerStrStream.str(); @@ -159,8 +159,9 @@ void ConsumerImpl::start() { void ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) { Lock lock(mutex_); - if (state_ == Closed) { - lock.unlock(); + const auto state = state_; + lock.unlock(); + if (state == Closed) { LOG_DEBUG(getName() << "connectionOpened : Consumer is already closed"); return; } @@ -169,23 +170,24 @@ void ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) { // sending the subscribe request. cnx->registerConsumer(consumerId_, shared_from_this()); + Lock lockForMessageId(mutexForMessageId_); Optional firstMessageInQueue = clearReceiveQueue(); - unAckedMessageTrackerPtr_->clear(); - batchAcknowledgementTracker_.clear(); - if (subscriptionMode_ == Commands::SubscriptionModeNonDurable) { // Update startMessageId so that we can discard messages after delivery // restarts startMessageId_ = firstMessageInQueue; } + const auto startMessageId = startMessageId_; + lockForMessageId.unlock(); - lock.unlock(); + unAckedMessageTrackerPtr_->clear(); + batchAcknowledgementTracker_.clear(); ClientImplPtr client = client_.lock(); uint64_t requestId = client->newRequestId(); SharedBuffer cmd = Commands::newSubscribe( topic_, subscription_, consumerId_, requestId, getSubType(), consumerName_, subscriptionMode_, - startMessageId_, readCompacted_, config_.getProperties(), config_.getSchema(), getInitialPosition(), + startMessageId, readCompacted_, config_.getProperties(), config_.getSchema(), getInitialPosition(), config_.isReplicateSubscriptionStateEnabled(), config_.getKeySharedPolicy(), config_.getPriorityLevel()); cnx->sendRequestWithId(cmd, requestId) @@ -440,6 +442,9 @@ uint32_t ConsumerImpl::receiveIndividualMessagesFromBatch(const ClientConnection batchAcknowledgementTracker_.receivedMessage(batchedMessage); LOG_DEBUG("Received Batch messages of size - " << batchSize << " -- msgId: " << batchedMessage.getMessageId()); + Lock lock(mutexForMessageId_); + const auto startMessageId = startMessageId_; + lock.unlock(); int skippedMessages = 0; @@ -449,14 +454,14 @@ uint32_t ConsumerImpl::receiveIndividualMessagesFromBatch(const ClientConnection msg.impl_->setRedeliveryCount(redeliveryCount); msg.impl_->setTopicName(batchedMessage.getTopicName()); - if (startMessageId_.is_present()) { + if (startMessageId.is_present()) { const MessageId& msgId = msg.getMessageId(); // If we are receiving a batch message, we need to discard messages that were prior // to the startMessageId - if (msgId.ledgerId() == startMessageId_.value().ledgerId() && - msgId.entryId() == startMessageId_.value().entryId() && - msgId.batchIndex() <= startMessageId_.value().batchIndex()) { + if (msgId.ledgerId() == startMessageId.value().ledgerId() && + msgId.entryId() == startMessageId.value().entryId() && + msgId.batchIndex() <= startMessageId.value().batchIndex()) { LOG_DEBUG(getName() << "Ignoring message from before the startMessageId" << msg.getMessageId()); ++skippedMessages; @@ -587,7 +592,7 @@ void ConsumerImpl::internalListener() { trackMessage(msg); try { consumerStatsBasePtr_->receivedMessage(msg, ResultOk); - lastDequedMessage_ = Optional::of(msg.getMessageId()); + lastDequedMessageId_ = msg.getMessageId(); messageListener_(Consumer(shared_from_this()), msg); } catch (const std::exception& e) { LOG_ERROR(getName() << "Exception thrown from listener" << e.what()); @@ -721,8 +726,9 @@ Result ConsumerImpl::receiveHelper(Message& msg, int timeout) { } void ConsumerImpl::messageProcessed(Message& msg, bool track) { - Lock lock(mutex_); - lastDequedMessage_ = Optional::of(msg.getMessageId()); + Lock lock(mutexForMessageId_); + lastDequedMessageId_ = msg.getMessageId(); + lock.unlock(); ClientConnectionPtr currentCnx = getCnx().lock(); if (currentCnx && msg.impl_->cnx_ != currentCnx.get()) { @@ -754,11 +760,11 @@ Optional ConsumerImpl::clearReceiveQueue() { previousMessageId = MessageId(-1, nextMessageId.ledgerId(), nextMessageId.entryId() - 1, -1); } return Optional::of(previousMessageId); - } else if (lastDequedMessage_.is_present()) { + } else if (lastDequedMessageId_ != MessageId::earliest()) { // If the queue was empty we need to restart from the message just after the last one that has been // dequeued // in the past - return lastDequedMessage_; + return Optional::of(lastDequedMessageId_); } else { // No message was received or dequeued by this consumer. Next message would still be the // startMessageId @@ -1094,6 +1100,9 @@ void ConsumerImpl::brokerConsumerStatsListener(Result res, BrokerConsumerStatsIm void ConsumerImpl::handleSeek(Result result, ResultCallback callback) { if (result == ResultOk) { + Lock lock(mutexForMessageId_); + lastDequedMessageId_ = MessageId::earliest(); + lock.unlock(); LOG_INFO(getName() << "Seek successfully"); } else { LOG_ERROR(getName() << "Failed to seek: " << strResult(result)); @@ -1168,37 +1177,42 @@ void ConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { bool ConsumerImpl::isReadCompacted() { return readCompacted_; } +inline bool hasMoreMessages(const MessageId& lastMessageIdInBroker, const MessageId& messageId) { + return lastMessageIdInBroker > messageId && lastMessageIdInBroker.entryId() != -1; +} + void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback) { - MessageId lastDequed = this->lastMessageIdDequed(); - MessageId lastInBroker = this->lastMessageIdInBroker(); - if (lastInBroker > lastDequed && lastInBroker.entryId() != -1) { - callback(ResultOk, true); - return; - } + Lock lock(mutexForMessageId_); + const auto messageId = + (lastDequedMessageId_ == MessageId::earliest()) ? startMessageId_.value() : lastDequedMessageId_; - getLastMessageIdAsync([lastDequed, callback](Result result, MessageId messageId) { - if (result == ResultOk) { - if (messageId > lastDequed && messageId.entryId() != -1) { - callback(ResultOk, true); + if (messageId == MessageId::latest()) { + lock.unlock(); + getLastMessageIdAsync([callback](Result result, const GetLastMessageIdResponse& response) { + if (result != ResultOk) { + callback(result, {}); + return; + } + if (response.hasMarkDeletePosition() && response.getLastMessageId().entryId() >= 0) { + // We only care about comparing ledger ids and entry ids as mark delete position doesn't have + // other ids such as batch index + callback(ResultOk, compareLedgerAndEntryId(response.getMarkDeletePosition(), + response.getLastMessageId()) < 0); } else { callback(ResultOk, false); } - } else { - callback(result, false); - } - }); -} - -void ConsumerImpl::brokerGetLastMessageIdListener(Result res, MessageId messageId, - BrokerGetLastMessageIdCallback callback) { - Lock lock(mutex_); - if (messageId > lastMessageIdInBroker()) { - lastMessageInBroker_ = Optional::of(messageId); - lock.unlock(); - callback(res, messageId); + }); } else { + if (hasMoreMessages(lastMessageIdInBroker_, messageId)) { + lock.unlock(); + callback(ResultOk, true); + return; + } lock.unlock(); - callback(res, lastMessageIdInBroker()); + + getLastMessageIdAsync([callback, messageId](Result result, const GetLastMessageIdResponse& response) { + callback(result, (result == ResultOk) && hasMoreMessages(response.getLastMessageId(), messageId)); + }); } } @@ -1222,9 +1236,19 @@ void ConsumerImpl::getLastMessageIdAsync(BrokerGetLastMessageIdCallback callback LOG_DEBUG(getName() << " Sending getLastMessageId Command for Consumer - " << getConsumerId() << ", requestId - " << requestId); + auto self = shared_from_this(); cnx->newGetLastMessageId(consumerId_, requestId) - .addListener(std::bind(&ConsumerImpl::brokerGetLastMessageIdListener, shared_from_this(), - std::placeholders::_1, std::placeholders::_2, callback)); + .addListener([this, self, callback](Result result, const GetLastMessageIdResponse& response) { + if (result == ResultOk) { + LOG_DEBUG(getName() << "getLastMessageId: " << response); + Lock lock(mutexForMessageId_); + lastMessageIdInBroker_ = response.getLastMessageId(); + lock.unlock(); + } else { + LOG_ERROR(getName() << "Failed to getLastMessageId: " << result); + } + callback(result, response); + }); } else { LOG_ERROR(getName() << " Operation not supported since server protobuf version " << cnx->getServerProtocolVersion() << " is older than proto::v12"); diff --git a/pulsar-client-cpp/lib/ConsumerImpl.h b/pulsar-client-cpp/lib/ConsumerImpl.h index 0754a89fdf5fc..80d96d039156d 100644 --- a/pulsar-client-cpp/lib/ConsumerImpl.h +++ b/pulsar-client-cpp/lib/ConsumerImpl.h @@ -33,6 +33,7 @@ #include "lib/UnAckedMessageTrackerDisabled.h" #include "MessageCrypto.h" #include "AckGroupingTracker.h" +#include "GetLastMessageIdResponse.h" #include "CompressionCodec.h" #include @@ -53,7 +54,7 @@ class ExecutorService; class ConsumerImpl; class BatchAcknowledgementTracker; typedef std::shared_ptr MessageCryptoPtr; -typedef std::function BrokerGetLastMessageIdCallback; +typedef std::function BrokerGetLastMessageIdCallback; enum ConsumerTopicType { @@ -191,10 +192,8 @@ class ConsumerImpl : public ConsumerImplBase, bool hasParent_; ConsumerTopicType consumerTopicType_; - Commands::SubscriptionMode subscriptionMode_; - Optional startMessageId_; + const Commands::SubscriptionMode subscriptionMode_; - Optional lastDequedMessage_; UnboundedBlockingQueue incomingMessages_; std::queue pendingReceives_; std::atomic_int availablePermits_; @@ -215,17 +214,11 @@ class ConsumerImpl : public ConsumerImplBase, MessageCryptoPtr msgCrypto_; const bool readCompacted_; - Optional lastMessageInBroker_; - void brokerGetLastMessageIdListener(Result res, MessageId messageId, - BrokerGetLastMessageIdCallback callback); - - const MessageId& lastMessageIdDequed() { - return lastDequedMessage_.is_present() ? lastDequedMessage_.value() : MessageId::earliest(); - } - - const MessageId& lastMessageIdInBroker() { - return lastMessageInBroker_.is_present() ? lastMessageInBroker_.value() : MessageId::earliest(); - } + // Make the access to `startMessageId_`, `lastDequedMessageId_` and `lastMessageIdInBroker_` thread safe + mutable std::mutex mutexForMessageId_; + Optional startMessageId_; + MessageId lastDequedMessageId_{MessageId::earliest()}; + MessageId lastMessageIdInBroker_{MessageId::earliest()}; friend class PulsarFriend; diff --git a/pulsar-client-cpp/lib/GetLastMessageIdResponse.h b/pulsar-client-cpp/lib/GetLastMessageIdResponse.h new file mode 100644 index 0000000000000..0acb78394e115 --- /dev/null +++ b/pulsar-client-cpp/lib/GetLastMessageIdResponse.h @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +namespace pulsar { + +class GetLastMessageIdResponse { + friend std::ostream& operator<<(std::ostream& os, const GetLastMessageIdResponse& response) { + os << "lastMessageId: " << response.lastMessageId_; + if (response.hasMarkDeletePosition_) { + os << ", markDeletePosition: " << response.markDeletePosition_; + } + return os; + } + + public: + GetLastMessageIdResponse() = default; + + GetLastMessageIdResponse(const MessageId& lastMessageId) + : lastMessageId_(lastMessageId), hasMarkDeletePosition_{false} {} + + GetLastMessageIdResponse(const MessageId& lastMessageId, const MessageId& markDeletePosition) + : lastMessageId_(lastMessageId), + markDeletePosition_(markDeletePosition), + hasMarkDeletePosition_(true) {} + + const MessageId& getLastMessageId() const noexcept { return lastMessageId_; } + const MessageId& getMarkDeletePosition() const noexcept { return markDeletePosition_; } + bool hasMarkDeletePosition() const noexcept { return hasMarkDeletePosition_; } + + private: + MessageId lastMessageId_; + MessageId markDeletePosition_; + bool hasMarkDeletePosition_; +}; + +} // namespace pulsar diff --git a/pulsar-client-cpp/lib/MessageIdUtil.h b/pulsar-client-cpp/lib/MessageIdUtil.h new file mode 100644 index 0000000000000..d6f80a10ea015 --- /dev/null +++ b/pulsar-client-cpp/lib/MessageIdUtil.h @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include "PulsarApi.pb.h" + +namespace pulsar { + +inline MessageId toMessageId(const proto::MessageIdData& messageIdData) { + return MessageId{messageIdData.partition(), static_cast(messageIdData.ledgerid()), + static_cast(messageIdData.entryid()), messageIdData.batch_index()}; +} + +namespace internal { +template +static int compare(T lhs, T rhs) { + return (lhs < rhs) ? -1 : ((lhs == rhs) ? 0 : 1); +} +} // namespace internal + +inline int compareLedgerAndEntryId(const MessageId& lhs, const MessageId& rhs) { + auto result = internal::compare(lhs.ledgerId(), rhs.ledgerId()); + if (result != 0) { + return result; + } + return internal::compare(lhs.entryId(), rhs.entryId()); +} + +} // namespace pulsar diff --git a/pulsar-client-cpp/lib/ReaderImpl.cc b/pulsar-client-cpp/lib/ReaderImpl.cc index 48f5d5866f453..0a7b3215437bf 100644 --- a/pulsar-client-cpp/lib/ReaderImpl.cc +++ b/pulsar-client-cpp/lib/ReaderImpl.cc @@ -139,7 +139,9 @@ void ReaderImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { } void ReaderImpl::getLastMessageIdAsync(GetLastMessageIdCallback callback) { - consumer_->getLastMessageIdAsync(callback); + consumer_->getLastMessageIdAsync([callback](Result result, const GetLastMessageIdResponse& response) { + callback(result, response.getLastMessageId()); + }); } ReaderImplWeakPtr ReaderImpl::getReaderImplWeakPtr() { return readerImplWeakPtr_; } diff --git a/pulsar-client-cpp/tests/MessageIdTest.cc b/pulsar-client-cpp/tests/MessageIdTest.cc index 06c25283794aa..55fa181da05f4 100644 --- a/pulsar-client-cpp/tests/MessageIdTest.cc +++ b/pulsar-client-cpp/tests/MessageIdTest.cc @@ -17,6 +17,7 @@ * under the License. */ #include +#include "lib/MessageIdUtil.h" #include "PulsarFriend.h" #include @@ -35,3 +36,24 @@ TEST(MessageIdTest, testSerialization) { ASSERT_EQ(msgId, deserialized); } + +TEST(MessageIdTest, testCompareLedgerAndEntryId) { + MessageId id1(-1, 2L, 1L, 0); + MessageId id2(-1, 2L, 1L, 1); + MessageId id3(-1, 2L, 2L, 0); + MessageId id4(-1, 3L, 0L, 0); + ASSERT_EQ(compareLedgerAndEntryId(id1, id2), 0); + ASSERT_EQ(compareLedgerAndEntryId(id1, id2), 0); + + ASSERT_EQ(compareLedgerAndEntryId(id1, id3), -1); + ASSERT_EQ(compareLedgerAndEntryId(id3, id1), 1); + + ASSERT_EQ(compareLedgerAndEntryId(id1, id4), -1); + ASSERT_EQ(compareLedgerAndEntryId(id4, id1), 1); + + ASSERT_EQ(compareLedgerAndEntryId(id2, id4), -1); + ASSERT_EQ(compareLedgerAndEntryId(id4, id2), 1); + + ASSERT_EQ(compareLedgerAndEntryId(id3, id4), -1); + ASSERT_EQ(compareLedgerAndEntryId(id4, id3), 1); +} diff --git a/pulsar-client-cpp/tests/ReaderTest.cc b/pulsar-client-cpp/tests/ReaderTest.cc index d95038b3cef3c..8cd535caf7f5e 100644 --- a/pulsar-client-cpp/tests/ReaderTest.cc +++ b/pulsar-client-cpp/tests/ReaderTest.cc @@ -26,6 +26,7 @@ #include #include +#include #include DECLARE_LOG_OBJECT() @@ -577,3 +578,47 @@ TEST(ReaderTest, testIsConnected) { ASSERT_EQ(ResultOk, reader.close()); ASSERT_FALSE(reader.isConnected()); } + +TEST(ReaderTest, testHasMessageAvailableWhenCreated) { + const std::string topic = "testHasMessageAvailableWhenCreated-" + std::to_string(time(nullptr)); + Client client(serviceUrl); + + ProducerConfiguration producerConf; + producerConf.setBatchingMaxMessages(3); + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, producerConf, producer)); + + std::vector messageIds; + constexpr int numMessages = 7; + Latch latch(numMessages); + for (int i = 0; i < numMessages; i++) { + producer.sendAsync(MessageBuilder().setContent("msg-" + std::to_string(i)).build(), + [i, &messageIds, &latch](Result result, const MessageId& messageId) { + if (result == ResultOk) { + LOG_INFO("Send " << i << " to " << messageId); + messageIds.emplace_back(messageId); + } else { + LOG_ERROR("Failed to send " << i << ": " << messageId); + } + latch.countdown(); + }); + } + latch.wait(std::chrono::seconds(3)); + ASSERT_EQ(messageIds.size(), numMessages); + + Reader reader; + bool hasMessageAvailable; + + for (size_t i = 0; i < messageIds.size() - 1; i++) { + ASSERT_EQ(ResultOk, client.createReader(topic, messageIds[i], {}, reader)); + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); + EXPECT_TRUE(hasMessageAvailable); + } + + // The start message ID is exclusive by default, so when we start at the last message, there should be no + // message available. + ASSERT_EQ(ResultOk, client.createReader(topic, messageIds.back(), {}, reader)); + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); + EXPECT_FALSE(hasMessageAvailable); + client.close(); +} From affd0476866abd6a50ae41bc4ee8a661f829d68d Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Sat, 29 Jan 2022 01:43:53 +0800 Subject: [PATCH 297/823] Fixes NPE - ``ReplicatedSubscriptionsController`` send marker message when enable deduplicated. (#14017) * Fixes marker message cause NPE. * Fixes marker message cause NPE. --- .../persistent/MessageDeduplication.java | 7 +- .../BrokerMessageDeduplicationTest.java | 73 +++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index e2436bb2a10c5..9913ddf91cf8f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -79,7 +79,8 @@ enum Status { Failed, } - enum MessageDupStatus { + @VisibleForTesting + public enum MessageDupStatus { // whether a message is a definitely a duplicate or not cannot be determined at this time Unknown, // message is definitely NOT a duplicate @@ -312,7 +313,7 @@ public boolean isEnabled() { * @return true if the message should be published or false if it was recognized as a duplicate */ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf headersAndPayload) { - if (!isEnabled()) { + if (!isEnabled() || publishContext.isMarkerMessage()) { return MessageDupStatus.NotDup; } @@ -365,7 +366,7 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade * Call this method whenever a message is persisted to get the chance to trigger a snapshot. */ public void recordMessagePersisted(PublishContext publishContext, PositionImpl position) { - if (!isEnabled()) { + if (!isEnabled() || publishContext.isMarkerMessage()) { return; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java new file mode 100644 index 0000000000000..f2d7a21a40f0b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.MessageDeduplication; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.testng.annotations.Test; + +public class BrokerMessageDeduplicationTest { + + @Test + public void markerMessageNotDeduplicated() { + PulsarService pulsarService = mock(PulsarService.class); + ServiceConfiguration configuration = new ServiceConfiguration(); + doReturn(configuration).when(pulsarService).getConfiguration(); + MessageDeduplication deduplication = spy(new MessageDeduplication(pulsarService, + mock(PersistentTopic.class), mock(ManagedLedger.class))); + doReturn(true).when(deduplication).isEnabled(); + Topic.PublishContext context = mock(Topic.PublishContext.class); + doReturn(true).when(context).isMarkerMessage(); + MessageDeduplication.MessageDupStatus status = deduplication.isDuplicate(context, null); + assertEquals(status, MessageDeduplication.MessageDupStatus.NotDup); + } + + @Test + public void markerMessageNotRecordPersistent() { + PulsarService pulsarService = mock(PulsarService.class); + ServiceConfiguration configuration = new ServiceConfiguration(); + doReturn(configuration).when(pulsarService).getConfiguration(); + MessageDeduplication deduplication = spy(new MessageDeduplication(pulsarService, + mock(PersistentTopic.class), mock(ManagedLedger.class))); + doReturn(true).when(deduplication).isEnabled(); + Topic.PublishContext context = mock(Topic.PublishContext.class); + // marker message don't record message persisted. + doReturn(true).when(context).isMarkerMessage(); + deduplication.recordMessagePersisted(context, null); + + // if is not a marker message, we will get NPE. because context is mocked with null value fields. + doReturn(false).when(context).isMarkerMessage(); + try { + deduplication.recordMessagePersisted(context, null); + fail(); + } catch (Exception npe) { + assertTrue(npe instanceof NullPointerException); + } + } + + +} From 32a9d52409a330e10002c6c4d45d1f88fcac0ee6 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Tue, 25 Jan 2022 04:59:30 -0800 Subject: [PATCH 298/823] [pulsar-broker] clean up active consumer on already closed connection (#13196) ### Motivation We have been frequently seeing an issue with an exclusive subscription where the broker doesn't clean up consumers on closed on connection and because of that exclusive consumer keeps failing with `ConsumerBusyException` even though none of the consumers is actually connected to the broker. Below log example shows that connection is closed but still broker couldn't clean up consumer in race condition when consumer quickly disconnects after creating the subscription and subsequent consumer creation requests are keep failing. ``` 23:30:16.896 [pulsar-io-23-42] INFO org.apache.pulsar.broker.service.ServerCnx - [/10.11.12.13:60851] Subscribing on topic persistent://my-prop/my-cluster/ns/topic / sub1 : 23:30:17.223 [pulsar-io-23-42] INFO org.apache.pulsar.broker.service.ServerCnx - [/10.11.12.13:60851] Closed consumer, consumerId=20 : 23:30:17.291 [bookkeeper-ml-workers-OrderedExecutor-1-0] INFO org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://my-prop/my-cluster/ns/topic][sub1] Created new subscription for 25 : 23:30:17.301 [pulsar-io-23-42] INFO org.apache.pulsar.broker.service.ServerCnx - Closed connection from /10.11.12.13:60851 : 23:30:17.302 [pulsar-io-23-42] INFO org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer - Removing consumer Consumer{subscription=PersistentSubscription{topic=persistent://my-prop/my-cluster/ns/topic, name=sub1}, consumerId=21, consumerName=c2302, address=/10.11.12.13:60851} : 23:30:17.302 [bookkeeper-ml-workers-OrderedExecutor-1-0] INFO org.apache.pulsar.broker.service.ServerCnx - [/10.11.12.13:60851] Created subscription on topic persistent://my-prop/my-cluster/ns/topic / sub1 : : : 23:30:17.496 [pulsar-io-23-36] WARN org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://my-prop/my-cluster/ns/topic][sub1] Consumer 25 7a977 already connected 23:30:17.885 [pulsar-io-23-36] INFO org.apache.pulsar.broker.service.ServerCnx - [/10.11.12.13:60853] Subscribing on topic persistent://my-prop/my-cluster/ns/topic / sub1 23:30:17.886 [pulsar-io-23-36] WARN org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://my-prop/my-cluster/ns/topic][sub1] Consumer 25 7a977 already connected 23:30:18.637 [pulsar-io-23-36] INFO org.apache.pulsar.broker.service.ServerCnx - [/10.11.12.13:60853] Subscribing on topic persistent://my-prop/my-cluster/ns/topic / sub1 ``` ### Modification Broker should clean up consumer if connection is already closed and allow consumer to reconnect as an active consumer. (cherry picked from commit 496afa7bd583c4b88a9fdc89a101be462e5c856f) --- .../pulsar/broker/service/AbstractTopic.java | 8 +++ .../service/persistent/PersistentTopic.java | 10 ++++ .../impl/BrokerClientIntegrationTest.java | 54 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index ba00a20ee6608..183fb05acb2d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -250,6 +250,14 @@ protected CompletableFuture addConsumerToSubscription(Subscription subscri return subscription.addConsumer(consumer); } + protected Consumer getActiveConsumer(Subscription subscription) { + Dispatcher dispatcher = subscription.getDispatcher(); + if (dispatcher instanceof AbstractDispatcherSingleActiveConsumer) { + return ((AbstractDispatcherSingleActiveConsumer) dispatcher).getActiveConsumer(); + } + return null; + } + @Override public void disableCnxAutoRead() { producers.values().forEach(producer -> producer.getCnx().disableCnxAutoRead()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 3c08bd9ea10f7..00934ad13a2d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -794,6 +794,16 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs if (ex.getCause() instanceof ConsumerBusyException) { log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName); + Consumer consumer = null; + try { + consumer = subscriptionFuture.isDone() ? getActiveConsumer(subscriptionFuture.get()) : null; + // cleanup consumer if connection is already closed + if (consumer != null && !consumer.cnx().isActive()) { + consumer.close(); + } + } catch (Exception be) { + log.error("Failed to clean up consumer on closed connection {}, {}", consumer, be.getMessage()); + } } else if (ex.getCause() instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, ex.getMessage()); } else { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java index 8e385a62b1b9b..2be6067c7b7fc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java @@ -69,6 +69,8 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.OwnershipCache; import org.apache.pulsar.broker.resources.BaseResources; +import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer; +import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -1005,4 +1007,56 @@ public void testConsumerWithPooledMessagesWithReader(boolean isBatchingEnabled) reader.close(); producer.close(); } + + @Test + public void testActiveConsumerCleanup() throws Exception { + log.info("-- Starting {} test --", methodName); + + int numMessages = 100; + final CountDownLatch latch = new CountDownLatch(numMessages); + String topic = "persistent://my-property/my-ns/closed-cnx-topic"; + String sub = "my-subscriber-name"; + + PulsarClient pulsarClient = newPulsarClient(lookupUrl.toString(), 0); + pulsarClient.newConsumer().topic(topic).subscriptionName(sub).messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + c1.acknowledgeAsync(msg); + latch.countDown(); + }).subscribe(); + + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + + AbstractDispatcherSingleActiveConsumer dispatcher = (AbstractDispatcherSingleActiveConsumer) topicRef + .getSubscription(sub).getDispatcher(); + ServerCnx cnx = (ServerCnx) dispatcher.getActiveConsumer().cnx(); + Field field = ServerCnx.class.getDeclaredField("isActive"); + field.setAccessible(true); + field.set(cnx, false); + + assertNotNull(dispatcher.getActiveConsumer()); + + pulsarClient = newPulsarClient(lookupUrl.toString(), 0); + Consumer consumer = null; + for (int i = 0; i < 2; i++) { + try { + consumer = pulsarClient.newConsumer().topic(topic).subscriptionName(sub).messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + c1.acknowledgeAsync(msg); + latch.countDown(); + }).subscribe(); + if (i == 0) { + fail("Should failed with ConsumerBusyException!"); + } + } catch (PulsarClientException.ConsumerBusyException ignore) { + // It's ok. + } + } + assertNotNull(consumer); + log.info("-- Exiting {} test --", methodName); + } + } From be26fadea0576d29e4098721c6219bf8dc404392 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 26 Jan 2022 16:05:16 +0800 Subject: [PATCH 299/823] Fix batch message ack does not decrease the unacked-msg count. (#13383) (cherry picked from commit d64f2916e0feee1b09459b4db6094db1143067a1) --- .../pulsar/broker/service/Consumer.java | 129 ++++++++++++++++-- .../broker/service/BatchMessageTest.java | 2 +- .../BatchMessageWithBatchIndexLevelTest.java | 108 +++++++++++++++ ...sistentDispatcherFailoverConsumerTest.java | 79 +++++------ 4 files changed, 265 insertions(+), 53 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 9f4214481588c..49b5704eb8ed0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -24,6 +24,7 @@ import io.netty.util.concurrent.Future; import io.netty.util.concurrent.Promise; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.List; import java.util.Map; @@ -56,6 +57,7 @@ import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,6 +128,7 @@ public class Consumer { private PositionImpl readPositionWhenJoining; private final String clientAddress; // IP address only, no port number included private final MessageId startMessageId; + private final boolean isAcknowledgmentAtBatchIndexLevelEnabled; public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, @@ -182,6 +185,8 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo } this.clientAddress = cnx.clientSourceAddress(); + this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() + .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); } public SubType subType() { @@ -233,7 +238,7 @@ public Future sendMessages(final List entries, EntryBatchSizes batc writePromise.setSuccess(null); return writePromise; } - + int unackedMessages = totalMessages; // Note // Must ensure that the message is written to the pendingAcks before sent is first , because this consumer // is possible to disconnect at this time. @@ -243,11 +248,16 @@ public Future sendMessages(final List entries, EntryBatchSizes batc if (entry != null) { int batchSize = batchSizes.getBatchSize(i); int stickyKeyHash = getStickyKeyHash(entry); + long[] ackSet = getCursorAckSet(PositionImpl.get(entry.getLedgerId(), entry.getEntryId())); + if (ackSet != null) { + unackedMessages -= (batchSize - BitSet.valueOf(ackSet).cardinality()); + } pendingAcks.put(entry.getLedgerId(), entry.getEntryId(), batchSize, stickyKeyHash); - if (log.isDebugEnabled()){ + if (log.isDebugEnabled()) { log.debug("[{}-{}] Added {}:{} ledger entry with batchSize of {} to pendingAcks in" + " broker.service.Consumer for consumerId: {}", - topicName, subscription, entry.getLedgerId(), entry.getEntryId(), batchSize, consumerId); + topicName, subscription, entry.getLedgerId(), entry.getEntryId(), batchSize, + consumerId); } } } @@ -267,7 +277,7 @@ public Future sendMessages(final List entries, EntryBatchSizes batc + " for consumerId: {}; avgMessagesPerEntry is {}", topicName, subscription, ackedCount, totalMessages, consumerId, tmpAvgMessagesPerEntry); } - incrementUnackedMessages(totalMessages); + incrementUnackedMessages(unackedMessages); msgOut.recordMultipleEvents(totalMessages, totalBytes); msgOutCounter.add(totalMessages); bytesOutCounter.add(totalBytes); @@ -381,16 +391,19 @@ public CompletableFuture messageAcked(CommandAck ack) { //this method is for individual ack not carry the transaction private CompletableFuture individualAckNormal(CommandAck ack, Map properties) { List positionsAcked = new ArrayList<>(); - for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); PositionImpl position; + long ackedCount = 0; + long batchSize = getBatchSize(msgId); + Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId()); if (msgId.getAckSetsCount() > 0) { long[] ackSets = new long[msgId.getAckSetsCount()]; for (int j = 0; j < msgId.getAckSetsCount(); j++) { ackSets[j] = msgId.getAckSetAt(j); } position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets); + ackedCount = getAckedCountForBatchIndexLevelEnabled(position, batchSize, ackSets); if (isTransactionEnabled()) { //sync the batch position bit set point, in order to delete the position in pending acks if (Subscription.isIndividualAckMode(subType)) { @@ -400,7 +413,20 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map individualAckNormal(CommandAck ack, Map individualAckWithTransaction(CommandAck ack) { // Individual ack List> positionsAcked = new ArrayList<>(); - if (!isTransactionEnabled()) { return FutureUtil.failedFuture( new BrokerServiceException.NotAllowedException("Server don't support transaction ack!")); @@ -476,6 +501,56 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { return completableFuture; } + private long getBatchSize(MessageIdData msgId) { + long batchSize = 1; + if (Subscription.isIndividualAckMode(subType)) { + LongPair longPair = pendingAcks.get(msgId.getLedgerId(), msgId.getEntryId()); + // Consumer may ack the msg that not belongs to it. + if (longPair == null) { + Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId()); + longPair = ackOwnerConsumer.getPendingAcks().get(msgId.getLedgerId(), msgId.getEntryId()); + if (longPair != null) { + batchSize = longPair.first; + } + } else { + batchSize = longPair.first; + } + } + return batchSize; + } + + private long getAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize, long[] ackSets) { + long ackedCount = 0; + if (isAcknowledgmentAtBatchIndexLevelEnabled) { + long[] cursorAckSet = getCursorAckSet(position); + if (cursorAckSet != null) { + BitSetRecyclable cursorBitSet = BitSetRecyclable.create().resetWords(cursorAckSet); + int lastCardinality = cursorBitSet.cardinality(); + BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(ackSets); + cursorBitSet.and(givenBitSet); + givenBitSet.recycle(); + int currentCardinality = cursorBitSet.cardinality(); + ackedCount = lastCardinality - currentCardinality; + cursorBitSet.recycle(); + } else if (pendingAcks.get(position.getLedgerId(), position.getEntryId()) != null) { + ackedCount = batchSize - BitSet.valueOf(ackSets).cardinality(); + } + } + return ackedCount; + } + + private long getUnAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize) { + long unAckedCount = batchSize; + if (isAcknowledgmentAtBatchIndexLevelEnabled) { + long[] cursorAckSet = getCursorAckSet(position); + if (cursorAckSet != null) { + BitSetRecyclable cursorBitSet = BitSetRecyclable.create().resetWords(cursorAckSet); + unAckedCount = cursorBitSet.cardinality(); + } + } + return unAckedCount; + } + private void checkAckValidationError(CommandAck ack, PositionImpl position) { if (ack.hasValidationError()) { log.error("[{}] [{}] Received ack for corrupted message at {} - Reason: {}", subscription, @@ -489,6 +564,26 @@ private void checkCanRemovePendingAcksAndHandle(PositionImpl position, MessageId } } + private Consumer getAckOwnerConsumer(long ledgerId, long entryId) { + Consumer ackOwnerConsumer = this; + if (Subscription.isIndividualAckMode(subType)) { + for (Consumer consumer : subscription.getConsumers()) { + if (consumer != this && consumer.getPendingAcks().containsKey(ledgerId, entryId)) { + ackOwnerConsumer = consumer; + break; + } + } + } + return ackOwnerConsumer; + } + + private long[] getCursorAckSet(PositionImpl position) { + if (!(subscription instanceof PersistentSubscription)) { + return null; + } + return (((PersistentSubscription) subscription).getCursor()).getDeletedBatchIndexesAsLongArray(position); + } + private boolean isTransactionEnabled() { return subscription instanceof PersistentSubscription && ((PersistentTopic) subscription.getTopic()) @@ -711,7 +806,6 @@ private void removePendingAcks(PositionImpl position) { ? ackOwnedConsumer.getPendingAcks().get(position.getLedgerId(), position.getEntryId()) : null; if (ackedPosition != null) { - int totalAckedMsgs = (int) ackedPosition.first; if (!ackOwnedConsumer.getPendingAcks().remove(position.getLedgerId(), position.getEntryId())) { // Message was already removed by the other consumer return; @@ -721,7 +815,7 @@ private void removePendingAcks(PositionImpl position) { } // unblock consumer-throttling when limit check is disabled or receives half of maxUnackedMessages => // consumer can start again consuming messages - int unAckedMsgs = addAndGetUnAckedMsgs(ackOwnedConsumer, -totalAckedMsgs); + int unAckedMsgs = UNACKED_MESSAGES_UPDATER.get(ackOwnedConsumer); if ((((unAckedMsgs <= maxUnackedMessages / 2) && ackOwnedConsumer.blockedConsumerOnUnackedMsgs) && ackOwnedConsumer.shouldBlockConsumerOnUnackMsgs()) || !shouldBlockConsumerOnUnackMsgs()) { @@ -751,7 +845,9 @@ public void redeliverUnacknowledgedMessages() { List pendingPositions = new ArrayList<>((int) pendingAcks.size()); MutableInt totalRedeliveryMessages = new MutableInt(0); pendingAcks.forEach((ledgerId, entryId, batchSize, stickyKeyHash) -> { - totalRedeliveryMessages.add((int) batchSize); + int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(PositionImpl.get(ledgerId, entryId), + batchSize); + totalRedeliveryMessages.add(unAckedCount); pendingPositions.add(new PositionImpl(ledgerId, entryId)); }); @@ -775,9 +871,9 @@ public void redeliverUnacknowledgedMessages(List messageIds) { PositionImpl position = PositionImpl.get(msg.getLedgerId(), msg.getEntryId()); LongPair longPair = pendingAcks.get(position.getLedgerId(), position.getEntryId()); if (longPair != null) { - long batchSize = longPair.first; + int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(position, longPair.first); pendingAcks.remove(position.getLedgerId(), position.getEntryId()); - totalRedeliveryMessages += batchSize; + totalRedeliveryMessages += unAckedCount; pendingPositions.add(position); } } @@ -811,8 +907,15 @@ public Subscription getSubscription() { } private int addAndGetUnAckedMsgs(Consumer consumer, int ackedMessages) { - subscription.addUnAckedMessages(ackedMessages); - return UNACKED_MESSAGES_UPDATER.addAndGet(consumer, ackedMessages); + int unackedMsgs = 0; + if (Subscription.isIndividualAckMode(subType)) { + subscription.addUnAckedMessages(ackedMessages); + unackedMsgs = UNACKED_MESSAGES_UPDATER.addAndGet(consumer, ackedMessages); + } + if (unackedMsgs < 0) { + log.error("unackedMsgs is : {}, ackedMessages : {}, consumer : {}", unackedMsgs, ackedMessages, consumer); + } + return unackedMsgs; } private void clearUnAckedMsgs() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java index 02fa3e29e5eee..6d8b31c4586cb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java @@ -673,7 +673,7 @@ public void testBatchAndNonBatchCumulativeAcks(BatcherBuilder builder) throws Ex * * @throws Exception */ - @Test(dataProvider = "containerBuilder", timeOut = 3000) + @Test(dataProvider = "containerBuilder") public void testConcurrentBatchMessageAck(BatcherBuilder builder) throws Exception { int numMsgs = 10; final String topicName = "persistent://prop/ns-abc/testConcurrentAck-" + UUID.randomUUID(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java new file mode 100644 index 0000000000000..20ba4f15b57b5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Lists; +import lombok.SneakyThrows; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import static org.testng.Assert.assertEquals; + +@Test(groups = "broker") +public class BatchMessageWithBatchIndexLevelTest extends BatchMessageTest { + + @BeforeClass + @Override + protected void setup() throws Exception { + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); + super.baseSetup(); + } + + @Test + @SneakyThrows + public void testBatchMessageAck() { + int numMsgs = 40; + final String topicName = "persistent://prop/ns-abc/batchMessageAck-" + UUID.randomUUID(); + final String subscriptionName = "sub-batch-1"; + + ConsumerImpl consumer = (ConsumerImpl) pulsarClient + .newConsumer() + .topic(topicName) + .subscriptionName(subscriptionName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .subscribe(); + + Producer producer = pulsarClient + .newProducer() + .topic(topicName) + .batchingMaxMessages(20) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .enableBatching(true) + .create(); + + List> sendFutureList = Lists.newArrayList(); + for (int i = 0; i < numMsgs; i++) { + byte[] message = ("batch-message-" + i).getBytes(); + sendFutureList.add(producer.newMessage().value(message).sendAsync()); + } + FutureUtil.waitForAll(sendFutureList).get(); + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) topic + .getSubscription(subscriptionName).getDispatcher(); + Message receive1 = consumer.receive(); + Message receive2 = consumer.receive(); + consumer.acknowledge(receive1); + consumer.acknowledge(receive2); + Awaitility.await().untilAsserted(() -> { + assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 18); + }); + Message receive3 = consumer.receive(); + Message receive4 = consumer.receive(); + consumer.acknowledge(receive3); + consumer.acknowledge(receive4); + Awaitility.await().untilAsserted(() -> { + assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 16); + }); + Message receive5 = consumer.receive(); + consumer.negativeAcknowledge(receive5); + Awaitility.await().pollInterval(1, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 0); + }); + consumer.receive(); + Awaitility.await().untilAsserted(() -> { + assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 16); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index 84cf739ebcf12..b886adee15ef4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -512,15 +512,15 @@ public void testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel() th PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock, null); - Consumer consumer1 = createConsumer(0, 2, false, 1); - Consumer consumer2 = createConsumer(0, 2, false, 2); - Consumer consumer3 = createConsumer(0, 2, false, 3); - Consumer consumer4 = createConsumer(1, 2, false, 4); - Consumer consumer5 = createConsumer(1, 1, false, 5); - Consumer consumer6 = createConsumer(1, 2, false, 6); - Consumer consumer7 = createConsumer(2, 1, false, 7); - Consumer consumer8 = createConsumer(2, 1, false, 8); - Consumer consumer9 = createConsumer(2, 1, false, 9); + Consumer consumer1 = createConsumer(topic, 0, 2, false, 1); + Consumer consumer2 = createConsumer(topic, 0, 2, false, 2); + Consumer consumer3 = createConsumer(topic, 0, 2, false, 3); + Consumer consumer4 = createConsumer(topic, 1, 2, false, 4); + Consumer consumer5 = createConsumer(topic, 1, 1, false, 5); + Consumer consumer6 = createConsumer(topic, 1, 2, false, 6); + Consumer consumer7 = createConsumer(topic, 2, 1, false, 7); + Consumer consumer8 = createConsumer(topic, 2, 1, false, 8); + Consumer consumer9 = createConsumer(topic, 2, 1, false, 9); dispatcher.addConsumer(consumer1); dispatcher.addConsumer(consumer2); dispatcher.addConsumer(consumer3); @@ -544,7 +544,7 @@ public void testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel() th Assert.assertEquals(getNextConsumer(dispatcher), consumer7); Assert.assertEquals(getNextConsumer(dispatcher), consumer8); // in between add upper priority consumer with more permits - Consumer consumer10 = createConsumer(0, 2, false, 10); + Consumer consumer10 = createConsumer(topic, 0, 2, false, 10); dispatcher.addConsumer(consumer10); Assert.assertEquals(getNextConsumer(dispatcher), consumer10); Assert.assertEquals(getNextConsumer(dispatcher), consumer10); @@ -556,12 +556,12 @@ public void testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel() th public void testFewBlockedConsumerSamePriority() throws Exception{ PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock, null); - Consumer consumer1 = createConsumer(0, 2, false, 1); - Consumer consumer2 = createConsumer(0, 2, false, 2); - Consumer consumer3 = createConsumer(0, 2, false, 3); - Consumer consumer4 = createConsumer(0, 2, false, 4); - Consumer consumer5 = createConsumer(0, 1, true, 5); - Consumer consumer6 = createConsumer(0, 2, true, 6); + Consumer consumer1 = createConsumer(topic, 0, 2, false, 1); + Consumer consumer2 = createConsumer(topic, 0, 2, false, 2); + Consumer consumer3 = createConsumer(topic, 0, 2, false, 3); + Consumer consumer4 = createConsumer(topic, 0, 2, false, 4); + Consumer consumer5 = createConsumer(topic, 0, 1, true, 5); + Consumer consumer6 = createConsumer(topic, 0, 2, true, 6); dispatcher.addConsumer(consumer1); dispatcher.addConsumer(consumer2); dispatcher.addConsumer(consumer3); @@ -583,18 +583,18 @@ public void testFewBlockedConsumerSamePriority() throws Exception{ public void testFewBlockedConsumerDifferentPriority() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock, null); - Consumer consumer1 = createConsumer(0, 2, false, 1); - Consumer consumer2 = createConsumer(0, 2, false, 2); - Consumer consumer3 = createConsumer(0, 2, false, 3); - Consumer consumer4 = createConsumer(0, 2, false, 4); - Consumer consumer5 = createConsumer(0, 1, true, 5); - Consumer consumer6 = createConsumer(0, 2, true, 6); - Consumer consumer7 = createConsumer(1, 2, false, 7); - Consumer consumer8 = createConsumer(1, 10, true, 8); - Consumer consumer9 = createConsumer(1, 2, false, 9); - Consumer consumer10 = createConsumer(2, 2, false, 10); - Consumer consumer11 = createConsumer(2, 10, true, 11); - Consumer consumer12 = createConsumer(2, 2, false, 12); + Consumer consumer1 = createConsumer(topic, 0, 2, false, 1); + Consumer consumer2 = createConsumer(topic, 0, 2, false, 2); + Consumer consumer3 = createConsumer(topic, 0, 2, false, 3); + Consumer consumer4 = createConsumer(topic, 0, 2, false, 4); + Consumer consumer5 = createConsumer(topic, 0, 1, true, 5); + Consumer consumer6 = createConsumer(topic, 0, 2, true, 6); + Consumer consumer7 = createConsumer(topic, 1, 2, false, 7); + Consumer consumer8 = createConsumer(topic, 1, 10, true, 8); + Consumer consumer9 = createConsumer(topic, 1, 2, false, 9); + Consumer consumer10 = createConsumer(topic, 2, 2, false, 10); + Consumer consumer11 = createConsumer(topic, 2, 10, true, 11); + Consumer consumer12 = createConsumer(topic, 2, 2, false, 12); dispatcher.addConsumer(consumer1); dispatcher.addConsumer(consumer2); dispatcher.addConsumer(consumer3); @@ -622,8 +622,8 @@ public void testFewBlockedConsumerDifferentPriority() throws Exception { Assert.assertEquals(getNextConsumer(dispatcher), consumer10); Assert.assertEquals(getNextConsumer(dispatcher), consumer12); // add consumer with lower priority again - Consumer consumer13 = createConsumer(0, 2, false, 13); - Consumer consumer14 = createConsumer(0, 2, true, 14); + Consumer consumer13 = createConsumer(topic, 0, 2, false, 13); + Consumer consumer14 = createConsumer(topic, 0, 2, true, 14); dispatcher.addConsumer(consumer13); dispatcher.addConsumer(consumer14); Assert.assertEquals(getNextConsumer(dispatcher), consumer13); @@ -637,13 +637,13 @@ public void testFewBlockedConsumerDifferentPriority() throws Exception { public void testFewBlockedConsumerDifferentPriority2() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock, null); - Consumer consumer1 = createConsumer(0, 2, true, 1); - Consumer consumer2 = createConsumer(0, 2, true, 2); - Consumer consumer3 = createConsumer(0, 2, true, 3); - Consumer consumer4 = createConsumer(1, 2, false, 4); - Consumer consumer5 = createConsumer(1, 1, false, 5); - Consumer consumer6 = createConsumer(2, 1, false, 6); - Consumer consumer7 = createConsumer(2, 2, true, 7); + Consumer consumer1 = createConsumer(topic, 0, 2, true, 1); + Consumer consumer2 = createConsumer(topic, 0, 2, true, 2); + Consumer consumer3 = createConsumer(topic, 0, 2, true, 3); + Consumer consumer4 = createConsumer(topic, 1, 2, false, 4); + Consumer consumer5 = createConsumer(topic, 1, 1, false, 5); + Consumer consumer6 = createConsumer(topic, 2, 1, false, 6); + Consumer consumer7 = createConsumer(topic, 2, 2, true, 7); dispatcher.addConsumer(consumer1); dispatcher.addConsumer(consumer2); dispatcher.addConsumer(consumer3); @@ -673,9 +673,10 @@ private Consumer getNextConsumer(PersistentDispatcherMultipleConsumers dispatche return null; } - private Consumer createConsumer(int priority, int permit, boolean blocked, int id) throws Exception { + private Consumer createConsumer(PersistentTopic topic, int priority, int permit, boolean blocked, int id) throws Exception { + PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); Consumer consumer = - new Consumer(null, SubType.Shared, "test-topic", id, priority, ""+id, 5000, + new Consumer(sub, SubType.Shared, "test-topic", id, priority, ""+id, 5000, serverCnx, "appId", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null, MessageId.latest); try { consumer.flowPermits(permit); From e9b951232418188013fcc9671e6d5e67f6ed8e3f Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 26 Jan 2022 15:42:29 +0800 Subject: [PATCH 300/823] [Transaction] PendingAckHandleImpl handle isInCacheRequest (#13481) ### Motivation When PendingAck is not Ready, requests will be put into the queue first. When the PendingAck recovery is completed, the new request and the request in the queue are processed at the same time, which may cause the ack to be out of order. ### Modification Use a single thread to handle requests. After the recovery is completed, the callback is processed in a single thread, which is mutually exclusive with the processing of new requests. (cherry picked from commit 97f0030ea4651c01227072b3a7522dc9ff198623) --- .../apache/pulsar/broker/PulsarService.java | 17 +- .../persistent/PersistentSubscription.java | 8 +- .../buffer/impl/TopicTransactionBuffer.java | 5 +- .../pendingack/PendingAckHandle.java | 15 +- .../impl/MLPendingAckReplyCallBack.java | 6 +- .../impl/PendingAckHandleDisabled.java | 12 +- .../pendingack/impl/PendingAckHandleImpl.java | 331 +++++++++--------- .../PersistentSubscriptionTest.java | 5 +- .../pendingack/PendingAckPersistentTest.java | 1 + 9 files changed, 185 insertions(+), 215 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 28ccbb1013426..79aba8173bfb4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -258,7 +258,7 @@ public class PulsarService implements AutoCloseable, ShutdownService { private PulsarResources pulsarResources; private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider; - private final ScheduledExecutorService transactionReplayExecutor; + private final ExecutorProvider transactionExecutorProvider; public enum State { Init, Started, Closing, Closed @@ -314,11 +314,10 @@ public PulsarService(ServiceConfiguration config, new DefaultThreadFactory("zk-cache-callback")); if (config.isTransactionCoordinatorEnabled()) { - this.transactionReplayExecutor = Executors.newScheduledThreadPool( - config.getNumTransactionReplayThreadPoolSize(), - new DefaultThreadFactory("transaction-replay")); + this.transactionExecutorProvider = new ExecutorProvider(this.getConfiguration() + .getNumTransactionReplayThreadPoolSize(), "pulsar-transaction-executor"); } else { - this.transactionReplayExecutor = null; + this.transactionExecutorProvider = null; } this.ioEventLoopGroup = EventLoopUtil.newEventLoopGroup(config.getNumIOThreads(), config.isEnableBusyWait(), @@ -494,8 +493,8 @@ public CompletableFuture closeAsync() { configurationMetadataStore.close(); } - if (transactionReplayExecutor != null) { - transactionReplayExecutor.shutdown(); + if (transactionExecutorProvider != null) { + transactionExecutorProvider.shutdownNow(); } ioEventLoopGroup.shutdownGracefully(); @@ -1246,8 +1245,8 @@ public ScheduledExecutorService getCacheExecutor() { return cacheExecutor; } - public ScheduledExecutorService getTransactionReplayExecutor() { - return transactionReplayExecutor; + public ExecutorProvider getTransactionExecutorProvider() { + return transactionExecutorProvider; } public ScheduledExecutorService getLoadManagerExecutor() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 8d75ea7b9a3ce..cd9f462e7ea10 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -499,11 +499,11 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public CompletableFuture transactionIndividualAcknowledge( TxnID txnId, List> positions) { - return pendingAckHandle.individualAcknowledgeMessage(txnId, positions, false); + return pendingAckHandle.individualAcknowledgeMessage(txnId, positions); } public CompletableFuture transactionCumulativeAcknowledge(TxnID txnId, List positions) { - return pendingAckHandle.cumulativeAcknowledgeMessage(txnId, positions, false); + return pendingAckHandle.cumulativeAcknowledgeMessage(txnId, positions); } private final MarkDeleteCallback markDeleteCallback = new MarkDeleteCallback() { @@ -1173,14 +1173,14 @@ public void processReplicatedSubscriptionSnapshot(ReplicatedSubscriptionsSnapsho public CompletableFuture endTxn(long txnidMostBits, long txnidLeastBits, int txnAction, long lowWaterMark) { TxnID txnID = new TxnID(txnidMostBits, txnidLeastBits); if (TxnAction.COMMIT.getValue() == txnAction) { - return pendingAckHandle.commitTxn(txnID, Collections.emptyMap(), lowWaterMark, false); + return pendingAckHandle.commitTxn(txnID, Collections.emptyMap(), lowWaterMark); } else if (TxnAction.ABORT.getValue() == txnAction) { Consumer redeliverConsumer = null; if (getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) { redeliverConsumer = ((PersistentDispatcherSingleActiveConsumer) getDispatcher()).getActiveConsumer(); } - return pendingAckHandle.abortTxn(txnID, redeliverConsumer, lowWaterMark, false); + return pendingAckHandle.abortTxn(txnID, redeliverConsumer, lowWaterMark); } else { return FutureUtil.failedFuture(new NotAllowedException("Unsupported txnAction " + txnAction)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 9a324fbc82b32..12818b7acbd7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -104,7 +104,7 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - this.topic.getBrokerService().getPulsar().getTransactionReplayExecutor() + this.topic.getBrokerService().getPulsar().getTransactionExecutorProvider().getExecutor(this) .execute(new TopicTransactionBufferRecover(new TopicTransactionBufferRecoverCallBack() { @Override public void recoverComplete() { @@ -607,7 +607,8 @@ public void run() { closeCursor(managedCursor); callBack.recoverComplete(); - }, topic.getBrokerService().getPulsar().getTransactionReplayExecutor()).exceptionally(e -> { + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider().getExecutor(this)) + .exceptionally(e -> { callBack.recoverExceptionally(new Exception(e)); log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index dc64cbe3b16a5..620db5c4b4814 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -50,15 +50,13 @@ public interface PendingAckHandle { * * @param txnID {@link TxnID} TransactionID of an ongoing transaction trying to sck message. * @param positions {@link MutablePair} the pair of positions and these batch size. - * @param isInCacheRequest {@link Boolean} the boolean of the request in cache whether or not. * @return the future of this operation. * @throws TransactionConflictException if the ack with transaction is conflict with pending ack. * @throws NotAllowedException if Use this method incorrectly eg. not use * PositionImpl or cumulative ack with a list of positions. */ CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions, boolean isInCacheRequest); - + Integer>> positions); /** * Acknowledge message(s) for an ongoing transaction. *

    @@ -76,14 +74,12 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, List cumulativeAcknowledgeMessage(TxnID txnID, List positions, - boolean isInCacheRequest); + CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions); /** * Commit a transaction. @@ -92,11 +88,9 @@ CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List commitTxn(TxnID txnID, Map properties, - long lowWaterMark, boolean isInCacheRequest); + CompletableFuture commitTxn(TxnID txnID, Map properties, long lowWaterMark); /** * Abort a transaction. @@ -104,10 +98,9 @@ CompletableFuture commitTxn(TxnID txnID, Map properties, * @param txnId {@link TxnID} to identify the transaction. * @param consumer {@link Consumer} which aborting transaction. * @param lowWaterMark the low water mark of this transaction - * @param isInCacheRequest {@link Boolean} the boolean of the request in cache whether or not. * @return the future of this operation. */ - CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark, boolean isInCacheRequest); + CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark); /** * Sync the position ack set, in order to clean up the cache of this position for pending ack handle. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java index b98e64ef9f5c9..728e7f0476b34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -44,7 +44,7 @@ public MLPendingAckReplyCallBack(PendingAckHandleImpl pendingAckHandle) { @Override public void replayComplete() { - synchronized (pendingAckHandle) { + pendingAckHandle.getInternalPinnedExecutor().execute(() -> { log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); @@ -56,8 +56,8 @@ public void replayComplete() { log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail!", pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); } - } - pendingAckHandle.handleCacheRequest(); + pendingAckHandle.handleCacheRequest(); + }); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index 634655e4ae7bd..119072700dfaa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -42,26 +42,22 @@ public class PendingAckHandleDisabled implements PendingAckHandle { @Override public CompletableFuture individualAcknowledgeMessage(TxnID txnID, - List> positions, - boolean isInCacheRequest) { + List> positions) { return FutureUtil.failedFuture(new NotAllowedException("The transaction is disabled")); } @Override - public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions, - boolean isInCacheRequest) { + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { return FutureUtil.failedFuture(new NotAllowedException("The transaction is disabled")); } @Override - public CompletableFuture commitTxn(TxnID txnID, Map properties, long lowWaterMark, - boolean isInCacheRequest) { + public CompletableFuture commitTxn(TxnID txnID, Map properties, long lowWaterMark) { return FutureUtil.failedFuture(new NotAllowedException("The transaction is disabled")); } @Override - public CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark, - boolean isInCacheRequest) { + public CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark) { return FutureUtil.failedFuture(new NotAllowedException("The transaction is disabled")); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index ba74d78c8ee9e..1c0b10fbd57f6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -29,7 +29,10 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ScheduledExecutorService; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; @@ -42,7 +45,6 @@ import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; @@ -111,13 +113,23 @@ public class PendingAckHandleImpl extends PendingAckHandleState implements Pendi private final BlockingQueue acceptQueue = new LinkedBlockingDeque<>(); + @Getter + private final ExecutorService internalPinnedExecutor; + + public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { super(State.None); this.topicName = persistentSubscription.getTopicName(); this.subName = persistentSubscription.getName(); this.persistentSubscription = persistentSubscription; - - this.pendingAckStoreProvider = ((PersistentTopic) this.persistentSubscription.getTopic()) + internalPinnedExecutor = persistentSubscription + .getTopic() + .getBrokerService() + .getPulsar() + .getTransactionExecutorProvider() + .getExecutor(this); + + this.pendingAckStoreProvider = this.persistentSubscription.getTopic() .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); pendingAckStoreProvider.checkInitializedBefore(persistentSubscription).thenAccept(init -> { if (init) { @@ -130,22 +142,19 @@ public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { private void initPendingAckStore() { if (changeToInitializingState()) { - synchronized (PendingAckHandleImpl.this) { - if (!checkIfClose()) { - this.pendingAckStoreFuture = - pendingAckStoreProvider.newPendingAckStore(persistentSubscription); - this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { - pendingAckStore.replayAsync(this, - ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService() - .getPulsar().getTransactionReplayExecutor()); - }).exceptionally(e -> { - acceptQueue.clear(); - changeToErrorState(); - exceptionHandleFuture(e.getCause()); - log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); - return null; - }); - } + if (!checkIfClose()) { + this.pendingAckStoreFuture = + pendingAckStoreProvider.newPendingAckStore(persistentSubscription); + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { + pendingAckStore.replayAsync(this, + (ScheduledExecutorService) internalPinnedExecutor); + }).exceptionally(e -> { + acceptQueue.clear(); + changeToErrorState(); + log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); + exceptionHandleFuture(e.getCause()); + return null; + }); } } } @@ -153,50 +162,20 @@ private void initPendingAckStore() { private void addIndividualAcknowledgeMessageRequest(TxnID txnID, List> positions, CompletableFuture completableFuture) { - acceptQueue.add(() -> individualAcknowledgeMessage(txnID, positions, true).thenAccept(v -> - completableFuture.complete(null)).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); + acceptQueue.add(() -> internalIndividualAcknowledgeMessage(txnID, positions, completableFuture)); } - @Override - public CompletableFuture individualAcknowledgeMessage(TxnID txnID, - List> positions, - boolean isInCacheRequest) { - - if (!checkIfReady()) { - CompletableFuture completableFuture = new CompletableFuture<>(); - synchronized (PendingAckHandleImpl.this) { - switch (state) { - case Initializing: - addIndividualAcknowledgeMessageRequest(txnID, positions, completableFuture); - return completableFuture; - case None: - addIndividualAcknowledgeMessageRequest(txnID, positions, completableFuture); - initPendingAckStore(); - return completableFuture; - case Error: - completableFuture.completeExceptionally( - new ServiceUnitNotReadyException("PendingAckHandle not replay error!")); - return completableFuture; - case Close: - completableFuture.completeExceptionally( - new ServiceUnitNotReadyException("PendingAckHandle have been closed!")); - return completableFuture; - default: - break; - } - } - } - + public void internalIndividualAcknowledgeMessage(TxnID txnID, List> positions, + CompletableFuture completableFuture) { if (txnID == null) { - return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); + completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null.")); + return; + } if (positions == null) { - return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); + completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null.")); + return; } - CompletableFuture completableFuture = new CompletableFuture<>(); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> { @@ -289,67 +268,67 @@ && isAckSetOverlap(individualAckPositions completableFuture.completeExceptionally(e); return null; }); - return completableFuture; - } - - private void addCumulativeAcknowledgeMessageRequest(TxnID txnID, - List positions, - CompletableFuture completableFuture) { - acceptQueue.add(() -> cumulativeAcknowledgeMessage(txnID, positions, true).thenAccept(v -> - completableFuture.complete(null)).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); } @Override - public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, - List positions, - boolean isInCacheRequest) { - if (!checkIfReady()) { - CompletableFuture completableFuture = new CompletableFuture<>(); - synchronized (PendingAckHandleImpl.this) { + public CompletableFuture individualAcknowledgeMessage(TxnID txnID, + List> positions) { + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { switch (state) { case Initializing: - addCumulativeAcknowledgeMessageRequest(txnID, positions, completableFuture); - return completableFuture; + addIndividualAcknowledgeMessageRequest(txnID, positions, completableFuture); + return; case None: - addCumulativeAcknowledgeMessageRequest(txnID, positions, completableFuture); + addIndividualAcknowledgeMessageRequest(txnID, positions, completableFuture); initPendingAckStore(); - return completableFuture; + return; case Error: completableFuture.completeExceptionally( new ServiceUnitNotReadyException("PendingAckHandle not replay error!")); - return completableFuture; + return; case Close: completableFuture.completeExceptionally( new ServiceUnitNotReadyException("PendingAckHandle have been closed!")); - return completableFuture; + return; default: break; - } } - } + internalIndividualAcknowledgeMessage(txnID, positions, completableFuture); + }); + return completableFuture; + } + + private void addCumulativeAcknowledgeMessageRequest(TxnID txnID, + List positions, + CompletableFuture completableFuture) { + acceptQueue.add(() -> internalCumulativeAcknowledgeMessage(txnID, positions, completableFuture)); + } + public void internalCumulativeAcknowledgeMessage(TxnID txnID, + List positions, + CompletableFuture completableFuture) { if (txnID == null) { - return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); + completableFuture.completeExceptionally(new NotAllowedException("TransactionID can not be null.")); + return; } if (positions == null) { - return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); + completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null.")); + return; } if (positions.size() != 1) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " invalid cumulative ack received with multiple message ids."; log.error(errorMsg); - return FutureUtil.failedFuture(new NotAllowedException(errorMsg)); + completableFuture.completeExceptionally(new NotAllowedException(errorMsg)); + return; } PositionImpl position = positions.get(0); - CompletableFuture completableFuture = new CompletableFuture<>(); - this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> { if (log.isDebugEnabled()) { @@ -392,59 +371,46 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, completableFuture.completeExceptionally(e); return null; }); - return completableFuture; - } - - private void addCommitTxnRequest(TxnID txnId, Map properties, long lowWaterMark, - CompletableFuture completableFuture) { - acceptQueue.add(() -> commitTxn(txnId, properties, lowWaterMark, true).thenAccept(v -> - completableFuture.complete(null)).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); } @Override - public synchronized CompletableFuture commitTxn(TxnID txnID, Map properties, - long lowWaterMark, boolean isInCacheRequest) { - if (!checkIfReady()) { - synchronized (PendingAckHandleImpl.this) { - if (state == State.Initializing) { - CompletableFuture completableFuture = new CompletableFuture<>(); - addCommitTxnRequest(txnID, properties, lowWaterMark, completableFuture); - return completableFuture; - } else if (state == State.None) { - CompletableFuture completableFuture = new CompletableFuture<>(); - addCommitTxnRequest(txnID, properties, lowWaterMark, completableFuture); - initPendingAckStore(); - return completableFuture; - } else if (checkIfReady()) { - - } else { - if (state == State.Error) { - return FutureUtil.failedFuture( + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + switch (state) { + case Initializing: + addCumulativeAcknowledgeMessageRequest(txnID, positions, completableFuture); + return; + case None: + addCumulativeAcknowledgeMessageRequest(txnID, positions, completableFuture); + initPendingAckStore(); + return; + case Error: + completableFuture.completeExceptionally( new ServiceUnitNotReadyException("PendingAckHandle not replay error!")); - } else { - return FutureUtil.failedFuture( + return; + case Close: + completableFuture.completeExceptionally( new ServiceUnitNotReadyException("PendingAckHandle have been closed!")); - } - + return; + default: + break; } } - } + internalCumulativeAcknowledgeMessage(txnID, positions, completableFuture); + }); - if (!acceptQueue.isEmpty() && !isInCacheRequest) { - synchronized (PendingAckHandleImpl.this) { - if (!acceptQueue.isEmpty()) { - CompletableFuture completableFuture = new CompletableFuture<>(); - addCommitTxnRequest(txnID, properties, lowWaterMark, completableFuture); - return completableFuture; - } - } - } + return completableFuture; + } - CompletableFuture commitFuture = new CompletableFuture<>(); + private void addCommitTxnRequest(TxnID txnId, Map properties, long lowWaterMark, + CompletableFuture completableFuture) { + acceptQueue.add(() -> internalCommitTxn(txnId, properties, lowWaterMark, completableFuture)); + } + private void internalCommitTxn(TxnID txnID, Map properties, long lowWaterMark, + CompletableFuture commitFuture) { // It's valid to create transaction then commit without doing any operation, which will cause // pendingAckMessagesMap to be null. if (this.cumulativeAckOfTransaction != null) { @@ -500,58 +466,44 @@ public synchronized CompletableFuture commitTxn(TxnID txnID, Map completableFuture) { - acceptQueue.add(() -> abortTxn(txnId, consumer, lowWaterMark, true).thenAccept(v -> - completableFuture.complete(null)).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); } @Override - public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consumer, - long lowWaterMark, boolean isInCacheRequest) { - if (!checkIfReady()) { - synchronized (PendingAckHandleImpl.this) { - if (state == State.Initializing) { - CompletableFuture completableFuture = new CompletableFuture<>(); - addAbortTxnRequest(txnId, consumer, lowWaterMark, completableFuture); - return completableFuture; - } else if (state == State.None) { - CompletableFuture completableFuture = new CompletableFuture<>(); - addAbortTxnRequest(txnId, consumer, lowWaterMark, completableFuture); - initPendingAckStore(); - return completableFuture; - } else if (checkIfReady()) { - - } else { - if (state == State.Error) { - return FutureUtil.failedFuture( - new ServiceUnitNotReadyException("PendingAckHandle not replay error!")); - } else { - return FutureUtil.failedFuture( - new ServiceUnitNotReadyException("PendingAckHandle have been closed!")); - } + public CompletableFuture commitTxn(TxnID txnID, Map properties, long lowWaterMark) { + CompletableFuture commitFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + switch (state) { + case Initializing: + addCommitTxnRequest(txnID, properties, lowWaterMark, commitFuture); + return; + case None: + addCommitTxnRequest(txnID, properties, lowWaterMark, commitFuture); + initPendingAckStore(); + return; + case Error: + if (state == State.Error) { + commitFuture.completeExceptionally( + new ServiceUnitNotReadyException("PendingAckHandle not replay error!")); + } else { + commitFuture.completeExceptionally( + new ServiceUnitNotReadyException("PendingAckHandle have been closed!")); + } + return; } } - } - + internalCommitTxn(txnID, properties, lowWaterMark, commitFuture); + }); + return commitFuture; + } - if (!acceptQueue.isEmpty() && !isInCacheRequest) { - synchronized (PendingAckHandleImpl.this) { - if (!acceptQueue.isEmpty()) { - CompletableFuture completableFuture = new CompletableFuture<>(); - addAbortTxnRequest(txnId, consumer, lowWaterMark, completableFuture); - return completableFuture; - } - } - } + private void addAbortTxnRequest(TxnID txnId, Consumer consumer, long lowWaterMark, + CompletableFuture completableFuture) { + acceptQueue.add(() -> internalAbortTxn(txnId, consumer, lowWaterMark, completableFuture)); + } - CompletableFuture abortFuture = new CompletableFuture<>(); + public CompletableFuture internalAbortTxn(TxnID txnId, Consumer consumer, + long lowWaterMark, CompletableFuture abortFuture) { if (this.cumulativeAckOfTransaction != null) { pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendAbortMark(txnId, AckType.Cumulative).thenAccept(v -> { @@ -611,6 +563,35 @@ public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consu return abortFuture; } + @Override + public CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark) { + CompletableFuture abortFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + switch (state) { + case Initializing: + addAbortTxnRequest(txnId, consumer, lowWaterMark, abortFuture); + return; + case None: + addAbortTxnRequest(txnId, consumer, lowWaterMark, abortFuture); + initPendingAckStore(); + return; + default: + if (state == State.Error) { + abortFuture.completeExceptionally( + new ServiceUnitNotReadyException("PendingAckHandle not replay error!")); + } else { + abortFuture.completeExceptionally( + new ServiceUnitNotReadyException("PendingAckHandle have been closed!")); + } + return; + } + } + internalAbortTxn(txnId, consumer, lowWaterMark, abortFuture); + }); + return abortFuture; + } + private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { if (individualAckOfTransaction != null && !individualAckOfTransaction.isEmpty()) { TxnID firstTxn = individualAckOfTransaction.firstKey(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index a76c6374e5c68..b72fe76f14caa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -56,7 +56,6 @@ import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferProvider; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; @@ -264,7 +263,7 @@ public void testCanAcknowledgeAndCommitForTransaction() throws ExecutionExceptio } @Test - public void testCanAcknowledgeAndAbortForTransaction() throws BrokerServiceException, InterruptedException { + public void testCanAcknowledgeAndAbortForTransaction() throws Exception { List> positionsPair = new ArrayList<>(); positionsPair.add(new MutablePair<>(new PositionImpl(2, 1), 0)); positionsPair.add(new MutablePair<>(new PositionImpl(2, 3), 0)); @@ -293,7 +292,7 @@ public void testCanAcknowledgeAndAbortForTransaction() throws BrokerServiceExcep positions.add(new PositionImpl(1, 100)); // Cumulative ack for txn1 - persistentSubscription.transactionCumulativeAcknowledge(txnID1, positions); + persistentSubscription.transactionCumulativeAcknowledge(txnID1, positions).get(); positions.clear(); positions.add(new PositionImpl(2, 1)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index 97f8f51d3e89b..affaf457e1c13 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -60,6 +60,7 @@ * Test for consuming transaction messages. */ @Slf4j +@Test(groups = "broker") public class PendingAckPersistentTest extends TransactionTestBase { private static final String PENDING_ACK_REPLAY_TOPIC = NAMESPACE1 + "/pending-ack-replay"; From 1aecf92ea2cbe1b8e15785257af8d6ee84499489 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 3 Nov 2021 22:10:39 -0700 Subject: [PATCH 301/823] Allow to configure schema compatibility policy for system topics (#12598) (cherry picked from commit 7aea58d293ba2ca29e0acbf4cfd5733d84846120) --- conf/broker.conf | 3 +++ .../pulsar/broker/ServiceConfiguration.java | 7 ++++++ .../pulsar/broker/service/AbstractTopic.java | 8 +++++- ...NamespaceEventsSystemTopicServiceTest.java | 25 +++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/conf/broker.conf b/conf/broker.conf index 2d7df90c611ef..8e518abc28cfd 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -534,6 +534,9 @@ zookeeperSessionExpiredPolicy=shutdown # Enable or disable system topic systemTopicEnabled=false +# The schema compatibility strategy to use for system topics +systemTopicSchemaCompatibilityStrategy=ALWAYS_COMPATIBLE + # Enable or disable topic level policies, topic level policies depends on the system topic # Please enable the system topic first. topicLevelPoliciesEnabled=false diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 0d578f4da369c..ce8ab98c7be67 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1043,6 +1043,13 @@ public class ServiceConfiguration implements PulsarConfiguration { doc = "Enable or disable system topic.") private boolean systemTopicEnabled = false; + @FieldContext( + category = CATEGORY_SCHEMA, + doc = "The schema compatibility strategy to use for system topics" + ) + private SchemaCompatibilityStrategy systemTopicSchemaCompatibilityStrategy = + SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE; + @FieldContext( category = CATEGORY_SERVER, doc = "Enable or disable topic level policies, topic level policies depends on the system topic, " + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 183fb05acb2d8..be4f904c135d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -47,6 +47,7 @@ import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; +import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; @@ -92,6 +93,8 @@ public abstract class AbstractTopic implements Topic { // Whether messages published must be encrypted or not in this topic protected volatile boolean isEncryptionRequired = false; + + @Getter protected volatile SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.FULL; protected volatile Boolean isAllowAutoUpdateSchema; @@ -537,7 +540,10 @@ public void recordAddLatency(long latency, TimeUnit unit) { } protected void setSchemaCompatibilityStrategy(Policies policies) { - if (policies.schema_compatibility_strategy == SchemaCompatibilityStrategy.UNDEFINED) { + if (SystemTopicClient.isSystemTopic(TopicName.get(this.topic))) { + schemaCompatibilityStrategy = + brokerService.pulsar().getConfig().getSystemTopicSchemaCompatibilityStrategy(); + } else if (policies.schema_compatibility_strategy == SchemaCompatibilityStrategy.UNDEFINED) { schemaCompatibilityStrategy = brokerService.pulsar() .getConfig().getSchemaCompatibilityStrategy(); if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java index b524e1a2148c6..2daca67520320 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java @@ -19,9 +19,14 @@ package org.apache.pulsar.broker.systopic; import com.google.common.collect.Sets; +import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.ActionType; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.EventsTopicNames; @@ -31,6 +36,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.slf4j.Logger; @@ -64,6 +70,25 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testSchemaCompatibility() throws Exception { + TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory + .createTopicPoliciesSystemTopicClient(NamespaceName.get(NAMESPACE1)); + String topicName = systemTopicClientForNamespace1.getTopicName().toString(); + @Cleanup + Reader reader = pulsarClient.newReader(Schema.BYTES) + .topic(topicName) + .startMessageId(MessageId.earliest) + .create(); + + PersistentTopic topic = + (PersistentTopic) pulsar.getBrokerService() + .getTopic(topicName, false) + .join().get(); + + Assert.assertEquals(SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE, topic.getSchemaCompatibilityStrategy()); + } + @Test public void testSendAndReceiveNamespaceEvents() throws Exception { TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory From 44d3c59ce41e370c7a2c22f55d111ec7b458e900 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 26 Jan 2022 15:46:18 +0800 Subject: [PATCH 302/823] [Broker] Fix read schema compatibility strategy priority (#13938) Signed-off-by: Zixuan Liu ### Motivation When we defined the `schemaCompatibilityStrategy=ALWAYS_INCOMPATIBLE` in `broker.conf`, and a namespace policies so like: ``` schema_auto_update_compatibility_strategy = SchemaAutoUpdateCompatibilityStrategy.Full schema_compatibility_strategy = null ``` We should get `SchemaCompatibilityStrategy.FULL` by `pulsar-admin namespaces get-schema-compatibility-strategy `, but got `SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE`, this is incorrect response. ### Modifications - Use `null` as `Policies#schema_auto_update_compatibility_strategy` value - Use `FULL` as `schemaCompatibilityStrategy` value in `broker.conf` and update configuration code - Only return `Policies#schema_compatibility_strategy` when get schema compatibility strategy of namespace - Change the read schema compatibility strategy priority (cherry picked from commit c18d64526708c6dd051217feba3c31a73507620c) --- conf/broker.conf | 8 ++- .../pulsar/broker/ServiceConfiguration.java | 11 ++-- .../pulsar/broker/admin/AdminResource.java | 15 ++++++ .../broker/admin/impl/NamespacesBase.java | 11 +--- .../admin/impl/SchemasResourceBase.java | 42 ++++----------- .../pulsar/broker/service/AbstractTopic.java | 18 ++++--- .../admin/AdminApiSchemaAutoUpdateTest.java | 20 +++---- .../broker/admin/AdminApiSchemaTest.java | 53 +++++++++++++++++++ .../SchemaCompatibilityCheckTest.java | 6 +-- .../SchemaTypeCompatibilityCheckTest.java | 8 +-- .../pulsar/common/policies/data/Policies.java | 3 +- .../data/SchemaCompatibilityStrategy.java | 5 +- 12 files changed, 121 insertions(+), 79 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 8e518abc28cfd..1cb10a06f606a 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1232,12 +1232,10 @@ schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.Bookkeepe # if you enable this setting, it will cause non-java clients failed to produce. isSchemaValidationEnforced=false -# The schema compatibility strategy in broker level. If this config in namespace policy is `UNDEFINED`, -# broker will use it in broker level. If schemaCompatibilityStrategy is `UNDEFINED` will use `FULL`. -# SchemaCompatibilityStrategy : UNDEFINED, ALWAYS_INCOMPATIBLE, ALWAYS_COMPATIBLE, BACKWARD, FORWARD, +# The schema compatibility strategy in broker level. +# SchemaCompatibilityStrategy : ALWAYS_INCOMPATIBLE, ALWAYS_COMPATIBLE, BACKWARD, FORWARD, # FULL, BACKWARD_TRANSITIVE, FORWARD_TRANSITIVE, FULL_TRANSITIVE -# default : UNDEFINED -schemaCompatibilityStrategy= +schemaCompatibilityStrategy=FULL ### --- Ledger Offloading --- ### diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index ce8ab98c7be67..349c46bc9fc80 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2011,10 +2011,9 @@ public class ServiceConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_SCHEMA, - doc = "The schema compatibility strategy in broker level. If this config in namespace policy is `UNDEFINED`" - + ", schema compatibility strategy check will use it in broker level." + doc = "The schema compatibility strategy in broker level" ) - private SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.UNDEFINED; + private SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.FULL; /**** --- WebSocket --- ****/ @FieldContext( @@ -2436,4 +2435,10 @@ public int getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds() { } } + public SchemaCompatibilityStrategy getSchemaCompatibilityStrategy() { + if (SchemaCompatibilityStrategy.isUndefined(schemaCompatibilityStrategy)) { + return SchemaCompatibilityStrategy.FULL; + } + return schemaCompatibilityStrategy; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 6a9311ca4ab5e..242f3a320f0ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -56,6 +56,7 @@ import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.TopicPolicies; @@ -766,6 +767,20 @@ protected void resumeAsyncResponseExceptionally(AsyncResponse asyncResponse, Thr } } + protected CompletableFuture getSchemaCompatibilityStrategyAsync() { + return getNamespacePoliciesAsync(namespaceName).thenApply(policies -> { + SchemaCompatibilityStrategy schemaCompatibilityStrategy = policies.schema_compatibility_strategy; + if (SchemaCompatibilityStrategy.isUndefined(schemaCompatibilityStrategy)) { + schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( + policies.schema_auto_update_compatibility_strategy); + if (SchemaCompatibilityStrategy.isUndefined(schemaCompatibilityStrategy)) { + schemaCompatibilityStrategy = pulsar().getConfig().getSchemaCompatibilityStrategy(); + } + } + return schemaCompatibilityStrategy; + }); + } + @CanIgnoreReturnValue public static T checkNotNull(T reference) { return com.google.common.base.Preconditions.checkNotNull(reference); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 7f4b288bfc22f..30941df5351f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2337,15 +2337,8 @@ protected SchemaCompatibilityStrategy internalGetSchemaCompatibilityStrategy() { validateNamespacePolicyOperation(namespaceName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); Policies policies = getNamespacePolicies(namespaceName); - SchemaCompatibilityStrategy schemaCompatibilityStrategy = policies.schema_compatibility_strategy; - if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = pulsar().getConfig().getSchemaCompatibilityStrategy(); - if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = SchemaCompatibilityStrategy - .fromAutoUpdatePolicy(policies.schema_auto_update_compatibility_strategy); - } - } - return schemaCompatibilityStrategy; + + return policies.schema_compatibility_strategy; } @Deprecated diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index dfd870a053d99..5b119ec881db4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -38,8 +38,6 @@ import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.DeleteSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetAllVersionsSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetSchemaResponse; @@ -136,16 +134,7 @@ public void deleteSchema(boolean authoritative, AsyncResponse response) { public void postSchema(PostSchemaPayload payload, boolean authoritative, AsyncResponse response) { validateDestinationAndAdminOperation(authoritative); - getNamespacePoliciesAsync(namespaceName).thenAccept(policies -> { - SchemaCompatibilityStrategy schemaCompatibilityStrategy = policies.schema_compatibility_strategy; - if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = - pulsar().getConfig().getSchemaCompatibilityStrategy(); - if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = SchemaCompatibilityStrategy - .fromAutoUpdatePolicy(policies.schema_auto_update_compatibility_strategy); - } - } + getSchemaCompatibilityStrategyAsync().thenAccept(schemaCompatibilityStrategy -> { byte[] data; if (SchemaType.KEY_VALUE.name().equals(payload.getType())) { try { @@ -199,26 +188,17 @@ public void testCompatibility(PostSchemaPayload payload, boolean authoritative, validateDestinationAndAdminOperation(authoritative); String schemaId = getSchemaId(); - Policies policies = getNamespacePolicies(namespaceName); - - SchemaCompatibilityStrategy schemaCompatibilityStrategy; - if (policies.schema_compatibility_strategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = SchemaCompatibilityStrategy - .fromAutoUpdatePolicy(policies.schema_auto_update_compatibility_strategy); - } else { - schemaCompatibilityStrategy = policies.schema_compatibility_strategy; - } - pulsar().getSchemaRegistryService() - .isCompatible(schemaId, - SchemaData.builder().data(payload.getSchema().getBytes(Charsets.UTF_8)).isDeleted(false) - .timestamp(clock.millis()).type(SchemaType.valueOf(payload.getType())) - .user(defaultIfEmpty(clientAppId(), "")).props(payload.getProperties()).build(), - schemaCompatibilityStrategy) - .thenAccept(isCompatible -> response.resume(Response.accepted() - .entity(IsCompatibilityResponse.builder().isCompatibility(isCompatible) - .schemaCompatibilityStrategy(schemaCompatibilityStrategy.name()).build()) - .build())) + getSchemaCompatibilityStrategyAsync().thenCompose(schemaCompatibilityStrategy -> pulsar() + .getSchemaRegistryService().isCompatible(schemaId, + SchemaData.builder().data(payload.getSchema().getBytes(Charsets.UTF_8)).isDeleted(false) + .timestamp(clock.millis()).type(SchemaType.valueOf(payload.getType())) + .user(defaultIfEmpty(clientAppId(), "")).props(payload.getProperties()).build(), + schemaCompatibilityStrategy) + .thenAccept(isCompatible -> response.resume(Response.accepted() + .entity(IsCompatibilityResponse.builder().isCompatibility(isCompatible) + .schemaCompatibilityStrategy(schemaCompatibilityStrategy.name()).build()) + .build()))) .exceptionally(error -> { response.resume(new RestException(error)); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index be4f904c135d4..620d35eaf91f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -543,17 +543,19 @@ protected void setSchemaCompatibilityStrategy(Policies policies) { if (SystemTopicClient.isSystemTopic(TopicName.get(this.topic))) { schemaCompatibilityStrategy = brokerService.pulsar().getConfig().getSystemTopicSchemaCompatibilityStrategy(); - } else if (policies.schema_compatibility_strategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = brokerService.pulsar() - .getConfig().getSchemaCompatibilityStrategy(); - if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { - schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( - policies.schema_auto_update_compatibility_strategy); + return; + } + + schemaCompatibilityStrategy = policies.schema_compatibility_strategy; + if (SchemaCompatibilityStrategy.isUndefined(schemaCompatibilityStrategy)) { + schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( + policies.schema_auto_update_compatibility_strategy); + if (SchemaCompatibilityStrategy.isUndefined(schemaCompatibilityStrategy)) { + schemaCompatibilityStrategy = brokerService.pulsar().getConfig().getSchemaCompatibilityStrategy(); } - } else { - schemaCompatibilityStrategy = policies.schema_compatibility_strategy; } } + private static final Summary PUBLISH_LATENCY = Summary.build("pulsar_broker_publish_latency", "-") .quantile(0.0) .quantile(0.50) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java index 891e7ac533d2c..3b3a8c9826a97 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java @@ -32,8 +32,6 @@ import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -42,9 +40,6 @@ @Slf4j @Test(groups = "broker") public class AdminApiSchemaAutoUpdateTest extends MockedPulsarServiceBaseTest { - - private static final Logger LOG = LoggerFactory.getLogger(AdminApiSchemaAutoUpdateTest.class); - @BeforeMethod @Override public void setup() throws Exception { @@ -67,8 +62,8 @@ public void cleanup() throws Exception { } private void testAutoUpdateBackward(String namespace, String topicName) throws Exception { - Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace), - SchemaAutoUpdateCompatibilityStrategy.Full); + Assert.assertNull(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace)); + admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace, SchemaAutoUpdateCompatibilityStrategy.Backward); @@ -91,8 +86,8 @@ private void testAutoUpdateBackward(String namespace, String topicName) throws E } private void testAutoUpdateForward(String namespace, String topicName) throws Exception { - Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace), - SchemaAutoUpdateCompatibilityStrategy.Full); + Assert.assertNull(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace)); + admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace, SchemaAutoUpdateCompatibilityStrategy.Forward); @@ -114,8 +109,7 @@ private void testAutoUpdateForward(String namespace, String topicName) throws Ex } private void testAutoUpdateFull(String namespace, String topicName) throws Exception { - Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace), - SchemaAutoUpdateCompatibilityStrategy.Full); + Assert.assertNull(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace)); try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) { p.send(new V1Data("test1", 1)); @@ -142,8 +136,8 @@ private void testAutoUpdateFull(String namespace, String topicName) throws Excep } private void testAutoUpdateDisabled(String namespace, String topicName) throws Exception { - Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace), - SchemaAutoUpdateCompatibilityStrategy.Full); + Assert.assertNull(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace)); + admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace, SchemaAutoUpdateCompatibilityStrategy.AutoUpdateDisabled); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index dce60e0d38b1f..64e4251cb01c2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -20,6 +20,7 @@ import static java.lang.String.format; import static java.nio.charset.StandardCharsets.US_ASCII; +import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; @@ -45,9 +46,11 @@ import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaInfoWithVersion; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -61,6 +64,8 @@ public class AdminApiSchemaTest extends MockedPulsarServiceBaseTest { final String cluster = "test"; + private final String schemaCompatibilityNamespace = "schematest/test-schema-compatibility-ns"; + @BeforeMethod @Override public void setup() throws Exception { @@ -72,6 +77,7 @@ public void setup() throws Exception { admin.tenants().createTenant("schematest", tenantInfo); admin.namespaces().createNamespace("schematest/test", Sets.newHashSet("test")); admin.namespaces().createNamespace("schematest/"+cluster+"/test", Sets.newHashSet("test")); + admin.namespaces().createNamespace(schemaCompatibilityNamespace, Sets.newHashSet("test")); } @AfterMethod(alwaysRun = true) @@ -349,4 +355,51 @@ public long getCToken() { assertEquals(ledgerInfo.entries, entryId + 1); assertEquals(ledgerInfo.size, length); } + + @Test + public void testGetSchemaCompatibilityStrategy() throws PulsarAdminException { + assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), + SchemaCompatibilityStrategy.UNDEFINED); + } + + @Test + public void testGetSchemaAutoUpdateCompatibilityStrategy() throws PulsarAdminException { + assertNull(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(schemaCompatibilityNamespace)); + } + + @Test + public void testGetSchemaCompatibilityStrategyWhenSetSchemaAutoUpdateCompatibilityStrategy() + throws PulsarAdminException { + assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), + SchemaCompatibilityStrategy.UNDEFINED); + + admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(schemaCompatibilityNamespace, + SchemaAutoUpdateCompatibilityStrategy.Forward); + Awaitility.await().untilAsserted(() -> assertEquals(SchemaAutoUpdateCompatibilityStrategy.Forward, + admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(schemaCompatibilityNamespace) + )); + + assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), + SchemaCompatibilityStrategy.UNDEFINED); + + admin.namespaces().setSchemaCompatibilityStrategy(schemaCompatibilityNamespace, + SchemaCompatibilityStrategy.BACKWARD); + Awaitility.await().untilAsserted(() -> assertEquals(SchemaCompatibilityStrategy.BACKWARD, + admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace))); + } + + @Test + public void testGetSchemaCompatibilityStrategyWhenSetBrokerLevelAndSchemaAutoUpdateCompatibilityStrategy() + throws PulsarAdminException { + pulsar.getConfiguration().setSchemaCompatibilityStrategy(SchemaCompatibilityStrategy.FORWARD); + + assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), + SchemaCompatibilityStrategy.UNDEFINED); + + admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(schemaCompatibilityNamespace, + SchemaAutoUpdateCompatibilityStrategy.AlwaysCompatible); + Awaitility.await().untilAsserted(() -> assertEquals( + admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), + SchemaCompatibilityStrategy.UNDEFINED)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 80168b9ae4c9f..5b12f37d40060 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -240,7 +240,7 @@ public void testBrokerAllowAutoUpdateSchemaDisabled(SchemaCompatibilityStrategy ); assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()), - SchemaCompatibilityStrategy.FULL); + SchemaCompatibilityStrategy.UNDEFINED); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); @@ -320,7 +320,7 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili ); assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()), - SchemaCompatibilityStrategy.FULL); + SchemaCompatibilityStrategy.UNDEFINED); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); @@ -399,7 +399,7 @@ public void testSchemaComparison() throws Exception { ); assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()), - SchemaCompatibilityStrategy.FULL); + SchemaCompatibilityStrategy.UNDEFINED); byte[] changeSchemaBytes = (new String(Schema.AVRO(Schemas.PersonOne.class) .getSchemaInfo().getSchema(), UTF_8) + "/n /n /n").getBytes(); SchemaInfo schemaInfo = SchemaInfo.builder().type(SchemaType.AVRO).schema(changeSchemaBytes).build(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaTypeCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaTypeCompatibilityCheckTest.java index 1367d4dbccd80..3fb047a2215d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaTypeCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaTypeCompatibilityCheckTest.java @@ -37,8 +37,8 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.schema.Schemas; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.util.Collections; @@ -57,7 +57,7 @@ public class SchemaTypeCompatibilityCheckTest extends MockedPulsarServiceBaseTes private static final String namespace = "test-namespace"; private static final String namespaceName = PUBLIC_TENANT + "/" + namespace; - @BeforeClass + @BeforeMethod @Override public void setup() throws Exception { super.internalSetup(); @@ -73,7 +73,7 @@ public void setup() throws Exception { } - @AfterClass(alwaysRun = true) + @AfterMethod(alwaysRun = true) @Override public void cleanup() throws Exception { super.internalCleanup(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index d26c61eefe838..dca72da69a8f2 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -101,8 +101,7 @@ public class Policies { @SuppressWarnings("checkstyle:MemberName") @Deprecated - public SchemaAutoUpdateCompatibilityStrategy schema_auto_update_compatibility_strategy = - SchemaAutoUpdateCompatibilityStrategy.Full; + public SchemaAutoUpdateCompatibilityStrategy schema_auto_update_compatibility_strategy = null; @SuppressWarnings("checkstyle:MemberName") public SchemaCompatibilityStrategy schema_compatibility_strategy = SchemaCompatibilityStrategy.UNDEFINED; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java index 9a4f74c437b14..f3b4569bad8f3 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java @@ -71,10 +71,13 @@ public enum SchemaCompatibilityStrategy { FULL_TRANSITIVE; + public static boolean isUndefined(SchemaCompatibilityStrategy strategy) { + return strategy == null || strategy == SchemaCompatibilityStrategy.UNDEFINED; + } public static SchemaCompatibilityStrategy fromAutoUpdatePolicy(SchemaAutoUpdateCompatibilityStrategy strategy) { if (strategy == null) { - return SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE; + return null; } switch (strategy) { case Backward: From 35497645437e9459f8358cf14757d99980d2036b Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:34:50 +0800 Subject: [PATCH 303/823] [Transaction] RetryException should not be return (#13828) Fix https://github.com/apache/pulsar/issues/13792 when TC get a RetryException, it will try that operation again and return the exception to client.But if the operation is executed again, we don`t need to return this RetryException to client. (cherry picked from commit 0f1913250d962f1763ebb586711a8d9be5ba5d16) --- .../apache/pulsar/broker/PulsarService.java | 1 + .../TransactionMetadataStoreService.java | 21 +++++++--- .../apache/pulsar/broker/service/Topic.java | 5 +++ .../TransactionRecoverTrackerImpl.java | 6 ++- .../TransactionMetadataStoreServiceTest.java | 14 ++++--- .../broker/stats/TransactionMetricsTest.java | 6 ++- .../broker/transaction/TransactionTest.java | 41 +++++++++++++++++++ 7 files changed, 79 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 79aba8173bfb4..adeec4c8561c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -127,6 +127,7 @@ import org.apache.pulsar.client.api.transaction.TransactionBufferClient; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 3f3e8d83d937d..9b60679b938af 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -321,6 +321,11 @@ public CompletableFuture updateTxnStatus(TxnID txnId, TxnStatus newStatus, public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolean isTimeout) { CompletableFuture completableFuture = new CompletableFuture<>(); + return endTransaction(txnID, txnAction, isTimeout, completableFuture); + } + + public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolean isTimeout, + CompletableFuture completableFuture) { TxnStatus newStatus; switch (txnAction) { case TxnAction.COMMIT_VALUE: @@ -352,8 +357,9 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea + "TxnAction : {}", txnID, txnAction, e); } transactionOpRetryTimer.newTimeout(timeout -> - endTransaction(txnID, txnAction, isTimeout), + endTransaction(txnID, txnAction, isTimeout, completableFuture), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + return null; } completableFuture.completeExceptionally(e.getCause()); @@ -367,8 +373,9 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea LOG.debug("EndTransaction UpdateTxnStatus op retry! TxnId : {}, " + "TxnAction : {}", txnID, txnAction, e); } - transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, isTimeout), - endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, + isTimeout, completableFuture), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + return null; } completableFuture.completeExceptionally(e.getCause()); @@ -385,8 +392,9 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea + "TxnAction : {}", txnID, txnAction, e); } transactionOpRetryTimer.newTimeout(timeout -> - endTransaction(txnID, txnAction, isTimeout), + endTransaction(txnID, txnAction, isTimeout, completableFuture), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + return null; } else { LOG.error("EndTxnInTransactionBuffer fail! TxnId : {}, " + "TxnAction : {}", txnID, txnAction, e); @@ -406,8 +414,9 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea if (LOG.isDebugEnabled()) { LOG.debug("End transaction op retry! TxnId : {}, TxnAction : {}", txnID, txnAction, e); } - transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, isTimeout), - endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, isTimeout, + completableFuture), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + return null; } completableFuture.completeExceptionally(e.getCause()); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 0e7589ade92ef..9dd945f78da33 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -288,4 +288,9 @@ default boolean isSystemTopic() { */ CompletableFuture truncate(); + /** + * Get BrokerService. + * @return + */ + BrokerService getBrokerService(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/recover/TransactionRecoverTrackerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/recover/TransactionRecoverTrackerImpl.java index 3667e666313c0..05b61fd637019 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/recover/TransactionRecoverTrackerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/recover/TransactionRecoverTrackerImpl.java @@ -123,9 +123,11 @@ public void appendOpenTransactionToTimeoutTracker() { @Override public void handleCommittingAndAbortingTransaction() { committingTransactions.forEach(k -> - transactionMetadataStoreService.endTransaction(new TxnID(tcId, k), TxnAction.COMMIT_VALUE, false)); + transactionMetadataStoreService.endTransaction(new TxnID(tcId, k), TxnAction.COMMIT_VALUE, + false)); abortingTransactions.forEach(k -> - transactionMetadataStoreService.endTransaction(new TxnID(tcId, k), TxnAction.ABORT_VALUE, false)); + transactionMetadataStoreService.endTransaction(new TxnID(tcId, k), TxnAction.ABORT_VALUE, + false)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java index 983f0d1dacf57..e50a39c5dfd91 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java @@ -25,11 +25,13 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import com.google.common.collect.Sets; +import java.util.concurrent.TimeoutException; import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.ServiceConfiguration; @@ -351,13 +353,15 @@ public void testEndTransactionOpRetry(TxnStatus txnStatus) throws Exception { Field field = TransactionMetadataStoreState.class.getDeclaredField("state"); field.setAccessible(true); field.set(transactionMetadataStore, TransactionMetadataStoreState.State.None); - + CompletableFuture completableFuture = null; try { - pulsar.getTransactionMetadataStoreService().endTransaction(txnID, TxnAction.COMMIT.getValue(), false).get(); + completableFuture = pulsar.getTransactionMetadataStoreService().endTransaction(txnID, TxnAction.COMMIT.getValue(), + false); + completableFuture.get(5, TimeUnit.SECONDS); fail(); } catch (Exception e) { if (txnStatus == TxnStatus.OPEN || txnStatus == TxnStatus.COMMITTING) { - assertTrue(e.getCause() instanceof CoordinatorException.TransactionMetadataStoreStateException); + assertTrue(e instanceof TimeoutException); } else if (txnStatus == TxnStatus.ABORTING) { assertTrue(e.getCause() instanceof CoordinatorException.InvalidTxnStatusException); } else { @@ -370,9 +374,9 @@ public void testEndTransactionOpRetry(TxnStatus txnStatus) throws Exception { field = TransactionMetadataStoreState.class.getDeclaredField("state"); field.setAccessible(true); field.set(transactionMetadataStore, TransactionMetadataStoreState.State.Ready); - if (txnStatus == TxnStatus.ABORTING) { - pulsar.getTransactionMetadataStoreService().endTransaction(txnID, TxnAction.ABORT.getValue(), false).get(); + pulsar.getTransactionMetadataStoreService() + .endTransaction(txnID, TxnAction.ABORT.getValue(), false).get(); } Awaitility.await().atMost(timeOut, TimeUnit.MILLISECONDS).until(() -> { try { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java index 7c48e81449813..0c9f877150a8e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java @@ -163,9 +163,11 @@ public void testTransactionCoordinatorRateMetrics() throws Exception{ for (int i = 0; i < txnCount; i++) { if (i % 2 == 0) { - pulsar.getTransactionMetadataStoreService().endTransaction(list.get(i), TxnAction.COMMIT_VALUE, false).get(); + pulsar.getTransactionMetadataStoreService().endTransaction(list.get(i), TxnAction.COMMIT_VALUE, + false).get(); } else { - pulsar.getTransactionMetadataStoreService().endTransaction(list.get(i), TxnAction.ABORT_VALUE, false).get(); + pulsar.getTransactionMetadataStoreService().endTransaction(list.get(i), TxnAction.ABORT_VALUE, + false).get(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index d7d90891624de..4aaadb2229691 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -45,6 +45,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -93,6 +94,7 @@ import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; import org.apache.pulsar.transaction.coordinator.TransactionRecoverTracker; import org.apache.pulsar.transaction.coordinator.TransactionTimeoutTracker; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; @@ -733,4 +735,43 @@ public void testNoEntryCanBeReadWhenRecovery() throws Exception { TopicTransactionBuffer transactionBuffer = new TopicTransactionBuffer(persistentTopic); Awaitility.await().untilAsserted(() -> Assert.assertTrue(transactionBuffer.checkIfReady())); } + + @Test + public void testRetryExceptionOfEndTxn() throws Exception{ + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS) + .build() + .get(); + Class transactionMetadataStoreStateClass = TransactionMetadataStoreState.class; + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores() + .values() + .forEach((transactionMetadataStore -> { + try { + Field field = transactionMetadataStoreStateClass.getDeclaredField("state"); + field.setAccessible(true); + field.set(transactionMetadataStore, TransactionMetadataStoreState.State.Initializing); + } catch (Exception e) { + e.printStackTrace(); + } + })); + CompletableFuture completableFuture = transaction.commit(); + try { + completableFuture.get(5, TimeUnit.SECONDS); + fail(); + } catch (TimeoutException ignored) { + } + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores() + .values() + .stream() + .forEach((transactionMetadataStore -> { + try { + Field field = transactionMetadataStoreStateClass.getDeclaredField("state"); + field.setAccessible(true); + field.set(transactionMetadataStore, TransactionMetadataStoreState.State.Ready); + } catch (Exception e) { + e.printStackTrace(); + } + })); + completableFuture.get(5, TimeUnit.SECONDS); + } } From 137200ba130a4f7d0bb437156df4a6878f1dcea6 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 28 Jan 2022 02:08:25 +0800 Subject: [PATCH 304/823] Trim configuration value string which contains blank prefix or suffix string (#13984) (cherry picked from commit fa47a9fe3c9bf4d3dde4a34c683644e49d8830a0) --- .../pulsar/common/util/FieldParser.java | 8 ++-- .../pulsar/common/util/FieldParserTest.java | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index a17045adfd2cc..0aa19b272a3c5 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -141,7 +141,7 @@ public static void update(Map properties, T obj) throws Ille f.setAccessible(true); String v = properties.get(f.getName()); if (!StringUtils.isBlank(v)) { - f.set(obj, value(v, f)); + f.set(obj, value(trim(v), f)); } else { setEmptyValue(v, f, obj); } @@ -316,7 +316,7 @@ public static Float stringToFloat(String val) { public static List stringToList(String val, Class type) { String[] tokens = trim(val).split(","); return Arrays.stream(tokens).map(t -> { - return convert(t, type); + return convert(trim(t), type); }).collect(Collectors.toList()); } @@ -332,7 +332,7 @@ public static List stringToList(String val, Class type) { public static Set stringToSet(String val, Class type) { String[] tokens = trim(val).split(","); return Arrays.stream(tokens).map(t -> { - return convert(t, type); + return convert(trim(t), type); }).collect(Collectors.toSet()); } @@ -343,7 +343,7 @@ private static Map stringToMap(String strValue, Class keyType, C String[] keyValue = trim(token).split("="); checkArgument(keyValue.length == 2, strValue + " map-value is not in correct format key1=value,key2=value2"); - map.put(convert(keyValue[0], keyType), convert(keyValue[1], valueType)); + map.put(convert(trim(keyValue[0]), keyType), convert(trim(keyValue[1]), valueType)); } return map; } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java index a0853116295ea..a8a3fee5cfdca 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java @@ -19,9 +19,13 @@ package org.apache.pulsar.common.util; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import org.testng.annotations.Test; import com.google.common.collect.Maps; @@ -35,6 +39,33 @@ public void testMap() { properties.put("stringStringMap", "key1=value1,key2=value2"); properties.put("stringIntMap", "key1=1,key2=2"); properties.put("longStringMap", "1=value1,2=value2"); + + MyConfig config = new MyConfig(); + FieldParser.update(properties, config); + assertEquals(config.name, "config"); + assertEquals(config.stringStringMap.get("key1"), "value1"); + assertEquals(config.stringStringMap.get("key2"), "value2"); + + assertEquals((int) config.stringIntMap.get("key1"), 1); + assertEquals((int) config.stringIntMap.get("key2"), 2); + + assertEquals(config.longStringMap.get(1L), "value1"); + assertEquals(config.longStringMap.get(2L), "value2"); + + } + + @Test + public void testWithBlankVallueConfig() { + Map properties = new HashMap<>(); + properties.put("name", " config "); + properties.put("stringStringMap", "key1=value1 , key2= value2 "); + properties.put("stringIntMap", "key1 = 1, key2 = 2 "); + properties.put("longStringMap", " 1 =value1 ,2 =value2 "); + properties.put("longList", " 1, 3, 8 , 0 ,9 "); + properties.put("stringList", " aa, bb , cc, ee "); + properties.put("longSet", " 1, 3, 8 , 0 , 3, 1 ,9 "); + properties.put("stringSet", " aa, bb , cc, ee , bb, aa "); + MyConfig config = new MyConfig(); FieldParser.update(properties, config); assertEquals(config.name, "config"); @@ -47,6 +78,11 @@ public void testMap() { assertEquals(config.longStringMap.get(1L), "value1"); assertEquals(config.longStringMap.get(2L), "value2"); + assertEquals((long)config.longList.get(2), 8); + assertEquals(config.stringList.get(1), "bb"); + + assertTrue(config.longSet.contains(3L)); + assertTrue(config.stringSet.contains("bb")); } public static class MyConfig { @@ -54,6 +90,10 @@ public static class MyConfig { public Map stringStringMap; public Map stringIntMap; public Map longStringMap; + public List longList; + public List stringList; + public Set longSet; + public Set stringSet; } } From fa549f6db79f035a6ec74174dbf00aaedf175ec6 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 28 Jan 2022 04:10:12 +0800 Subject: [PATCH 305/823] [Transaction] Resolve the performance bottleneck of TransactionBufferHandle (#13988) ### Motivation Previously, synchronization locks were frequently used in TransactionBufferHandleImpl in order to achieve request timeouts. For this reason, the performance of TC has been affected. ### Modification Each request uses a Timeout to do a timeout check (cherry picked from commit f48b53d33ee1be9b8436593119dfbce38be2c81f) --- .../impl/TransactionBufferHandlerImpl.java | 55 +++++-------------- 1 file changed, 15 insertions(+), 40 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 54f77a331647c..cb74bf3bf2e14 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -25,9 +25,6 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Recycler; import io.netty.util.ReferenceCountUtil; -import io.netty.util.Timeout; -import io.netty.util.TimerTask; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; @@ -49,12 +46,11 @@ import org.apache.pulsar.common.protocol.Commands; @Slf4j -public class TransactionBufferHandlerImpl implements TransactionBufferHandler, TimerTask { +public class TransactionBufferHandlerImpl implements TransactionBufferHandler { private final ConcurrentSkipListMap pendingRequests; private final AtomicLong requestIdGenerator = new AtomicLong(); private final long operationTimeoutInMills; - private Timeout requestTimeout; private final HashedWheelTimer timer; private final Semaphore semaphore; private final boolean blockIfReachMaxPendingOps; @@ -84,11 +80,10 @@ public TransactionBufferHandlerImpl(PulsarClient pulsarClient, this.semaphore = new Semaphore(10000); this.blockIfReachMaxPendingOps = true; this.timer = timer; - this.requestTimeout = timer.newTimeout(this, operationTimeoutInMills, TimeUnit.MILLISECONDS); } @Override - public synchronized CompletableFuture endTxnOnTopic(String topic, long txnIdMostBits, long txnIdLeastBits, + public CompletableFuture endTxnOnTopic(String topic, long txnIdMostBits, long txnIdLeastBits, TxnAction action, long lowWaterMark) { if (log.isDebugEnabled()) { log.debug("[{}] endTxnOnTopic txnId: [{}], txnAction: [{}]", @@ -105,7 +100,7 @@ public synchronized CompletableFuture endTxnOnTopic(String topic, long tx } @Override - public synchronized CompletableFuture endTxnOnSubscription(String topic, String subscription, + public CompletableFuture endTxnOnSubscription(String topic, String subscription, long txnIdMostBits, long txnIdLeastBits, TxnAction action, long lowWaterMark) { if (log.isDebugEnabled()) { @@ -129,10 +124,16 @@ private CompletableFuture endTxn(long requestId, String topic, ByteBuf cm if (throwable == null) { if (clientCnx.ctx().channel().isActive()) { clientCnx.registerTransactionBufferHandler(TransactionBufferHandlerImpl.this); - synchronized (TransactionBufferHandlerImpl.this) { - pendingRequests.put(requestId, op); - cmd.retain(); - } + pendingRequests.put(requestId, op); + timer.newTimeout(timeout -> { + OpRequestSend peek = pendingRequests.remove(requestId); + if (peek != null && !peek.cb.isDone() && !peek.cb.isCompletedExceptionally()) { + peek.cb.completeExceptionally(new TransactionBufferClientException + .RequestTimeoutException()); + onResponse(peek); + } + }, operationTimeoutInMills, TimeUnit.MILLISECONDS); + cmd.retain(); clientCnx.ctx().writeAndFlush(cmd, clientCnx.ctx().voidPromise()); } else { cache.invalidate(topic); @@ -158,7 +159,7 @@ private CompletableFuture endTxn(long requestId, String topic, ByteBuf cm } @Override - public synchronized void handleEndTxnOnTopicResponse(long requestId, CommandEndTxnOnPartitionResponse response) { + public void handleEndTxnOnTopicResponse(long requestId, CommandEndTxnOnPartitionResponse response) { OpRequestSend op = pendingRequests.remove(requestId); if (op == null) { if (log.isDebugEnabled()) { @@ -183,7 +184,7 @@ public synchronized void handleEndTxnOnTopicResponse(long requestId, CommandEndT } @Override - public synchronized void handleEndTxnOnSubscriptionResponse(long requestId, + public void handleEndTxnOnSubscriptionResponse(long requestId, CommandEndTxnOnSubscriptionResponse response) { OpRequestSend op = pendingRequests.remove(requestId); if (op == null) { @@ -227,32 +228,6 @@ private boolean canSendRequest(CompletableFuture callback) { return true; } - public synchronized void run(Timeout timeout) throws Exception { - if (timeout.isCancelled()) { - return; - } - long timeToWaitMs; - OpRequestSend peeked; - Map.Entry firstEntry = pendingRequests.firstEntry(); - peeked = firstEntry == null ? null : firstEntry.getValue(); - while (peeked != null && peeked.createdAt + operationTimeoutInMills - System.currentTimeMillis() <= 0) { - if (!peeked.cb.isDone()) { - peeked.cb.completeExceptionally(new TransactionBufferClientException.RequestTimeoutException()); - onResponse(peeked); - } else { - break; - } - firstEntry = pendingRequests.firstEntry(); - pendingRequests.remove(pendingRequests.firstKey()); - peeked = firstEntry == null ? null : firstEntry.getValue(); - } - if (peeked == null) { - timeToWaitMs = operationTimeoutInMills; - } else { - timeToWaitMs = (peeked.createdAt + operationTimeoutInMills) - System.currentTimeMillis(); - } - requestTimeout = timer.newTimeout(this, timeToWaitMs, TimeUnit.MILLISECONDS); - } void onResponse(OpRequestSend op) { ReferenceCountUtil.safeRelease(op.byteBuf); From 84f5de65eff4234de99b8aafe6cf14e5d9335e90 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 23 Dec 2021 11:09:43 +0800 Subject: [PATCH 306/823] [Transaction] Add transaction admin (#13447) ## Motivation Add the CmdTransactions to PulsarAdminTool (cherry picked from commit f97d6e5a50355e1de628569c6de68cd7210094f0) --- .../main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index d0f514d6f1f03..8992e923d1d76 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -158,6 +158,7 @@ public class PulsarAdminTool { commandMap.put("sink", CmdSinks.class); commandMap.put("packages", CmdPackages.class); + commandMap.put("transactions", CmdTransactions.class); } private static class PulsarAdminSupplier implements Supplier { From df232a03c66ba88723a814170c60503d3aada75f Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 27 Jan 2022 17:54:10 -0800 Subject: [PATCH 307/823] Fixed handling of consumers with equal names on on key shared selector with consistent hashing (#13991) Fixes #10750 When removing consumers from the key-shared selector based on the consistent hashing, we were removing by consumer name, although there can be duplicated consumer names (even though is not recommended). (cherry picked from commit 8360d45ed88ba551ecd16ce910ca82181cf6dc1b) --- ...stentHashingStickyKeyConsumerSelector.java | 2 +- .../client/api/KeySharedSubscriptionTest.java | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java index 7b7a8307baa45..ac255b59464d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java @@ -91,7 +91,7 @@ public void removeConsumer(Consumer consumer) { if (v == null) { return null; } else { - v.removeIf(c -> c.consumerName().equals(consumer.consumerName())); + v.removeIf(c -> c.equals(consumer)); if (v.isEmpty()) { v = null; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index d1f007537ebe2..8fc89806289ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -40,7 +40,10 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -1069,6 +1072,75 @@ public void testSelectorChangedAfterAllConsumerDisconnected(String topicDomain) consumer1.close(); } + @Test(timeOut = 30_000) + public void testCheckConsumersWithSameName() throws Exception { + final String topicName = "persistent://public/default/same-name-" + UUID.randomUUID(); + final String subName = "my-sub"; + final String consumerName = "name"; + + ConsumerBuilder cb = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .consumerName(consumerName) + .subscriptionType(SubscriptionType.Key_Shared); + + // Create 3 consumers with same name + Consumer c1 = cb.subscribe(); + + @Cleanup + Consumer c2 = cb.subscribe(); + @Cleanup + Consumer c3 = cb.subscribe(); + + Producer p = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + for (int i = 0; i < 100; i++) { + p.newMessage() + .key(Integer.toString(i)) + .value("msg-" + i) + .send(); + } + + // C1 receives some messages and won't ack + for (int i = 0; i < 5; i++) { + c1.receive(); + } + + // Close C1, now all messages should go to c2 & c3 + c1.close(); + + CountDownLatch l = new CountDownLatch(100); + + @Cleanup("shutdownNow") + ExecutorService e = Executors.newCachedThreadPool(); + e.submit(() -> { + while (l.getCount() > 0) { + try { + Message msg = c2.receive(1, TimeUnit.SECONDS); + c2.acknowledge(msg); + l.countDown(); + } catch (PulsarClientException ex) { + ex.printStackTrace(); + } + } + }); + + e.submit(() -> { + while (l.getCount() > 0) { + try { + Message msg = c3.receive(1, TimeUnit.SECONDS); + c3.acknowledge(msg); + l.countDown(); + } catch (PulsarClientException ex) { + ex.printStackTrace(); + } + } + }); + + l.await(); + } + private KeySharedMode getKeySharedModeOfSubscription(Topic topic, String subscription) { if (TopicName.get(topic.getName()).getDomain().equals(TopicDomain.persistent)) { return ((PersistentStickyKeyDispatcherMultipleConsumers) topic.getSubscription(subscription) From 16cf802d4089a573d4b652d53122886d35b25c2c Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 24 Nov 2021 15:31:39 +0800 Subject: [PATCH 308/823] fix flaky in step2 and step3 (#12954) (cherry picked from commit 94736a43f1b9a6d1db75936032b94eb9a11b9c0d) --- .../api/DispatcherBlockConsumerTest.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java index d43a759bf3622..00f52ad090e1f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java @@ -25,7 +25,11 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Queues; +import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.util.Iterator; import java.util.List; @@ -41,8 +45,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; - - import lombok.Cleanup; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; @@ -50,8 +52,8 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.SubscriptionStats; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,12 +63,6 @@ import org.testng.annotations.Test; import org.testng.collections.Lists; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Queues; -import com.google.common.collect.Sets; - @Test(groups = "flaky") public class DispatcherBlockConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(DispatcherBlockConsumerTest.class); @@ -688,6 +684,7 @@ public void testBlockBrokerDispatching() { ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); try { + final int waitMills = 500; final int maxUnAckPerBroker = 200; final double unAckMsgPercentagePerDispatcher = 10; int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100); // 200 * @@ -745,7 +742,7 @@ public void testBlockBrokerDispatching() { Message msg = null; Set messages1 = Sets.newHashSet(); for (int j = 0; j < totalProducedMsgs; j++) { - msg = consumer1Sub1.receive(100, TimeUnit.MILLISECONDS); + msg = consumer1Sub1.receive(waitMills, TimeUnit.MILLISECONDS); if (msg != null) { messages1.add(msg.getMessageId()); } else { @@ -754,7 +751,7 @@ public void testBlockBrokerDispatching() { // once consumer receives maxUnAckPerBroker-msgs then sleep to give a chance to scheduler to block the // subscription if (j == maxUnAckPerBroker) { - Thread.sleep(200); + Thread.sleep(waitMills); } } // client must receive number of messages = maxUnAckPerbroker rather all produced messages @@ -767,7 +764,7 @@ public void testBlockBrokerDispatching() { .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); int consumer2Msgs = 0; for (int j = 0; j < totalProducedMsgs; j++) { - msg = consumer2Sub1.receive(100, TimeUnit.MILLISECONDS); + msg = consumer2Sub1.receive(waitMills, TimeUnit.MILLISECONDS); if (msg != null) { consumer2Msgs++; } else { @@ -792,7 +789,7 @@ public void testBlockBrokerDispatching() { .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); Set messages2 = Sets.newHashSet(); for (int j = 0; j < totalProducedMsgs; j++) { - msg = consumerSub2.receive(100, TimeUnit.MILLISECONDS); + msg = consumerSub2.receive(waitMills, TimeUnit.MILLISECONDS); if (msg != null) { messages2.add(msg.getMessageId()); } else { @@ -809,7 +806,7 @@ public void testBlockBrokerDispatching() { .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); int consumedMsgsSub3 = 0; for (int j = 0; j < totalProducedMsgs; j++) { - msg = consumer1Sub3.receive(100, TimeUnit.MILLISECONDS); + msg = consumer1Sub3.receive(); if (msg != null) { consumedMsgsSub3++; consumer1Sub3.acknowledge(msg); From 9c24f79478e154e4a344d2c7046813ebda0955d6 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Sat, 29 Jan 2022 05:14:29 +0100 Subject: [PATCH 309/823] KeyShared stickyHashRange subscription: prevent stuck subscription in case of consumer restart (#14014) ### Motivation When using KeyShared subscription with `stickyHashRange` it is possible to a stuck subscription while restarting the consumers. This bug is not a regression in 2.10, the problem is present also in Pulsar 2.8 (and probably older versions) ### Modifications add the entry to the list of messaged to be redelivered (cherry picked from commit da9e80650b1cc3d573008c38585ec2e6ed4a00e9) --- ...tStickyKeyDispatcherMultipleConsumers.java | 1 + .../client/api/KeySharedSubscriptionTest.java | 128 ++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 420795cad2313..21c98b9ab3399 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -213,6 +213,7 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { groupedEntries.computeIfAbsent(c, k -> new ArrayList<>()).add(entry); consumerStickyKeyHashesMap.computeIfAbsent(c, k -> new HashSet<>()).add(stickyKeyHash); } else { + addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); entry.release(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index 8fc89806289ba..057f0c6392a93 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -40,6 +40,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -1372,4 +1373,131 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe return null; } } + + @Test + public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, InterruptedException { + final String topic = TopicName.get("persistent", "public", "default", + "testStickyKeyRangesRestartConsumers" + UUID.randomUUID()).toString(); + + final String subscriptionName = "my-sub"; + + final int numMessages = 100; + // start 2 consumers + Set sentMessages = new ConcurrentSkipListSet<>(); + + CountDownLatch count1 = new CountDownLatch(2); + CountDownLatch count2 = new CountDownLatch(13); // consumer 2 usually receive the fix messages + CountDownLatch count3 = new CountDownLatch(numMessages); + Consumer consumer1 = pulsarClient.newConsumer( + Schema.STRING) + .topic(topic) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.stickyHashRange().ranges(Range.of(0, 65536 / 2))) + .messageListener((consumer, msg) -> { + consumer.acknowledgeAsync(msg).whenComplete((m, e) -> { + if (e != null) { + log.error("error", e); + } else { + sentMessages.remove(msg.getKey()); + count1.countDown(); + count3.countDown(); + } + }); + }) + .subscribe(); + + Consumer consumer2 = pulsarClient.newConsumer( + Schema.STRING) + .topic(topic) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.stickyHashRange().ranges(Range.of(65536 / 2 + 1, 65535))) + .messageListener((consumer, msg) -> { + consumer.acknowledgeAsync(msg).whenComplete((m, e) -> { + if (e != null) { + log.error("error", e); + } else { + sentMessages.remove(msg.getKey()); + count2.countDown(); + count3.countDown(); + } + }); + }) + .subscribe(); + + pulsar.getExecutor().submit(() -> { + try + { + try (Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .enableBatching(false) + .create();) { + for (int i = 0; i < numMessages; i++) + { + String key = "test" + i; + sentMessages.add(key); + producer.newMessage() + .key(key) + .value("test" + i). + send(); + Thread.sleep(100); + } + } + } catch (Throwable t) { + log.error("error", t); + }}); + + // wait for some messages to be received by both of the consumers + count1.await(); + count2.await(); + consumer1.close(); + consumer2.close(); + + // this sleep is to trigger a race condition that happens + // when there are some messages that cannot be dispatched while consuming + Thread.sleep(3000); + + // start consuming again... + + pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.stickyHashRange().ranges(Range.of(0, 65536 / 2))) + .messageListener((consumer, msg) -> { + consumer.acknowledgeAsync(msg).whenComplete((m, e) -> { + if (e != null) { + log.error("error", e); + } else { + sentMessages.remove(msg.getKey()); + count3.countDown(); + } + }); + }) + .subscribe(); + pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.stickyHashRange().ranges(Range.of(65536 / 2 + 1, 65535))) + .messageListener((consumer, msg) -> { + consumer.acknowledgeAsync(msg).whenComplete((m, e) -> { + if (e != null) { + log.error("error", e); + } else { + sentMessages.remove(msg.getKey()); + count3.countDown(); + } + }); + }) + .subscribe(); + // wait for all the messages to be delivered + count3.await(); + assertTrue(sentMessages.isEmpty(), "didn't receive " + sentMessages); + } } From a9ffed9e00a70dc06c1582b4f94cb39c5aa49a07 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Sat, 29 Jan 2022 15:34:57 +0800 Subject: [PATCH 310/823] [Schema] Fix parse BigDecimal (#14019) ### Motivation I can use Avro schema with BigDecimal in Pulsar 2.8.0, but this doesn't work on Pulsar 2.8.1, so I check this codebase and PR, found https://github.com/apache/pulsar/pull/10428 breaks this. The following is error log: ``` org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:154) ~[spring-beans-5.3.4.jar:5.3.4] ... 34 common frames omitted Caused by: java.lang.UnsupportedOperationException: No recommended schema for decimal (scale is required) at org.apache.pulsar.shade.org.apache.avro.Conversions$DecimalConversion.getRecommendedSchema(Conversions.java:73) ~[pulsar-client-2.8.1.jar:2.8.1] at org.apache.pulsar.shade.org.apache.avro.reflect.ReflectData.createSchema(ReflectData.java:696) ~[pulsar-client-2.8.1.jar:2.8.1] at org.apache.pulsar.shade.org.apache.avro.reflect.ReflectData.createFieldSchema(ReflectData.java:873) ~[pulsar-client-2.8.1.jar:2.8.1] at org.apache.pulsar.shade.org.apache.avro.reflect.ReflectData$AllowNull.createFieldSchema(ReflectData.java:92) ~[pulsar-client-2.8.1.jar:2.8.1] at org.apache.pulsar.shade.org.apache.avro.reflect.ReflectData.createSchema(ReflectData.java:736) ~[pulsar-client-2.8.1.jar:2.8.1] at org.apache.pulsar.shade.org.apache.avro.specific.SpecificData$3.computeValue(SpecificData.java:328) ~[pulsar-client-2.8.1.jar:2.8.1] at org.apache.pulsar.shade.org.apache.avro.specific.SpecificData$3.computeValue(SpecificData.java:325) ~[pulsar-client-2.8.1.jar:2.8.1] at java.base/java.lang.ClassValue.getFromHashMap(ClassValue.java:228) ~[na:na] at java.base/java.lang.ClassValue.getFromBackup(ClassValue.java:210) ~[na:na] at java.base/java.lang.ClassValue.get(ClassValue.java:116) ~[na:na] at org.apache.pulsar.shade.org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:339) ~[pulsar-client-2.8.1.jar:2.8.1] ... 52 common frames omitted ``` I think that Avro cannot work on using the ReflectData with Conversions.DecimalConversion to parse the BigDecimal field without AvroSchema when getting schema, but the Avro ReflectDatumWriter and ReflectDatumReader are working, it seems that Avro support for BigDecimal is not enough. Affected version: 2.8.1...2.8.x, 2.9.x ### Modifications - Skip add the DecimalConversion in `extractAvroSchema()` (cherry picked from commit 2ca8e8a7a94c7d2cbba771fe8bb1ce5b1f4445bd) --- .../pulsar/client/impl/schema/AvroSchema.java | 9 +++++++- .../client/impl/schema/util/SchemaUtil.java | 2 +- .../client/impl/schema/AvroSchemaTest.java | 23 ++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index cc31b1970a2fc..b34017e20aa1d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -106,7 +106,14 @@ public static AvroSchema of(Class pojo, Map properties } public static void addLogicalTypeConversions(ReflectData reflectData, boolean jsr310ConversionEnabled) { - reflectData.addLogicalTypeConversion(new Conversions.DecimalConversion()); + addLogicalTypeConversions(reflectData, jsr310ConversionEnabled, true); + } + + public static void addLogicalTypeConversions(ReflectData reflectData, boolean jsr310ConversionEnabled, + boolean decimalConversionEnabled) { + if (decimalConversionEnabled) { + reflectData.addLogicalTypeConversion(new Conversions.DecimalConversion()); + } reflectData.addLogicalTypeConversion(new TimeConversions.DateConversion()); reflectData.addLogicalTypeConversion(new TimeConversions.TimeMillisConversion()); reflectData.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java index 2d0c810d8f029..abf5208628daf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java @@ -94,7 +94,7 @@ public static Schema extractAvroSchema(SchemaDefinition schemaDefinition, Class ReflectData reflectData = schemaDefinition.getAlwaysAllowNull() ? new ReflectData.AllowNull() : new ReflectData(); - AvroSchema.addLogicalTypeConversions(reflectData, schemaDefinition.isJsr310ConversionEnabled()); + AvroSchema.addLogicalTypeConversions(reflectData, schemaDefinition.isJsr310ConversionEnabled(), false); return reflectData.getSchema(pojo); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 00cbbdd6ad201..9e707af836796 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -36,7 +36,6 @@ import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.UUID; - import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; @@ -438,4 +437,26 @@ public void testAvroUUID() { assertEquals(pojo1.uid, pojo2.uid); } + static class MyBigDecimalPojo { + public BigDecimal value1; + @org.apache.avro.reflect.AvroSchema("{\n" + + " \"type\": \"bytes\",\n" + + " \"logicalType\": \"decimal\",\n" + + " \"precision\": 4,\n" + + " \"scale\": 2\n" + + "}") + public BigDecimal value2; + } + + @Test + public void testAvroBigDecimal() { + org.apache.pulsar.client.api.Schema schema = + org.apache.pulsar.client.api.Schema.AVRO(MyBigDecimalPojo.class); + MyBigDecimalPojo myBigDecimalPojo = new MyBigDecimalPojo(); + myBigDecimalPojo.value1 = new BigDecimal("10.21"); + myBigDecimalPojo.value2 = new BigDecimal("10.22"); + MyBigDecimalPojo pojo2 = schema.decode(schema.encode(myBigDecimalPojo)); + assertEquals(pojo2.value1, myBigDecimalPojo.value1); + assertEquals(pojo2.value2, myBigDecimalPojo.value2); + } } From b5160f7b9fce15cc8ce7a8633e102d5e7ada0089 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Sun, 30 Jan 2022 01:08:11 +0800 Subject: [PATCH 311/823] Fix NPE of cumulative ack mode and incorrect unack message count (#14021) link https://github.com/apache/pulsar/pull/13383 ## Motivation #13383 has fixed the batch message ack does not decrease the unacked-msg count, but in cumulative ack mode also decrease, it will use pendingAcks, but in cumulative ack, this will not init. ![image](https://user-images.githubusercontent.com/39078850/151622041-7fb0acc5-32fd-4140-82d7-8c75d2a6aef5.png) ![image](https://user-images.githubusercontent.com/39078850/151622106-bf75f3fa-84d5-4099-99f4-50f4dddd43a2.png) If ack the batch index one by one, the last ack of a batch will decrease unack message with `batchSize` ``` ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 1 ================ batch size -> 10 ================ message id -> 3:1 ================ acked count -> 9 ================ batch size -> 10 ``` ### Modifications add judge `Subscription.isIndividualAckMode(subType)` when get ackCount. If the ack from consumer don't have ackset, we should treat it as empty ackset to calculate the ack count with the currently ackset. (cherry picked from commit 618f17c155a8e28a99bbcc5e26f9ec3a6c7d9b08) --- .../pulsar/broker/service/Consumer.java | 7 +- .../BatchMessageWithBatchIndexLevelTest.java | 68 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 49b5704eb8ed0..d3fa495a233ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -122,6 +122,7 @@ public class Consumer { private static final AtomicIntegerFieldUpdater AVG_MESSAGES_PER_ENTRY = AtomicIntegerFieldUpdater.newUpdater(Consumer.class, "avgMessagesPerEntry"); private volatile int avgMessagesPerEntry = 1000; + private static final long [] EMPTY_ACK_SET = new long[0]; private static final double avgPercent = 0.9; private boolean preciseDispatcherFlowControl; @@ -413,10 +414,10 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map consumer = (ConsumerImpl) pulsarClient + .newConsumer(Schema.BYTES) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName(subscriptionName) + .subscriptionType(subType) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient + .newProducer() + .enableBatching(enableBatch) + .topic(topicName) + .batchingMaxMessages(10) + .create(); + + CountDownLatch countDownLatch = new CountDownLatch(messageCount); + for (int i = 0; i < messageCount; i++) { + producer.sendAsync((i + "").getBytes()).thenRun(countDownLatch::countDown); + } + + countDownLatch.await(); + + for (int i = 0; i < messageCount; i++) { + Message message = consumer.receive(); + // wait for receipt + if (i < messageCount / 2) { + consumer.acknowledgeAsync(message.getMessageId()).get(); + } + } + + String topic = TopicName.get(topicName).toString(); + PersistentSubscription persistentSubscription = (PersistentSubscription) pulsar.getBrokerService() + .getTopic(topic, false).get().get().getSubscription(subscriptionName); + + Awaitility.await().untilAsserted(() -> { + if (subType == SubscriptionType.Shared) { + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), messageCount / 2); + } else { + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), 0); + } + }); + } } From 735f700f93a529a0b150057bbf46d72c93536d46 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Sun, 30 Jan 2022 11:09:23 +0800 Subject: [PATCH 312/823] [Transaction] Fix individual ack with transaction decrease unAckMessageCounnt (#14020) link https://github.com/apache/pulsar/pull/13383 ## Motivation #13383 has fixed the batch message ack does not decrease the unacked-msg count, but ack with transaction don't fix because decrease unAckMessageCount move to another method. ack with transaction can't decrease unackMessageCount. (cherry picked from commit 1e2ff8a3941b7cc6d583f528ceedc393b7e607fb) --- .../pulsar/broker/service/Consumer.java | 34 +++++---- .../client/impl/TransactionEndToEndTest.java | 70 ++++++++++++++++--- 2 files changed, 83 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index d3fa495a233ec..630caac413e85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -414,16 +414,7 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map individualAckWithTransaction(CommandAck ack) { for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); PositionImpl position; + long ackedCount = 0; + long batchSize = getBatchSize(msgId); + Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId()); if (msgId.getAckSetsCount() > 0) { - long[] acksSets = new long[msgId.getAckSetsCount()]; + long[] ackSets = new long[msgId.getAckSetsCount()]; for (int j = 0; j < msgId.getAckSetsCount(); j++) { - acksSets[j] = msgId.getAckSetAt(j); + ackSets[j] = msgId.getAckSetAt(j); } - position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), acksSets); + position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets); + ackedCount = getAckedCountForBatchIndexLevelEnabled(position, batchSize, ackSets); } else { position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId()); + ackedCount = getAckedCountForMsgIdNoAckSets(batchSize, position); } if (msgId.hasBatchIndex()) { @@ -481,6 +477,8 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { positionsAcked.add(new MutablePair<>(position, 0)); } + addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); + checkCanRemovePendingAcksAndHandle(position, msgId); checkAckValidationError(ack, position); @@ -520,6 +518,16 @@ private long getBatchSize(MessageIdData msgId) { return batchSize; } + private long getAckedCountForMsgIdNoAckSets(long batchSize, PositionImpl position) { + if (Subscription.isIndividualAckMode(subType) && isAcknowledgmentAtBatchIndexLevelEnabled) { + long[] cursorAckSet = getCursorAckSet(position); + if (cursorAckSet != null) { + return getAckedCountForBatchIndexLevelEnabled(position, batchSize, EMPTY_ACK_SET); + } + } + return batchSize; + } + private long getAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize, long[] ackSets) { long ackedCount = 0; if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index f52d3193050cd..1f2bd06b52fda 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -30,6 +30,7 @@ import java.util.concurrent.CompletableFuture; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -97,16 +98,12 @@ protected void cleanup() { super.internalCleanup(); } - @Test - public void noBatchProduceCommitTest() throws Exception { - produceCommitTest(false); - } - - @Test - public void batchProduceCommitTest() throws Exception { - produceCommitTest(true); + @DataProvider(name = "enableBatch") + public Object[][] enableBatch() { + return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } + @Test(dataProvider="enableBatch") private void produceCommitTest(boolean enableBatch) throws Exception { @Cleanup Consumer consumer = pulsarClient @@ -249,6 +246,63 @@ public void produceAbortTest() throws Exception { log.info("finished test partitionAbortTest"); } + @Test(dataProvider="enableBatch") + private void testAckWithTransactionReduceUnAckMessageCount(boolean enableBatch) throws Exception { + + final int messageCount = 50; + final String subName = "testAckWithTransactionReduceUnAckMessageCount"; + final String topicName = NAMESPACE1 + "/testAckWithTransactionReduceUnAckMessageCount-" + enableBatch; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer() + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true) + .subscribe(); + Awaitility.await().until(consumer::isConnected); + + Producer producer = pulsarClient + .newProducer() + .topic(topicName) + .enableBatching(enableBatch) + .batchingMaxMessages(10) + .create(); + + CountDownLatch countDownLatch = new CountDownLatch(messageCount); + for (int i = 0; i < messageCount; i++) { + producer.sendAsync((i + "").getBytes()).thenRun(countDownLatch::countDown); + } + + countDownLatch.await(); + + Transaction txn = getTxn(); + + for (int i = 0; i < messageCount / 2; i++) { + Message message = consumer.receive(); + consumer.acknowledgeAsync(message.getMessageId(), txn).get(); + } + + txn.commit().get(); + boolean flag = false; + String topic = TopicName.get(topicName).toString(); + for (int i = 0; i < getPulsarServiceList().size(); i++) { + CompletableFuture> topicFuture = getPulsarServiceList().get(i) + .getBrokerService().getTopic(topic, false); + + if (topicFuture != null) { + Optional topicOptional = topicFuture.get(); + if (topicOptional.isPresent()) { + PersistentSubscription persistentSubscription = + (PersistentSubscription) topicOptional.get().getSubscription(subName); + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), messageCount / 2); + flag = true; + } + } + } + assertTrue(flag); + } + @Test public void txnIndividualAckTestNoBatchAndSharedSub() throws Exception { txnAckTest(false, 1, SubscriptionType.Shared); From 1d3a40e8c9379f01c782637485970e9ff2bc4fcc Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 23 Nov 2021 22:17:16 +0800 Subject: [PATCH 313/823] Increase timeout in PersistentStreamingDispatcherBlockConsumerTest.testBlockBrokerDispatching (#12943) (cherry picked from commit 5abf42cf24b43f7f1874eb0b385e093926ea0b21) --- .../apache/pulsar/client/api/DispatcherBlockConsumerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java index 00f52ad090e1f..0f21e211d92ce 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java @@ -672,7 +672,7 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw * * */ - @Test(timeOut = 10000) + @Test(timeOut = 60000) public void testBlockBrokerDispatching() { log.info("-- Starting {} test --", methodName); From 4691d3e80c0afc2b31e15023263247d171b7a696 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 28 Jan 2022 22:01:21 +0200 Subject: [PATCH 314/823] Add null check to workaround NPE in unit tests with Mockito/PowerMock (#14006) - Fixes #13620 (cherry picked from commit e45b4f211db43cdbf604a6dfc5bf305b541b0702) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 9f317be3280ff..189c079823de3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -230,7 +230,10 @@ public ServerCnx(PulsarService pulsar) { } public ServerCnx(PulsarService pulsar, String listenerName) { - super(pulsar.getBrokerService().getKeepAliveIntervalSeconds(), TimeUnit.SECONDS); + // pulsar.getBrokerService() can sometimes be null in unit tests when using mocks + // the null check is a workaround for #13620 + super(pulsar.getBrokerService() != null ? pulsar.getBrokerService().getKeepAliveIntervalSeconds() : 0, + TimeUnit.SECONDS); this.service = pulsar.getBrokerService(); this.schemaService = pulsar.getSchemaRegistryService(); this.listenerName = listenerName; From 2636aa486e041f49d72821145ebd9cd1a2e63778 Mon Sep 17 00:00:00 2001 From: Addison Higham Date: Sun, 30 Jan 2022 00:51:44 -0700 Subject: [PATCH 315/823] Add a default timeout for OAuth2 Metadata Resolver (#14056) ## Motivation Currently, there is no timeout on the metadata resolver in oauth2 clients. This can result in a connection hanging indefintely. We should add sane defaults here. ## Modification These defaults match the defaults of the TokenClient and generally the Pulsar defaults of 10s connect timeout and 30s read timeout ## Test This seems to primarily depend on testing timeout of the java HTTP lib, so no test is likely needed. ## Doc This is a trivial change and doesn't require docs --- .../impl/auth/oauth2/protocol/DefaultMetadataResolver.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/DefaultMetadataResolver.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/DefaultMetadataResolver.java index 2c09113055ba0..7d6ca1e5efabd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/DefaultMetadataResolver.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/DefaultMetadataResolver.java @@ -33,6 +33,9 @@ */ public class DefaultMetadataResolver implements MetadataResolver { + protected static final int DEFAULT_CONNECT_TIMEOUT_IN_SECONDS = 10; + protected static final int DEFAULT_READ_TIMEOUT_IN_SECONDS = 30; + private final URL metadataUrl; private final ObjectReader objectReader; private Duration connectTimeout; @@ -41,6 +44,9 @@ public class DefaultMetadataResolver implements MetadataResolver { public DefaultMetadataResolver(URL metadataUrl) { this.metadataUrl = metadataUrl; this.objectReader = new ObjectMapper().readerFor(Metadata.class); + // set a default timeout to ensure that this doesn't block + this.connectTimeout = Duration.ofSeconds(DEFAULT_CONNECT_TIMEOUT_IN_SECONDS); + this.readTimeout = Duration.ofSeconds(DEFAULT_READ_TIMEOUT_IN_SECONDS); } public DefaultMetadataResolver withConnectTimeout(Duration connectTimeout) { From e8d65a8fc56f3bb07bdcdda0d3019e1bb48dc294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 2 Feb 2022 05:55:21 +0100 Subject: [PATCH 316/823] [branch-2.9] Bump version to 2.9.3-SNAPSHOT (#14089) --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- deployment/terraform-ansible/deploy-pulsar.yaml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 127 files changed, 127 insertions(+), 127 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index ddca4e6494ada..9b36a306b979c 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index fb84e6fe63d32..616aa02c01a43 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 4fa3ead0eadd4..389cfb7f2e90c 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 979803c96f58c..49bc0d85da445 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 2a6002297c1cb..1d549ff0deece 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.2 + 2.9.3-SNAPSHOT jar Pulsar Build Tools diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml index cdde04d8aab5c..a06b73b99852e 100644 --- a/deployment/terraform-ansible/deploy-pulsar.yaml +++ b/deployment/terraform-ansible/deploy-pulsar.yaml @@ -39,7 +39,7 @@ zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" service_url: "{{ pulsar_service_url }}" http_url: "{{ pulsar_web_url }}" - pulsar_version: "2.9.2" + pulsar_version: "2.9.3-SNAPSHOT" - name: Download Pulsar binary package unarchive: src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 155b7da8a080d..4ee3328cc4d8b 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index ff4055cafd909..1248ea1682609 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 72afff04ad6ab..f79f26d45fb26 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 5fa966d476187..ed5401ad39650 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index df9617f314ec9..adb423253ef31 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 94fabef4d82be..a3534d2c0c83d 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index a897b00b648cb..b9e4f8d158a49 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 9ee0334651e1c..f772a4403bf98 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 3e51419304178..4b46838a3ec3f 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 85d189f87b5b5..f32bc87c46e27 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 63525923b0003..509d00380abc0 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 2a14a858dfe2f..ac79ca3c8dba2 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index c6ff0570e1bde..fdd52671868fe 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 660097361abad..92f27eb385ca7 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 77757b42fa6d8..9e755b12161f1 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 289105f78f2f8..68a53c806ffef 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 60b175c3d4ecd..2d4605a7a0153 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 6078062dc1e69..ac7d7ec5354da 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 59cf04834338d..b17e509d89a9f 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 1f3346fd0da53..6c0bde533061e 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 60840c9f37f44..e5147c4837a8a 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index faebb94d05622..0a3dc027dfb86 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index f6db9d39ee693..74f20a15f4caf 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 2722b42911976..37d12891d049c 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index c9ef5d87e4106..e7808b61a03ea 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 40f5f8407b8d5..11f5fac560a32 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 020af46f5f855..8e7a0b4ecad6e 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 9f9528b58e80e..3b24ced062b6e 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 97002332870b7..891bd2f7f8b88 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 2397289149c1e..593d5842448e7 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 9252d603f32b4..27838ecddf998 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 089ac49c5d579..f5c6cd2e06392 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 42c823a7f8615..c4b5a4b87ba4d 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index afbc14ec0eec1..759e6acccf810 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 6e42f63c66acc..1392e9badc7ba 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index efbb8fb6229c4..6790d1d6b7886 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index ad37c3588e0fe..f6cf87853f7d2 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 254ed1437d129..732dee3f597ea 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index b5c22b5cff2af..d05780478a168 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 36591e6f2f194..727304d30e78a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 871a07a9658dc..2506936143b58 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index bf01c32367513..14be823874009 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 2037b5d3f6194..4278a6553ab11 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 30c8375a0c7b9..247a6393a81fd 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 676a983f20dc2..b16f364689af8 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 8bc57600a8759..7b76ab13e098a 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 8bc73937e2029..83395d20966b3 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 176d920631e2f..f06c5a609b1f3 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 6ce7e8c415c03..70778879c23df 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index b66f5bddd275b..3833961d94c57 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 4c5b57d67fbbd..889f0ab150cb2 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 311197c9f2756..bf92cea841c28 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index d4a711ed01186..71606c0548153 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 917981f55658c..f1c47fb5dfa2a 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 1a49c3eb32b6e..81dad1f245cee 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 5401ace104fc4..21423a08143bd 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index c6c8c58f95e27..53910d0cff22f 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 0a0f831bef961..229623deee7a0 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index c963ae626c7e4..a68c942d97180 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index cde53aeb6359b..e5badf75a677e 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 79b92b89999c2..f7005c635e9ff 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index ad331e30847d1..32e3ff17bfcd0 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 38c8b53a6bd0b..e8f597224e582 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 0a173aa8650e8..fbdaa6345abb0 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 180e338099f7f..bdd7bfc55758b 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 4a7a2a335e4b7..0fb45c9a5c009 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index b259cbb1de580..a760ee9584693 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index cde101487ff3e..5025e47bf2dbc 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 2f3e178a9de09..beec6c6ab96e5 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index d6dabf60355ba..8a23962c7388d 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 3fa60cd1807d7..ef84dd16c2e00 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index b1fcbc010f996..5e48ab354e27b 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 2db6fb2ef7d99..6bc9c0a025cd3 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 483a855523e08..67b887c1e07ff 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 5f95ff1e50211..aad83eaeeaa06 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 127c6f72ef6fa..b53a8ed9282df 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index e1d65e87302bf..67327a8f4d46c 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index af7f025330897..8199c1fe3df18 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 4df689dfff5d2..c22f704565290 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index f3060d71166bc..a6222fda0a7e0 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 5d6166e13ffad..76e8aa5fbd05a 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 1678a9294dc89..d37dd1e8f9f08 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 2bf4d0baf3452..739f1acade64f 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index bdf1081106964..6a4d3df44430b 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 5974289444586..350bcfd05a345 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index e573833baba2d..8824f35891b16 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 7291f3e88f29e..bb10dec5cdc53 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 76fcf52c380fa..72a9162706f89 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 780e216c96484..f8f8a2a6491ee 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 809def86ed1dc..468a8debdb903 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index e688a104a16f2..eb93d9900c971 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 168397dacb44d..c205dcad6d505 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index f6f776ce015a3..ad838c8a96bab 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 39a783c27ac7e..14e7cd0f04a92 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 31ec8436754ff..206e926dec9b7 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index e3f0ae3b51475..034997d40f7a2 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2 + 2.9.3-SNAPSHOT pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index b1f9486286769..f49d541cdd570 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2 + 2.9.3-SNAPSHOT pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 48376baa59270..a67a7debc8d77 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2 + 2.9.3-SNAPSHOT pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index c31872944da0d..9e686cde439c5 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index ddc76bebb24e9..58ca5ad6fc17a 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index b4ddf32ae19de..b274931049df1 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index e0b566a0c3434..7aeb860a9eaf3 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 2fff29d1c0165..8a85e3f8d641d 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index e2c29fe00ae0b..4b21f3e8ed25c 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index e52e5b7350f2c..1bead7e397c59 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index b18cd2eb75c9b..c4db8a1ade2de 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 80f166bb0c575..acb01809c8f13 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 9a61cb41d4f6e..e0ac02e7f4747 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 559bc5fe6f658..9cc9619d2a388 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 20e71e217ce44..06d6f18a678d2 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 46558ca492aa2..063a4c1a50075 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index c9e16e115f056..40518114309b2 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index f2681751f3eda..e114b1c6bd945 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 1314c15dcb45c..16a0eb7d6bd5a 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT integration diff --git a/tests/pom.xml b/tests/pom.xml index 512c1529b195f..e34f6678186bb 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index a84660e326864..dbb9be2d0d670 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 9bdd8fea40493..664c56cfd5044 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 281cf4cd64c02..191c31527877e 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index f839ad30810ee..3e3c1985f0b6d 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 716aed18b98f0..b776161662dc8 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 5b8ae94a8e323..132083817932c 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. From 9799bb0cd37220f9025f8f274b2e68801abd2d98 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 4 Feb 2022 15:29:23 -0800 Subject: [PATCH 317/823] Use `scheduleWithFixedDelay` instead of `scheduleAtFixedRate` for java producer batch timer (#14125) Fixes #11100 ### Motivation We believe that the use of `scheduleAtFixedRate` in the java producer's batch timer can result in unnecessarily high thread usage, which can become especially problematic for applications that start many producers. ### Modifications Replaced the use of `scheduleAtFixedRate` with `scheduleWithFixedDelay`, which is the same behavior as previously in 2.6.x. The producer's parameter `batchingMaxPublishDelay` implies the use of the "delay" method instead of "rate" method as well. ### Verifying this change - [x] Make sure that the change passes the CI checks. This change is already covered by existing tests, such as existing pulsar client producer tests. Testing of the performance regression can be demonstrated by using [this](https://github.com/klevy-toast/dropwizard-pulsar-test) artifact and comparing a recent release of pulsar client with a manually built SNAPSHOT version with this change: #### Version 2.7.1 CPU & thread behavior - While sending messages image - While running idle producers image - 30 second profile while sending messages image #### Version 2.10.0-SNAPSHOT CPU & thread behavior - While sending messages image - While running idle producers image - 30 second profile while sending messages image These samples show fewer threads running with this change compared to 2.7.1, less time spend in `batchMessageAndSend`, and overall lower CPU usage -- note that this testing was done on a laptop machine, and we have observed, along with [other users](https://github.com/apache/pulsar/issues/11100#issuecomment-1007487433) that this CPU regression can be much worse with more producers, smaller batch intervals, and on deployed cloud applications. ### Does this pull request potentially affect one of the following parts: *If `yes` was chosen, please highlight the changes* - Dependencies (does it add or upgrade a dependency): (no) - The public API: (no) - The schema: (no) - The default values of configurations: (no) - The wire protocol: (no) - The rest endpoints: (no) - The admin cli options: (no) - Anything that affects deployment: (no) ### Documentation Check the box below or label this PR directly (if you have committer privilege). Need to update docs? - [x] `no-need-doc` The general behavior of the batch timer feature should not be changing (cherry picked from commit f9e69f08c21a57980ef6f6fbb9b8c974621d9bab) --- .../main/java/org/apache/pulsar/client/impl/ProducerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 95adbcc6f34b7..877d88bd9508d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1479,7 +1479,7 @@ public void connectionOpened(final ClientCnx cnx) { if (!producerCreatedFuture.isDone() && isBatchMessagingEnabled()) { // schedule the first batch message task batchTimerTask = cnx.ctx().executor() - .scheduleAtFixedRate(catchingAndLoggingThrowables(() -> { + .scheduleWithFixedDelay(catchingAndLoggingThrowables(() -> { if (log.isTraceEnabled()) { log.trace( "[{}] [{}] Batching the messages from the batch container from " From 68165c8a7066860bf6b874aea24a4e509ba6cff1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Jan 2022 19:34:44 +0200 Subject: [PATCH 318/823] [Proxy] Remove unnecessary Pulsar Client usage from Pulsar Proxy (#13836) (cherry picked from commit 324aa1bf14d89a93b66ed3613a16c118cf9d4c0f) --- .../pulsar/client/impl/ConnectionPool.java | 67 +++++------ .../pulsar/proxy/server/ProxyConnection.java | 105 +++++++++--------- .../pulsar/proxy/server/ProxyService.java | 11 -- 3 files changed, 83 insertions(+), 100 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 9cf2da1765342..3df52eaed2106 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -18,11 +18,9 @@ */ package org.apache.pulsar.client.impl; -import static org.apache.pulsar.common.util.netty.ChannelFutures.toCompletableFuture; import static org.apache.pulsar.client.util.MathUtils.signSafeMod; - +import static org.apache.pulsar.common.util.netty.ChannelFutures.toCompletableFuture; import com.google.common.annotations.VisibleForTesting; - import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelException; @@ -31,9 +29,6 @@ import io.netty.resolver.dns.DnsNameResolver; import io.netty.resolver.dns.DnsNameResolverBuilder; import io.netty.util.concurrent.Future; - -import java.io.Closeable; -import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; @@ -45,9 +40,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; - import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.InvalidServiceURL; @@ -58,7 +51,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ConnectionPool implements Closeable { +public class ConnectionPool implements AutoCloseable { protected final ConcurrentHashMap>> pool; private final Bootstrap bootstrap; @@ -222,7 +215,7 @@ private CompletableFuture createConnection(InetSocketAddress logicalA } /** - * Resolve DNS asynchronously and attempt to connect to any IP address returned by DNS server + * Resolve DNS asynchronously and attempt to connect to any IP address returned by DNS server. */ private CompletableFuture createConnection(InetSocketAddress unresolvedAddress) { int port; @@ -247,27 +240,32 @@ private CompletableFuture createConnection(InetSocketAddress unresolved } /** - * Try to connect to a sequence of IP addresses until a successfull connection can be made, or fail if no address is - * working + * Try to connect to a sequence of IP addresses until a successful connection can be made, or fail if no + * address is working. */ - private CompletableFuture connectToResolvedAddresses(Iterator unresolvedAddresses, int port, InetSocketAddress sniHost) { + private CompletableFuture connectToResolvedAddresses(Iterator unresolvedAddresses, + int port, + InetSocketAddress sniHost) { CompletableFuture future = new CompletableFuture<>(); // Successfully connected to server - connectToAddress(unresolvedAddresses.next(), port, sniHost).thenAccept(future::complete).exceptionally(exception -> { - if (unresolvedAddresses.hasNext()) { - // Try next IP address - connectToResolvedAddresses(unresolvedAddresses, port, sniHost).thenAccept(future::complete).exceptionally(ex -> { - // This is already unwinding the recursive call - future.completeExceptionally(ex); + connectToAddress(unresolvedAddresses.next(), port, sniHost) + .thenAccept(future::complete) + .exceptionally(exception -> { + if (unresolvedAddresses.hasNext()) { + // Try next IP address + connectToResolvedAddresses(unresolvedAddresses, port, sniHost).thenAccept(future::complete) + .exceptionally(ex -> { + // This is already unwinding the recursive call + future.completeExceptionally(ex); + return null; + }); + } else { + // Failed to connect to any IP address + future.completeExceptionally(exception); + } return null; }); - } else { - // Failed to connect to any IP address - future.completeExceptionally(exception); - } - return null; - }); return future; } @@ -285,7 +283,7 @@ CompletableFuture> resolveName(String hostname) { } /** - * Attempt to establish a TCP connection to an already resolved single IP address + * Attempt to establish a TCP connection to an already resolved single IP address. */ private CompletableFuture connectToAddress(InetAddress ipAddress, int port, InetSocketAddress sniHost) { InetSocketAddress remoteAddress = new InetSocketAddress(ipAddress, port); @@ -293,12 +291,11 @@ private CompletableFuture connectToAddress(InetAddress ipAddress, int p return toCompletableFuture(bootstrap.register()) .thenCompose(channel -> channelInitializerHandler .initTls(channel, sniHost != null ? sniHost : remoteAddress)) - .thenCompose(channel -> channelInitializerHandler - .initSocks5IfConfig(channel)) + .thenCompose(channelInitializerHandler::initSocks5IfConfig) .thenCompose(channel -> toCompletableFuture(channel.connect(remoteAddress))); } else { return toCompletableFuture(bootstrap.register()) - .thenCompose(channel -> channelInitializerHandler.initSocks5IfConfig(channel)) + .thenCompose(channelInitializerHandler::initSocks5IfConfig) .thenCompose(channel -> toCompletableFuture(channel.connect(remoteAddress))); } } @@ -307,7 +304,7 @@ public void releaseConnection(ClientCnx cnx) { if (maxConnectionsPerHosts == 0) { //Disable pooling if (cnx.channel().isActive()) { - if(log.isDebugEnabled()) { + if (log.isDebugEnabled()) { log.debug("close connection due to pooling disabled."); } cnx.close(); @@ -316,14 +313,8 @@ public void releaseConnection(ClientCnx cnx) { } @Override - public void close() throws IOException { - try { - if (!eventLoopGroup.isShutdown()) { - eventLoopGroup.shutdownGracefully(0, 10, TimeUnit.SECONDS).await(); - } - } catch (InterruptedException e) { - log.warn("EventLoopGroup shutdown was interrupted", e); - } + public void close() throws Exception { + closeAllConnections(); dnsResolver.close(); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index f1b780726489c..ff392ca0f09e3 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -21,8 +21,9 @@ import static com.google.common.base.Preconditions.checkArgument; import java.net.SocketAddress; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; - +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; @@ -34,11 +35,9 @@ import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarChannelInitializer; -import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.protocol.Commands; @@ -68,8 +67,9 @@ */ public class ProxyConnection extends PulsarHandler implements FutureListener { // ConnectionPool is used by the proxy to issue lookup requests - private PulsarClientImpl client; private ConnectionPool connectionPool; + private final AtomicLong requestIdGenerator = + new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2)); private ProxyService service; AuthenticationDataSource authenticationData; private State state; @@ -113,7 +113,7 @@ enum State { } ConnectionPool getConnectionPool() { - return client.getCnxPool(); + return connectionPool; } public ProxyConnection(ProxyService proxyService, Supplier sslHandlerSupplier) { @@ -130,7 +130,6 @@ public void channelRegistered(ChannelHandlerContext ctx) throws Exception { if (ProxyService.activeConnections.get() > service.getConfiguration().getMaxConcurrentInboundConnections()) { ctx.close(); ProxyService.rejectedConnections.inc(); - return; } } @@ -149,26 +148,27 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { } @Override - public void channelInactive(ChannelHandlerContext ctx) throws Exception { + public synchronized void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); if (directProxyHandler != null && directProxyHandler.outboundChannel != null) { directProxyHandler.outboundChannel.close(); + directProxyHandler = null; } - if (client != null) { - client.close(); - } service.getClientCnxs().remove(this); LOG.info("[{}] Connection closed", remoteAddress); if (connectionPool != null) { try { connectionPool.close(); + connectionPool = null; } catch (Exception e) { LOG.error("Failed to close connection pool {}", e.getMessage(), e); } } + + state = State.Closed; } @Override @@ -222,7 +222,30 @@ public void operationComplete(Future future) throws Exception { } } - private void completeConnect() { + private synchronized void completeConnect(AuthData clientData) throws PulsarClientException { + if (service.getConfiguration().isAuthenticationEnabled()) { + if (service.getConfiguration().isForwardAuthorizationCredentials()) { + this.clientAuthData = clientData; + this.clientAuthMethod = authMethod; + } + if (this.connectionPool == null) { + this.connectionPool = new ProxyConnectionPool(clientConf, service.getWorkerGroup(), + () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, clientAuthData, + clientAuthMethod, protocolVersionToAdvertise)); + } else { + LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", + remoteAddress, state, clientAuthRole); + } + } else { + if (this.connectionPool == null) { + this.connectionPool = new ProxyConnectionPool(clientConf, service.getWorkerGroup(), + () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise)); + } else { + LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {}", + remoteAddress, state); + } + } + LOG.info("[{}] complete connection, init proxy handler. authenticated with {} role {}, hasProxyToBrokerUrl: {}", remoteAddress, authMethod, clientAuthRole, hasProxyToBrokerUrl); if (hasProxyToBrokerUrl) { @@ -242,17 +265,6 @@ private void completeConnect() { } } - private void createClientAndCompleteConnect(AuthData clientData) - throws PulsarClientException { - if (service.getConfiguration().isForwardAuthorizationCredentials()) { - this.clientAuthData = clientData; - this.clientAuthMethod = authMethod; - } - this.client = createClient(clientConf, this.clientAuthData, this.clientAuthMethod, protocolVersionToAdvertise); - - completeConnect(); - } - // According to auth result, send newConnected or newAuthChallenge command. private void doAuthentication(AuthData clientData) throws Exception { AuthData brokerData = authState.authenticate(clientData); @@ -263,7 +275,7 @@ private void doAuthentication(AuthData clientData) throws Exception { LOG.debug("[{}] Client successfully authenticated with {} role {}", remoteAddress, authMethod, clientAuthRole); } - createClientAndCompleteConnect(clientData); + completeConnect(clientData); return; } @@ -274,7 +286,6 @@ private void doAuthentication(AuthData clientData) throws Exception { remoteAddress, authMethod); } state = State.Connecting; - return; } @Override @@ -302,16 +313,10 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), try { // init authn this.clientConf = createClientConfiguration(); - int protocolVersion = getProtocolVersionToAdvertise(connect); // authn not enabled, complete if (!service.getConfiguration().isAuthenticationEnabled()) { - this.connectionPool = new ProxyConnectionPool(clientConf, service.getWorkerGroup(), - () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersion)); - this.client = - new PulsarClientImpl(clientConf, service.getWorkerGroup(), connectionPool, service.getTimer()); - - completeConnect(); + completeConnect(null); return; } @@ -336,7 +341,7 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), .orElseThrow(() -> new AuthenticationException("No anonymous role, and no authentication provider configured")); - createClientAndCompleteConnect(clientData); + completeConnect(clientData); return; } @@ -354,7 +359,6 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), LOG.warn("[{}] Unable to authenticate: ", remoteAddress, e); ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, "Failed to authenticate")); close(); - return; } } @@ -409,19 +413,26 @@ protected void handleLookup(CommandLookupTopic lookup) { lookupProxyHandler.handleLookup(lookup); } - private void close() { - state = State.Closed; - ctx.close(); - try { - if (client != null) { - client.close(); + private synchronized void close() { + if (state != State.Closed) { + state = State.Closed; + if (directProxyHandler != null && directProxyHandler.outboundChannel != null) { + directProxyHandler.outboundChannel.close(); + directProxyHandler = null; + } + if (connectionPool != null) { + try { + connectionPool.close(); + connectionPool = null; + } catch (Exception e) { + LOG.error("Error closing connection pool", e); + } } - } catch (PulsarClientException e) { - LOG.error("Unable to close pulsar client - {}. Error - {}", client, e.getMessage()); + ctx.close(); } } - ClientConfigurationData createClientConfiguration() throws UnsupportedAuthenticationException { + ClientConfigurationData createClientConfiguration() { ClientConfigurationData clientConf = new ClientConfigurationData(); clientConf.setServiceUrl(service.getServiceUrl()); ProxyConfiguration proxyConfig = service.getConfiguration(); @@ -441,20 +452,12 @@ ClientConfigurationData createClientConfiguration() throws UnsupportedAuthentica return clientConf; } - private PulsarClientImpl createClient(final ClientConfigurationData clientConf, final AuthData clientAuthData, - final String clientAuthMethod, final int protocolVersion) throws PulsarClientException { - this.connectionPool = new ProxyConnectionPool(clientConf, service.getWorkerGroup(), - () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, clientAuthData, - clientAuthMethod, protocolVersion)); - return new PulsarClientImpl(clientConf, service.getWorkerGroup(), connectionPool, service.getTimer()); - } - private static int getProtocolVersionToAdvertise(CommandConnect connect) { return Math.min(connect.getProtocolVersion(), Commands.getCurrentProtocolVersion()); } long newRequestId() { - return client.newRequestId(); + return requestIdGenerator.getAndIncrement(); } public Authentication getClientAuthentication() { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index af5b2a8ec9409..fc4791b3aa40a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -30,8 +30,6 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.util.concurrent.DefaultThreadFactory; -import io.netty.util.HashedWheelTimer; -import io.netty.util.Timer; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import java.io.Closeable; @@ -73,7 +71,6 @@ public class ProxyService implements Closeable { private final ProxyConfiguration proxyConfig; private final Authentication proxyClientAuthentication; - private final Timer timer; private String serviceUrl; private String serviceUrlTls; private final AuthenticationService authenticationService; @@ -133,7 +130,6 @@ public ProxyService(ProxyConfiguration proxyConfig, AuthenticationService authenticationService) throws Exception { checkNotNull(proxyConfig); this.proxyConfig = proxyConfig; - this.timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), 1, TimeUnit.MILLISECONDS); this.clientCnxs = Sets.newConcurrentHashSet(); this.topicStats = Maps.newConcurrentMap(); @@ -314,9 +310,6 @@ public void close() throws IOException { } acceptorGroup.shutdownGracefully(); workerGroup.shutdownGracefully(); - if (timer != null) { - timer.stop(); - } } public String getServiceUrl() { @@ -331,10 +324,6 @@ public ProxyConfiguration getConfiguration() { return proxyConfig; } - public Timer getTimer() { - return timer; - } - public AuthenticationService getAuthenticationService() { return authenticationService; } From b4733d1cbd147ed9d3615887b9aa97e7fb1dbfd1 Mon Sep 17 00:00:00 2001 From: Addison Higham Date: Sun, 30 Jan 2022 13:04:26 -0700 Subject: [PATCH 319/823] Allow config of IO and acceptor threads in proxy (#14054) * Allow config of IO and acceptor threads in proxy Previously, the Pulasr Proxy did not allow configuration of the number of IO threads and acceptor threads in the proxy. These options can be very important to tune, as is tuneable in the broker, so this change simply matches the brokers perspective. Also, we increase the default number of IO threads to 2x number of processors instead of 1x, as in a single CPU config, it still makes sense to have 2 threads, at least for now, where some blocking operatings can happen (such as authn/authz plugins) * fix checkstyle (cherry picked from commit f455418c8e1efcd3895d4dab593aadb36a682165) --- .../pulsar/proxy/server/ProxyConfiguration.java | 14 ++++++++++++++ .../apache/pulsar/proxy/server/ProxyService.java | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 0573670e993ae..a49caae60bc21 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -505,6 +505,20 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private int httpNumThreads = Math.max(8, 2 * Runtime.getRuntime().availableProcessors()); + @FieldContext( + category = CATEGORY_SERVER, + doc = "Number of threads to use for Netty IO." + + " Default is set to `2 * Runtime.getRuntime().availableProcessors()`" + ) + private int numIOThreads = 2 * Runtime.getRuntime().availableProcessors(); + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Number of threads to use for Netty Acceptor." + + " Default is set to `1`" + ) + private int numAcceptorThreads = 1; + @Deprecated @FieldContext( category = CATEGORY_PLUGIN, diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index fc4791b3aa40a..73beda2b72e94 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -99,8 +99,6 @@ public class ProxyService implements Closeable { private final ScheduledExecutorService statsExecutor; - private static final int numThreads = Runtime.getRuntime().availableProcessors(); - static final Gauge activeConnections = Gauge .build("pulsar_proxy_active_connections", "Number of connections currently active in the proxy").create() .register(); @@ -141,8 +139,10 @@ public ProxyService(ProxyConfiguration proxyConfig, } else { proxyLogLevel = 0; } - this.acceptorGroup = EventLoopUtil.newEventLoopGroup(1, false, acceptorThreadFactory); - this.workerGroup = EventLoopUtil.newEventLoopGroup(numThreads, false, workersThreadFactory); + this.acceptorGroup = EventLoopUtil.newEventLoopGroup(proxyConfig.getNumAcceptorThreads(), + false, acceptorThreadFactory); + this.workerGroup = EventLoopUtil.newEventLoopGroup(proxyConfig.getNumIOThreads(), + false, workersThreadFactory); this.authenticationService = authenticationService; // Initialize the message protocol handlers From be806ea68e69ced4408370581496ca1a583526f3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 9 Feb 2022 20:35:45 +0200 Subject: [PATCH 320/823] [Proxy] Fix port exhaustion and connection issues in Pulsar Proxy (#14078) (cherry picked from commit 640b4e6ec14d9b812da608037c58b664e6778637) --- .../server/src/assemble/LICENSE.bin.txt | 2 + pom.xml | 1 + .../ProxySaslAuthenticationTest.java | 1 + .../pulsar/common/protocol/PulsarHandler.java | 2 +- pulsar-proxy/pom.xml | 17 +- .../proxy/server/BrokerProxyValidator.java | 181 ++++++++++++++++++ .../proxy/server/DirectProxyHandler.java | 154 ++++++++------- .../proxy/server/ProxyConfiguration.java | 39 ++++ .../pulsar/proxy/server/ProxyConnection.java | 101 ++++++++-- .../pulsar/proxy/server/ProxyService.java | 17 ++ .../proxy/server/ProxyServiceStarter.java | 2 + .../server/ServiceChannelInitializer.java | 8 + .../server/TargetAddressDeniedException.java | 26 +++ .../server/AuthedAdminProxyHandlerTest.java | 1 + .../server/BrokerProxyValidatorTest.java | 102 ++++++++++ .../server/ProxyAdditionalServletTest.java | 1 + ...roxyAuthenticatedProducerConsumerTest.java | 1 + .../proxy/server/ProxyAuthenticationTest.java | 1 + .../proxy/server/ProxyConnectionTest.java | 38 ++++ .../server/ProxyConnectionThrottlingTest.java | 1 + .../ProxyEnableHAProxyProtocolTest.java | 1 + .../server/ProxyForwardAuthDataTest.java | 1 + .../server/ProxyKeyStoreTlsTestWithAuth.java | 1 + .../ProxyKeyStoreTlsTestWithoutAuth.java | 1 + .../server/ProxyLookupThrottlingTest.java | 1 + .../pulsar/proxy/server/ProxyParserTest.java | 1 + .../server/ProxyRolesEnforcementTest.java | 1 + .../proxy/server/ProxyServiceStarterTest.java | 5 +- .../server/ProxyServiceTlsStarterTest.java | 15 +- .../pulsar/proxy/server/ProxyStatsTest.java | 1 + .../apache/pulsar/proxy/server/ProxyTest.java | 1 + .../pulsar/proxy/server/ProxyTlsTest.java | 1 + .../proxy/server/ProxyTlsTestWithAuth.java | 1 + .../server/ProxyWithAuthorizationNegTest.java | 1 + .../server/ProxyWithAuthorizationTest.java | 2 + .../server/ProxyWithJwtAuthorizationTest.java | 1 + .../ProxyWithoutServiceDiscoveryTest.java | 1 + .../SuperUserAuthedAdminProxyHandlerTest.java | 1 + .../server/UnauthedAdminProxyHandlerTest.java | 1 + 39 files changed, 645 insertions(+), 89 deletions(-) create mode 100644 pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java create mode 100644 pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/TargetAddressDeniedException.java create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionTest.java diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 68297da5fc1e2..a4c643a765305 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -526,6 +526,8 @@ The Apache Software License, Version 2.0 - com.google.http-client-google-http-client-1.38.0.jar - com.google.auto.value-auto-value-annotations-1.7.4.jar - com.google.re2j-re2j-1.5.jar + * IPAddress + - com.github.seancfoley-ipaddress-5.3.3.jar BSD 3-clause "New" or "Revised" License * Google auth library diff --git a/pom.xml b/pom.xml index ac79ca3c8dba2..88e1635d9273b 100644 --- a/pom.xml +++ b/pom.xml @@ -201,6 +201,7 @@ flexible messaging model and an intuitive client API. 9.1.6 5.3.15 4.5.13 + 5.3.3 3.6.0 diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java index aabdcf81df387..e7b2c4411fdbe 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java @@ -224,6 +224,7 @@ void testAuthentication() throws Exception { ProxyConfiguration proxyConfig = new ProxyConfiguration(); proxyConfig.setAuthenticationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setSaslJaasClientAllowedIds(".*" + localHostname + ".*"); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index cdf372dca02c8..48e7ffa7b887b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -118,7 +118,7 @@ private void handleKeepAliveTimeout() { } } - protected void cancelKeepAliveTask() { + public void cancelKeepAliveTask() { if (keepAliveTask != null) { keepAliveTask.cancel(false); keepAliveTask = null; diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 14e7cd0f04a92..0faafd53034dd 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -19,7 +19,7 @@ --> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 org.apache.pulsar @@ -174,10 +174,17 @@ com.beust jcommander - - org.apache.logging.log4j - log4j-core - + + + org.apache.logging.log4j + log4j-core + + + + com.github.seancfoley + ipaddress + ${seancfoley.ipaddress.version} + diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java new file mode 100644 index 0000000000000..debe1f7fcac87 --- /dev/null +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.proxy.server; + +import inet.ipaddr.IPAddress; +import inet.ipaddr.IPAddressString; +import inet.ipaddr.ipv4.IPv4Address; +import inet.ipaddr.ipv6.IPv6Address; +import io.netty.resolver.AddressResolver; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.StringTokenizer; +import java.util.concurrent.CompletableFuture; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.netty.NettyFutureUtil; + +@Slf4j +public class BrokerProxyValidator { + private static final String SEPARATOR = "\\s*,\\s*"; + private static final String ALLOW_ANY = "*"; + private final int[] allowedTargetPorts; + private final boolean allowAnyTargetPort; + private final List allowedIPAddresses; + private final boolean allowAnyIPAddress; + private final AddressResolver inetSocketAddressResolver; + private final List allowedHostNames; + private final boolean allowAnyHostName; + + public BrokerProxyValidator(AddressResolver inetSocketAddressResolver, String allowedHostNames, + String allowedIPAddresses, String allowedTargetPorts) { + this.inetSocketAddressResolver = inetSocketAddressResolver; + List allowedHostNamesStrings = parseCommaSeparatedConfigValue(allowedHostNames); + if (allowedHostNamesStrings.contains(ALLOW_ANY)) { + this.allowAnyHostName = true; + this.allowedHostNames = Collections.emptyList(); + } else { + this.allowAnyHostName = false; + this.allowedHostNames = allowedHostNamesStrings.stream() + .map(BrokerProxyValidator::parseWildcardPattern).collect(Collectors.toList()); + } + List allowedIPAddressesStrings = parseCommaSeparatedConfigValue(allowedIPAddresses); + if (allowedIPAddressesStrings.contains(ALLOW_ANY)) { + allowAnyIPAddress = true; + this.allowedIPAddresses = Collections.emptyList(); + } else { + allowAnyIPAddress = false; + this.allowedIPAddresses = allowedIPAddressesStrings.stream().map(IPAddressString::new) + .filter(ipAddressString -> { + if (ipAddressString.isValid()) { + return true; + } else { + throw new IllegalArgumentException("Invalid IP address filter '" + ipAddressString + "'", + ipAddressString.getAddressStringException()); + } + }).map(IPAddressString::getAddress) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + List allowedTargetPortsStrings = parseCommaSeparatedConfigValue(allowedTargetPorts); + if (allowedTargetPortsStrings.contains(ALLOW_ANY)) { + allowAnyTargetPort = true; + this.allowedTargetPorts = new int[0]; + } else { + allowAnyTargetPort = false; + this.allowedTargetPorts = + allowedTargetPortsStrings.stream().mapToInt(Integer::parseInt).toArray(); + } + } + + private static Pattern parseWildcardPattern(String wildcardPattern) { + String regexPattern = + Collections.list(new StringTokenizer(wildcardPattern, "*", true)) + .stream() + .map(String::valueOf) + .map(token -> { + if ("*".equals(token)) { + return ".*"; + } else { + return Pattern.quote(token); + } + }).collect(Collectors.joining()); + return Pattern.compile( + "^" + regexPattern + "$", + Pattern.CASE_INSENSITIVE); + } + + private static List parseCommaSeparatedConfigValue(String configValue) { + return Arrays.stream(configValue.split(SEPARATOR)).map(String::trim).filter(s -> s.length() > 0) + .collect(Collectors.toList()); + } + + public CompletableFuture resolveAndCheckTargetAddress(String hostAndPort) { + int pos = hostAndPort.indexOf(':'); + String host = hostAndPort.substring(0, pos); + int port = Integer.parseInt(hostAndPort.substring(pos + 1)); + if (!isPortAllowed(port)) { + return FutureUtil.failedFuture( + new TargetAddressDeniedException("Given port in '" + hostAndPort + "' isn't allowed.")); + } else if (!isHostAllowed(host)) { + return FutureUtil.failedFuture( + new TargetAddressDeniedException("Given host in '" + hostAndPort + "' isn't allowed.")); + } else { + return NettyFutureUtil.toCompletableFuture( + inetSocketAddressResolver.resolve(InetSocketAddress.createUnresolved(host, port))) + .thenCompose(resolvedAddress -> { + CompletableFuture result = new CompletableFuture(); + if (isIPAddressAllowed(resolvedAddress)) { + result.complete(resolvedAddress); + } else { + result.completeExceptionally(new TargetAddressDeniedException( + "The IP address of the given host and port '" + hostAndPort + "' isn't allowed.")); + } + return result; + }); + } + } + + private boolean isPortAllowed(int port) { + if (allowAnyTargetPort) { + return true; + } + for (int allowedPort : allowedTargetPorts) { + if (allowedPort == port) { + return true; + } + } + return false; + } + + private boolean isIPAddressAllowed(InetSocketAddress resolvedAddress) { + if (allowAnyIPAddress) { + return true; + } + byte[] addressBytes = resolvedAddress.getAddress().getAddress(); + IPAddress candidateAddress = + addressBytes.length == 4 ? new IPv4Address(addressBytes) : new IPv6Address(addressBytes); + for (IPAddress allowedAddress : allowedIPAddresses) { + if (allowedAddress.contains(candidateAddress)) { + return true; + } + } + return false; + } + + private boolean isHostAllowed(String host) { + if (allowAnyHostName) { + return true; + } + boolean matched = false; + for (Pattern allowedHostName : allowedHostNames) { + if (allowedHostName.matcher(host).matches()) { + matched = true; + break; + } + } + return matched; + } +} diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index c896be5a62993..37fc3d5a8a298 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -39,6 +39,7 @@ import io.netty.handler.codec.haproxy.HAProxyProtocolVersion; import io.netty.handler.codec.haproxy.HAProxyProxiedProtocol; import io.netty.handler.ssl.SslHandler; +import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.FutureListener; @@ -49,6 +50,7 @@ import java.util.Arrays; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import javax.net.ssl.SSLSession; @@ -73,25 +75,25 @@ public class DirectProxyHandler { @Getter - private Channel inboundChannel; + private final Channel inboundChannel; @Getter Channel outboundChannel; @Getter private final Rate inboundChannelRequestsRate; protected static Map inboundOutboundChannelMap = new ConcurrentHashMap<>(); - private String originalPrincipal; - private AuthData clientAuthData; - private String clientAuthMethod; - private int protocolVersion; + private final String originalPrincipal; + private final AuthData clientAuthData; + private final String clientAuthMethod; public static final String TLS_HANDLER = "tls"; private final Authentication authentication; - private final Supplier sslHandlerSupplier; private AuthenticationDataProvider authenticationDataProvider; - private ProxyService service; + private final ProxyService service; + private final Runnable onHandshakeCompleteAction; public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, String targetBrokerUrl, - int protocolVersion, Supplier sslHandlerSupplier) { + InetSocketAddress targetBrokerAddress, int protocolVersion, + Supplier sslHandlerSupplier) { this.service = service; this.authentication = proxyConnection.getClientAuthentication(); this.inboundChannel = proxyConnection.ctx().channel(); @@ -99,8 +101,7 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, this.originalPrincipal = proxyConnection.clientAuthRole; this.clientAuthData = proxyConnection.clientAuthData; this.clientAuthMethod = proxyConnection.clientAuthMethod; - this.protocolVersion = protocolVersion; - this.sslHandlerSupplier = sslHandlerSupplier; + this.onHandshakeCompleteAction = proxyConnection::cancelKeepAliveTask; ProxyConfiguration config = service.getConfiguration(); // Start the connection attempt. @@ -109,13 +110,22 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, // switches when passing data between the 2 // connections b.option(ChannelOption.ALLOCATOR, PulsarByteBufAllocator.DEFAULT); + int brokerProxyConnectTimeoutMs = service.getConfiguration().getBrokerProxyConnectTimeoutMs(); + if (brokerProxyConnectTimeoutMs > 0) { + b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, brokerProxyConnectTimeoutMs); + } b.group(inboundChannel.eventLoop()).channel(inboundChannel.getClass()).option(ChannelOption.AUTO_READ, false); b.handler(new ChannelInitializer() { @Override - protected void initChannel(SocketChannel ch) throws Exception { + protected void initChannel(SocketChannel ch) { if (sslHandlerSupplier != null) { ch.pipeline().addLast(TLS_HANDLER, sslHandlerSupplier.get()); } + int brokerProxyReadTimeoutMs = service.getConfiguration().getBrokerProxyReadTimeoutMs(); + if (brokerProxyReadTimeoutMs > 0) { + ch.pipeline().addLast("readTimeoutHandler", + new ReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS)); + } ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder( Commands.DEFAULT_MAX_MESSAGE_SIZE + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, 4)); ch.pipeline().addLast("proxyOutboundHandler", new ProxyBackendHandler(config, protocolVersion)); @@ -133,7 +143,7 @@ protected void initChannel(SocketChannel ch) throws Exception { return; } - ChannelFuture f = b.connect(targetBroker.getHost(), targetBroker.getPort()); + ChannelFuture f = b.connect(targetBrokerAddress); outboundChannel = f.channel(); f.addListener(future -> { if (!future.isSuccess()) { @@ -211,8 +221,8 @@ public class ProxyBackendHandler extends PulsarDecoder implements FutureListener private BackendState state = BackendState.Init; private String remoteHostName; protected ChannelHandlerContext ctx; - private ProxyConfiguration config; - private int protocolVersion; + private final ProxyConfiguration config; + private final int protocolVersion; public ProxyBackendHandler(ProxyConfiguration config, int protocolVersion) { this.config = config; @@ -225,7 +235,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { // Send the Connect command to broker authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); - ByteBuf command = null; + ByteBuf command; command = Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion, "Pulsar proxy", null /* target broker */, originalPrincipal, clientAuthData, clientAuthMethod); outboundChannel.writeAndFlush(command); @@ -293,12 +303,11 @@ protected void handleAuthChallenge(CommandAuthChallenge authChallenge) { outboundChannel.read(); } catch (Exception e) { log.error("Error mutual verify", e); - return; } } @Override - public void operationComplete(Future future) throws Exception { + public void operationComplete(Future future) { // This is invoked when the write operation on the paired connection // is completed if (future.isSuccess()) { @@ -317,6 +326,7 @@ protected void messageReceived() { @Override protected void handleConnected(CommandConnected connected) { + checkArgument(state == BackendState.Init, "Unexpected state %s. BackendState.Init was expected.", state); if (log.isDebugEnabled()) { log.debug("[{}] [{}] Received Connected from broker", inboundChannel, outboundChannel); } @@ -332,58 +342,68 @@ protected void handleConnected(CommandConnected connected) { state = BackendState.HandshakeCompleted; - ChannelFuture channelFuture; - if (connected.hasMaxMessageSize()) { - channelFuture = inboundChannel.writeAndFlush( - Commands.newConnected(connected.getProtocolVersion(), connected.getMaxMessageSize())); - } else { - channelFuture = inboundChannel.writeAndFlush(Commands.newConnected(connected.getProtocolVersion())); - } + onHandshakeCompleteAction.run(); + startDirectProxying(connected); + + int maxMessageSize = + connected.hasMaxMessageSize() ? connected.getMaxMessageSize() : Commands.INVALID_MAX_MESSAGE_SIZE; + inboundChannel.writeAndFlush(Commands.newConnected(connected.getProtocolVersion(), maxMessageSize)) + .addListener(future -> { + if (future.isSuccess()) { + // Start reading from both connections + inboundChannel.read(); + outboundChannel.read(); + } else { + log.warn("[{}] [{}] Failed to write to inbound connection. Closing both connections.", + inboundChannel, + outboundChannel, future.cause()); + inboundChannel.close(); + } + }); + } - channelFuture.addListener(future -> { - if (service.getProxyLogLevel() == 0) { - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Removing decoder from pipeline", inboundChannel, outboundChannel); - } - // direct tcp proxy - inboundChannel.pipeline().remove("frameDecoder"); - outboundChannel.pipeline().remove("frameDecoder"); + private void startDirectProxying(CommandConnected connected) { + if (service.getProxyLogLevel() == 0) { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Removing decoder from pipeline", inboundChannel, outboundChannel); + } + // direct tcp proxy + inboundChannel.pipeline().remove("frameDecoder"); + outboundChannel.pipeline().remove("frameDecoder"); + } else { + // Enable parsing feature, proxyLogLevel(1 or 2) + // Add parser handler + if (connected.hasMaxMessageSize()) { + inboundChannel.pipeline() + .replace("frameDecoder", "newFrameDecoder", + new LengthFieldBasedFrameDecoder(connected.getMaxMessageSize() + + Commands.MESSAGE_SIZE_FRAME_PADDING, + 0, 4, 0, 4)); + outboundChannel.pipeline().replace("frameDecoder", "newFrameDecoder", + new LengthFieldBasedFrameDecoder( + connected.getMaxMessageSize() + + Commands.MESSAGE_SIZE_FRAME_PADDING, + 0, 4, 0, 4)); + + inboundChannel.pipeline().addBefore("handler", "inboundParser", + new ParserProxyHandler(service, inboundChannel, + ParserProxyHandler.FRONTEND_CONN, + connected.getMaxMessageSize())); + outboundChannel.pipeline().addBefore("proxyOutboundHandler", "outboundParser", + new ParserProxyHandler(service, outboundChannel, + ParserProxyHandler.BACKEND_CONN, + connected.getMaxMessageSize())); } else { - // Enable parsing feature, proxyLogLevel(1 or 2) - // Add parser handler - if (connected.hasMaxMessageSize()) { - inboundChannel.pipeline().replace("frameDecoder", "newFrameDecoder", - new LengthFieldBasedFrameDecoder(connected.getMaxMessageSize() - + Commands.MESSAGE_SIZE_FRAME_PADDING, - 0, 4, 0, 4)); - outboundChannel.pipeline().replace("frameDecoder", "newFrameDecoder", - new LengthFieldBasedFrameDecoder( - connected.getMaxMessageSize() - + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, 4)); - - inboundChannel.pipeline().addBefore("handler", "inboundParser", - new ParserProxyHandler(service, inboundChannel, - ParserProxyHandler.FRONTEND_CONN, - connected.getMaxMessageSize())); - outboundChannel.pipeline().addBefore("proxyOutboundHandler", "outboundParser", - new ParserProxyHandler(service, outboundChannel, - ParserProxyHandler.BACKEND_CONN, - connected.getMaxMessageSize())); - } else { - inboundChannel.pipeline().addBefore("handler", "inboundParser", - new ParserProxyHandler(service, inboundChannel, - ParserProxyHandler.FRONTEND_CONN, - Commands.DEFAULT_MAX_MESSAGE_SIZE)); - outboundChannel.pipeline().addBefore("proxyOutboundHandler", "outboundParser", - new ParserProxyHandler(service, outboundChannel, - ParserProxyHandler.BACKEND_CONN, - Commands.DEFAULT_MAX_MESSAGE_SIZE)); - } + inboundChannel.pipeline().addBefore("handler", "inboundParser", + new ParserProxyHandler(service, inboundChannel, + ParserProxyHandler.FRONTEND_CONN, + Commands.DEFAULT_MAX_MESSAGE_SIZE)); + outboundChannel.pipeline().addBefore("proxyOutboundHandler", "outboundParser", + new ParserProxyHandler(service, outboundChannel, + ParserProxyHandler.BACKEND_CONN, + Commands.DEFAULT_MAX_MESSAGE_SIZE)); } - // Start reading from both connections - inboundChannel.read(); - outboundChannel.read(); - }); + } } @Override @@ -404,7 +424,7 @@ public void setRemoteHostName(String remoteHostName) { private boolean verifyTlsHostName(String hostname, ChannelHandlerContext ctx) { ChannelHandler sslHandler = ctx.channel().pipeline().get("tls"); - SSLSession sslSession = null; + SSLSession sslSession; if (sslHandler != null) { sslSession = ((SslHandler) sslHandler).engine().getSession(); return (new TlsHostnameVerifier()).verify(hostname, sslSession); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index a49caae60bc21..0231251e3f79c 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -49,6 +49,8 @@ public class ProxyConfiguration implements PulsarConfiguration { @Category private static final String CATEGORY_BROKER_DISCOVERY = "Broker Discovery"; @Category + private static final String CATEGORY_BROKER_PROXY = "Broker Proxy"; + @Category private static final String CATEGORY_AUTHENTICATION = "Proxy Authentication"; @Category private static final String CATEGORY_AUTHORIZATION = "Proxy Authorization"; @@ -136,6 +138,43 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private String functionWorkerWebServiceURLTLS; + @FieldContext(category = CATEGORY_BROKER_PROXY, + doc = "When enabled, checks that the target broker is active before connecting. " + + "zookeeperServers and configurationStoreServers must be configured in proxy configuration " + + "for retrieving the active brokers.") + private boolean checkActiveBrokers = false; + + @FieldContext( + category = CATEGORY_BROKER_PROXY, + doc = "Broker proxy connect timeout.\n" + + "The timeout value for Broker proxy connect timeout is in millisecond. Set to 0 to disable." + ) + private int brokerProxyConnectTimeoutMs = 10000; + + @FieldContext( + category = CATEGORY_BROKER_PROXY, + doc = "Broker proxy read timeout.\n" + + "The timeout value for Broker proxy read timeout is in millisecond. Set to 0 to disable." + ) + private int brokerProxyReadTimeoutMs = 75000; + + @FieldContext( + category = CATEGORY_BROKER_PROXY, + doc = "Allowed broker target host names. " + + "Supports multiple comma separated entries and a wildcard.") + private String brokerProxyAllowedHostNames = "*"; + + @FieldContext( + category = CATEGORY_BROKER_PROXY, + doc = "Allowed broker target ip addresses or ip networks / netmasks. " + + "Supports multiple comma separated entries.") + private String brokerProxyAllowedIPAddresses = "*"; + + @FieldContext( + category = CATEGORY_BROKER_PROXY, + doc = "Allowed broker target ports") + private String brokerProxyAllowedTargetPorts = "6650,6651"; + @FieldContext( category = CATEGORY_SERVER, doc = "Hostname or IP address the service binds on" diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index ff392ca0f09e3..df99acacbef5d 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -20,7 +20,10 @@ import static com.google.common.base.Preconditions.checkArgument; +import io.netty.handler.codec.haproxy.HAProxyMessage; import java.net.SocketAddress; +import java.util.Collections; +import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -28,7 +31,7 @@ import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; -import io.netty.handler.codec.haproxy.HAProxyMessage; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; @@ -50,6 +53,7 @@ import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.api.proto.ServerError; +import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,11 +70,12 @@ * */ public class ProxyConnection extends PulsarHandler implements FutureListener { + private static final Logger LOG = LoggerFactory.getLogger(ProxyConnection.class); // ConnectionPool is used by the proxy to issue lookup requests private ConnectionPool connectionPool; private final AtomicLong requestIdGenerator = new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2)); - private ProxyService service; + private final ProxyService service; AuthenticationDataSource authenticationData; private State state; private final Supplier sslHandlerSupplier; @@ -78,6 +83,7 @@ public class ProxyConnection extends PulsarHandler implements FutureListener sslHandle this.service = proxyService; this.state = State.Init; this.sslHandlerSupplier = sslHandlerSupplier; + this.brokerProxyValidator = service.getBrokerProxyValidator(); } @Override @@ -128,6 +137,7 @@ public void channelRegistered(ChannelHandlerContext ctx) throws Exception { super.channelRegistered(ctx); ProxyService.activeConnections.inc(); if (ProxyService.activeConnections.get() > service.getConfiguration().getMaxConcurrentInboundConnections()) { + state = State.Closing; ctx.close(); ProxyService.rejectedConnections.inc(); } @@ -173,6 +183,7 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + state = State.Closing; super.exceptionCaught(ctx, cause); LOG.warn("[{}] Got exception {} : {} {}", remoteAddress, cause.getClass().getSimpleName(), cause.getMessage(), ClientCnx.isKnownException(cause) ? null : cause); @@ -211,7 +222,7 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce } @Override - public void operationComplete(Future future) throws Exception { + public void operationComplete(Future future) { // This is invoked when the write operation on the paired connection is // completed if (future.isSuccess()) { @@ -247,14 +258,51 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie } LOG.info("[{}] complete connection, init proxy handler. authenticated with {} role {}, hasProxyToBrokerUrl: {}", - remoteAddress, authMethod, clientAuthRole, hasProxyToBrokerUrl); + remoteAddress, authMethod, clientAuthRole, hasProxyToBrokerUrl); if (hasProxyToBrokerUrl) { - // Client already knows which broker to connect. Let's open a - // connection there and just pass bytes in both directions - state = State.ProxyConnectionToBroker; - directProxyHandler = new DirectProxyHandler(service, this, proxyToBrokerUrl, - protocolVersionToAdvertise, sslHandlerSupplier); - cancelKeepAliveTask(); + // Optimize proxy connection to fail-fast if the target broker isn't active + // Pulsar client will retry connecting after a back off timeout + if (service.getConfiguration().isCheckActiveBrokers() + && !isBrokerActive(proxyToBrokerUrl)) { + state = State.Closing; + LOG.warn("[{}] Target broker '{}' isn't available. authenticated with {} role {}.", + remoteAddress, proxyToBrokerUrl, authMethod, clientAuthRole); + ctx() + .writeAndFlush( + Commands.newError(-1, ServerError.ServiceNotReady, "Target broker isn't available.")) + .addListener(future -> ctx().close()); + return; + } + + brokerProxyValidator.resolveAndCheckTargetAddress(proxyToBrokerUrl) + .thenAccept(address -> ctx().executor().submit(() -> { + // Client already knows which broker to connect. Let's open a + // connection there and just pass bytes in both directions + state = State.ProxyConnectionToBroker; + directProxyHandler = new DirectProxyHandler(service, this, proxyToBrokerUrl, address, + protocolVersionToAdvertise, sslHandlerSupplier); + })) + .exceptionally(throwable -> { + if (throwable instanceof TargetAddressDeniedException + || throwable.getCause() instanceof TargetAddressDeniedException) { + TargetAddressDeniedException targetAddressDeniedException = + (TargetAddressDeniedException) (throwable instanceof TargetAddressDeniedException + ? throwable : throwable.getCause()); + + LOG.warn("[{}] Target broker '{}' cannot be validated. {}. authenticated with {} role {}.", + remoteAddress, proxyToBrokerUrl, targetAddressDeniedException.getMessage(), + authMethod, clientAuthRole); + } else { + LOG.error("[{}] Error validating target broker '{}'. authenticated with {} role {}.", + remoteAddress, proxyToBrokerUrl, authMethod, clientAuthRole, throwable); + } + ctx() + .writeAndFlush( + Commands.newError(-1, ServerError.ServiceNotReady, + "Target broker cannot be validated.")) + .addListener(future -> ctx().close()); + return null; + }); } else { // Client is doing a lookup, we can consider the handshake complete // and we'll take care of just topics and @@ -306,6 +354,7 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), if (getRemoteEndpointProtocolVersion() < ProtocolVersion.v10.getValue()) { LOG.warn("[{}] Client doesn't support connecting through proxy", remoteAddress); + state = State.Closing; ctx.close(); return; } @@ -485,6 +534,36 @@ public HAProxyMessage getHAProxyMessage() { return haProxyMessage; } - private static final Logger LOG = LoggerFactory.getLogger(ProxyConnection.class); + private boolean isBrokerActive(String targetBrokerHostPort) { + for (ServiceLookupData serviceLookupData : getAvailableBrokers()) { + if (matchesHostAndPort("pulsar://", serviceLookupData.getPulsarServiceUrl(), targetBrokerHostPort) + || matchesHostAndPort("pulsar+ssl://", serviceLookupData.getPulsarServiceUrlTls(), + targetBrokerHostPort)) { + return true; + } + } + return false; + } + + private List getAvailableBrokers() { + if (service.getDiscoveryProvider() == null) { + LOG.warn("Unable to retrieve active brokers. service.getDiscoveryProvider() is null." + + "zookeeperServers and configurationStoreServers must be configured in proxy configuration " + + "when checkActiveBrokers is enabled."); + return Collections.emptyList(); + } + try { + return service.getDiscoveryProvider().getAvailableBrokers(); + } catch (PulsarServerException e) { + LOG.error("Unable to get available brokers", e); + return Collections.emptyList(); + } + } + static boolean matchesHostAndPort(String expectedPrefix, String pulsarServiceUrl, String brokerHostPort) { + return pulsarServiceUrl != null + && pulsarServiceUrl.length() == expectedPrefix.length() + brokerHostPort.length() + && pulsarServiceUrl.startsWith(expectedPrefix) + && pulsarServiceUrl.startsWith(brokerHostPort, expectedPrefix.length()); + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index 73beda2b72e94..abefe49f0597d 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -29,6 +29,8 @@ import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.resolver.dns.DnsNameResolver; +import io.netty.resolver.dns.DnsNameResolverBuilder; import io.netty.util.concurrent.DefaultThreadFactory; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; @@ -71,6 +73,10 @@ public class ProxyService implements Closeable { private final ProxyConfiguration proxyConfig; private final Authentication proxyClientAuthentication; + @Getter + private final DnsNameResolver dnsNameResolver; + @Getter + private final BrokerProxyValidator brokerProxyValidator; private String serviceUrl; private String serviceUrlTls; private final AuthenticationService authenticationService; @@ -145,6 +151,15 @@ public ProxyService(ProxyConfiguration proxyConfig, false, workersThreadFactory); this.authenticationService = authenticationService; + DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder(workerGroup.next()) + .channelType(EventLoopUtil.getDatagramChannelClass(workerGroup)); + dnsNameResolver = dnsNameResolverBuilder.build(); + + brokerProxyValidator = new BrokerProxyValidator(dnsNameResolver.asAddressResolver(), + proxyConfig.getBrokerProxyAllowedHostNames(), + proxyConfig.getBrokerProxyAllowedIPAddresses(), + proxyConfig.getBrokerProxyAllowedTargetPorts()); + // Initialize the message protocol handlers proxyExtensions = ProxyExtensions.load(proxyConfig); proxyExtensions.initialize(proxyConfig); @@ -273,6 +288,8 @@ public BrokerDiscoveryProvider getDiscoveryProvider() { } public void close() throws IOException { + dnsNameResolver.close(); + if (discoveryProvider != null) { discoveryProvider.close(); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index 315607e9d3457..c966868bcbed8 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -27,6 +27,7 @@ import static org.slf4j.bridge.SLF4JBridgeHandler.removeHandlersForRootLogger; import com.google.common.annotations.VisibleForTesting; +import lombok.Getter; import org.apache.logging.log4j.core.util.datetime.FixedDateFormat; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; @@ -93,6 +94,7 @@ public class ProxyServiceStarter { private ProxyConfiguration config; + @Getter private ProxyService proxyService; private WebServer server; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 658dd8762a7f2..a033a87912d87 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -21,6 +21,8 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import io.netty.handler.ssl.SslHandler; +import io.netty.handler.timeout.ReadTimeoutHandler; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.AuthenticationFactory; @@ -46,6 +48,7 @@ public class ServiceChannelInitializer extends ChannelInitializer private final ProxyService proxyService; private final boolean enableTls; private final boolean tlsEnabledWithKeyStore; + private final int brokerProxyReadTimeoutMs; private SslContextAutoRefreshBuilder serverSslCtxRefresher; private SslContextAutoRefreshBuilder clientSslCtxRefresher; @@ -58,6 +61,7 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s this.proxyService = proxyService; this.enableTls = enableTls; this.tlsEnabledWithKeyStore = serviceConfig.isTlsEnabledWithKeyStore(); + this.brokerProxyReadTimeoutMs = serviceConfig.getBrokerProxyReadTimeoutMs(); if (enableTls) { if (tlsEnabledWithKeyStore) { @@ -127,6 +131,10 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast(TLS_HANDLER, new SslHandler(serverSSLContextAutoRefreshBuilder.get().createSSLEngine())); } + if (brokerProxyReadTimeoutMs > 0) { + ch.pipeline().addLast("readTimeoutHandler", + new ReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS)); + } if (proxyService.getConfiguration().isHaProxyProtocolEnabled()) { ch.pipeline().addLast(OptionalProxyProtocolDecoder.NAME, new OptionalProxyProtocolDecoder()); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/TargetAddressDeniedException.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/TargetAddressDeniedException.java new file mode 100644 index 0000000000000..e62525fbca175 --- /dev/null +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/TargetAddressDeniedException.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.proxy.server; + +class TargetAddressDeniedException extends RuntimeException { + public TargetAddressDeniedException(String message) { + super(message); + } +} diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java index 545912dcdc030..88de4b37469fb 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java @@ -83,6 +83,7 @@ protected void setup() throws Exception { proxyConfig.setAuthenticationEnabled(true); proxyConfig.setAuthorizationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java new file mode 100644 index 0000000000000..8e457554cf5ad --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.proxy.server; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import io.netty.resolver.AddressResolver; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.SucceededFuture; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.concurrent.ExecutionException; +import org.apache.curator.shaded.com.google.common.net.InetAddresses; +import org.testng.annotations.Test; + +public class BrokerProxyValidatorTest { + + @Test + public void shouldAllowValidInput() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("1.2.3.4"), + "myhost" + , "1.2.0.0/16" + , "6650"); + InetSocketAddress inetSocketAddress = brokerProxyValidator.resolveAndCheckTargetAddress("myhost:6650").get(); + assertNotNull(inetSocketAddress); + assertEquals(inetSocketAddress.getAddress().getHostAddress(), "1.2.3.4"); + assertEquals(inetSocketAddress.getPort(), 6650); + } + + @Test(expectedExceptions = ExecutionException.class, + expectedExceptionsMessageRegExp = ".*Given host in 'myhost:6650' isn't allowed.") + public void shouldPreventInvalidHostName() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("1.2.3.4"), + "allowedhost" + , "1.2.0.0/16" + , "6650"); + brokerProxyValidator.resolveAndCheckTargetAddress("myhost:6650").get(); + } + + @Test(expectedExceptions = ExecutionException.class, + expectedExceptionsMessageRegExp = ".* The IP address of the given host and port 'myhost:6650' isn't allowed.") + public void shouldPreventInvalidIPAddress() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("1.2.3.4"), + "myhost" + , "1.3.0.0/16" + , "6650"); + brokerProxyValidator.resolveAndCheckTargetAddress("myhost:6650").get(); + } + + @Test + public void shouldSupportHostNamePattern() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("1.2.3.4"), + "*.mydomain" + , "1.2.0.0/16" + , "6650"); + brokerProxyValidator.resolveAndCheckTargetAddress("myhost.mydomain:6650").get(); + } + + @Test + public void shouldAllowAllWithWildcard() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("1.2.3.4"), + "*" + , "*" + , "6650"); + brokerProxyValidator.resolveAndCheckTargetAddress("myhost.mydomain:6650").get(); + } + + private AddressResolver createMockedAddressResolver(String ipAddressResult) { + AddressResolver inetSocketAddressResolver = mock(AddressResolver.class); + when(inetSocketAddressResolver.resolve(any())).then(invocationOnMock -> { + InetSocketAddress address = (InetSocketAddress) invocationOnMock.getArgument(0); + return new SucceededFuture(mock(EventExecutor.class), + new InetSocketAddress(InetAddresses.forString(ipAddressResult), address.getPort())); + }); + return inetSocketAddressResolver; + } +} diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java index a909a9ff3b818..94009c84f0e64 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java @@ -72,6 +72,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java index e63d3aeb4cb96..b37dedfe6a93a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java @@ -106,6 +106,7 @@ protected void setup() throws Exception { proxyConfig.setAuthenticationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java index f6d53c8ec5357..9ebe0ab65c820 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java @@ -215,6 +215,7 @@ void testAuthentication() throws Exception { ProxyConfiguration proxyConfig = new ProxyConfiguration(); proxyConfig.setAuthenticationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionTest.java new file mode 100644 index 0000000000000..5f533e37d3594 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionTest.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import org.testng.annotations.Test; + +public class ProxyConnectionTest { + + @Test + public void testMatchesHostAndPort() { + assertTrue(ProxyConnection + .matchesHostAndPort("pulsar://", "pulsar://1.2.3.4:6650", "1.2.3.4:6650")); + assertTrue(ProxyConnection + .matchesHostAndPort("pulsar+ssl://", "pulsar+ssl://1.2.3.4:6650", "1.2.3.4:6650")); + assertFalse(ProxyConnection + .matchesHostAndPort("pulsar://", "pulsar://1.2.3.4:12345", "5.6.7.8:1234")); + assertFalse(ProxyConnection + .matchesHostAndPort("pulsar://", "pulsar://1.2.3.4:12345", "1.2.3.4:1234")); + } +} diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java index 062db184e0682..128d33fbf19db 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java @@ -53,6 +53,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); proxyConfig.setMaxConcurrentLookupRequests(NUM_CONCURRENT_LOOKUP); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java index 44403fbb39b16..496b3ca5c4d4a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java @@ -56,6 +56,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.ofNullable(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); proxyConfig.setHaProxyProtocolEnabled(true); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java index cf61dac0e6b68..aa8475565155b 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java @@ -104,6 +104,7 @@ public void testForwardAuthData() throws Exception { proxyConfig.setAuthenticationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java index af76bfaeb2bb4..f1cb69f782da2 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java @@ -78,6 +78,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java index 9b0e9b427e56c..03d0b2b2a8fcc 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java @@ -73,6 +73,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java index fa3c485335581..51450264c8d15 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java @@ -52,6 +52,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); proxyConfig.setMaxConcurrentLookupRequests(NUM_CONCURRENT_LOOKUP); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java index 905ca2066c738..654686dedf2f4 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java @@ -71,6 +71,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); //enable full parsing feature diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java index 9ae3fbc09f3ff..39446af99a577 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java @@ -209,6 +209,7 @@ public void testIncorrectRoles() throws Exception { proxyConfig.setAuthenticationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java index bdba8d35c5515..62b65d32e8c2b 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java @@ -51,6 +51,7 @@ public class ProxyServiceStarterTest extends MockedPulsarServiceBaseTest { static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; private ProxyServiceStarter serviceStarter; + private String serviceUrl; @Override @BeforeClass @@ -62,7 +63,9 @@ protected void setup() throws Exception { serviceStarter.getConfig().setWebServicePort(Optional.of(0)); serviceStarter.getConfig().setServicePort(Optional.of(0)); serviceStarter.getConfig().setWebSocketServiceEnabled(true); + serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*"); serviceStarter.start(); + serviceUrl = serviceStarter.getProxyService().getServiceUrl(); } @Override @@ -92,7 +95,7 @@ public void testEnableWebSocketServer() throws Exception { @Test public void testProducer() throws Exception { @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:" + this.pulsar.getBrokerService().getListenPort().get()) + PulsarClient client = PulsarClient.builder().serviceUrl(serviceUrl) .build(); @Cleanup diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java index 7e6c0f5f25f6c..742cfbb6581ee 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java @@ -52,6 +52,8 @@ public class ProxyServiceTlsStarterTest extends MockedPulsarServiceBaseTest { private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem"; private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem"; private ProxyServiceStarter serviceStarter; + private String serviceUrl; + private int webPort; @Override @BeforeClass @@ -62,12 +64,17 @@ protected void setup() throws Exception { serviceStarter.getConfig().setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress()); serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); - serviceStarter.getConfig().setServicePortTls(Optional.of(11043)); + serviceStarter.getConfig().setServicePort(Optional.empty()); + serviceStarter.getConfig().setServicePortTls(Optional.of(0)); + serviceStarter.getConfig().setWebServicePort(Optional.of(0)); serviceStarter.getConfig().setTlsEnabledWithBroker(true); serviceStarter.getConfig().setWebSocketServiceEnabled(true); serviceStarter.getConfig().setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH); serviceStarter.getConfig().setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH); + serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*"); serviceStarter.start(); + serviceUrl = serviceStarter.getProxyService().getServiceUrlTls(); + webPort = serviceStarter.getServer().getListenPortHTTP().get(); } protected void doInitConf() throws Exception { @@ -86,7 +93,7 @@ protected void cleanup() throws Exception { @Test public void testProducer() throws Exception { @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl("pulsar+ssl://localhost:11043") + PulsarClient client = PulsarClient.builder().serviceUrl(serviceUrl) .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .build(); @@ -106,7 +113,7 @@ public void testProduceAndConsumeMessageWithWebsocket() throws Exception { WebSocketClient producerWebSocketClient = new WebSocketClient(producerClient); producerWebSocketClient.start(); MyWebSocket producerSocket = new MyWebSocket(); - String produceUri = "ws://localhost:8080/ws/producer/persistent/sample/test/local/websocket-topic"; + String produceUri = "ws://localhost:" + webPort + "/ws/producer/persistent/sample/test/local/websocket-topic"; Future producerSession = producerWebSocketClient.connect(producerSocket, URI.create(produceUri)); ProducerMessage produceRequest = new ProducerMessage(); @@ -117,7 +124,7 @@ public void testProduceAndConsumeMessageWithWebsocket() throws Exception { WebSocketClient consumerWebSocketClient = new WebSocketClient(consumerClient); consumerWebSocketClient.start(); MyWebSocket consumerSocket = new MyWebSocket(); - String consumeUri = "ws://localhost:8080/ws/consumer/persistent/sample/test/local/websocket-topic/my-sub"; + String consumeUri = "ws://localhost:" + webPort + "/ws/consumer/persistent/sample/test/local/websocket-topic/my-sub"; Future consumerSession = consumerWebSocketClient.connect(consumerSocket, URI.create(consumeUri)); consumerSession.get().getRemote().sendPing(ByteBuffer.wrap("ping".getBytes())); producerSession.get().getRemote().sendString(ObjectMapperFactory.getThreadLocal().writeValueAsString(produceRequest)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java index 2b1c22c22d012..1859c243436f4 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java @@ -67,6 +67,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java index 92f6a63d0f185..a90243fe019c7 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java @@ -90,6 +90,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.ofNullable(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setZookeeperServers(DUMMY_VALUE); proxyConfig.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java index 59beb94712c7d..5081d0e3bb596 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java @@ -56,6 +56,7 @@ protected void setup() throws Exception { internalSetup(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java index 0d3d3a041f75c..ece35cf7b22f6 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java @@ -58,6 +58,7 @@ protected void setup() throws Exception { writer.close(); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java index 5d05867d4fffd..b9d9b04ae3d74 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java @@ -113,6 +113,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index 14c72881b2994..d813777f7eb2d 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -184,6 +184,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); @@ -402,6 +403,7 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java index 693e4ca5db9d6..6178454dd1900 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java @@ -98,6 +98,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); // enable auth&auth and use JWT at proxy diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java index f20401c33aebf..59c50deafec1a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java @@ -104,6 +104,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java index 7dc927a8a5cc2..342df28c31079 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java @@ -80,6 +80,7 @@ protected void setup() throws Exception { proxyConfig.setAuthenticationEnabled(true); proxyConfig.setAuthorizationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java index 628f7cc05224a..ee18b60a11e5f 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java @@ -68,6 +68,7 @@ protected void setup() throws Exception { // start proxy service proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerWebServiceURL(brokerUrl.toString()); proxyConfig.setStatusFilePath(STATUS_FILE_PATH); From 96f6f7b5d2c4011d7f1ba212d137fd5165474208 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 31 Jan 2022 23:56:39 +0800 Subject: [PATCH 321/823] [C++] Fix pulsar client cpp build fail in gcc-4.8.5 (#14053) * Fix pulsar client cpp build fail in gcc-4.8.5 * Enable test * Speedup make speed * Add a new line at the tail Add a new line at the tail of a file (cherry picked from commit caf5acf0d5e1971e4a9958da582b905793f75f1f) --- pulsar-client-cpp/docker-build-centos7.sh | 2 +- pulsar-client-cpp/docker/centos-7/Dockerfile | 7 +++++++ pulsar-client-cpp/tests/ConsumerTest.cc | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pulsar-client-cpp/docker-build-centos7.sh b/pulsar-client-cpp/docker-build-centos7.sh index 5ceeca9a1ab55..e97e374181b7c 100755 --- a/pulsar-client-cpp/docker-build-centos7.sh +++ b/pulsar-client-cpp/docker-build-centos7.sh @@ -32,7 +32,7 @@ cd - VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"} COMMAND="cd /pulsar/pulsar-client-cpp && mkdir -p _builds && cd _builds && - /opt/cmake/cmake-3.4.0-Linux-x86_64/bin/cmake .. -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=OFF && make" + /opt/cmake/cmake-3.4.0-Linux-x86_64/bin/cmake .. -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=ON && make -j8" DOCKER_CMD="docker run -i ${VOLUME_OPTION} ${IMAGE}" diff --git a/pulsar-client-cpp/docker/centos-7/Dockerfile b/pulsar-client-cpp/docker/centos-7/Dockerfile index 53ce9bdbba4f8..690e8f1f73f94 100644 --- a/pulsar-client-cpp/docker/centos-7/Dockerfile +++ b/pulsar-client-cpp/docker/centos-7/Dockerfile @@ -35,3 +35,10 @@ RUN mkdir -p /opt/cmake WORKDIR /opt/cmake RUN curl -L -O https://cmake.org/files/v3.4/cmake-3.4.0-Linux-x86_64.tar.gz \ && tar zxf cmake-3.4.0-Linux-x86_64.tar.gz + +# googletest +RUN curl -O -L https://github.com/google/googletest/archive/refs/tags/release-1.10.0.tar.gz \ + && tar zxf release-1.10.0.tar.gz \ + && cd googletest-release-1.10.0 \ + && mkdir build && cd build \ + && /opt/cmake/cmake-3.4.0-Linux-x86_64/bin/cmake .. && make install diff --git a/pulsar-client-cpp/tests/ConsumerTest.cc b/pulsar-client-cpp/tests/ConsumerTest.cc index 2747b2dcedff8..100086e2a01c1 100644 --- a/pulsar-client-cpp/tests/ConsumerTest.cc +++ b/pulsar-client-cpp/tests/ConsumerTest.cc @@ -660,7 +660,7 @@ TEST(ConsumerTest, testGetTopicNameFromReceivedMessage) { // 2. MultiTopicsConsumerImpl Consumer consumer2; - ASSERT_EQ(ResultOk, client.subscribe({topic1, topic2}, "sub-2", consumer2)); + ASSERT_EQ(ResultOk, client.subscribe(std::vector{topic1, topic2}, "sub-2", consumer2)); sendMessage(topic1, true); validateTopicName(consumer1, topic1); From dafc8385719c4d7451140290267f86bc52599f01 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 2 Feb 2022 20:29:23 +0200 Subject: [PATCH 322/823] Upgrade commons-cli to 1.5.0 (#14094) - commons-cli 1.3+ is required by Zookeeper 3.7.0+ - https://github.com/apache/zookeeper/commit/492fd79b - commons-cli versions <1.3 fail with error: java.lang.NoClassDefFoundError: org/apache/commons/cli/DefaultParser (cherry picked from commit 20af454af44d979fc43c77de3b8d0e0114411db7) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 6 ++++++ pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a4c643a765305..5ca8eee1a2e64 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -341,7 +341,7 @@ The Apache Software License, Version 2.0 - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar * Apache Commons - - commons-cli-commons-cli-1.2.jar + - commons-cli-commons-cli-1.5.0.jar - commons-codec-commons-codec-1.15.jar - commons-collections-commons-collections-3.2.2.jar - commons-configuration-commons-configuration-1.10.jar diff --git a/pom.xml b/pom.xml index 88e1635d9273b..41ef32bf45449 100644 --- a/pom.xml +++ b/pom.xml @@ -105,6 +105,7 @@ flexible messaging model and an intuitive client API. 4.14.4 3.6.3 + 1.5.0 1.1.7 3.2.5 5.1.0 @@ -341,6 +342,11 @@ flexible messaging model and an intuitive client API. zookeeper-jute ${zookeeper.version} + + commons-cli + commons-cli + ${commons-cli.version} + io.dropwizard.metrics metrics-core diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index c6840b8346428..2aaf1755e1f6d 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -428,7 +428,7 @@ The Apache Software License, Version 2.0 - prometheus-metrics-provider-4.14.4.jar - codahale-metrics-provider-4.14.4.jar * Apache Commons - - commons-cli-1.2.jar + - commons-cli-1.5.0.jar - commons-codec-1.15.jar - commons-collections4-4.1.jar - commons-configuration-1.10.jar From e2c94b8976ecaf6582684cde1584a81c80ef7abd Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Sun, 16 Jan 2022 09:48:59 -0800 Subject: [PATCH 323/823] Updated dependencies to get rid of pulsar-io/jdbc related CVE-2020-13692 (#13753) * Updated dependencies to get rid of pulsar-io/jdbc related CVE-2020-13692 Also upgraded clickhouse lib and suppressed wrongly detected clickhouse CVEs (client lib matched to server CVEs) * CR feedback (cherry picked from commit 8214da86b2bd2213a7d97e1d174e8d4e53c1b669) --- pom.xml | 4 +- src/owasp-dependency-check-suppressions.xml | 74 ++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 41ef32bf45449..76cf36ddaaf8f 100644 --- a/pom.xml +++ b/pom.xml @@ -148,8 +148,8 @@ flexible messaging model and an intuitive client API. 2.3.0 3.8.11.2 8.0.11 - 42.2.12 - 0.2.4 + 42.2.24 + 0.3.2 2.6.0 3.3.0 7.9.1 diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 139365d98985f..838e142b65d55 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -41,4 +41,76 @@ org\.apache\.zookeeper:.*:3\.6\.2 .* - \ No newline at end of file + + + + + ^pkg:maven/org\.apache\.avro/avro@.*$ + CVE-2021-43045 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2018-14668 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2018-14669 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2018-14670 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2018-14671 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2018-14672 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2019-15024 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2019-16535 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2019-18657 + + + + ^pkg:maven/ru\.yandex\.clickhouse/clickhouse\-jdbc@.*$ + CVE-2021-25263 + + From d78e315c01056ab1c3886b19ca3029c2ccaed89a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 6 Feb 2022 23:54:54 -0800 Subject: [PATCH 324/823] Fixed deadlock on txn semaphore permit exhaustion (#14131) ### Motivation Removing semaphore on the end of transactions operations. The semaphore is not very useful here as we are already closing the transactions (backpressure should eventually be applied at the starting of the transactions). The semaphore here is being acquired from a BK callback thread and it causes a deadlock in broker when the semaphore is full, because the response that will release the permits on the semaphore are coming from either the same thread or a thread in the same condition. ``` sun.misc.Unsafe.park(Unsafe.java) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) java.util.concurrent.Semaphore.acquire(Semaphore.java:312) org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl.canSendRequest(TransactionBufferHandlerImpl.java:216) org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl.endTxnOnTopic(TransactionBufferHandlerImpl.java:93) org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl.commitTxnOnTopic(TransactionBufferClientImpl.java:50) org.apache.pulsar.broker.TransactionMetadataStoreService.lambda$null$23(TransactionMetadataStoreService.java:484) org.apache.pulsar.broker.TransactionMetadataStoreService$$Lambda$1253.accept() java.util.ArrayList.forEach(ArrayList.java:1257) org.apache.pulsar.broker.TransactionMetadataStoreService.lambda$endTxnInTransactionBuffer$25(TransactionMetadataStoreService.java:481) org.apache.pulsar.broker.TransactionMetadataStoreService$$Lambda$1251.accept() java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:778) java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2140) org.apache.pulsar.broker.TransactionMetadataStoreService.endTxnInTransactionBuffer(TransactionMetadataStoreService.java:458) org.apache.pulsar.broker.TransactionMetadataStoreService.lambda$null$11(TransactionMetadataStoreService.java:349) org.apache.pulsar.broker.TransactionMetadataStoreService$$Lambda$1309.accept() java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:656) java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:632) java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl$3.addComplete(MLTransactionLogImpl.java:160) org.apache.bookkeeper.mledger.impl.OpAddEntry.safeRun(OpAddEntry.java:228) org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:748) ``` (cherry picked from commit bea5bb875c60173e5d365655617185b0df783051) --- .../impl/TransactionBufferHandlerImpl.java | 32 ------------------- .../buffer/TransactionBufferClientTest.java | 11 ------- 2 files changed, 43 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index cb74bf3bf2e14..552ee275dd543 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -28,14 +28,12 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionBufferClientException; -import org.apache.pulsar.client.api.transaction.TransactionBufferClientException.ReachMaxPendingOpsException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.PulsarClientImpl; @@ -52,8 +50,6 @@ public class TransactionBufferHandlerImpl implements TransactionBufferHandler { private final AtomicLong requestIdGenerator = new AtomicLong(); private final long operationTimeoutInMills; private final HashedWheelTimer timer; - private final Semaphore semaphore; - private final boolean blockIfReachMaxPendingOps; private final PulsarClient pulsarClient; private final LoadingCache> cache = CacheBuilder.newBuilder() @@ -77,8 +73,6 @@ public TransactionBufferHandlerImpl(PulsarClient pulsarClient, this.pulsarClient = pulsarClient; this.pendingRequests = new ConcurrentSkipListMap<>(); this.operationTimeoutInMills = 3000L; - this.semaphore = new Semaphore(10000); - this.blockIfReachMaxPendingOps = true; this.timer = timer; } @@ -90,9 +84,6 @@ public CompletableFuture endTxnOnTopic(String topic, long txnIdMostBits, topic, new TxnID(txnIdMostBits, txnIdLeastBits), action.getValue()); } CompletableFuture cb = new CompletableFuture<>(); - if (!canSendRequest(cb)) { - return cb; - } long requestId = requestIdGenerator.getAndIncrement(); ByteBuf cmd = Commands.newEndTxnOnPartition(requestId, txnIdLeastBits, txnIdMostBits, topic, action, lowWaterMark); @@ -108,9 +99,6 @@ public CompletableFuture endTxnOnSubscription(String topic, String subscr topic, new TxnID(txnIdMostBits, txnIdLeastBits), action.getValue()); } CompletableFuture cb = new CompletableFuture<>(); - if (!canSendRequest(cb)) { - return cb; - } long requestId = requestIdGenerator.getAndIncrement(); ByteBuf cmd = Commands.newEndTxnOnSubscription(requestId, txnIdLeastBits, txnIdMostBits, topic, subscription, action, lowWaterMark); @@ -210,29 +198,9 @@ public void handleEndTxnOnSubscriptionResponse(long requestId, onResponse(op); } - private boolean canSendRequest(CompletableFuture callback) { - try { - if (blockIfReachMaxPendingOps) { - semaphore.acquire(); - } else { - if (!semaphore.tryAcquire()) { - callback.completeExceptionally(new ReachMaxPendingOpsException("Reach max pending ops.")); - return false; - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - callback.completeExceptionally(TransactionBufferClientException.unwrap(e)); - return false; - } - return true; - } - - void onResponse(OpRequestSend op) { ReferenceCountUtil.safeRelease(op.byteBuf); op.recycle(); - semaphore.release(); } private static final class OpRequestSend { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index 0082d81f5fe41..c25447bfb0915 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -30,7 +30,6 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Semaphore; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -248,16 +247,6 @@ public void testTransactionBufferLookUp() throws Exception { @Test public void testTransactionBufferHandlerSemaphore() throws Exception { - - Field field = TransactionBufferClientImpl.class.getDeclaredField("tbHandler"); - field.setAccessible(true); - TransactionBufferHandlerImpl transactionBufferHandler = (TransactionBufferHandlerImpl) field.get(tbClient); - - field = TransactionBufferHandlerImpl.class.getDeclaredField("semaphore"); - field.setAccessible(true); - field.set(transactionBufferHandler, new Semaphore(2)); - - String topic = "persistent://" + namespace + "/testTransactionBufferHandlerSemaphore"; String subName = "test"; From 33448a74f0357befd8d11056a77757eb08fc3240 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Mon, 7 Feb 2022 18:28:41 +0800 Subject: [PATCH 325/823] Fix OpBase.callback is not called in failPendingRequest (#14133) (cherry picked from commit 49490dbd2439328628b62a0d7c9ebcb152c6c5ab) --- .../impl/TransactionMetaStoreHandler.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 3c6286b0fb26f..a30c2350908c0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -123,7 +123,6 @@ public void connectionOpened(ClientCnx cnx) { if (getState() == State.Closing || getState() == State.Closed) { setState(State.Closed); failPendingRequest(); - this.pendingRequests.clear(); return; } @@ -173,17 +172,16 @@ public void connectionOpened(ClientCnx cnx) { } private void failPendingRequest() { - internalPinnedExecutor.execute(() -> { - pendingRequests.keys().forEach(k -> { - OpBase op = pendingRequests.remove(k); - if (op != null && !op.callback.isDone()) { - op.callback.completeExceptionally(new PulsarClientException.AlreadyClosedException( - "Could not get response from transaction meta store when " + - "the transaction meta store has already close.")); - onResponse(op); - } - }); + // this method is executed in internalPinnedExecutor. + pendingRequests.forEach((k, op) -> { + if (op != null && !op.callback.isDone()) { + op.callback.completeExceptionally(new PulsarClientException.AlreadyClosedException( + "Could not get response from transaction meta store when " + + "the transaction meta store has already close.")); + onResponse(op); + } }); + this.pendingRequests.clear(); } public CompletableFuture newTransactionAsync(long timeout, TimeUnit unit) { From befdb84fb9466e5b406cc77ea69d7467dbd54a78 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 8 Feb 2022 23:20:48 +0800 Subject: [PATCH 326/823] [Issue 14105] Avoid creating any topics in `NamespaceService#checkTopicExists` during topic lookup. (#14134) (cherry picked from commit 6d03880aabd11e614a29148859aa71678376c7aa) --- .../broker/namespace/NamespaceService.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 3c31f711d6a25..058354cd01ac3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -55,6 +55,7 @@ import org.apache.pulsar.broker.loadbalance.ResourceUnit; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.broker.web.PulsarWebResource; @@ -1121,9 +1122,25 @@ public CompletableFuture checkTopicExists(TopicName topic) { if (topic.isPersistent()) { return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topic); } else { - return pulsar.getBrokerService() - .getTopicIfExists(topic.toString()) - .thenApply(optTopic -> optTopic.isPresent()); + if (topic.isPartitioned()) { + final TopicName partitionedTopicName = TopicName.get(topic.getPartitionedTopicName()); + return pulsar.getBrokerService() + .fetchPartitionedTopicMetadataAsync(partitionedTopicName) + .thenApply((metadata) -> topic.getPartitionIndex() < metadata.partitions); + } else { + // only checks and don't do any topic creating and loading. + CompletableFuture> topicFuture = + pulsar.getBrokerService().getTopics().get(topic.toString()); + if (topicFuture == null) { + return CompletableFuture.completedFuture(false); + } else { + return topicFuture.thenApply(Optional::isPresent).exceptionally(throwable -> { + LOG.warn("[{}] topicFuture completed with exception when checkTopicExists, {}", + topic, throwable.getMessage()); + return false; + }); + } + } } } From 12ee1a1d6e01e69dda7ce55ea3e304bb78abd6bf Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Tue, 8 Feb 2022 23:21:22 +0800 Subject: [PATCH 327/823] fix consume failure when BatchReceivePolicy#maxNumBytes < message size (#14139) (cherry picked from commit 88fc8445213650f3ab8eb4e3a8cc6fbe24545d07) --- .../client/api/ConsumerBatchReceiveTest.java | 22 +++++++++++++++++++ .../pulsar/client/impl/MessagesImpl.java | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java index 5522dd7e495a2..1473f28075316 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java @@ -403,6 +403,28 @@ public void verifyBatchSizeIsEqualToPolicyConfiguration() throws Exception { receiveAllBatchesAndVerifyBatchSizeIsEqualToMaxNumMessages(consumer, batchReceivePolicy, messagesToSend / muxNumMessages); } + @Test + public void verifyNumBytesSmallerThanMessageSize() throws Exception { + final int messagesToSend = 500; + + final String topic = "persistent://my-property/my-ns/batch-receive-" + UUID.randomUUID(); + BatchReceivePolicy batchReceivePolicy = BatchReceivePolicy.builder().maxNumBytes(10).build(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("s2") + .batchReceivePolicy(batchReceivePolicy) + .subscribe(); + + sendMessagesAsyncAndWait(producer, messagesToSend); + CountDownLatch latch = new CountDownLatch(messagesToSend+1); + receiveAsync(consumer, messagesToSend, latch); + latch.await(); + } + private void receiveAllBatchesAndVerifyBatchSizeIsEqualToMaxNumMessages(Consumer consumer, BatchReceivePolicy batchReceivePolicy, diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java index 4ff23eb46f5b6..bb0335b1f158b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java @@ -45,6 +45,10 @@ protected MessagesImpl(int maxNumberOfMessages, long maxSizeOfMessages) { } protected boolean canAdd(Message message) { + if (currentNumberOfMessages == 0) { + // It's ok to add at least one message into a batch. + return true; + } if (maxNumberOfMessages > 0 && currentNumberOfMessages + 1 > maxNumberOfMessages) { return false; } From fafdb6d5cff00201d61ce07ee47b9ef2b319ac48 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Tue, 8 Feb 2022 15:11:30 +0800 Subject: [PATCH 328/823] [Doc] Fix doc for the wrong default value of `maxPendingChunkedMessage` (#14144) (cherry picked from commit ef405ceac89cf23077ae01f8a795faaa61422a88) --- .../java/org/apache/pulsar/client/api/ConsumerBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java index 3c3ce177177f3..4b822a0cd1281 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java @@ -677,7 +677,7 @@ public interface ConsumerBuilder extends Cloneable { * the outstanding unchunked-messages by silently acking or asking broker to redeliver later by marking it unacked. * This behavior can be controlled by configuration: @autoAckOldestChunkedMessageOnQueueFull * - * @default 100 + * The default value is 10. * * @param maxPendingChuckedMessage * @return @@ -702,7 +702,7 @@ public interface ConsumerBuilder extends Cloneable { * the outstanding unchunked-messages by silently acking or asking broker to redeliver later by marking it unacked. * This behavior can be controlled by configuration: @autoAckOldestChunkedMessageOnQueueFull * - * @default 100 + * The default value is 10. * * @param maxPendingChunkedMessage * @return From 5ebd5b6e56c21ec7f3b12c15dbdef076a21887e6 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Wed, 9 Feb 2022 15:41:48 +0800 Subject: [PATCH 329/823] [Transaction] Fix subscription ack transaction marker. (#14170) (cherry picked from commit 603d252ed6e405d2015d2d0fb73047bb9a96b268) --- .../service/AbstractBaseDispatcher.java | 4 + .../persistent/PersistentSubscription.java | 83 -------- .../apache/pulsar/broker/BrokerTestUtil.java | 15 ++ .../service/TransactionMarkerDeleteTest.java | 184 ++++++++---------- 4 files changed, 104 insertions(+), 182 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index f98cfe59c0813..ba757b529fe0b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -126,6 +126,10 @@ public void filterEntriesForConsumer(Optional entryWrapper, int if (!isReplayRead && msgMetadata != null && msgMetadata.hasTxnidMostBits() && msgMetadata.hasTxnidLeastBits()) { if (Markers.isTxnMarker(msgMetadata)) { + // because consumer can receive message is smaller than maxReadPosition, + // so this marker is useless for this subscription + subscription.acknowledgeMessage(Collections.singletonList(entry.getPosition()), AckType.Individual, + Collections.emptyMap()); entries.set(i, null); entry.release(); continue; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index cd9f462e7ea10..f69db6825e907 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -69,7 +69,6 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; -import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.TopicName; @@ -77,8 +76,6 @@ import org.apache.pulsar.common.policies.data.TransactionPendingAckStats; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,26 +111,15 @@ public class PersistentSubscription implements Subscription { private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Collections.emptyMap(); private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; - private volatile Position lastMarkDeleteForTransactionMarker; private final PendingAckHandle pendingAckHandle; private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); - private DeleteTransactionMarkerState deleteTransactionMarkerState = DeleteTransactionMarkerState.None; - - private final Object waitObject = new Object(); - static { REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); } - public enum DeleteTransactionMarkerState { - Process, - Wait, - None - } - static Map getBaseCursorProperties(boolean isReplicated) { return isReplicated ? REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES : NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES; } @@ -419,8 +405,6 @@ public void acknowledgeMessage(List positions, AckType ackType, Map positions, AckType ackType, Map properties) { - - if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled()) { - PositionImpl currentMarkDeletePosition = (PositionImpl) cursor.getMarkDeletedPosition(); - if ((lastMarkDeleteForTransactionMarker == null - || ((PositionImpl) lastMarkDeleteForTransactionMarker) - .compareTo(currentMarkDeletePosition) < 0)) { - if (currentMarkDeletePosition != null) { - ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) cursor.getManagedLedger()); - PositionImpl nextPosition = managedLedger.getNextValidPosition(currentMarkDeletePosition); - if (nextPosition != null - && nextPosition.compareTo((PositionImpl) managedLedger.getLastConfirmedEntry()) <= 0) { - synchronized (waitObject) { - if (deleteTransactionMarkerState == DeleteTransactionMarkerState.None) { - deleteTransactionMarkerState = DeleteTransactionMarkerState.Process; - managedLedger.asyncReadEntry(nextPosition, new ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - try { - MessageMetadata messageMetadata = - Commands.parseMessageMetadata(entry.getDataBuffer()); - if (Markers.isTxnCommitMarker(messageMetadata) - || Markers.isTxnAbortMarker(messageMetadata)) { - synchronized (waitObject) { - deleteTransactionMarkerState = DeleteTransactionMarkerState.None; - } - lastMarkDeleteForTransactionMarker = currentMarkDeletePosition; - acknowledgeMessage(Collections.singletonList(nextPosition), - AckType.Individual, properties); - } else { - synchronized (waitObject) { - if (deleteTransactionMarkerState - == DeleteTransactionMarkerState.Wait) { - deleteTransactionMarkerState = - DeleteTransactionMarkerState.None; - deleteTransactionMarker(properties); - } else { - deleteTransactionMarkerState = - DeleteTransactionMarkerState.None; - } - } - } - } finally { - entry.release(); - } - } - - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - synchronized (waitObject) { - deleteTransactionMarkerState = - DeleteTransactionMarkerState.None; - } - log.error("Fail to read transaction marker! Position : {}", - currentMarkDeletePosition, exception); - } - }, null); - } else if (deleteTransactionMarkerState == DeleteTransactionMarkerState.Process) { - deleteTransactionMarkerState = DeleteTransactionMarkerState.Wait; - } - } - } - } - } - } - } - public CompletableFuture transactionIndividualAcknowledge( TxnID txnId, List> positions) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java index ff03e425ccbcd..224060c9d912e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker; import java.util.UUID; +import org.mockito.Mockito; /** * Holds util methods used in test. @@ -29,4 +30,18 @@ public static String newUniqueName(String prefix) { return prefix + "-" + UUID.randomUUID(); } + /** + * Creates a Mockito spy directly without an intermediate instance to spy. + * This is to address flaky test issue where a spy created with a given instance fails with + * {@link org.mockito.exceptions.misusing.WrongTypeOfReturnValue} exception. + * + * @param classToSpy the class to spy + * @param args the constructor arguments to use when creating the spy instance + * @return a spy of the provided class created with given constructor arguments + */ + public static T spyWithClassAndConstructorArgs(Class classToSpy, Object... args) { + return Mockito.mock(classToSpy, Mockito.withSettings() + .useConstructor(args) + .defaultAnswer(Mockito.CALLS_REAL_METHODS)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java index f25b346a3360a..aa2a8d49e9c95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -25,45 +26,39 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; +import static org.testng.Assert.assertNull; import java.util.Collections; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandAck.AckType; -import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.protocol.Markers; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import org.testng.collections.Sets; @Test(groups = "broker") -public class TransactionMarkerDeleteTest extends BrokerTestBase { +public class TransactionMarkerDeleteTest extends TransactionTestBase { + private static final int TOPIC_PARTITION = 3; + private static final String TOPIC_OUTPUT = NAMESPACE1 + "/output"; + private static final int NUM_PARTITIONS = 16; @BeforeMethod - @Override protected void setup() throws Exception { - conf.setTransactionCoordinatorEnabled(true); - super.baseSetup(); - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet("test"))); - - admin.namespaces().createNamespace("public/default"); + setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); } @AfterMethod(alwaysRun = true) @@ -74,7 +69,8 @@ protected void cleanup() throws Exception { @Test public void testMarkerDeleteTimes() throws Exception { - ManagedLedgerImpl managedLedger = spy((ManagedLedgerImpl) pulsar.getManagedLedgerFactory().open("test")); + ManagedLedgerImpl managedLedger = + spy((ManagedLedgerImpl) getPulsarServiceList().get(0).getManagedLedgerFactory().open("test")); PersistentTopic topic = mock(PersistentTopic.class); BrokerService brokerService = mock(BrokerService.class); PulsarService pulsarService = mock(PulsarService.class); @@ -85,8 +81,8 @@ public void testMarkerDeleteTimes() throws Exception { doReturn(false).when(configuration).isTransactionCoordinatorEnabled(); doReturn(managedLedger).when(topic).getManagedLedger(); ManagedCursor cursor = managedLedger.openCursor("test"); - PersistentSubscription persistentSubscription = spy(new PersistentSubscription(topic, "test", - cursor, false)); + PersistentSubscription persistentSubscription = + spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "test", cursor, false); Position position = managedLedger.addEntry("test".getBytes()); persistentSubscription.acknowledgeMessage(Collections.singletonList(position), AckType.Individual, Collections.emptyMap()); @@ -96,84 +92,74 @@ public void testMarkerDeleteTimes() throws Exception { @Test public void testMarkerDelete() throws Exception { - - MessageMetadata msgMetadata = new MessageMetadata().clear() - .setPublishTime(1) - .setProducerName("test") - .setSequenceId(0); - - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(0); - - payload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, - msgMetadata, payload); - - ManagedLedger managedLedger = pulsar.getManagedLedgerFactory().open("test"); - PersistentTopic topic = mock(PersistentTopic.class); - doReturn(pulsar.getBrokerService()).when(topic).getBrokerService(); - doReturn(managedLedger).when(topic).getManagedLedger(); - doReturn("test").when(topic).getName(); - ManagedCursor cursor = managedLedger.openCursor("test"); - PersistentSubscription persistentSubscription = new PersistentSubscription(topic, "test", - managedLedger.openCursor("test"), false); - - byte[] payloadBytes = toBytes(payload); - Position position1 = managedLedger.addEntry(payloadBytes); - Position markerPosition1 = managedLedger.addEntry(toBytes(Markers - .newTxnCommitMarker(1, 1, 1))); - - Position position2 = managedLedger.addEntry(payloadBytes); - Position markerPosition2 = managedLedger.addEntry(toBytes(Markers - .newTxnAbortMarker(1, 1, 1))); - - Position position3 = managedLedger.addEntry(payloadBytes); - - assertEquals(cursor.getNumberOfEntriesInBacklog(true), 5); - assertTrue(((PositionImpl) cursor.getMarkDeletedPosition()).compareTo((PositionImpl) position1) < 0); - - // ack position1, markerDeletePosition to markerPosition1 - persistentSubscription.acknowledgeMessage(Collections.singletonList(position1), - AckType.Individual, Collections.emptyMap()); - - // ack position1, markerDeletePosition to markerPosition1 - Awaitility.await().during(1, TimeUnit.SECONDS).until(() -> - ((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) - .compareTo((PositionImpl) markerPosition1) == 0); - - // ack position2, markerDeletePosition to markerPosition2 - persistentSubscription.acknowledgeMessage(Collections.singletonList(position2), - AckType.Individual, Collections.emptyMap()); - - Awaitility.await().until(() -> - ((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) - .compareTo((PositionImpl) markerPosition2) == 0); - - // add consequent marker - managedLedger.addEntry(toBytes(Markers - .newTxnCommitMarker(1, 1, 1))); - - managedLedger.addEntry(toBytes(Markers - .newTxnAbortMarker(1, 1, 1))); - - Position markerPosition3 = managedLedger.addEntry(toBytes(Markers - .newTxnAbortMarker(1, 1, 1))); - - // ack with transaction, then commit this transaction - persistentSubscription.transactionIndividualAcknowledge(new TxnID(0, 0), - Collections.singletonList(MutablePair.of((PositionImpl) position3, 0))).get(); - - persistentSubscription.endTxn(0, 0, 0, 0).get(); - - // ack with transaction, then commit this transaction - Awaitility.await().until(() -> - ((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) - .compareTo((PositionImpl) markerPosition3) == 0); - + final String subName = "testMarkerDelete"; + final String topicName = NAMESPACE1 + "/testMarkerDelete"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer() + .topic(topicName) + .subscriptionName(subName) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + Producer producer = pulsarClient + .newProducer() + .sendTimeout(0, TimeUnit.SECONDS) + .topic(topicName) + .create(); + + Transaction txn1 = getTxn(); + Transaction txn2 = getTxn(); + Transaction txn3 = getTxn(); + Transaction txn4 = getTxn(); + + MessageIdImpl msgId1 = (MessageIdImpl) producer.newMessage(txn1).send(); + MessageIdImpl msgId2 = (MessageIdImpl) producer.newMessage(txn2).send(); + assertNull(consumer.receive(1, TimeUnit.SECONDS)); + txn1.commit().get(); + + consumer.acknowledgeAsync(consumer.receive()).get(); + assertNull(consumer.receive(1, TimeUnit.SECONDS)); + + // maxReadPosition move to msgId1, msgId2 have not be committed + assertEquals(admin.topics().getInternalStats(topicName).cursors.get(subName).markDeletePosition, + PositionImpl.get(msgId1.getLedgerId(), msgId1.getEntryId()).toString()); + + MessageIdImpl msgId3 = (MessageIdImpl) producer.newMessage(txn3).send(); + txn2.commit().get(); + + consumer.acknowledgeAsync(consumer.receive()).get(); + assertNull(consumer.receive(1, TimeUnit.SECONDS)); + + // maxReadPosition move to txn1 marker, so entryId is msgId2.getEntryId() + 1, + // because send msgId2 before commit txn1 + assertEquals(admin.topics().getInternalStats(topicName).cursors.get(subName).markDeletePosition, + PositionImpl.get(msgId2.getLedgerId(), msgId2.getEntryId() + 1).toString()); + + MessageIdImpl msgId4 = (MessageIdImpl) producer.newMessage(txn4).send(); + txn3.commit().get(); + + consumer.acknowledgeAsync(consumer.receive()).get(); + assertNull(consumer.receive(1, TimeUnit.SECONDS)); + + // maxReadPosition move to txn2 marker, because msgId4 have not be committed + assertEquals(admin.topics().getInternalStats(topicName).cursors.get(subName).markDeletePosition, + PositionImpl.get(msgId3.getLedgerId(), msgId3.getEntryId() + 1).toString()); + + txn4.abort().get(); + + // maxReadPosition move to txn4 abort marker, so entryId is msgId4.getEntryId() + 2 + Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getInternalStats(topicName) + .cursors.get(subName).markDeletePosition, PositionImpl.get(msgId4.getLedgerId(), + msgId4.getEntryId() + 2).toString())); } - static byte[] toBytes(ByteBuf byteBuf) { - byte[] buf = new byte[byteBuf.readableBytes()]; - byteBuf.readBytes(buf); - byteBuf.release(); - return buf; + private Transaction getTxn() throws Exception { + return pulsarClient + .newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS) + .build() + .get(); } } From de5dec4239fbd1c3a595e013a81f262059444489 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 9 Feb 2022 15:34:25 +0800 Subject: [PATCH 330/823] [Transaction] Optimize transaction timeout (#14172) (cherry picked from commit 5ee210a5a12573bf8d047962bbac82528091216c) --- .../broker/transaction/TransactionTest.java | 27 ++++++++++++++++++- .../transaction/TransactionBuilderImpl.java | 4 +-- .../impl/transaction/TransactionImpl.java | 6 +++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 4aaadb2229691..b627438eb824a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -34,6 +34,7 @@ import static org.testng.Assert.fail; import io.netty.buffer.Unpooled; +import io.netty.util.Timeout; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.HashMap; @@ -774,4 +775,28 @@ public void testRetryExceptionOfEndTxn() throws Exception{ })); completableFuture.get(5, TimeUnit.SECONDS); } -} + + @Test + public void testCancelTxnTimeout() throws Exception{ + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS) + .build() + .get(); + + transaction.commit().get(); + + Field field = TransactionImpl.class.getDeclaredField("timeout"); + field.setAccessible(true); + Timeout timeout = (Timeout) field.get(transaction); + Assert.assertTrue(timeout.isCancelled()); + + transaction = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS) + .build() + .get(); + + transaction.abort().get(); + timeout = (Timeout) field.get(transaction); + Assert.assertTrue(timeout.isCancelled()); + } +} \ No newline at end of file diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java index 3ac8676283a1b..9878264ba49c3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java @@ -69,10 +69,8 @@ public CompletableFuture build() { future.completeExceptionally(throwable); return; } - TransactionImpl transaction = new TransactionImpl(client, txnTimeout, + TransactionImpl transaction = new TransactionImpl(client, timeUnit.toMillis(txnTimeout), txnID.getLeastSigBits(), txnID.getMostSigBits()); - client.getTimer().newTimeout(transaction, - txnTimeout, timeUnit); future.complete(transaction); }); return future; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index ebcb20e8c91ad..8adc162584469 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -27,6 +27,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import com.google.common.collect.Lists; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -68,6 +69,7 @@ public class TransactionImpl implements Transaction , TimerTask { private volatile State state; private static final AtomicReferenceFieldUpdater STATE_UPDATE = AtomicReferenceFieldUpdater.newUpdater(TransactionImpl.class, State.class, "state"); + private final Timeout timeout; @Override public void run(Timeout timeout) throws Exception { @@ -100,6 +102,8 @@ public enum State { this.sendFutureList = new ArrayList<>(); this.ackFutureList = new ArrayList<>(); + this.timeout = client.getTimer().newTimeout(this, transactionTimeoutMs, TimeUnit.MILLISECONDS); + } // register the topics that will be modified by this transaction @@ -164,6 +168,7 @@ public CompletableFuture commit() { return checkIfOpenOrCommitting().thenCompose((value) -> { CompletableFuture commitFuture = new CompletableFuture<>(); this.state = State.COMMITTING; + timeout.cancel(); allOpComplete().whenComplete((v, e) -> { if (e != null) { abort().whenComplete((vx, ex) -> commitFuture.completeExceptionally(e)); @@ -192,6 +197,7 @@ public CompletableFuture abort() { return checkIfOpenOrAborting().thenCompose(value -> { CompletableFuture abortFuture = new CompletableFuture<>(); this.state = State.ABORTING; + timeout.cancel(); allOpComplete().whenComplete((v, e) -> { if (e != null) { log.error(e.getMessage()); From 8a360da29601a849ca15c00d361684e4ccfc7589 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 9 Feb 2022 01:41:24 -0600 Subject: [PATCH 331/823] Add test to ensure correct zk children cache invalidation (#14178) (cherry picked from commit 5886327b721b5af265d6a83d06b8a721ecde54d5) --- .../MultiBrokerMetadataConsistencyTest.java | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java new file mode 100644 index 0000000000000..15849a6773a1b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.zookeeper; + +import static org.testng.Assert.assertTrue; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.metadata.TestZKServer; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class MultiBrokerMetadataConsistencyTest extends MultiBrokerBaseTest { + @Override + protected int numberOfAdditionalBrokers() { + return 2; + } + + TestZKServer testZKServer; + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + testZKServer = new TestZKServer(); + } + + @Override + protected void onCleanup() { + super.onCleanup(); + if (testZKServer != null) { + try { + testZKServer.close(); + } catch (Exception e) { + log.error("Error in stopping ZK server", e); + } + } + } + + @Override + protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException { + return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build()); + } + + @Override + protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { + return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build()); + } + + @Test + public void newTopicShouldBeInTopicsList() throws PulsarAdminException { + List admins = getAllAdmins(); + PulsarAdmin first = admins.get(0); + PulsarAdmin second = admins.get(1); + List cacheMiss = second.topics().getList("public/default"); + assertTrue(cacheMiss.isEmpty()); + first.topics().createNonPartitionedTopic("persistent://public/default/my-topic"); + List topics = second.topics().getList("public/default"); + assertTrue(topics.contains("persistent://public/default/my-topic")); + } +} From 0b516b8672ee9987f8dac791717f170fd4391ca5 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 10 Feb 2022 09:15:19 +0800 Subject: [PATCH 332/823] [Transaction] Fix send normal message can't change MaxReadPosition (#14192) link https://github.com/apache/pulsar/pull/14097 When disable transaction producer connect to broker and the TopicTransactionBuffer is recovering, the TopicTransactionBuffer state is None or Initializing, then send normal message can't change the MaxReadPosition. If recover success and then producer don't send message to this topic. The maxReadPosition will not be change and consumer will not receive message when the disable transaction producer sent. 1. recover to Ready state, if no ongoing txns, change maxReadPosition to LAC 2. recover to NoSnapshot state, change maxReadPosition to LAC (cherry picked from commit 0287f7f4278c8bf892d869d81d5d1982be39a516) --- .../buffer/impl/TopicTransactionBuffer.java | 38 +++++-- .../buffer/TransactionStablePositionTest.java | 100 ++++++++++++++++-- 2 files changed, 119 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 12818b7acbd7e..89c77d6313484 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -104,25 +104,43 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + this.recover(); + } + + private void recover() { this.topic.getBrokerService().getPulsar().getTransactionExecutorProvider().getExecutor(this) .execute(new TopicTransactionBufferRecover(new TopicTransactionBufferRecoverCallBack() { @Override public void recoverComplete() { - if (!changeToReadyState()) { - log.error("[{}]Transaction buffer recover fail", topic.getName()); - } else { - timer.newTimeout(TopicTransactionBuffer.this, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - transactionBufferFuture.complete(null); + synchronized (TopicTransactionBuffer.this) { + // sync maxReadPosition change to LAC when TopicTransaction buffer have not recover + // completely the normal message have been sent to broker and state is + // not Ready can't sync maxReadPosition when no ongoing transactions + if (ongoingTxns.isEmpty()) { + maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + } + if (!changeToReadyState()) { + log.error("[{}]Transaction buffer recover fail", topic.getName()); + } else { + timer.newTimeout(TopicTransactionBuffer.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + transactionBufferFuture.complete(null); + } } } @Override public void noNeedToRecover() { - if (!changeToNoSnapshotState()) { - log.error("[{}]Transaction buffer recover fail", topic.getName()); - } else { - transactionBufferFuture.complete(null); + synchronized (TopicTransactionBuffer.this) { + // sync maxReadPosition change to LAC when TopicTransaction buffer have not recover + // completely the normal message have been sent to broker and state is + // not NoSnapshot can't sync maxReadPosition + maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + if (!changeToNoSnapshotState()) { + log.error("[{}]Transaction buffer recover fail", topic.getName()); + } else { + transactionBufferFuture.complete(null); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java index ef1c76182467d..0184b2725d733 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java @@ -18,33 +18,37 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import static org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState.State.NoSnapshot; +import static org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState.State.Ready; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; - -import com.google.common.collect.Sets; - +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.concurrent.TimeUnit; - +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; - +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** @@ -157,4 +161,82 @@ public void abortTxnTest() throws Exception { assertNull(message); } + @DataProvider(name = "enableTransactionAndState") + public static Object[][] enableTransactionAndState() { + return new Object[][] { + { true, TopicTransactionBufferState.State.None }, + { false, TopicTransactionBufferState.State.None }, + { true, TopicTransactionBufferState.State.Initializing }, + { false, TopicTransactionBufferState.State.Initializing } + }; + } + + @Test(dataProvider = "enableTransactionAndState") + public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, + TopicTransactionBufferState.State state) throws Exception { + + final String topicName = NAMESPACE1 + "/testSyncNormalPositionWhenTBRecover-" + + clientEnableTransaction + state.name(); + pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(clientEnableTransaction) + .build(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .sendTimeout(0, TimeUnit.SECONDS) + .topic(topicName) + .create(); + + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(topicName).toString(), false).get().get(); + + TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); + + // wait topic transaction buffer recover success + checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer); + + Field field = TopicTransactionBufferState.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(topicTransactionBuffer, state); + + // init maxReadPosition is PositionImpl.EARLIEST + Position position = topicTransactionBuffer.getMaxReadPosition(); + assertEquals(position, PositionImpl.earliest); + + MessageIdImpl messageId = (MessageIdImpl) producer.send("test".getBytes()); + + // send normal message can't change MaxReadPosition when state is None or Initializing + position = topicTransactionBuffer.getMaxReadPosition(); + assertEquals(position, PositionImpl.earliest); + + // invoke recover + Method method = TopicTransactionBuffer.class.getDeclaredMethod("recover"); + method.setAccessible(true); + method.invoke(topicTransactionBuffer); + + // change to None state can recover + field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); + + // recover success again + checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer); + + // change MaxReadPosition to normal message position + assertEquals(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()), + topicTransactionBuffer.getMaxReadPosition()); + } + + private void checkTopicTransactionBufferState(boolean clientEnableTransaction, + TopicTransactionBuffer topicTransactionBuffer) { + // recover success + Awaitility.await().until(() -> { + if (clientEnableTransaction) { + // recover success, client enable transaction will change to Ready State + return topicTransactionBuffer.getStats().state.equals(Ready.name()); + } else { + // recover success, client disable transaction will change to NoSnapshot State + return topicTransactionBuffer.getStats().state.equals(NoSnapshot.name()); + } + }); + } } From 69de8d7550859854b066927e9e88bff5ce3b3021 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 10 Feb 2022 10:56:20 +0800 Subject: [PATCH 333/823] Fix unack message count for transaction Ack while disabled batch index ack (#14071) * Fix unack message count for transaction Ack while disabled batch index ack. ### Motivation Fix unack message count for transaction Ack while disabled batch index ack. Transaction Ack is different with normal message ack for a batch message. For normal message, we are using a bitset to carry the batch index state, for example ``` 1. Ack with `00111111` means acks batch index 0 and 1 2. For ack batch index 2 and 3, the client will send `00001111` to broker 3. After all the batch been acked, send `00000000` to broker ``` The following is for transaction ack: ``` 1. `00111111` means acks batch index 0 and 1 1. `11001111` means acks batch index 2 and 3 ``` ### Verification Enabled transaction e2e test for batch index ack disabled (cherry picked from commit a64014627bcdce0a1584b592ee942435e119a2a7) --- .../pulsar/broker/service/Consumer.java | 19 ++-- .../pendingack/impl/PendingAckHandleImpl.java | 11 ++- .../broker/service/BatchMessageTest.java | 92 +++++++++++++++++++ .../BatchMessageWithBatchIndexLevelTest.java | 70 +------------- .../transaction/TransactionTestBase.java | 3 +- .../PendingAckInMemoryDeleteTest.java | 1 + .../client/impl/TransactionEndToEndTest.java | 11 ++- ...ctionEndToEndWithoutBatchIndexAckTest.java | 46 ++++++++++ 8 files changed, 167 insertions(+), 86 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 630caac413e85..846eb6efd6bff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -465,17 +465,13 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { ackSets[j] = msgId.getAckSetAt(j); } position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets); - ackedCount = getAckedCountForBatchIndexLevelEnabled(position, batchSize, ackSets); + ackedCount = getAckedCountForTransactionAck(batchSize, ackSets); } else { position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId()); - ackedCount = getAckedCountForMsgIdNoAckSets(batchSize, position); + ackedCount = batchSize; } - if (msgId.hasBatchIndex()) { - positionsAcked.add(new MutablePair<>(position, msgId.getBatchSize())); - } else { - positionsAcked.add(new MutablePair<>(position, 0)); - } + positionsAcked.add(new MutablePair<>(position, (int) batchSize)); addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); @@ -519,7 +515,7 @@ private long getBatchSize(MessageIdData msgId) { } private long getAckedCountForMsgIdNoAckSets(long batchSize, PositionImpl position) { - if (Subscription.isIndividualAckMode(subType) && isAcknowledgmentAtBatchIndexLevelEnabled) { + if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType)) { long[] cursorAckSet = getCursorAckSet(position); if (cursorAckSet != null) { return getAckedCountForBatchIndexLevelEnabled(position, batchSize, EMPTY_ACK_SET); @@ -548,6 +544,13 @@ private long getAckedCountForBatchIndexLevelEnabled(PositionImpl position, long return ackedCount; } + private long getAckedCountForTransactionAck(long batchSize, long[] ackSets) { + BitSetRecyclable bitset = BitSetRecyclable.create().resetWords(ackSets); + long ackedCount = batchSize - bitset.cardinality(); + bitset.recycle(); + return ackedCount; + } + private long getUnAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize) { long unAckedCount = batchSize; if (isAcknowledgmentAtBatchIndexLevelEnabled) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 1c0b10fbd57f6..6fafc686169d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -674,13 +674,18 @@ private void individualAckAbortCommon(TxnID txnID, HashMap consumer = (ConsumerImpl) pulsarClient + .newConsumer(Schema.BYTES) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName(subscriptionName) + .subscriptionType(subType) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient + .newProducer() + .enableBatching(enableBatch) + .topic(topicName) + .batchingMaxPublishDelay(Integer.MAX_VALUE, TimeUnit.MILLISECONDS) + .create(); + + CountDownLatch countDownLatch = new CountDownLatch(messageCount); + for (int i = 0; i < messageCount; i++) { + producer.sendAsync((i + "").getBytes()).thenAccept(msgId -> { + log.info("Published message with msgId: {}", msgId); + countDownLatch.countDown(); + }); + // To generate batch message with different batch size + // 31 total batches, 5 batches with 3 messages, 8 batches with 2 messages and 37 batches with 1 message + if (((i / 3) % (i % 3 + 1)) == 0) { + producer.flush(); + } + } + + countDownLatch.await(); + + for (int i = 0; i < messageCount; i++) { + Message message = consumer.receive(); + if (enableBatch) { + // only ack messages which batch index < 2, which means we will not to ack the + // whole batch for the batch that with more than 2 messages + if (((BatchMessageIdImpl) message.getMessageId()).getBatchIndex() < 2) { + consumer.acknowledgeAsync(message).get(); + } + } else { + if (i % 2 == 0) { + consumer.acknowledgeAsync(message).get(); + } + } + } + + String topic = TopicName.get(topicName).toString(); + PersistentSubscription persistentSubscription = (PersistentSubscription) pulsar.getBrokerService() + .getTopic(topic, false).get().get().getSubscription(subscriptionName); + + Awaitility.await().untilAsserted(() -> { + if (subType == SubscriptionType.Shared) { + if (enableBatch) { + if (conf.isAcknowledgmentAtBatchIndexLevelEnabled()) { + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), 5 * 1); + } else { + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), 5 * 3); + } + } else { + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), messageCount / 2); + } + } else { + assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), 0); + } + }); + } + private static final Logger LOG = LoggerFactory.getLogger(BatchMessageTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index 5e09def401663..c3785c7d3cb40 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -19,28 +19,24 @@ package org.apache.pulsar.broker.service; import com.google.common.collect.Lists; -import lombok.Cleanup; import lombok.SneakyThrows; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; -import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.ConsumerImpl; -import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; + import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; + import static org.testng.Assert.assertEquals; @Test(groups = "broker") @@ -111,66 +107,4 @@ public void testBatchMessageAck() { assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 16); }); } - - @DataProvider(name = "testSubTypeAndEnableBatch") - public Object[][] testSubTypeAndEnableBatch() { - return new Object[][] { { SubscriptionType.Shared, Boolean.TRUE }, - { SubscriptionType.Failover, Boolean.TRUE }, - { SubscriptionType.Shared, Boolean.FALSE }, - { SubscriptionType.Failover, Boolean.FALSE }}; - } - - - @Test(dataProvider="testSubTypeAndEnableBatch") - private void testDecreaseUnAckMessageCountWithAckReceipt(SubscriptionType subType, - boolean enableBatch) throws Exception { - - final int messageCount = 50; - final String topicName = "persistent://prop/ns-abc/testDecreaseWithAckReceipt" + UUID.randomUUID(); - final String subscriptionName = "sub-batch-1"; - @Cleanup - ConsumerImpl consumer = (ConsumerImpl) pulsarClient - .newConsumer(Schema.BYTES) - .topic(topicName) - .isAckReceiptEnabled(true) - .subscriptionName(subscriptionName) - .subscriptionType(subType) - .enableBatchIndexAcknowledgment(true) - .subscribe(); - - @Cleanup - Producer producer = pulsarClient - .newProducer() - .enableBatching(enableBatch) - .topic(topicName) - .batchingMaxMessages(10) - .create(); - - CountDownLatch countDownLatch = new CountDownLatch(messageCount); - for (int i = 0; i < messageCount; i++) { - producer.sendAsync((i + "").getBytes()).thenRun(countDownLatch::countDown); - } - - countDownLatch.await(); - - for (int i = 0; i < messageCount; i++) { - Message message = consumer.receive(); - // wait for receipt - if (i < messageCount / 2) { - consumer.acknowledgeAsync(message.getMessageId()).get(); - } - } - - String topic = TopicName.get(topicName).toString(); - PersistentSubscription persistentSubscription = (PersistentSubscription) pulsar.getBrokerService() - .getTopic(topic, false).get().get().getSubscription(subscriptionName); - - Awaitility.await().untilAsserted(() -> { - if (subType == SubscriptionType.Shared) { - assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), messageCount / 2); - } else { - assertEquals(persistentSubscription.getConsumers().get(0).getUnackedMessages(), 0); - } - }); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index fe7a813567efa..d7a0d3dd7ad2a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -85,6 +85,7 @@ public abstract class TransactionTestBase extends TestRetrySupport { public static final String TENANT = "tnx"; protected static final String NAMESPACE1 = TENANT + "/ns1"; + protected ServiceConfiguration conf = new ServiceConfiguration(); public void internalSetup() throws Exception { incrementSetupNumber(); @@ -146,7 +147,6 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int protected void startBroker() throws Exception { for (int i = 0; i < brokerCount; i++) { - ServiceConfiguration conf = new ServiceConfiguration(); conf.setClusterName(CLUSTER_NAME); conf.setAdvertisedAddress("localhost"); conf.setManagedLedgerCacheSizeMB(8); @@ -156,7 +156,6 @@ protected void startBroker() throws Exception { conf.setConfigurationStoreServers("localhost:3181"); conf.setAllowAutoTopicCreationType("non-partitioned"); conf.setBookkeeperClientExposeStatsToPrometheus(true); - conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); conf.setBrokerShutdownTimeoutMs(0L); conf.setBrokerServicePort(Optional.of(0)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index bc22473e285f0..da2a3a940bd2c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -62,6 +62,7 @@ public class PendingAckInMemoryDeleteTest extends TransactionTestBase { private static final int NUM_PARTITIONS = 16; @BeforeMethod protected void setup() throws Exception { + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); setUpBase(1, NUM_PARTITIONS, NAMESPACE1 +"/test", 0); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 1f2bd06b52fda..b4f43f8e83839 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -83,12 +83,13 @@ @Test(groups = "flaky") public class TransactionEndToEndTest extends TransactionTestBase { - private static final int TOPIC_PARTITION = 3; - private static final String TOPIC_OUTPUT = NAMESPACE1 + "/output"; - private static final String TOPIC_MESSAGE_ACK_TEST = NAMESPACE1 + "/message-ack-test"; - private static final int NUM_PARTITIONS = 16; + protected static final int TOPIC_PARTITION = 3; + protected static final String TOPIC_OUTPUT = NAMESPACE1 + "/output"; + protected static final String TOPIC_MESSAGE_ACK_TEST = NAMESPACE1 + "/message-ack-test"; + protected static final int NUM_PARTITIONS = 16; @BeforeMethod protected void setup() throws Exception { + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); admin.topics().createPartitionedTopic(TOPIC_MESSAGE_ACK_TEST, 1); } @@ -323,7 +324,7 @@ public void txnIndividualAckTestBatchAndFailoverSub() throws Exception { txnAckTest(true, 200, SubscriptionType.Failover); } - private void txnAckTest(boolean batchEnable, int maxBatchSize, + protected void txnAckTest(boolean batchEnable, int maxBatchSize, SubscriptionType subscriptionType) throws Exception { String normalTopic = NAMESPACE1 + "/normal-topic"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java new file mode 100644 index 0000000000000..1ef3998c3467d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.SubscriptionType; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * End to end transaction test. + */ +@Slf4j +@Test(groups = "flaky") +public class TransactionEndToEndWithoutBatchIndexAckTest extends TransactionEndToEndTest { + + @BeforeMethod + protected void setup() throws Exception { + conf.setAcknowledgmentAtBatchIndexLevelEnabled(false); + setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); + admin.topics().createPartitionedTopic(TOPIC_MESSAGE_ACK_TEST, 1); + } + + // TODO need to fix which using transaction with individual ack for failover subscription + @Test + public void txnIndividualAckTestBatchAndFailoverSub() throws Exception { + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); + txnAckTest(true, 200, SubscriptionType.Failover); + } +} From d7ecdfac0c9695ee37e325a2e04b9190f26b79da Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 11 Feb 2022 19:54:52 -0800 Subject: [PATCH 334/823] Fixed detecting number of NICs in EC2 (#14252) --- .../loadbalance/impl/LinuxBrokerHostUsageImpl.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 5bc7bf952b129..fa18795000316 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -23,6 +23,7 @@ import com.sun.management.OperatingSystemMXBean; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -231,8 +232,15 @@ private boolean isPhysicalNic(Path path) { Files.readAllBytes(path.resolve("speed")); return true; } catch (Exception e) { - // wireless nics don't report speed, ignore them. - return false; + // In some cases, VMs in EC2 won't have the speed reported on the NIC and will give a read-error. + // Check the type to make sure it's ethernet (type "1") + try { + String type = new String(Files.readAllBytes(path.resolve("type")), StandardCharsets.UTF_8).trim(); + return Integer.parseInt(type) == 1; + } catch (IOException ioe) { + // wireless nics don't report speed, ignore them. + return false; + } } } return false; From 1c3c3f40b0d58108a4d0f53489f272f8468bed65 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sat, 12 Feb 2022 13:26:42 +0800 Subject: [PATCH 335/823] Fix race condition of OpSendMsgQueue when publishing messages (#14231) * Add synchronized for getPendingQueueSize(); * Use iterator instead. * Use counter to keep track of messages count * Changed to int Co-authored-by: Matteo Merli --- .../pulsar/client/impl/ProducerImpl.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 877d88bd9508d..8486283128444 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -57,7 +57,6 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Message; @@ -994,7 +993,7 @@ void ackReceived(ClientCnx cnx, long sequenceId, long highestSequenceId, long le if (sequenceId > op.sequenceId) { log.warn("[{}] [{}] Got ack for msg. expecting: {} - {} - got: {} - {} - queue-size: {}", topic, producerName, - op.sequenceId, op.highestSequenceId, sequenceId, highestSequenceId, pendingMessages.size()); + op.sequenceId, op.highestSequenceId, sequenceId, highestSequenceId, pendingMessages.messagesCount()); // Force connection closing so that messages can be re-transmitted in a new connection cnx.channel().close(); return; @@ -1016,7 +1015,7 @@ void ackReceived(ClientCnx cnx, long sequenceId, long highestSequenceId, long le releaseSemaphoreForSendOp(op); } else { log.warn("[{}] [{}] Got ack for batch msg error. expecting: {} - {} - got: {} - {} - queue-size: {}", topic, producerName, - op.sequenceId, op.highestSequenceId, sequenceId, highestSequenceId, pendingMessages.size()); + op.sequenceId, op.highestSequenceId, sequenceId, highestSequenceId, pendingMessages.messagesCount()); // Force connection closing so that messages can be re-transmitted in a new connection cnx.channel().close(); return; @@ -1332,6 +1331,7 @@ protected static class OpSendMsgQueue implements Iterable { private final Queue delegate = new ArrayDeque<>(); private int forEachDepth = 0; private List postponedOpSendMgs; + private final AtomicInteger messagesCount = new AtomicInteger(0); @Override public void forEach(Consumer action) { @@ -1352,6 +1352,7 @@ public void forEach(Consumer action) { public boolean add(OpSendMsg o) { // postpone adding to the queue while forEach iteration is in progress + messagesCount.addAndGet(o.numMessagesInBatch); if (forEachDepth > 0) { if (postponedOpSendMgs == null) { postponedOpSendMgs = new ArrayList<>(); @@ -1364,18 +1365,22 @@ public boolean add(OpSendMsg o) { public void clear() { delegate.clear(); + messagesCount.set(0); } public void remove() { - delegate.remove(); + OpSendMsg op = delegate.remove(); + if (op != null) { + messagesCount.addAndGet(-op.numMessagesInBatch); + } } public OpSendMsg peek() { return delegate.peek(); } - public int size() { - return delegate.size(); + public int messagesCount() { + return messagesCount.get(); } @Override @@ -1548,7 +1553,7 @@ public void connectionOpened(final ClientCnx cnx) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Pending messages: {}", topic, producerName, - pendingMessages.size()); + pendingMessages.messagesCount()); } PulsarClientException bqe = new PulsarClientException.ProducerBlockedQuotaExceededException( @@ -1649,7 +1654,7 @@ private void resendMessages(ClientCnx cnx, long expectedEpoch) { cnx.channel().close(); return; } - int messagesToResend = pendingMessages.size(); + int messagesToResend = pendingMessages.messagesCount(); if (messagesToResend == 0) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] No pending messages to resend {}", topic, producerName, messagesToResend); @@ -1755,7 +1760,7 @@ public void run(Timeout timeout) throws Exception { // The diff is less than or equal to zero, meaning that the message has been timed out. // Set the callback to timeout on every message, then clear the pending queue. log.info("[{}] [{}] Message send timed out. Failing {} messages", topic, producerName, - pendingMessages.size()); + pendingMessages.messagesCount()); PulsarClientException te = new PulsarClientException.TimeoutException( format("The producer %s can not send message to the topic %s within given timeout", @@ -1925,7 +1930,7 @@ private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from, long e // called again once the new connection registers the producer with the broker. log.info("[{}][{}] Producer epoch mismatch or the current connection is null. Skip re-sending the " + " {} pending messages since they will deliver using another connection.", topic, producerName, - pendingMessages.size()); + pendingMessages.messagesCount()); return; } final boolean stripChecksum = cnx.getRemoteEndpointProtocolVersion() < brokerChecksumSupportedVersion(); @@ -1997,14 +2002,7 @@ public String getConnectedSince() { } public int getPendingQueueSize() { - if (!isBatchMessagingEnabled()) { - return pendingMessages.size(); - } - MutableInt size = new MutableInt(0); - pendingMessages.forEach(op -> { - size.add(Math.max(op.numMessagesInBatch, 1)); - }); - return size.getValue(); + return pendingMessages.messagesCount(); } @Override From 1cea9909d695bf0c8a838f3ae600500d31371e95 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 13 Feb 2022 21:22:38 -0800 Subject: [PATCH 336/823] Clean up individually deleted messages before the mark-delete position (#14261) --- .../mledger/impl/ManagedCursorImpl.java | 17 +++++++++++- .../mledger/impl/ManagedCursorTest.java | 26 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 2f9578ced4d37..d37ecda789f14 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1580,7 +1580,9 @@ void initializeCursorPosition(Pair lastPositionCounter) { */ PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) { if (newMarkDeletePosition.compareTo(markDeletePosition) < 0) { - throw new IllegalArgumentException("Mark deleting an already mark-deleted position"); + throw new IllegalArgumentException( + "Mark deleting an already mark-deleted position. Current mark-delete: " + markDeletePosition + + " -- attempted mark delete: " + newMarkDeletePosition); } PositionImpl oldMarkDeletePosition = markDeletePosition; @@ -2000,6 +2002,19 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb // mark-delete to the upper bound of the first range segment Range range = individualDeletedMessages.firstRange(); + // If the upper bound is before the mark-delete position, we need to move ahead as these + // individualDeletedMessages are now irrelevant + if (range.upperEndpoint().compareTo(markDeletePosition) <= 0) { + individualDeletedMessages.removeAtMost(markDeletePosition.getLedgerId(), + markDeletePosition.getEntryId()); + range = individualDeletedMessages.firstRange(); + } + + if (range == null) { + // The set was completely cleaned up now + return; + } + // If the lowerBound is ahead of MarkDelete, verify if there are any entries in-between if (range.lowerEndpoint().compareTo(markDeletePosition) <= 0 || ledger .getNumberOfEntries(Range.openClosed(markDeletePosition, range.lowerEndpoint())) <= 0) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 2f80bc85b5821..a00a8173ea745 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -88,6 +88,7 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.apache.pulsar.metadata.api.extended.SessionEvent; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -3507,6 +3508,31 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { }); } + @Test + public void testConsistencyOfIndividualMessages() throws Exception { + ManagedLedger ledger1 = factory.open("testConsistencyOfIndividualMessages"); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger1.openCursor("c"); + + PositionImpl p1 = (PositionImpl) ledger1.addEntry(new byte[1024]); + c1.markDelete(p1); + + // Artificially add a position that is before the current mark-delete position + LongPairRangeSet idm = c1.getIndividuallyDeletedMessagesSet(); + idm.addOpenClosed(p1.getLedgerId() - 1, 0, p1.getLedgerId() - 1, 10); + + List positions = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + positions.add(ledger1.addEntry(new byte[1024])); + } + + for (int i = 0; i < 20; i++) { + c1.delete(positions.get(i)); + } + + assertEquals(c1.getTotalNonContiguousDeletedMessagesRange(), 0); + assertEquals(c1.getMarkDeletedPosition(), positions.get(positions.size() -1)); + } + @Test public void testCursorCheckReadPositionChanged() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig()); From 57bf019b34e87c6ba3be2e029363629a94de86b1 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 14 Feb 2022 10:20:24 +0800 Subject: [PATCH 337/823] Bump netty version to 4.1.74.Final (#14257) Changelog: https://netty.io/news/2022/02/08/4-1-74-Final.html Netty 4.1.74 had solved several dns resolver bug * Upgrade Netty from 4.1.73.Final to 4.1.74.Final * Netty 4.1.74.Final depends on netty-tc-native 2.0.48, also updates (cherry picked from commit 949c67c69cc8cb1c74d527f8f3bcf9ce8d3b5580) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 40 +++++++++---------- pom.xml | 4 +- pulsar-sql/presto-distribution/LICENSE | 36 ++++++++--------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 1d549ff0deece..4470f4214ffa7 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -105,7 +105,7 @@ io.netty netty-common - 4.1.72.Final + 4.1.74.Final test diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 5ca8eee1a2e64..358d61c3c0260 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -352,26 +352,26 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.11.jar * Netty - - io.netty-netty-buffer-4.1.72.Final.jar - - io.netty-netty-codec-4.1.72.Final.jar - - io.netty-netty-codec-dns-4.1.72.Final.jar - - io.netty-netty-codec-http-4.1.72.Final.jar - - io.netty-netty-codec-http2-4.1.72.Final.jar - - io.netty-netty-codec-socks-4.1.72.Final.jar - - io.netty-netty-codec-haproxy-4.1.72.Final.jar - - io.netty-netty-common-4.1.72.Final.jar - - io.netty-netty-handler-4.1.72.Final.jar - - io.netty-netty-handler-proxy-4.1.72.Final.jar - - io.netty-netty-resolver-4.1.72.Final.jar - - io.netty-netty-resolver-dns-4.1.72.Final.jar - - io.netty-netty-transport-4.1.72.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.72.Final.jar - - io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.72.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.72.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.72.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - - io.netty-netty-tcnative-classes-2.0.46.Final.jar + - io.netty-netty-buffer-4.1.74.Final.jar + - io.netty-netty-codec-4.1.74.Final.jar + - io.netty-netty-codec-dns-4.1.74.Final.jar + - io.netty-netty-codec-http-4.1.74.Final.jar + - io.netty-netty-codec-http2-4.1.74.Final.jar + - io.netty-netty-codec-socks-4.1.74.Final.jar + - io.netty-netty-codec-haproxy-4.1.74.Final.jar + - io.netty-netty-common-4.1.74.Final.jar + - io.netty-netty-handler-4.1.74.Final.jar + - io.netty-netty-handler-proxy-4.1.74.Final.jar + - io.netty-netty-resolver-4.1.74.Final.jar + - io.netty-netty-resolver-dns-4.1.74.Final.jar + - io.netty-netty-transport-4.1.74.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.74.Final.jar + - io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.74.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.74.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.74.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar + - io.netty-netty-tcnative-classes-2.0.48.Final.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar diff --git a/pom.xml b/pom.xml index 76cf36ddaaf8f..659c315ec0494 100644 --- a/pom.xml +++ b/pom.xml @@ -109,8 +109,8 @@ flexible messaging model and an intuitive client API. 1.1.7 3.2.5 5.1.0 - 4.1.72.Final - 2.0.46.Final + 4.1.74.Final + 2.0.48.Final 9.4.43.v20210629 2.5.2 2.34 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 2aaf1755e1f6d..ad17a62ce437d 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -233,25 +233,25 @@ The Apache Software License, Version 2.0 - commons-lang3-3.11.jar * Netty - netty-3.10.6.Final.jar - - netty-buffer-4.1.72.Final.jar - - netty-codec-4.1.72.Final.jar - - netty-codec-dns-4.1.72.Final.jar - - netty-codec-http-4.1.72.Final.jar - - netty-codec-haproxy-4.1.72.Final.jar - - netty-codec-socks-4.1.72.Final.jar - - netty-handler-proxy-4.1.72.Final.jar - - netty-common-4.1.72.Final.jar - - netty-handler-4.1.72.Final.jar + - netty-buffer-4.1.74.Final.jar + - netty-codec-4.1.74.Final.jar + - netty-codec-dns-4.1.74.Final.jar + - netty-codec-http-4.1.74.Final.jar + - netty-codec-haproxy-4.1.74.Final.jar + - netty-codec-socks-4.1.74.Final.jar + - netty-handler-proxy-4.1.74.Final.jar + - netty-common-4.1.74.Final.jar + - netty-handler-4.1.74.Final.jar - netty-reactive-streams-2.0.4.jar - - netty-resolver-4.1.72.Final.jar - - netty-resolver-dns-4.1.72.Final.jar - - netty-tcnative-boringssl-static-2.0.46.Final.jar - - netty-tcnative-classes-2.0.46.Final.jar - - netty-transport-4.1.72.Final.jar - - netty-transport-classes-epoll-4.1.72.Final.jar - - netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.72.Final.jar - - netty-transport-native-unix-common-4.1.72.Final-linux-x86_64.jar + - netty-resolver-4.1.74.Final.jar + - netty-resolver-dns-4.1.74.Final.jar + - netty-tcnative-boringssl-static-2.0.48.Final.jar + - netty-tcnative-classes-2.0.48.Final.jar + - netty-transport-4.1.74.Final.jar + - netty-transport-classes-epoll-4.1.74.Final.jar + - netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.74.Final.jar + - netty-transport-native-unix-common-4.1.74.Final-linux-x86_64.jar * Joda Time - joda-time-2.10.5.jar * Jetty From d692f8b25e3c943f06481b5bda1f337f4148c1b4 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 13 Feb 2022 03:05:52 -0800 Subject: [PATCH 338/823] If mark-delete operation fails, mark the cursor as "dirty" (#14256) (cherry picked from commit 8928c3496a61c588b50461d6adaab089dd421619) --- .../mledger/impl/ManagedCursorImpl.java | 1 + .../mledger/impl/ManagedCursorTest.java | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index d37ecda789f14..8c345026d08ab 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1835,6 +1835,7 @@ public void operationComplete() { @Override public void operationFailed(ManagedLedgerException exception) { + isDirty = true; log.warn("[{}] Failed to mark delete position for cursor={} position={}", ledger.getName(), ManagedCursorImpl.this, mdEntry.newPosition); if (log.isDebugEnabled()) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index a00a8173ea745..676e92fd09d23 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -3508,6 +3508,57 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { }); } + + + @Test + public void testFlushCursorAfterError() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setThrottleMarkDelete(1.0); + + ManagedLedgerFactoryConfig factoryConfig = new ManagedLedgerFactoryConfig(); + factoryConfig.setCursorPositionFlushSeconds(1); + + @Cleanup("shutdown") + ManagedLedgerFactory factory1 = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConfig); + ManagedLedger ledger1 = factory1.open("testFlushCursorAfterInactivity", config); + ManagedCursor c1 = ledger1.openCursor("c"); + List positions = new ArrayList<>(); + + for (int i = 0; i < 20; i++) { + positions.add(ledger1.addEntry(new byte[1024])); + } + + // Simulate BK write error + bkc.failNow(BKException.Code.NotEnoughBookiesException); + metadataStore.setAlwaysFail(new MetadataStoreException.BadVersionException("")); + + try { + c1.markDelete(positions.get(positions.size() - 1)); + fail("should have failed"); + } catch (ManagedLedgerException e) { + // Expected + } + + metadataStore.unsetAlwaysFail(); + + // In memory position is updated + assertEquals(c1.getMarkDeletedPosition(), positions.get(positions.size() - 1)); + + Awaitility.await() + // Give chance to the flush to be automatically triggered. + // NOTE: this can't be set too low, or it causes issues with ZK thread pool rejecting + .pollDelay(Duration.ofMillis(2000)) + .untilAsserted(() -> { + // Abruptly re-open the managed ledger without graceful close + @Cleanup("shutdown") + ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedger ledger2 = factory2.open("testFlushCursorAfterInactivity", config); + ManagedCursor c2 = ledger2.openCursor("c"); + + assertEquals(c2.getMarkDeletedPosition(), positions.get(positions.size() - 1)); + }); + } + @Test public void testConsistencyOfIndividualMessages() throws Exception { ManagedLedger ledger1 = factory.open("testConsistencyOfIndividualMessages"); From 9b37232a3a606b4062798f02680e88cbfc1f6faa Mon Sep 17 00:00:00 2001 From: Callum Duffy Date: Fri, 11 Feb 2022 22:08:05 +0000 Subject: [PATCH 339/823] [pulsar-client] Add conf backoff values (#12520) * add conf backoff values * Apply suggestions from code review Co-authored-by: Callum Duffy Co-authored-by: Michael Marshall ### Motivation Allowing services waiting on pulsar container etc to use the backoff values given in the config when the client is initialised. Using a service where i handle the future on my side, but i feel this should be in the service as is! ### Modifications Edit the values used for the backoff to use the client values. Defaults are the exact same as the values used previous to this change. (cherry picked from commit 970363c84d42b5d64ce55cb9eca3eb5bc716131b) --- .../java/org/apache/pulsar/client/impl/PulsarClientImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index c5195c9054d75..0ab7a1376081f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -885,9 +885,9 @@ public CompletableFuture getPartitionedTopicMetadata(S TopicName topicName = TopicName.get(topic); AtomicLong opTimeoutMs = new AtomicLong(conf.getLookupTimeoutMs()); Backoff backoff = new BackoffBuilder() - .setInitialTime(100, TimeUnit.MILLISECONDS) + .setInitialTime(conf.getInitialBackoffIntervalNanos(), TimeUnit.NANOSECONDS) .setMandatoryStop(opTimeoutMs.get() * 2, TimeUnit.MILLISECONDS) - .setMax(1, TimeUnit.MINUTES) + .setMax(conf.getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) .create(); getPartitionedTopicMetadata(topicName, backoff, opTimeoutMs, metadataFuture, new ArrayList<>()); From da86d9d062fcb30d60d9026719fd51d86ec8fc75 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 12 Feb 2022 01:34:53 +0800 Subject: [PATCH 340/823] Remove log unacked msg. (#14246) (cherry picked from commit 78bfaa27d0fdd0c97b90f1671c44b5c459bb93c1) --- .../main/java/org/apache/pulsar/broker/service/Consumer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 846eb6efd6bff..00511f469e470 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -924,9 +924,6 @@ private int addAndGetUnAckedMsgs(Consumer consumer, int ackedMessages) { subscription.addUnAckedMessages(ackedMessages); unackedMsgs = UNACKED_MESSAGES_UPDATER.addAndGet(consumer, ackedMessages); } - if (unackedMsgs < 0) { - log.error("unackedMsgs is : {}, ackedMessages : {}, consumer : {}", unackedMsgs, ackedMessages, consumer); - } return unackedMsgs; } From b1bf0022443f110ccbd5a726118e2fa21ad41b93 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 14 Feb 2022 11:18:11 +0800 Subject: [PATCH 341/823] Fix PersistentAcknowledgmentsGroupingTracker set bitSet issue. (#14260) When consumers set `enableBatchIndexAcknowledgment=true`, client will execute PersistentAcknowledgmentsGroupingTracker#doIndividualBatchAckAsync : https://github.com/apache/pulsar/blob/8928c3496a61c588b50461d6adaab089dd421619/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java#L357-L372 There is an error in line 367, it should be `value.set(0, batchMessageId.getBatchSize()); ` But batchMessageId.getBatchSize() always return acker.getBatchSize(): https://github.com/apache/pulsar/blob/8928c3496a61c588b50461d6adaab089dd421619/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java#L137-L139 If line 362 is false, BatchMessageIdImpl only has acker with BatchMessageAckerDisabled which batch is always 0. So I have added `getOriginalBatchSize` to return the user-specified batch size. Then, when print logs in line 556, `pendingIndividualBatchIndexAcks` is always empty. Should replace with `entriesToAck` (cherry picked from commit 816eaed900bbff1a8514f349cd60e439c6db97bc) --- .../client/impl/BatchMessageIdImpl.java | 4 ++ ...sistentAcknowledgmentsGroupingTracker.java | 2 +- .../AcknowledgementsGroupingTrackerTest.java | 41 +++++++++++++++++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java index fd8ea72c3aa56..75ab3a8806b66 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java @@ -138,6 +138,10 @@ public int getBatchSize() { return acker.getBatchSize(); } + public int getOriginalBatchSize() { + return this.batchSize; + } + public MessageIdImpl prevBatchMessageId() { return new MessageIdImpl( ledgerId, entryId - 1, partitionIndex); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index aa65c6119a38c..e737a2a6f161d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -363,7 +363,7 @@ private void doIndividualBatchAckAsync(BatchMessageIdImpl batchMessageId) { value = ConcurrentBitSetRecyclable.create(batchMessageId.getAcker().getBitSet()); } else { value = ConcurrentBitSetRecyclable.create(); - value.set(0, batchMessageId.getBatchIndex()); + value.set(0, batchMessageId.getOriginalBatchSize()); } return value; }); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index 9632a88793d20..c0b952a281a8e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -22,22 +22,27 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; - import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; - +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.BitSet; import java.util.Collections; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; - +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.util.TimedCompletableFuture; import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.apache.pulsar.common.util.collections.ConcurrentBitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.testng.annotations.AfterClass; @@ -381,6 +386,36 @@ public void testBatchAckTrackerMultiAck(boolean isNeedReceipt) throws Exception tracker.close(); } + @Test + public void testDoIndividualBatchAckAsync() throws Exception{ + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + AcknowledgmentsGroupingTracker tracker = new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup); + MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, BatchMessageAckerDisabled.INSTANCE); + BitSet bitSet = new BitSet(20); + for(int i = 0; i < 20; i ++) { + bitSet.set(i, true); + } + MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, BatchMessageAcker.newAcker(bitSet)); + Method doIndividualBatchAckAsync = PersistentAcknowledgmentsGroupingTracker.class + .getDeclaredMethod("doIndividualBatchAckAsync", BatchMessageIdImpl.class); + doIndividualBatchAckAsync.setAccessible(true); + doIndividualBatchAckAsync.invoke(tracker, messageId1); + doIndividualBatchAckAsync.invoke(tracker, messageId2); + Field pendingIndividualBatchIndexAcks = PersistentAcknowledgmentsGroupingTracker.class.getDeclaredField("pendingIndividualBatchIndexAcks"); + pendingIndividualBatchIndexAcks.setAccessible(true); + ConcurrentHashMap batchIndexAcks = + (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker); + MessageIdImpl position1 = new MessageIdImpl(5, 1, 0); + MessageIdImpl position2 = new MessageIdImpl(3, 2, 0); + assertTrue(batchIndexAcks.containsKey(position1)); + assertNotNull(batchIndexAcks.get(position1)); + assertEquals(batchIndexAcks.get(position1).cardinality(), 9); + assertTrue(batchIndexAcks.containsKey(position2)); + assertNotNull(batchIndexAcks.get(position2)); + assertEquals(batchIndexAcks.get(position2).cardinality(), 19); + tracker.close(); + } + public class ClientCnxTest extends ClientCnx { public ClientCnxTest(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { From 2bfa60152c3e9889d3f2b904b322603c7d81dd55 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Sun, 13 Feb 2022 12:23:29 +0800 Subject: [PATCH 342/823] Check ``getTlsTrustStorePath`` NPE when user forget to set it. (#14253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a user sets ``useKeyStoreTls=true`` then forget to set ``getTlsTrustStorePath``, we’re having NPE which becomes hard to debug for users. - Add NPE check and give use more clear error information. - [x] Make sure that the change passes the CI checks. *If `yes` was chosen, please highlight the changes* - Dependencies (does it add or upgrade a dependency): (no) - The public API: (no) - The schema: (no) - The default values of configurations: (no) - The wire protocol: (no) - The rest endpoints: (no) - The admin cli options: (no) - Anything that affects deployment: (no) - [x] `no-need-doc` (cherry picked from commit eca563e2cb4ebaadd2dc1762e490c05b30fbde18) --- .../pulsar/client/impl/PulsarChannelInitializer.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index 1353424b1b4ea..b7a5fbadb4237 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -23,9 +23,12 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; - import io.netty.handler.proxy.Socks5ProxyHandler; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.util.ObjectCache; import org.apache.pulsar.common.protocol.ByteBufPair; @@ -39,8 +42,6 @@ import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; @Slf4j public class PulsarChannelInitializer extends ChannelInitializer { @@ -74,7 +75,10 @@ public PulsarChannelInitializer(ClientConfigurationData conf, Supplier Date: Tue, 15 Feb 2022 22:49:17 +0800 Subject: [PATCH 343/823] Fix validateGlobalNamespaceOwnership wrap exception issue. (#14269) ### Motivation When Rest API call `AdminResource#validateGlobalNamespaceOwnership`, broker will execute `PulsarWebResource#checkLocalOrGetPeerReplicationCluster`. In `PulsarWebResource#checkLocalOrGetPeerReplicationCluster`: https://github.com/apache/pulsar/blob/6d717a08ef8cfcac032caee06105285594baf09f/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java#L773-L802 Line 780, 794, and 801 has thrown RestException. But `validateGlobalNamespaceOwnership ` has wrapped the exception : https://github.com/apache/pulsar/blob/6d717a08ef8cfcac032caee06105285594baf09f/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java#L202-L216 This could make the user confused that the log printed is not matched with the REST API. (cherry picked from commit 18d9f1b88c4ab8b3deb11b966a425da58ebd932c) --- .../pulsar/broker/admin/AdminResource.java | 5 +--- .../pulsar/broker/web/PulsarWebResource.java | 4 +-- .../broker/admin/PersistentTopicsTest.java | 30 +++++++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 242f3a320f0ef..547665f597428 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -197,10 +197,7 @@ protected void validateGlobalNamespaceOwnership() { } catch (IllegalArgumentException e) { throw new RestException(Status.PRECONDITION_FAILED, "Tenant name or namespace is not valid"); } catch (RestException re) { - if (re.getResponse().getStatus() == Status.NOT_FOUND.getStatusCode()) { - throw new RestException(Status.NOT_FOUND, "Namespace not found"); - } - throw new RestException(Status.PRECONDITION_FAILED, "Namespace does not have any clusters configured"); + throw re; } catch (Exception e) { log.warn("Failed to validate global cluster configuration : ns={} emsg={}", namespaceName, e.getMessage()); throw new RestException(Status.SERVICE_UNAVAILABLE, "Failed to validate global cluster configuration"); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 696e3d108b45d..8b6f30b464967 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -740,9 +740,9 @@ public static CompletableFuture checkLocalOrGetPeerReplicationC validationFuture.complete(null); } } else { - String msg = String.format("Policies not found for %s namespace", namespace.toString()); + String msg = String.format("Namespace %s not found", namespace.toString()); log.warn(msg); - validationFuture.completeExceptionally(new RestException(Status.NOT_FOUND, msg)); + validationFuture.completeExceptionally(new RestException(Status.NOT_FOUND, "Namespace not found")); } }).exceptionally(ex -> { String msg = String.format("Failed to validate global cluster configuration : cluster=%s ns=%s emsg=%s", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index e41db381f4084..00c4683c68673 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -52,6 +52,7 @@ import org.apache.pulsar.broker.admin.v2.PersistentTopics; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; +import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TopicResources; import org.apache.pulsar.broker.service.BrokerService; @@ -100,6 +101,7 @@ public class PersistentTopicsTest extends MockedPulsarServiceBaseTest { protected Field uriField; protected UriInfo uriInfo; private NonPersistentTopics nonPersistentTopic; + private NamespaceResources namespaceResources; @BeforeClass public void initPersistentTopics() throws Exception { @@ -125,6 +127,7 @@ protected void setup() throws Exception { nonPersistentTopic = spy(new NonPersistentTopics()); nonPersistentTopic.setServletContext(new MockServletContext()); nonPersistentTopic.setPulsar(pulsar); + namespaceResources = mock(NamespaceResources.class); doReturn(false).when(nonPersistentTopic).isRequestHttps(); doReturn(null).when(nonPersistentTopic).originalPrincipal(); doReturn("test").when(nonPersistentTopic).clientAppId(); @@ -406,6 +409,33 @@ public void testCreateNonPartitionedTopic() { Assert.assertEquals(metadata.partitions, 0); } + @Test + public void testCreateTopicWithReplicationCluster() { + final String topicName = "test-topic-ownership"; + NamespaceName namespaceName = NamespaceName.get(testTenant, testNamespace); + CompletableFuture> policyFuture = new CompletableFuture<>(); + Policies policies = new Policies(); + policyFuture.complete(Optional.of(policies)); + when(pulsar.getPulsarResources().getNamespaceResources()).thenReturn(namespaceResources); + doReturn(policyFuture).when(namespaceResources).getPoliciesAsync(namespaceName); + AsyncResponse response = mock(AsyncResponse.class); + ArgumentCaptor errCaptor = ArgumentCaptor.forClass(RestException.class); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); + verify(response, timeout(5000).times(1)).resume(errCaptor.capture()); + Assert.assertEquals(errCaptor.getValue().getResponse().getStatus(), Response.Status.PRECONDITION_FAILED.getStatusCode()); + Assert.assertTrue(errCaptor.getValue().getMessage().contains("Namespace does not have any clusters configured")); + // Test policy not exist and return 'Namespace not found' + CompletableFuture> policyFuture2 = new CompletableFuture<>(); + policyFuture2.complete(Optional.empty()); + doReturn(policyFuture2).when(namespaceResources).getPoliciesAsync(namespaceName); + response = mock(AsyncResponse.class); + errCaptor = ArgumentCaptor.forClass(RestException.class); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); + verify(response, timeout(5000).times(1)).resume(errCaptor.capture()); + Assert.assertEquals(errCaptor.getValue().getResponse().getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + Assert.assertTrue(errCaptor.getValue().getMessage().contains("Namespace not found")); + } + @Test(expectedExceptions = RestException.class) public void testCreateNonPartitionedTopicWithInvalidName() { final String topicName = "standard-topic-partition-10"; From 59fdd431bd09d29743657057032d550521b3b517 Mon Sep 17 00:00:00 2001 From: chenlin <1572139390@qq.com> Date: Tue, 15 Feb 2022 18:34:10 +0800 Subject: [PATCH 344/823] Fix IllegalArgumentException: Invalid period 0.0 to calculate rate (cherry picked from commit c331281bc021c43c7911fe8316931a123541d0cf) --- .../bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index e1761818b2ff6..f8806824d8f0c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -188,7 +188,7 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, this.config = config; this.mbean = new ManagedLedgerFactoryMBeanImpl(this); this.entryCacheManager = new EntryCacheManager(this); - this.statsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::refreshStats), + this.statsTask = scheduledExecutor.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::refreshStats), 0, config.getStatsPeriodSeconds(), TimeUnit.SECONDS); this.flushCursorsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::flushCursors), config.getCursorPositionFlushSeconds(), config.getCursorPositionFlushSeconds(), TimeUnit.SECONDS); From 48dbf01f3e981856fa3b2129ae8d2963f494dc52 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 15 Feb 2022 21:39:10 +0800 Subject: [PATCH 345/823] Fix batch ack count is negtive issue. (#14288) ### Motivation As #13383 fixed the batch ack issue. we find that the unack-msg count could be negative(#14246). At first, we think it was the normal case caused by msg redelivery. But after diving into the logic, we find it's a bug. The test is copy from #14246 : ``` for (int i = 0; i < 50; i++) { Message msg = consumer.receive(); if (i % 2 == 0) { consumer.acknowledgeAsync(msg); } else { consumer.negativeAcknowledge(msg); } } ``` When msg is `negativeAcknowledge`, Consumer#redeliverUnacknowledgedMessages will invoke: https://github.com/apache/pulsar/blob/b22f70658927e07e3726d32290065f47313070b9/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java#L900-L912 When calculating `totalRedeliveryMessages`, it must check `pendingAcks` contains this message. and remove from `pendingAcks` after that. (Dispatch messages will add messages to pendingAcks) So the above test may exist that when `negativeAcknowledge` first and then `acknowledgeAsync`. `acknowledgeAsync` mapped to `Consumer#individualAckNormal` and decrease unack-msg in : https://github.com/apache/pulsar/blob/b22f70658927e07e3726d32290065f47313070b9/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java#L543-L561 It doesn't check `pendingAcks`. this is the root cause. Should move line 556 to 545. (cherry picked from commit 6b828b41382e5a94f89d628aca38871ccff8df9d) --- .../pulsar/broker/service/Consumer.java | 6 +- .../BatchMessageWithBatchIndexLevelTest.java | 96 ++++++++++++++++++- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 00511f469e470..bfaa6608e774e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -526,7 +526,8 @@ private long getAckedCountForMsgIdNoAckSets(long batchSize, PositionImpl positio private long getAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize, long[] ackSets) { long ackedCount = 0; - if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType)) { + if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType) + && pendingAcks.get(position.getLedgerId(), position.getEntryId()) != null) { long[] cursorAckSet = getCursorAckSet(position); if (cursorAckSet != null) { BitSetRecyclable cursorBitSet = BitSetRecyclable.create().resetWords(cursorAckSet); @@ -537,7 +538,7 @@ private long getAckedCountForBatchIndexLevelEnabled(PositionImpl position, long int currentCardinality = cursorBitSet.cardinality(); ackedCount = lastCardinality - currentCardinality; cursorBitSet.recycle(); - } else if (pendingAcks.get(position.getLedgerId(), position.getEntryId()) != null) { + } else { ackedCount = batchSize - BitSet.valueOf(ackSets).cardinality(); } } @@ -558,6 +559,7 @@ private long getUnAckedCountForBatchIndexLevelEnabled(PositionImpl position, lon if (cursorAckSet != null) { BitSetRecyclable cursorBitSet = BitSetRecyclable.create().resetWords(cursorAckSet); unAckedCount = cursorBitSet.cardinality(); + cursorBitSet.recycle(); } } return unAckedCount; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index c3785c7d3cb40..b953772ad6703 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -19,24 +19,24 @@ package org.apache.pulsar.broker.service; import com.google.common.collect.Lists; +import lombok.Cleanup; import lombok.SneakyThrows; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; - import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - import static org.testng.Assert.assertEquals; @Test(groups = "broker") @@ -56,7 +56,8 @@ public void testBatchMessageAck() { final String topicName = "persistent://prop/ns-abc/batchMessageAck-" + UUID.randomUUID(); final String subscriptionName = "sub-batch-1"; - ConsumerImpl consumer = (ConsumerImpl) pulsarClient + @Cleanup + Consumer consumer = pulsarClient .newConsumer() .topic(topicName) .subscriptionName(subscriptionName) @@ -66,6 +67,7 @@ public void testBatchMessageAck() { .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) .subscribe(); + @Cleanup Producer producer = pulsarClient .newProducer() .topic(topicName) @@ -107,4 +109,90 @@ public void testBatchMessageAck() { assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 16); }); } + + @Test + public void testBatchMessageMultiNegtiveAck() throws Exception{ + final String topicName = "persistent://prop/ns-abc/batchMessageMultiNegtiveAck-" + UUID.randomUUID(); + final String subscriptionName = "sub-negtive-1"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .receiverQueueSize(10) + .enableBatchIndexAcknowledgment(true) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.STRING) + .topic(topicName) + .batchingMaxMessages(20) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .enableBatching(true) + .create(); + + final int N = 20; + for (int i = 0; i < N; i++) { + String value = "test-" + i; + producer.sendAsync(value); + } + producer.flush(); + for (int i = 0; i < N; i++) { + Message msg = consumer.receive(); + if (i % 2 == 0) { + consumer.acknowledgeAsync(msg); + } else { + consumer.negativeAcknowledge(msg); + } + } + Awaitility.await().untilAsserted(() -> { + long unackedMessages = admin.topics().getStats(topicName).getSubscriptions().get(subscriptionName) + .getUnackedMessages(); + assertEquals(unackedMessages, 10); + }); + + // Test negtive ack with sleep + final String topicName2 = "persistent://prop/ns-abc/batchMessageMultiNegtiveAck2-" + UUID.randomUUID(); + final String subscriptionName2 = "sub-negtive-2"; + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName2) + .subscriptionName(subscriptionName2) + .subscriptionType(SubscriptionType.Shared) + .receiverQueueSize(10) + .enableBatchIndexAcknowledgment(true) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .subscribe(); + @Cleanup + Producer producer2 = pulsarClient + .newProducer(Schema.STRING) + .topic(topicName2) + .batchingMaxMessages(20) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .enableBatching(true) + .create(); + + for (int i = 0; i < N; i++) { + String value = "test-" + i; + producer2.sendAsync(value); + } + producer2.flush(); + for (int i = 0; i < N; i++) { + Message msg = consumer2.receive(); + if (i % 2 == 0) { + consumer.acknowledgeAsync(msg); + } else { + consumer.negativeAcknowledge(msg); + Thread.sleep(100); + } + } + Awaitility.await().untilAsserted(() -> { + long unackedMessages = admin.topics().getStats(topicName).getSubscriptions().get(subscriptionName) + .getUnackedMessages(); + assertEquals(unackedMessages, 10); + }); + } } From 4c016b987e3f55d40307aa24f190427378545039 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 16 Feb 2022 00:07:23 +0800 Subject: [PATCH 346/823] Fix time unit error in errMsg when producer send fails. (#14299) (cherry picked from commit 5bcdb5ef40a24fefad042efac484f2be4b92c527) --- .../main/java/org/apache/pulsar/client/impl/ProducerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 8486283128444..dc651f36e01ff 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1249,7 +1249,8 @@ void sendComplete(final Exception e) { long sequenceId = te.getSequenceId(); long ns = System.nanoTime(); String errMsg = String.format( - "%s : createdAt %s ns ago, firstSentAt %s ns ago, lastSentAt %s ns ago, retryCount %s", + "%s : createdAt %s seconds ago, firstSentAt %s seconds ago, lastSentAt %s seconds ago, " + + "retryCount %s", te.getMessage(), RelativeTimeUtil.nsToSeconds(ns - this.createdAt), RelativeTimeUtil.nsToSeconds(this.firstSentAt <= 0 ? ns - this.lastSentAt : ns - this.firstSentAt), From 5615809ffd68599ae7fec1be0679ee83171f42dc Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 16 Feb 2022 20:21:07 +0800 Subject: [PATCH 347/823] Release 2.9.2 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- deployment/terraform-ansible/deploy-pulsar.yaml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 127 files changed, 127 insertions(+), 127 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 9b36a306b979c..ddca4e6494ada 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 616aa02c01a43..fb84e6fe63d32 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 389cfb7f2e90c..4fa3ead0eadd4 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 49bc0d85da445..979803c96f58c 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 4470f4214ffa7..eca2dadddf384 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.3-SNAPSHOT + 2.9.2 jar Pulsar Build Tools diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml index a06b73b99852e..cdde04d8aab5c 100644 --- a/deployment/terraform-ansible/deploy-pulsar.yaml +++ b/deployment/terraform-ansible/deploy-pulsar.yaml @@ -39,7 +39,7 @@ zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" service_url: "{{ pulsar_service_url }}" http_url: "{{ pulsar_web_url }}" - pulsar_version: "2.9.3-SNAPSHOT" + pulsar_version: "2.9.2" - name: Download Pulsar binary package unarchive: src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 4ee3328cc4d8b..155b7da8a080d 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 1248ea1682609..ff4055cafd909 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index f79f26d45fb26..72afff04ad6ab 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index ed5401ad39650..5fa966d476187 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index adb423253ef31..df9617f314ec9 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index a3534d2c0c83d..94fabef4d82be 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index b9e4f8d158a49..a897b00b648cb 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index f772a4403bf98..9ee0334651e1c 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 4b46838a3ec3f..3e51419304178 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index f32bc87c46e27..85d189f87b5b5 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 509d00380abc0..63525923b0003 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pom.xml b/pom.xml index 659c315ec0494..333f91279c394 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index fdd52671868fe..c6ff0570e1bde 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 92f27eb385ca7..660097361abad 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 9e755b12161f1..77757b42fa6d8 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 68a53c806ffef..289105f78f2f8 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 2d4605a7a0153..60b175c3d4ecd 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index ac7d7ec5354da..6078062dc1e69 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index b17e509d89a9f..59cf04834338d 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 6c0bde533061e..1f3346fd0da53 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index e5147c4837a8a..60840c9f37f44 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 0a3dc027dfb86..faebb94d05622 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 74f20a15f4caf..f6db9d39ee693 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 37d12891d049c..2722b42911976 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index e7808b61a03ea..c9ef5d87e4106 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 11f5fac560a32..40f5f8407b8d5 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 8e7a0b4ecad6e..020af46f5f855 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 3b24ced062b6e..9f9528b58e80e 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 891bd2f7f8b88..97002332870b7 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 593d5842448e7..2397289149c1e 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 27838ecddf998..9252d603f32b4 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index f5c6cd2e06392..089ac49c5d579 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index c4b5a4b87ba4d..42c823a7f8615 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 759e6acccf810..afbc14ec0eec1 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 1392e9badc7ba..6e42f63c66acc 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 6790d1d6b7886..efbb8fb6229c4 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index f6cf87853f7d2..ad37c3588e0fe 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 732dee3f597ea..254ed1437d129 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index d05780478a168..b5c22b5cff2af 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 727304d30e78a..36591e6f2f194 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 2506936143b58..871a07a9658dc 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 14be823874009..bf01c32367513 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 4278a6553ab11..2037b5d3f6194 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 247a6393a81fd..30c8375a0c7b9 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index b16f364689af8..676a983f20dc2 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 7b76ab13e098a..8bc57600a8759 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 83395d20966b3..8bc73937e2029 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index f06c5a609b1f3..176d920631e2f 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 70778879c23df..6ce7e8c415c03 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 3833961d94c57..b66f5bddd275b 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 889f0ab150cb2..4c5b57d67fbbd 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index bf92cea841c28..311197c9f2756 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 71606c0548153..d4a711ed01186 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index f1c47fb5dfa2a..917981f55658c 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 81dad1f245cee..1a49c3eb32b6e 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 21423a08143bd..5401ace104fc4 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 53910d0cff22f..c6c8c58f95e27 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 229623deee7a0..0a0f831bef961 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index a68c942d97180..c963ae626c7e4 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index e5badf75a677e..cde53aeb6359b 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index f7005c635e9ff..79b92b89999c2 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 32e3ff17bfcd0..ad331e30847d1 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index e8f597224e582..38c8b53a6bd0b 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index fbdaa6345abb0..0a173aa8650e8 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index bdd7bfc55758b..180e338099f7f 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 0fb45c9a5c009..4a7a2a335e4b7 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index a760ee9584693..b259cbb1de580 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 5025e47bf2dbc..cde101487ff3e 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index beec6c6ab96e5..2f3e178a9de09 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 8a23962c7388d..d6dabf60355ba 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index ef84dd16c2e00..3fa60cd1807d7 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 5e48ab354e27b..b1fcbc010f996 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 6bc9c0a025cd3..2db6fb2ef7d99 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 67b887c1e07ff..483a855523e08 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index aad83eaeeaa06..5f95ff1e50211 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index b53a8ed9282df..127c6f72ef6fa 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 67327a8f4d46c..e1d65e87302bf 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 8199c1fe3df18..af7f025330897 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index c22f704565290..4df689dfff5d2 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index a6222fda0a7e0..f3060d71166bc 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 76e8aa5fbd05a..5d6166e13ffad 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index d37dd1e8f9f08..1678a9294dc89 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 739f1acade64f..2bf4d0baf3452 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 6a4d3df44430b..bdf1081106964 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 350bcfd05a345..5974289444586 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 8824f35891b16..e573833baba2d 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index bb10dec5cdc53..7291f3e88f29e 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 72a9162706f89..76fcf52c380fa 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index f8f8a2a6491ee..780e216c96484 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.2 pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 468a8debdb903..809def86ed1dc 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index eb93d9900c971..e688a104a16f2 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index c205dcad6d505..168397dacb44d 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index ad838c8a96bab..f6f776ce015a3 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 0faafd53034dd..e132a5de6ace0 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 206e926dec9b7..31ec8436754ff 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 034997d40f7a2..e3f0ae3b51475 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.3-SNAPSHOT + 2.9.2 pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index f49d541cdd570..b1f9486286769 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.3-SNAPSHOT + 2.9.2 pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index a67a7debc8d77..48376baa59270 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.3-SNAPSHOT + 2.9.2 pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 9e686cde439c5..c31872944da0d 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 58ca5ad6fc17a..ddc76bebb24e9 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.3-SNAPSHOT + 2.9.2 pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index b274931049df1..b4ddf32ae19de 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.3-SNAPSHOT + 2.9.2 pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 7aeb860a9eaf3..e0b566a0c3434 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 8a85e3f8d641d..2fff29d1c0165 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index 4b21f3e8ed25c..e2c29fe00ae0b 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 1bead7e397c59..e52e5b7350f2c 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index c4db8a1ade2de..b18cd2eb75c9b 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.2 testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index acb01809c8f13..80f166bb0c575 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index e0ac02e7f4747..9a61cb41d4f6e 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 9cc9619d2a388..559bc5fe6f658 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 06d6f18a678d2..20e71e217ce44 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 063a4c1a50075..46558ca492aa2 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 40518114309b2..c9e16e115f056 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.3-SNAPSHOT + 2.9.2 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index e114b1c6bd945..f2681751f3eda 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 16a0eb7d6bd5a..1314c15dcb45c 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 integration diff --git a/tests/pom.xml b/tests/pom.xml index e34f6678186bb..512c1529b195f 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index dbb9be2d0d670..a84660e326864 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 664c56cfd5044..9bdd8fea40493 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 191c31527877e..281cf4cd64c02 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.2 pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 3e3c1985f0b6d..f839ad30810ee 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index b776161662dc8..716aed18b98f0 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.3-SNAPSHOT + 2.9.2 .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 132083817932c..5b8ae94a8e323 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.2 .. From 1e429b5f1311374d3a27d822bfc26c5aa25cacef Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Tue, 15 Feb 2022 23:02:46 -0800 Subject: [PATCH 348/823] [issue-14304] Check for blank advertised listener name (#14306) Fixes #14304 ### Modifications - [ServerCnx] check for blank advertised listener name - [ServerCnx] improved logging (cherry picked from commit 422efbb0528f54aab92d33732747b454016727aa) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 189c079823de3..beb713c313ed0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -417,10 +417,12 @@ protected void handleLookup(CommandLookupTopic lookup) { final boolean authoritative = lookup.isAuthoritative(); // use the connection-specific listener name by default. - final String advertisedListenerName = lookup.hasAdvertisedListenerName() ? lookup.getAdvertisedListenerName() - : this.listenerName; + final String advertisedListenerName = + lookup.hasAdvertisedListenerName() && StringUtils.isNotBlank(lookup.getAdvertisedListenerName()) + ? lookup.getAdvertisedListenerName() : this.listenerName; if (log.isDebugEnabled()) { - log.debug("[{}] Received Lookup from {} for {}", lookup.getTopic(), remoteAddress, requestId); + log.debug("[{}] Received Lookup from {} for {} requesting listener {}", lookup.getTopic(), remoteAddress, + requestId, StringUtils.isNotBlank(advertisedListenerName) ? advertisedListenerName : "(none)"); } TopicName topicName = validateTopicName(lookup.getTopic(), requestId, lookup); From fa58bfe89bb1b833ebabb5e1c8198d8ced5b1134 Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Fri, 18 Feb 2022 18:37:27 +0800 Subject: [PATCH 349/823] [CI]Upgrade Windows runner os to windows-2022 and generator to Visual Studio 17 2022 (#14368) ### Motivation As github windows runner latest upgrade to [2022](https://github.com/actions/virtual-environments/issues/4856), the `ci-cpp-build-windows.yaml` workflow should be changed otherwise the action would be failed such as [this one](https://github.com/apache/pulsar/actions/runs/1860147632) ### Modifications - Upgrade os version to 2022 - Change generator to `Visual Studio 17 2022` (cherry picked from commit 0facd24e8eec2df56f6f241ce2cf87eb98590a7e) --- .github/workflows/ci-cpp-build-windows.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml index cc99204c31076..d1e98339d43b6 100644 --- a/.github/workflows/ci-cpp-build-windows.yaml +++ b/.github/workflows/ci-cpp-build-windows.yaml @@ -47,18 +47,18 @@ jobs: matrix: include: - name: 'Windows x64' - os: windows-latest + os: windows-2022 triplet: x64-windows vcpkg_dir: 'C:\vcpkg' suffix: 'windows-win64' - generator: 'Visual Studio 16 2019' + generator: 'Visual Studio 17 2022' arch: '-A x64' - name: 'Windows x86' - os: windows-latest + os: windows-2022 triplet: x86-windows vcpkg_dir: 'C:\vcpkg' suffix: 'windows-win32' - generator: 'Visual Studio 16 2019' + generator: 'Visual Studio 17 2022' arch: '-A Win32' steps: @@ -66,7 +66,7 @@ jobs: uses: actions/checkout@v2 - name: Detect changed files - id: changes + id: changes uses: apache/pulsar-test-infra/paths-filter@master with: filters: .github/changes-filter.yaml From 23c6581dcdb7922c33d3397efefec73cca149d27 Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Wed, 23 Feb 2022 21:10:28 +0800 Subject: [PATCH 350/823] Make sure policies.is_allow_auto_update_schema not null (#14409) (cherry picked from commit 7d6079529b37890ddb42d91abe915dae6ebb22b7) --- .../pulsar/broker/admin/AdminResource.java | 19 +++++-------------- .../pulsar/broker/admin/AdminApiTest.java | 2 ++ .../broker/admin/v1/V1_AdminApiTest.java | 2 ++ .../SchemaCompatibilityCheckTest.java | 5 +++++ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 547665f597428..71c88834d3f69 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -286,6 +286,10 @@ protected Policies getNamespacePolicies(NamespaceName namespaceName) { BundlesData bundleData = pulsar().getNamespaceService().getNamespaceBundleFactory() .getBundles(namespaceName).getBundlesData(); policies.bundles = bundleData != null ? bundleData : policies.bundles; + if (policies.is_allow_auto_update_schema == null) { + // the type changed from boolean to Boolean. return broker value here for keeping compatibility. + policies.is_allow_auto_update_schema = pulsar().getConfig().isAllowAutoUpdateSchemaEnabled(); + } return policies; } catch (RestException re) { @@ -517,20 +521,7 @@ protected void validateClusterExists(String cluster) { protected Policies getNamespacePolicies(String tenant, String cluster, String namespace) { NamespaceName ns = NamespaceName.get(tenant, cluster, namespace); - try { - Policies policies = namespaceResources().getPolicies(ns) - .orElseThrow(() -> new RestException(Status.NOT_FOUND, "Namespace does not exist")); - // fetch bundles from LocalZK-policies - BundlesData bundleData = pulsar().getNamespaceService().getNamespaceBundleFactory() - .getBundles(ns).getBundlesData(); - policies.bundles = bundleData != null ? bundleData : policies.bundles; - return policies; - } catch (RestException re) { - throw re; - } catch (Exception e) { - log.error("[{}] Failed to get namespace policies {}", clientAppId(), ns, e); - throw new RestException(e); - } + return getNamespacePolicies(ns); } protected boolean isNamespaceReplicated(NamespaceName namespaceName) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 3e87f13c436f3..eee39b823e075 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -736,6 +736,7 @@ public void namespaces() throws Exception { policies.bundles = PoliciesUtil.defaultBundle(); policies.auth_policies.getNamespaceAuthentication().put("spiffe://developer/passport-role", EnumSet.allOf(AuthAction.class)); policies.auth_policies.getNamespaceAuthentication().put("my-role", EnumSet.allOf(AuthAction.class)); + policies.is_allow_auto_update_schema = conf.isAllowAutoUpdateSchemaEnabled(); assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies); assertEquals(admin.namespaces().getPermissions("prop-xyz/ns1"), policies.auth_policies.getNamespaceAuthentication()); @@ -746,6 +747,7 @@ public void namespaces() throws Exception { admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1", "my-role"); policies.auth_policies.getNamespaceAuthentication().remove("spiffe://developer/passport-role"); policies.auth_policies.getNamespaceAuthentication().remove("my-role"); + policies.is_allow_auto_update_schema = conf.isAllowAutoUpdateSchemaEnabled(); assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies); assertEquals(admin.namespaces().getPersistence("prop-xyz/ns1"), null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java index 06d066f4c38b2..16bf0553345da 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java @@ -650,6 +650,7 @@ public void namespaces() throws Exception { Policies policies = new Policies(); policies.bundles = PoliciesUtil.defaultBundle(); policies.auth_policies.getNamespaceAuthentication().put("my-role", EnumSet.allOf(AuthAction.class)); + policies.is_allow_auto_update_schema = conf.isAllowAutoUpdateSchemaEnabled(); assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies); assertEquals(admin.namespaces().getPermissions("prop-xyz/use/ns1"), policies.auth_policies.getNamespaceAuthentication()); @@ -658,6 +659,7 @@ public void namespaces() throws Exception { admin.namespaces().revokePermissionsOnNamespace("prop-xyz/use/ns1", "my-role"); policies.auth_policies.getNamespaceAuthentication().remove("my-role"); + policies.is_allow_auto_update_schema = conf.isAllowAutoUpdateSchemaEnabled(); assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies); assertNull(admin.namespaces().getPersistence("prop-xyz/use/ns1")); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 5b12f37d40060..1b5e4d67232a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -39,6 +39,7 @@ import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.schema.SchemaInfo; @@ -247,6 +248,7 @@ public void testBrokerAllowAutoUpdateSchemaDisabled(SchemaCompatibilityStrategy pulsar.getConfig().setAllowAutoUpdateSchemaEnabled(false); + ProducerBuilder producerThreeBuilder = pulsarClient .newProducer(Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). @@ -259,6 +261,9 @@ public void testBrokerAllowAutoUpdateSchemaDisabled(SchemaCompatibilityStrategy } pulsar.getConfig().setAllowAutoUpdateSchemaEnabled(true); + Policies policies = admin.namespaces().getPolicies(namespaceName.toString()); + Assert.assertTrue(policies.is_allow_auto_update_schema); + ConsumerBuilder comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO( SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). From 7792facfbff68e04273cc0aa32c3d25ea5f2d1d9 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 24 Feb 2022 21:00:01 +0800 Subject: [PATCH 351/823] [pulsar-io] pass client builder if no service url provided to debezium connector (#12145) (#14041) --- .../pulsar/io/debezium/DebeziumSource.java | 18 +++++++----------- .../io/debezium/PulsarDatabaseHistory.java | 6 +++--- .../debezium/DebeziumMySqlSourceTester.java | 7 +++++-- .../debezium/PulsarDebeziumSourcesTest.java | 15 +++++++++++---- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/DebeziumSource.java b/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/DebeziumSource.java index b9074b91bc7c5..eeb216b5d9d11 100644 --- a/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/DebeziumSource.java +++ b/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/DebeziumSource.java @@ -18,9 +18,8 @@ */ package org.apache.pulsar.io.debezium; -import io.debezium.relational.history.DatabaseHistory; import java.util.Map; - +import io.debezium.relational.history.DatabaseHistory; import io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.naming.TopicName; @@ -50,10 +49,7 @@ public static void throwExceptionIfConfigNotMatch(Map config, } public static void setConfigIfNull(Map config, String key, String value) { - Object orig = config.get(key); - if (orig == null) { - config.put(key, value); - } + config.putIfAbsent(key, value); } // namespace for output topics, default value is "tenant/namespace" @@ -81,9 +77,6 @@ public void open(Map config, SourceContext sourceContext) throws // database.history.pulsar.service.url String pulsarUrl = (String) config.get(PulsarDatabaseHistory.SERVICE_URL.name()); - if (StringUtils.isEmpty(pulsarUrl)) { - throw new IllegalArgumentException("Pulsar service URL for History Database not provided."); - } String topicNamespace = topicNamespace(sourceContext); // topic.namespace @@ -97,8 +90,11 @@ public void open(Map config, SourceContext sourceContext) throws setConfigIfNull(config, PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, topicNamespace + "/" + sourceName + "-" + DEFAULT_OFFSET_TOPIC); - config.put(DatabaseHistory.CONFIGURATION_FIELD_PREFIX_STRING + "pulsar.client.builder", - SerDeUtils.serialize(sourceContext.getPulsarClientBuilder())); + // pass pulsar.client.builder if database.history.pulsar.service.url is not provided + if (StringUtils.isEmpty(pulsarUrl)) { + String pulsarClientBuilder = SerDeUtils.serialize(sourceContext.getPulsarClientBuilder()); + config.put(PulsarDatabaseHistory.CLIENT_BUILDER.name(), pulsarClientBuilder); + } super.open(config, sourceContext); } diff --git a/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java b/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java index be152a6da8eb2..c97e101a39716 100644 --- a/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java +++ b/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java @@ -103,12 +103,12 @@ public void configure( } this.topicName = config.getString(TOPIC); - if (config.getString(CLIENT_BUILDER) == null && config.getString(SERVICE_URL) == null) { + String clientBuilderBase64Encoded = config.getString(CLIENT_BUILDER); + if (isBlank(clientBuilderBase64Encoded) && isBlank(config.getString(SERVICE_URL))) { throw new IllegalArgumentException("Neither Pulsar Service URL nor ClientBuilder provided."); } - String clientBuilderBase64Encoded = config.getString(CLIENT_BUILDER); this.clientBuilder = PulsarClient.builder(); - if (null != clientBuilderBase64Encoded) { + if (!isBlank(clientBuilderBase64Encoded)) { // deserialize the client builder to the same classloader this.clientBuilder = (ClientBuilder) SerDeUtils.deserialize(clientBuilderBase64Encoded, this.clientBuilder.getClass().getClassLoader()); } else { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java index 3cb64db8a7de0..7958fa019925f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java @@ -49,7 +49,8 @@ public class DebeziumMySqlSourceTester extends SourceTester Date: Mon, 7 Feb 2022 10:04:22 +0100 Subject: [PATCH 352/823] [security] Upgrade Postgre driver to 42.2.25 to get rid of CVE-2022-21724 (#14119) http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2022-21724 Upgrade both `jdbc` and `debezium` Postgre java driver dependency to 42.2.25 (from 42.2.24 and 42.2.22). Note: the version is not shared on purpose because we should leave the driver dependencies separated since the two connectors are used in different ways. (For example, when we'll upgrade Debezium to 1.8.x we'll need to remove the override and keep the 42.3.x version) For cherry-picks, branch-2.9 and branch-2.8 are compatible since: * branch-2.9 has the same debezium version * branch-2.8 has 1.0.0 but it uses [pg driver 42.2.x](https://search.maven.org/artifact/io.debezium/debezium-parent/1.0.0.Final/pom) as well - [x] `no-need-doc` (cherry picked from commit 64818458727df20384463bbedf9cb7c92c0f9216) --- pom.xml | 3 ++- pulsar-io/debezium/postgres/pom.xml | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 333f91279c394..30df8e64aadb3 100644 --- a/pom.xml +++ b/pom.xml @@ -148,7 +148,7 @@ flexible messaging model and an intuitive client API. 2.3.0 3.8.11.2 8.0.11 - 42.2.24 + 42.2.25 0.3.2 2.6.0 3.3.0 @@ -157,6 +157,7 @@ flexible messaging model and an intuitive client API. 2.13 2.13.6 1.7.1.Final + 42.2.25 0.11.1 0.18.0 2.3.0 diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index ad331e30847d1..5bcf07941e921 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -44,6 +44,13 @@ ${debezium.version} + + org.postgresql + postgresql + ${debezium.postgresql.version} + runtime + + From b70f6cdcd3a48550795c2e0e4de86669867b0586 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 16 Feb 2022 11:50:41 +0800 Subject: [PATCH 353/823] [Transaction] Optimize testEndTBRecoveringWhenManagerLedgerDisReadable (#14303) ### Motivation The time used by clean up is too long. This is because the broker service fails to execute the `unloadserviceunit`, and can only rely on the timeout mechanism to cancel the `unloadserviceunit` this time. As a result, the clean up takes too much time. The root cause is that there is a mock cursor in `ManagerLedger.cursors`. ### Modification Remove the cursor from `ManagerLedger.cursors` after test. (cherry picked from commit a43fab0045a93ee864da6bc386bcc0e8bf17bf11) --- .../org/apache/pulsar/broker/transaction/TransactionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index b627438eb824a..b882d26d17133 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -562,6 +562,7 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ TransactionBuffer buffer2 = new TopicTransactionBuffer(persistentTopic); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer2.getStats().state, "Ready")); + managedCursors.removeCursor("transaction-buffer-sub"); } @Test From 8279db990479d5ebc8c00c3bf7dde4cdcb05643d Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Wed, 16 Feb 2022 14:05:53 +0800 Subject: [PATCH 354/823] Fix the wrong parameter in the log. (#14309) ### Motivation Bad parameters in the log will always print "null", which can confuse users. ### Modifications - Correct exception parameter. (cherry picked from commit 9b003f96f1deb8e02a1654e30fbddd211d787e8e) --- .../broker/service/SystemTopicBasedTopicPoliciesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 838edfe7f05c8..ddf926dc925ea 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -323,7 +323,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp reader.readNextAsync().whenComplete((msg, e) -> { if (e != null) { log.error("[{}] Failed to read event from the system topic.", - reader.getSystemTopic().getTopicName(), ex); + reader.getSystemTopic().getTopicName(), e); future.completeExceptionally(e); readerCaches.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); policyCacheInitMap.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); From c16083e95a9c3407860a00c7cabe2d0db2708347 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 16 Feb 2022 22:57:00 +0800 Subject: [PATCH 355/823] [Broker] Change broker producer fence log level (#14196) (cherry picked from commit 141ea9b6dd3897d492cf468b6ded29ce9f7cf73f) --- .../org/apache/pulsar/broker/service/ServerCnx.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index beb713c313ed0..2ba51d2be342c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1335,8 +1335,15 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ producers.remove(producerId, producerFuture); }).exceptionally(ex -> { - log.error("[{}] Failed to add producer to topic {}: producerId={}, {}", - remoteAddress, topicName, producerId, ex.getMessage()); + if (ex.getCause() instanceof BrokerServiceException.ProducerFencedException) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to add producer to topic {}: producerId={}, {}", + remoteAddress, topicName, producerId, ex.getCause().getMessage()); + } + } else { + log.warn("[{}] Failed to add producer to topic {}: producerId={}, {}", + remoteAddress, topicName, producerId, ex.getCause().getMessage()); + } producer.closeNow(true); if (producerFuture.completeExceptionally(ex)) { From 0e438a982ef74328495f0cf6102546f545fa4b58 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Wed, 16 Feb 2022 22:58:42 +0800 Subject: [PATCH 356/823] [Flaky-Test] BacklogQuotaManagerTest#testProducerExceptionAndThenUnblockSizeQuota (#14213) (cherry picked from commit cd172e7455f90955ac951d713a28c4a580fdf75d) --- .../apache/pulsar/broker/service/BacklogQuotaManagerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index f531119386808..21fcffac1ea21 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -1123,6 +1123,8 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { } Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); // publish should work now + producer.close(); + producer = createProducer(client, topic1); Exception sendException = null; gotException = false; try { From d6867b44f2747c4fdb1b09e14b2a356931f49ff9 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 17 Feb 2022 13:23:03 +0800 Subject: [PATCH 357/823] [Websocket] Fix ``ClassCastException`` when user create ``MultiTopicReader``. (#14316) (cherry picked from commit 7a7cf54b01420aeac855eea91529ea13bd753e52) --- .../pulsar/websocket/ReaderHandler.java | 10 +- .../pulsar/websocket/ReaderHandlerTest.java | 214 ++++++++++++++++++ 2 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index ef0279dfaeea3..2b87802ed9ee6 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -36,6 +36,7 @@ import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.MultiTopicsReaderImpl; import org.apache.pulsar.client.impl.ReaderImpl; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -103,8 +104,13 @@ public ReaderHandler(WebSocketService service, HttpServletRequest request, Servl } this.reader = builder.create(); - - this.subscription = ((ReaderImpl) this.reader).getConsumer().getSubscription(); + if (reader instanceof MultiTopicsReaderImpl) { + this.subscription = ((MultiTopicsReaderImpl) reader).getMultiTopicsConsumer().getSubscription(); + } else if (reader instanceof ReaderImpl) { + this.subscription = ((ReaderImpl) reader).getConsumer().getSubscription(); + } else { + throw new IllegalArgumentException(String.format("Illegal Reader Type %s", reader.getClass())); + } if (!this.service.addReader(this)) { log.warn("[{}:{}] Failed to add reader handler for topic {}", request.getRemoteAddr(), request.getRemotePort(), topic); diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java new file mode 100644 index 0000000000000..0d2a13d1a7496 --- /dev/null +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket; + +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.ReaderBuilder; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; +import org.apache.pulsar.client.impl.MultiTopicsReaderImpl; +import org.apache.pulsar.client.impl.ReaderImpl; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.testng.Assert; +import org.testng.annotations.Test; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ReaderHandlerTest { + + @Test + @SuppressWarnings("unchecked") + public void testCreateReaderImp() throws IOException { + final String subName = "readerImpSubscription"; + // mock data + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.receiverQueueSize(anyInt())).thenReturn(mockedReaderBuilder); + ReaderImpl mockedReader = mock(ReaderImpl.class); + when(mockedReaderBuilder.create()).thenReturn(mockedReader); + ConsumerImpl consumerImp = mock(ConsumerImpl.class); + when(consumerImp.getSubscription()).thenReturn(subName); + when(mockedReader.getConsumer()).thenReturn(consumerImp); + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + // create reader handler + HttpServletResponse response = spy(HttpServletResponse.class); + ServletUpgradeResponse servletUpgradeResponse = new ServletUpgradeResponse(response); + ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); + // verify success + Assert.assertEquals(readerHandler.getSubscription(), subName); + } + + @Test + @SuppressWarnings("unchecked") + public void testCreateMultipleTopicReaderImp() throws IOException { + final String subName = "multipleTopicReaderImpSubscription"; + // mock data + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.receiverQueueSize(anyInt())).thenReturn(mockedReaderBuilder); + MultiTopicsReaderImpl mockedReader = mock(MultiTopicsReaderImpl.class); + when(mockedReaderBuilder.create()).thenReturn(mockedReader); + MultiTopicsConsumerImpl consumerImp = mock(MultiTopicsConsumerImpl.class); + when(consumerImp.getSubscription()).thenReturn(subName); + when(mockedReader.getMultiTopicsConsumer()).thenReturn(consumerImp); + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + // create reader handler + HttpServletResponse response = spy(HttpServletResponse.class); + ServletUpgradeResponse servletUpgradeResponse = new ServletUpgradeResponse(response); + ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); + // verify success + Assert.assertEquals(readerHandler.getSubscription(), subName); + } + + @Test + @SuppressWarnings("unchecked") + public void testCreateIllegalReaderImp() throws IOException { + // mock data + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.receiverQueueSize(anyInt())).thenReturn(mockedReaderBuilder); + IllegalReader illegalReader = new IllegalReader(); + when(mockedReaderBuilder.create()).thenReturn(illegalReader); + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + // create reader handler + HttpServletResponse response = spy(HttpServletResponse.class); + ServletUpgradeResponse servletUpgradeResponse = new ServletUpgradeResponse(response); + new ReaderHandler(wss, request, servletUpgradeResponse); + // verify get error + verify(response, times(1)).sendError(anyInt(), anyString()); + } + + + static class IllegalReader implements Reader { + + @Override + public String getTopic() { + return null; + } + + @Override + public Message readNext() throws PulsarClientException { + return null; + } + + @Override + public Message readNext(int timeout, TimeUnit unit) throws PulsarClientException { + return null; + } + + @Override + public CompletableFuture> readNextAsync() { + return null; + } + + @Override + public CompletableFuture closeAsync() { + return null; + } + + @Override + public boolean hasReachedEndOfTopic() { + return false; + } + + @Override + public boolean hasMessageAvailable() { + return false; + } + + @Override + public CompletableFuture hasMessageAvailableAsync() { + return null; + } + + @Override + public boolean isConnected() { + return false; + } + + @Override + public void seek(MessageId messageId) throws PulsarClientException { + + } + + @Override + public void seek(long timestamp) throws PulsarClientException { + + } + + @Override + public void seek(Function function) throws PulsarClientException { + + } + + @Override + public CompletableFuture seekAsync(Function function) { + return null; + } + + @Override + public CompletableFuture seekAsync(MessageId messageId) { + return null; + } + + @Override + public CompletableFuture seekAsync(long timestamp) { + return null; + } + + @Override + public void close() throws IOException { + + } + } +} From 44408bf3268c36abd5b8040ff7c1fcebd27134c8 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 19 Feb 2022 01:38:24 +0800 Subject: [PATCH 358/823] Fix adding message to list potential issue (#14377) (cherry picked from commit b22445f961da5cf2e7baaac4b3847007f4c6ed59) --- .../java/org/apache/pulsar/client/impl/ConsumerImpl.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 9042788eb2368..e5a5745f82450 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1771,18 +1771,17 @@ private CompletableFuture> getRedeliveryMessageIdData(List data = new ArrayList<>(messageIds.size()); - List> futures = new ArrayList<>(messageIds.size()); + List> futures = new ArrayList<>(messageIds.size()); messageIds.forEach(messageId -> { CompletableFuture future = processPossibleToDLQ(messageId); - futures.add(future); - future.thenAccept(sendToDLQ -> { + futures.add(future.thenAccept(sendToDLQ -> { if (!sendToDLQ) { data.add(new MessageIdData() .setPartition(messageId.getPartitionIndex()) .setLedgerId(messageId.getLedgerId()) .setEntryId(messageId.getEntryId())); } - }); + })); }); return FutureUtil.waitForAll(futures).thenCompose(v -> CompletableFuture.completedFuture(data)); } From e583b055915bcd9b295bbed0d1b99f6d139ec568 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 22 Feb 2022 02:18:56 +0800 Subject: [PATCH 359/823] [Broker] waitingCursors potential heap memory leak (#13939) (cherry picked from commit 478fd36227c2ede3e1162dd9a4361cffc5dbfceb) --- .../bookkeeper/mledger/ManagedLedger.java | 7 ++++++ .../mledger/impl/ManagedLedgerImpl.java | 4 ++++ .../persistent/PersistentSubscription.java | 1 + .../broker/admin/CreateSubscriptionTest.java | 22 +++++++++++++++++++ .../jcloud/impl/MockManagedLedger.java | 5 +++++ 5 files changed, 39 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index 0200e25cff132..cd39919a3b357 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -288,6 +288,13 @@ ManagedCursor newNonDurableCursor(Position startPosition, String subscriptionNam */ void deleteCursor(String name) throws InterruptedException, ManagedLedgerException; + /** + * Remove a ManagedCursor from this ManagedLedger's waitingCursors. + * + * @param cursor the ManagedCursor + */ + void removeWaitingCursor(ManagedCursor cursor); + /** * Open a ManagedCursor asynchronously. * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 528816047b709..87287530a54a8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3415,6 +3415,10 @@ public void deactivateCursor(ManagedCursor cursor) { } } + public void removeWaitingCursor(ManagedCursor cursor) { + this.waitingCursors.remove(cursor); + } + public boolean isCursorActive(ManagedCursor cursor) { return activeCursors.get(cursor.getName()) != null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index f69db6825e907..8637ecd1e15b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -299,6 +299,7 @@ public synchronized void removeConsumer(Consumer consumer, boolean isResetCursor if (dispatcher != null && dispatcher.getConsumers().isEmpty()) { deactivateCursor(); + topic.getManagedLedger().removeWaitingCursor(cursor); if (!cursor.isDurable()) { // If cursor is not durable, we need to clean up the subscription as well diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java index e0d1720dc1f9a..09f2c91cc2ab6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java @@ -19,15 +19,22 @@ package org.apache.pulsar.broker.admin; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Lists; import javax.ws.rs.ClientErrorException; import javax.ws.rs.core.Response.Status; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.TopicName; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -127,4 +134,19 @@ public void createSubscriptionOnPartitionedTopicWithPartialFailure() throws Exce Lists.newArrayList("sub-1")); } } + + @Test + public void testWaitingCurosrCausedMemoryLeak() throws Exception { + String topic = "persistent://my-property/my-ns/my-topic"; + for (int i = 0; i < 10; i ++) { + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topic) + .subscriptionType(SubscriptionType.Failover).subscriptionName("test" + i).subscribe(); + Awaitility.await().untilAsserted(() -> assertTrue(consumer.isConnected())); + consumer.close(); + } + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl)(topicRef.getManagedLedger()); + assertEquals(ml.getWaitingCursorsCount(), 0); + } + } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java index 229cd666acff8..907aba67b9ef3 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java @@ -137,6 +137,11 @@ public void deleteCursor(String name) throws InterruptedException, ManagedLedger } + @Override + public void removeWaitingCursor(ManagedCursor cursor) { + + } + @Override public void asyncOpenCursor(String name, AsyncCallbacks.OpenCursorCallback callback, Object ctx) { From 718a95b3f919465500dd15f6eacfca8352c9412c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Tue, 22 Feb 2022 17:17:47 +0800 Subject: [PATCH 360/823] Fix send to deadLetterTopic not working when reach maxRedeliverCount (#14317) If a message reached maxRedeliverCount, it will send to deadLetterTopic, since 2.8.0, this mechanism is broken, it was introduced in #9970 (cherry picked from commit 16beb9d97fdc64092c8f3fe6959d6bf20dd0aa13) --- .../pulsar/client/impl/schema/AbstractSchema.java | 7 +++---- .../org/apache/pulsar/client/impl/MessageTest.java | 13 ++++++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AbstractSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AbstractSchema.java index 8cf7a05cc1719..33c2ed17836ac 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AbstractSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AbstractSchema.java @@ -75,14 +75,13 @@ public Schema clone() { * @param schemaVersion the version * @return the schema at that specific version * @throws SchemaSerializationException in case of unknown schema version - * @throws NullPointerException in case of null schemaVersion + * @throws NullPointerException in case of null schemaVersion and supportSchemaVersioning is true */ public Schema atSchemaVersion(byte[] schemaVersion) throws SchemaSerializationException { - Objects.requireNonNull(schemaVersion); if (!supportSchemaVersioning()) { return this; - } else { - throw new SchemaSerializationException("Not implemented for " + this.getClass()); } + Objects.requireNonNull(schemaVersion); + throw new SchemaSerializationException("Not implemented for " + this.getClass()); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageTest.java index 6d633e793d738..13cf4f6b78865 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageTest.java @@ -22,8 +22,8 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import java.nio.ByteBuffer; +import java.util.Optional; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -81,4 +81,15 @@ public void testTopicMessageImplNoReplicatedInfo() { assertFalse(topicMessage.isReplicated()); assertNull(topicMessage.getReplicatedFrom()); } + + @Test + public void testMessageImplGetReaderSchema() { + MessageMetadata builder = new MessageMetadata(); + builder.hasSchemaVersion(); + ByteBuffer payload = ByteBuffer.wrap(new byte[0]); + Message msg = MessageImpl.create(builder, payload, Schema.BYTES, null); + + Optional> readerSchema = msg.getReaderSchema(); + assertTrue(readerSchema.isPresent()); + } } From 3101b662f04d113946c73a11940a3211ea809c65 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 23 Feb 2022 05:41:28 +0800 Subject: [PATCH 361/823] [C++] Fix GCC compilation failure caused by warning macro (#14402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation When I tried to build the C++ client with GCC 7.3 and when warnings are printed (because `BOOST_ARCH_X86_64` is not defined), the compilation failed with: ``` #warning “BOOST_ARCH_X86_64 is not defined, CRC32C SSE4.2 will be disabled” ^~~~~~~ cc1plus: error: unrecognized command line option '-Wno-stringop-truncation' [-Werror] cc1plus: all warnings being treated as errors ``` It seems to be a bug before GCC 8.1. I added `-DCMAKE_VERBOSE_MAKEFILE=ON` to CMake command and see the full compile command: ``` -Wno-error -Wall -Wformat-security -Wvla -Werror -Wno-sign-compare -Wno-deprecated-declarations -Wno-error=cpp -Wno-stringop-truncation ``` See https://github.com/apache/pulsar/blob/b829a4ce121268f55748bbdd6f19ac36129e7dab/pulsar-client-cpp/CMakeLists.txt#L105-L106 For GCC > 4.9, `-Wno-stringop-truncation` option was added. However, when a message was printed by `#warning` macro, it would fail with the strange error message. The simplest way to reproduce the bug is compiling following code: ```c++ #warnings "hello" int main() {} ``` You can paste the code above to https://godbolt.org/, select any GCC compiler whose version is lower than 8.0, then add the following options: ``` -Werror -Wno-error=cpp -Wno-stringop-truncation ``` The compilation failed for x86-64 gcc 7.5 while it succeeded for 8.1. ### Modifications Only add the `-Wno-stringop-truncation` option for GCC >= 8.1. (cherry picked from commit 958fc7820106c9b4da33f1b720d1dcce8ff772b1) --- pulsar-client-cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 3fadb05a092a7..2d98281a1809d 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -99,7 +99,7 @@ else() # GCC or Clang are mostly compatible: # Options unique to Clang or GCC: if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Qunused-arguments) - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9)) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.1)) add_compile_options(-Wno-stringop-truncation) endif() endif() From 982a11fe26be5adb8e1f368fb6736a95c3eadadf Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 24 Feb 2022 21:54:17 +0800 Subject: [PATCH 362/823] [Transaction] delete changeMaxReadPositionAndAddAbortTimes when checkIfNoSnapshot (#14276) (cherry picked from commit 0a9fd913528181951fd6ad97d3ba07e11e77cd70) --- .../buffer/impl/TopicTransactionBuffer.java | 1 - .../broker/transaction/TransactionTest.java | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 89c77d6313484..e0a6695ef2260 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -474,7 +474,6 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { synchronized (TopicTransactionBuffer.this) { if (checkIfNoSnapshot()) { maxReadPosition = position; - changeMaxReadPositionAndAddAbortTimes.incrementAndGet(); } else if (checkIfReady()) { if (ongoingTxns.isEmpty()) { maxReadPosition = position; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index b882d26d17133..28d4fcf94c958 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -47,6 +47,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -800,4 +801,28 @@ public void testCancelTxnTimeout() throws Exception{ timeout = (Timeout) field.get(transaction); Assert.assertTrue(timeout.isCancelled()); } + + @Test + public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService() + .getTopic(NAMESPACE1 + "/test", true) + .get().get(); + TransactionBuffer buffer = persistentTopic.getTransactionBuffer(); + Field field = TopicTransactionBuffer.class.getDeclaredField("changeMaxReadPositionAndAddAbortTimes"); + field.setAccessible(true); + AtomicLong changeMaxReadPositionAndAddAbortTimes = (AtomicLong) field.get(buffer); + Field field1 = TopicTransactionBufferState.class.getDeclaredField("state"); + field1.setAccessible(true); + + Awaitility.await().untilAsserted(() -> { + TopicTransactionBufferState.State state = (TopicTransactionBufferState.State) field1.get(buffer); + Assert.assertEquals(state, TopicTransactionBufferState.State.NoSnapshot); + }); + Assert.assertEquals(changeMaxReadPositionAndAddAbortTimes.get(), 0L); + + buffer.syncMaxReadPositionForNormalPublish(new PositionImpl(1, 1)); + Assert.assertEquals(changeMaxReadPositionAndAddAbortTimes.get(), 0L); + + } } \ No newline at end of file From 45c74fb6b46231acd8f584ca0d45c7f4d6992cc4 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 24 Feb 2022 21:57:38 +0800 Subject: [PATCH 363/823] [Transaction] Adopt single thread pool in TC (#14238) ### Motivation Optimize code and improve maintainability. ### Modification * Option 1 (the way I use) Create a thread pool at peer TC. * advantage Each TC has a single thread pool to perform its own tasks, and will not be blocked due to sharing a single thread with other TCs * disadvantage Too many thread pools may be created * Option 2 Create an ExecuteProvider in the TC service. It create some single-threaded pools when the TC Service is created, and then assign a single-threaded pool to TC when the TC is created * The advantages and disadvantages are opposite to the option one (cherry picked from commit ced57866700aaeae163bcc6670d9a8eb1ffe8c50) --- .../impl/MLTransactionMetadataStore.java | 328 ++++++++++-------- 1 file changed, 188 insertions(+), 140 deletions(-) diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index a71d2030ebf2b..f109ec4a3be55 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -18,12 +18,15 @@ */ package org.apache.pulsar.transaction.coordinator.impl; +import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; @@ -69,6 +72,7 @@ public class MLTransactionMetadataStore private final LongAdder transactionTimeoutCount; private final LongAdder appendLogCount; private final MLTransactionSequenceIdGenerator sequenceIdGenerator; + private final ExecutorService internalPinnedExecutor; public MLTransactionMetadataStore(TransactionCoordinatorID tcID, MLTransactionLogImpl mlTransactionLog, @@ -87,12 +91,16 @@ public MLTransactionMetadataStore(TransactionCoordinatorID tcID, this.abortedTransactionCount = new LongAdder(); this.transactionTimeoutCount = new LongAdder(); this.appendLogCount = new LongAdder(); + DefaultThreadFactory threadFactory = new DefaultThreadFactory("transaction_coordinator_" + + tcID.toString() + "thread_factory"); + this.internalPinnedExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); if (!changeToInitializingState()) { log.error("Managed ledger transaction metadata store change state error when init it"); return; } - new Thread(() -> transactionLog.replayAsync(new TransactionLogReplayCallback() { + + internalPinnedExecutor.execute(() -> transactionLog.replayAsync(new TransactionLogReplayCallback() { @Override public void replayComplete() { @@ -125,7 +133,8 @@ public void handleMetadataEntry(Position position, TransactionMetadataEntry tran long timeoutAt = transactionMetadataEntry.getTimeoutMs(); txnMetaMap.put(transactionId, MutablePair.of(new TxnMetaImpl(txnID, openTimestamp, timeoutAt), positions)); - recoverTracker.handleOpenStatusTransaction(txnSequenceId, timeoutAt + openTimestamp); + recoverTracker.handleOpenStatusTransaction(txnSequenceId, + timeoutAt + openTimestamp); } break; case ADD_PARTITION: @@ -174,7 +183,7 @@ public void handleMetadataEntry(Position position, TransactionMetadataEntry tran log.error(e.getMessage(), e); } } - })).start(); + })); } @Override @@ -195,167 +204,206 @@ public CompletableFuture getTxnMeta(TxnID txnID) { } @Override - public synchronized CompletableFuture newTransaction(long timeOut) { - if (!checkIfReady()) { - return FutureUtil.failedFuture( - new CoordinatorException - .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "new Transaction")); - } + public CompletableFuture newTransaction(long timeOut) { + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + completableFuture.completeExceptionally(new CoordinatorException + .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "new Transaction")); + return; + } - long mostSigBits = tcID.getId(); - long leastSigBits = sequenceIdGenerator.generateSequenceId(); - TxnID txnID = new TxnID(mostSigBits, leastSigBits); - long currentTimeMillis = System.currentTimeMillis(); - TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() - .setTxnidMostBits(mostSigBits) - .setTxnidLeastBits(leastSigBits) - .setStartTime(currentTimeMillis) - .setTimeoutMs(timeOut) - .setMetadataOp(TransactionMetadataEntry.TransactionMetadataOp.NEW) - .setLastModificationTime(currentTimeMillis) - .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - return transactionLog.append(transactionMetadataEntry) - .thenCompose(position -> { - appendLogCount.increment(); - TxnMeta txn = new TxnMetaImpl(txnID, currentTimeMillis, timeOut); - List positions = new ArrayList<>(); - positions.add(position); - Pair> pair = MutablePair.of(txn, positions); - txnMetaMap.put(leastSigBits, pair); - this.timeoutTracker.addTransaction(leastSigBits, timeOut); - createdTransactionCount.increment(); - return CompletableFuture.completedFuture(txnID); - }); + long mostSigBits = tcID.getId(); + long leastSigBits = sequenceIdGenerator.generateSequenceId(); + TxnID txnID = new TxnID(mostSigBits, leastSigBits); + long currentTimeMillis = System.currentTimeMillis(); + TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() + .setTxnidMostBits(mostSigBits) + .setTxnidLeastBits(leastSigBits) + .setStartTime(currentTimeMillis) + .setTimeoutMs(timeOut) + .setMetadataOp(TransactionMetadataEntry.TransactionMetadataOp.NEW) + .setLastModificationTime(currentTimeMillis) + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); + transactionLog.append(transactionMetadataEntry) + .whenComplete((position, throwable) -> { + if (throwable != null) { + completableFuture.completeExceptionally(throwable); + } else { + appendLogCount.increment(); + TxnMeta txn = new TxnMetaImpl(txnID, currentTimeMillis, timeOut); + List positions = new ArrayList<>(); + positions.add(position); + Pair> pair = MutablePair.of(txn, positions); + txnMetaMap.put(leastSigBits, pair); + this.timeoutTracker.addTransaction(leastSigBits, timeOut); + createdTransactionCount.increment(); + completableFuture.complete(txnID); + } + }); + }); + return completableFuture; } @Override - public synchronized CompletableFuture addProducedPartitionToTxn(TxnID txnID, List partitions) { - if (!checkIfReady()) { - return FutureUtil.failedFuture( - new CoordinatorException.TransactionMetadataStoreStateException(tcID, - State.Ready, getState(), "add produced partition")); - } - return getTxnPositionPair(txnID).thenCompose(txnMetaListPair -> { - TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() - .setTxnidMostBits(txnID.getMostSigBits()) - .setTxnidLeastBits(txnID.getLeastSigBits()) - .setMetadataOp(TransactionMetadataOp.ADD_PARTITION) - .addAllPartitions(partitions) - .setLastModificationTime(System.currentTimeMillis()) - .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); + public CompletableFuture addProducedPartitionToTxn(TxnID txnID, List partitions) { + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + completableFuture + .completeExceptionally(new CoordinatorException.TransactionMetadataStoreStateException(tcID, + State.Ready, getState(), "add produced partition")); + return; + } + getTxnPositionPair(txnID).thenAccept(txnMetaListPair -> { + TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() + .setTxnidMostBits(txnID.getMostSigBits()) + .setTxnidLeastBits(txnID.getLeastSigBits()) + .setMetadataOp(TransactionMetadataOp.ADD_PARTITION) + .addAllPartitions(partitions) + .setLastModificationTime(System.currentTimeMillis()) + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - return transactionLog.append(transactionMetadataEntry) - .thenCompose(position -> { - appendLogCount.increment(); - try { - synchronized (txnMetaListPair.getLeft()) { - txnMetaListPair.getLeft().addProducedPartitions(partitions); - txnMetaMap.get(txnID.getLeastSigBits()).getRight().add(position); + transactionLog.append(transactionMetadataEntry) + .whenComplete((position, exception) -> { + if (exception != null) { + completableFuture.completeExceptionally(exception); + return; } - return CompletableFuture.completedFuture(null); - } catch (InvalidTxnStatusException e) { - transactionLog.deletePosition(Collections.singletonList(position)); - log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() - + " add produced partition error with TxnStatus : " - + txnMetaListPair.getLeft().status().name(), e); - return FutureUtil.failedFuture(e); - } - }); + appendLogCount.increment(); + try { + synchronized (txnMetaListPair.getLeft()) { + txnMetaListPair.getLeft().addProducedPartitions(partitions); + txnMetaMap.get(txnID.getLeastSigBits()).getRight().add(position); + } + completableFuture.complete(null); + } catch (InvalidTxnStatusException e) { + transactionLog.deletePosition(Collections.singletonList(position)); + log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() + + " add produced partition error with TxnStatus : " + + txnMetaListPair.getLeft().status().name(), e); + completableFuture.completeExceptionally(e); + } + }); + }); }); + return completableFuture; } @Override - public synchronized CompletableFuture addAckedPartitionToTxn(TxnID txnID, + public CompletableFuture addAckedPartitionToTxn(TxnID txnID, List txnSubscriptions) { - if (!checkIfReady()) { - return FutureUtil.failedFuture( - new CoordinatorException.TransactionMetadataStoreStateException(tcID, - State.Ready, getState(), "add acked partition")); - } - return getTxnPositionPair(txnID).thenCompose(txnMetaListPair -> { - TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() - .setTxnidMostBits(txnID.getMostSigBits()) - .setTxnidLeastBits(txnID.getLeastSigBits()) - .setMetadataOp(TransactionMetadataOp.ADD_SUBSCRIPTION) - .addAllSubscriptions(txnSubscriptionToSubscription(txnSubscriptions)) - .setLastModificationTime(System.currentTimeMillis()) - .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + completableFuture.completeExceptionally(new CoordinatorException + .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "add acked partition")); + return; + } + getTxnPositionPair(txnID).thenAccept(txnMetaListPair -> { + TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() + .setTxnidMostBits(txnID.getMostSigBits()) + .setTxnidLeastBits(txnID.getLeastSigBits()) + .setMetadataOp(TransactionMetadataOp.ADD_SUBSCRIPTION) + .addAllSubscriptions(txnSubscriptionToSubscription(txnSubscriptions)) + .setLastModificationTime(System.currentTimeMillis()) + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - return transactionLog.append(transactionMetadataEntry) - .thenCompose(position -> { - appendLogCount.increment(); - try { - synchronized (txnMetaListPair.getLeft()) { - txnMetaListPair.getLeft().addAckedPartitions(txnSubscriptions); - txnMetaMap.get(txnID.getLeastSigBits()).getRight().add(position); + transactionLog.append(transactionMetadataEntry) + .whenComplete((position, exception) -> { + if (exception != null) { + completableFuture.completeExceptionally(exception); + return; } - return CompletableFuture.completedFuture(null); - } catch (InvalidTxnStatusException e) { - transactionLog.deletePosition(Collections.singletonList(position)); - log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() - + " add acked subscription error with TxnStatus : " - + txnMetaListPair.getLeft().status().name(), e); - return FutureUtil.failedFuture(e); - } - }); + appendLogCount.increment(); + try { + synchronized (txnMetaListPair.getLeft()) { + txnMetaListPair.getLeft().addAckedPartitions(txnSubscriptions); + txnMetaMap.get(txnID.getLeastSigBits()).getRight().add(position); + } + completableFuture.complete(null); + } catch (InvalidTxnStatusException e) { + transactionLog.deletePosition(Collections.singletonList(position)); + log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() + + " add acked subscription error with TxnStatus : " + + txnMetaListPair.getLeft().status().name(), e); + completableFuture.completeExceptionally(e); + } + }); + }); }); + return completableFuture; } @Override - public synchronized CompletableFuture updateTxnStatus(TxnID txnID, TxnStatus newStatus, + public CompletableFuture updateTxnStatus(TxnID txnID, TxnStatus newStatus, TxnStatus expectedStatus, boolean isTimeout) { - if (!checkIfReady()) { - return FutureUtil.failedFuture( - new CoordinatorException.TransactionMetadataStoreStateException(tcID, - State.Ready, getState(), "update transaction status")); - } - return getTxnPositionPair(txnID).thenCompose(txnMetaListPair -> { - if (txnMetaListPair.getLeft().status() == newStatus) { - return CompletableFuture.completedFuture(null); + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (!checkIfReady()) { + completableFuture.completeExceptionally(new CoordinatorException + .TransactionMetadataStoreStateException(tcID, + State.Ready, getState(), "update transaction status")); + return; } - TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() - .setTxnidMostBits(txnID.getMostSigBits()) - .setTxnidLeastBits(txnID.getLeastSigBits()) - .setExpectedStatus(expectedStatus) - .setMetadataOp(TransactionMetadataOp.UPDATE) - .setLastModificationTime(System.currentTimeMillis()) - .setNewStatus(newStatus) - .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); + getTxnPositionPair(txnID).thenAccept(txnMetaListPair -> { + if (txnMetaListPair.getLeft().status() == newStatus) { + completableFuture.complete(null); + return; + } + TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() + .setTxnidMostBits(txnID.getMostSigBits()) + .setTxnidLeastBits(txnID.getLeastSigBits()) + .setExpectedStatus(expectedStatus) + .setMetadataOp(TransactionMetadataOp.UPDATE) + .setLastModificationTime(System.currentTimeMillis()) + .setNewStatus(newStatus) + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - return transactionLog.append(transactionMetadataEntry).thenCompose(position -> { - appendLogCount.increment(); - try { - synchronized (txnMetaListPair.getLeft()) { - txnMetaListPair.getLeft().updateTxnStatus(newStatus, expectedStatus); - txnMetaListPair.getRight().add(position); + transactionLog.append(transactionMetadataEntry).whenComplete((position, throwable) -> { + if (throwable != null) { + completableFuture.completeExceptionally(throwable); + return; } - if (newStatus == TxnStatus.ABORTING && isTimeout) { - this.transactionTimeoutCount.increment(); - } - if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { - return transactionLog.deletePosition(txnMetaListPair.getRight()).thenCompose(v -> { - this.transactionMetadataStoreStats - .addTransactionExecutionLatencySample(System.currentTimeMillis() - - txnMetaListPair.getLeft().getOpenTimestamp()); - if (newStatus == TxnStatus.COMMITTED) { - committedTransactionCount.increment(); - } else { - abortedTransactionCount.increment(); - } - txnMetaMap.remove(txnID.getLeastSigBits()); - return CompletableFuture.completedFuture(null); - }); + appendLogCount.increment(); + try { + synchronized (txnMetaListPair.getLeft()) { + txnMetaListPair.getLeft().updateTxnStatus(newStatus, expectedStatus); + txnMetaListPair.getRight().add(position); + } + if (newStatus == TxnStatus.ABORTING && isTimeout) { + this.transactionTimeoutCount.increment(); + } + if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { + transactionLog.deletePosition(txnMetaListPair.getRight()).whenComplete((v, exception) -> { + if (exception != null) { + completableFuture.completeExceptionally(exception); + return; + } + this.transactionMetadataStoreStats + .addTransactionExecutionLatencySample(System.currentTimeMillis() + - txnMetaListPair.getLeft().getOpenTimestamp()); + if (newStatus == TxnStatus.COMMITTED) { + committedTransactionCount.increment(); + } else { + abortedTransactionCount.increment(); + } + txnMetaMap.remove(txnID.getLeastSigBits()); + completableFuture.complete(null); + }); + } + completableFuture.complete(null); + } catch (InvalidTxnStatusException e) { + transactionLog.deletePosition(Collections.singletonList(position)); + log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() + + " add update txn status error with TxnStatus : " + + txnMetaListPair.getLeft().status().name(), e); + completableFuture.completeExceptionally(e); } - return CompletableFuture.completedFuture(null); - } catch (InvalidTxnStatusException e) { - transactionLog.deletePosition(Collections.singletonList(position)); - log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() - + " add update txn status error with TxnStatus : " - + txnMetaListPair.getLeft().status().name(), e); - return FutureUtil.failedFuture(e); - } + }); }); }); + return completableFuture; } @Override From 6094879419001af2625d18555e55e267323a2b03 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 25 Feb 2022 15:23:40 +0800 Subject: [PATCH 364/823] Fix ConsumerBuilderImpl#subscribeAsync blocks calling thread. (#14433) (cherry picked from commit 7a58aeba0b439479e1d68fa67c57e120f85687b0) --- .../client/impl/ConsumerBuilderImpl.java | 86 +++++++++++-------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index cbfc27d2b0381..471d4bac007b2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -24,9 +24,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.AccessLevel; @@ -56,6 +54,7 @@ import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.util.RetryMessageUtil; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.util.FutureUtil; @Getter(AccessLevel.PUBLIC) @@ -117,48 +116,63 @@ public CompletableFuture> subscribeAsync() { return FutureUtil.failedFuture( new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription")); } - if(conf.isRetryEnable() && conf.getTopicNames().size() > 0 ) { + CompletableFuture applyDLQConfig; + if (conf.isRetryEnable() && conf.getTopicNames().size() > 0) { TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next()); - String retryLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; - String deadLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; - //Issue 9327: do compatibility check in case of the default retry and dead letter topic name changed - String oldRetryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; - String oldDeadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; - try { - if (client.getPartitionedTopicMetadata(oldRetryLetterTopic) - .get(client.conf.getOperationTimeoutMs(), TimeUnit.MILLISECONDS).partitions > 0) { - retryLetterTopic = oldRetryLetterTopic; - } - if (client.getPartitionedTopicMetadata(oldDeadLetterTopic) - .get(client.conf.getOperationTimeoutMs(), TimeUnit.MILLISECONDS).partitions > 0) { - deadLetterTopic = oldDeadLetterTopic; - } - } catch (InterruptedException | TimeoutException e) { - return FutureUtil.failedFuture(e); - } catch (ExecutionException e) { - return FutureUtil.failedFuture(e.getCause()); - } - - if(conf.getDeadLetterPolicy() == null) { - conf.setDeadLetterPolicy(DeadLetterPolicy.builder() + String oldRetryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; + String oldDeadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; + DeadLetterPolicy deadLetterPolicy = conf.getDeadLetterPolicy(); + if (deadLetterPolicy == null || StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic()) + || StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) { + CompletableFuture retryLetterTopicMetadata = + client.getPartitionedTopicMetadata(oldRetryLetterTopic); + CompletableFuture deadLetterTopicMetadata = + client.getPartitionedTopicMetadata(oldDeadLetterTopic); + applyDLQConfig = CompletableFuture.allOf(retryLetterTopicMetadata, deadLetterTopicMetadata) + .thenAccept(__ -> { + String retryLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; + String deadLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; + if (retryLetterTopicMetadata.join().partitions > 0) { + retryLetterTopic = oldRetryLetterTopic; + } + if (deadLetterTopicMetadata.join().partitions > 0) { + deadLetterTopic = oldDeadLetterTopic; + } + if (deadLetterPolicy == null) { + conf.setDeadLetterPolicy(DeadLetterPolicy.builder() .maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES) .retryLetterTopic(retryLetterTopic) .deadLetterTopic(deadLetterTopic) .build()); + } else { + if (StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic())) { + conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic); + } + if (StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) { + conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic); + } + } + conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic()); + }); } else { - if (StringUtils.isBlank(conf.getDeadLetterPolicy().getRetryLetterTopic())) { - conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic); - } - if (StringUtils.isBlank(conf.getDeadLetterPolicy().getDeadLetterTopic())) { - conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic); - } + conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic()); + applyDLQConfig = CompletableFuture.completedFuture(null); } - conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic()); + } else { + applyDLQConfig = CompletableFuture.completedFuture(null); } - return interceptorList == null || interceptorList.size() == 0 ? - client.subscribeAsync(conf, schema, null) : - client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList)); + return applyDLQConfig.thenCompose(__ -> { + if (interceptorList == null || interceptorList.size() == 0) { + return client.subscribeAsync(conf, schema, null); + } else { + return client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList)); + } + }); } @Override @@ -332,7 +346,7 @@ public ConsumerBuilder autoAckOldestChunkedMessageOnQueueFull(boolean autoAck conf.setAutoAckOldestChunkedMessageOnQueueFull(autoAckOldestChunkedMessageOnQueueFull); return this; } - + @Override public ConsumerBuilder property(String key, String value) { checkArgument(StringUtils.isNotBlank(key) && StringUtils.isNotBlank(value), From 86fa49620191fd265dc27055f6840a100253aea8 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 25 Feb 2022 16:07:42 +0800 Subject: [PATCH 365/823] [Pulsar SQL] Fix PulsarRecordCursor deserialize issue. (#14379) (cherry picked from commit a96a1584ba2e9d19c6919b7597a0f344a2af1a35) --- .../pulsar/sql/presto/PulsarRecordCursor.java | 202 ++++++++++-------- 1 file changed, 111 insertions(+), 91 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index d839e05def76f..56ae17be97a7a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -45,7 +45,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -110,6 +112,7 @@ public class PulsarRecordCursor implements RecordCursor { private final long splitSize; private long entriesProcessed = 0; private int partition = -1; + private volatile Throwable deserializingError; private PulsarSqlSchemaInfoProvider schemaInfoProvider; @@ -236,113 +239,125 @@ public void setPulsarSqlSchemaInfoProvider(PulsarSqlSchemaInfoProvider schemaInf } @VisibleForTesting - class DeserializeEntries implements Runnable { + class DeserializeEntries extends Thread { - protected boolean isRunning = false; + private final AtomicBoolean isRunning; - private final Thread thread; + private final CompletableFuture closeHandle; public DeserializeEntries() { - this.thread = new Thread(this, "derserialize-thread-split-" + pulsarSplit.getSplitId()); + super("deserialize-thread-split-" + pulsarSplit.getSplitId()); + this.isRunning = new AtomicBoolean(false); + this.closeHandle = new CompletableFuture<>(); } - public void interrupt() { - isRunning = false; - thread.interrupt(); + @Override + public void start() { + if (isRunning.compareAndSet(false, true)) { + super.start(); + } } - public void start() { - this.thread.start(); + public CompletableFuture close() { + if (isRunning.compareAndSet(true, false)) { + super.interrupt(); + } + return closeHandle; } @Override public void run() { - isRunning = true; - while (isRunning) { - - int read = entryQueue.drain(new MessagePassingQueue.Consumer() { - @Override - public void accept(Entry entry) { - - try { - entryQueueCacheSizeAllocator.release(entry.getLength()); - - long bytes = entry.getDataBuffer().readableBytes(); - completedBytes += bytes; - // register stats for bytes read - metricsTracker.register_BYTES_READ(bytes); - - // check if we have processed all entries in this split - // and no incomplete chunked messages exist - if (entryExceedSplitEndPosition(entry) && chunkedMessagesMap.isEmpty()) { - return; - } - - // set start time for time deserializing entries for stats - metricsTracker.start_ENTRY_DESERIALIZE_TIME(); + try { + while (isRunning.get()) { + int read = entryQueue.drain(new MessagePassingQueue.Consumer() { + @Override + public void accept(Entry entry) { try { - MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), - entry.getDataBuffer(), (message) -> { - try { - // start time for message queue read - metricsTracker.start_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); - - if (message.getNumChunksFromMsg() > 1) { - message = processChunkedMessages(message); - } else if (entryExceedSplitEndPosition(entry)) { - // skip no chunk or no multi chunk message - // that exceed split end position - message.release(); - message = null; - } - if (message != null) { - while (true) { - if (!haveAvailableCacheSize( - messageQueueCacheSizeAllocator, messageQueue) - || !messageQueue.offer(message)) { - Thread.sleep(1); - } else { - messageQueueCacheSizeAllocator.allocate( - message.getData().readableBytes()); - break; + entryQueueCacheSizeAllocator.release(entry.getLength()); + + long bytes = entry.getDataBuffer().readableBytes(); + completedBytes += bytes; + // register stats for bytes read + metricsTracker.register_BYTES_READ(bytes); + + // check if we have processed all entries in this split + // and no incomplete chunked messages exist + if (entryExceedSplitEndPosition(entry) && chunkedMessagesMap.isEmpty()) { + return; + } + + // set start time for time deserializing entries for stats + metricsTracker.start_ENTRY_DESERIALIZE_TIME(); + + try { + MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), + entry.getDataBuffer(), (message) -> { + try { + // start time for message queue read + metricsTracker.start_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); + + if (message.getNumChunksFromMsg() > 1) { + message = processChunkedMessages(message); + } else if (entryExceedSplitEndPosition(entry)) { + // skip no chunk or no multi chunk message + // that exceed split end position + message.release(); + message = null; + } + if (message != null) { + while (true) { + if (!haveAvailableCacheSize( + messageQueueCacheSizeAllocator, messageQueue) + || !messageQueue.offer(message)) { + Thread.sleep(1); + } else { + messageQueueCacheSizeAllocator.allocate( + message.getData().readableBytes()); + break; + } } } - } - // stats for how long a read from message queue took - metricsTracker.end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); - // stats for number of messages read - metricsTracker.incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY(); - - } catch (InterruptedException e) { - //no-op - } - }, pulsarConnectorConfig.getMaxMessageSize()); - } catch (IOException e) { - log.error(e, "Failed to parse message from pulsar topic %s", topicName.toString()); - throw new RuntimeException(e); - } - // stats for time spend deserializing entries - metricsTracker.end_ENTRY_DESERIALIZE_TIME(); + // stats for how long a read from message queue took + metricsTracker.end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); + // stats for number of messages read + metricsTracker.incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY(); - // stats for num messages per entry - metricsTracker.end_NUM_MESSAGES_DESERIALIZED_PER_ENTRY(); - - } finally { - entriesProcessed++; - entry.release(); + } catch (InterruptedException e) { + //no-op + } + }, pulsarConnectorConfig.getMaxMessageSize()); + } catch (IOException e) { + log.error(e, "Failed to parse message from pulsar topic %s", topicName.toString()); + throw new RuntimeException(e); + } + // stats for time spend deserializing entries + metricsTracker.end_ENTRY_DESERIALIZE_TIME(); + + // stats for num messages per entry + metricsTracker.end_NUM_MESSAGES_DESERIALIZED_PER_ENTRY(); + + } finally { + entriesProcessed++; + entry.release(); + } } - } - }); + }); - if (read <= 0) { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - return; + if (read <= 0) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + return; + } } } + closeHandle.complete(null); + } catch (Throwable ex) { + log.error(ex, "Stop running DeserializeEntries"); + closeHandle.completeExceptionally(ex); + throw ex; } } } @@ -468,6 +483,9 @@ public boolean advanceNextPosition() { if (readEntries == null) { // start deserialize thread deserializeEntries = new DeserializeEntries(); + deserializeEntries.setUncaughtExceptionHandler((t, ex) -> { + deserializingError = ex; + }); deserializeEntries.start(); readEntries = new ReadEntries(); @@ -492,6 +510,8 @@ public boolean advanceNextPosition() { if (currentMessage != null) { messageQueueCacheSizeAllocator.release(currentMessage.getData().readableBytes()); break; + } else if (deserializingError != null) { + throw new RuntimeException(deserializingError); } else { try { Thread.sleep(1); @@ -503,7 +523,7 @@ public boolean advanceNextPosition() { } } - //start time for deseralizing record + //start time for deserializing record metricsTracker.start_RECORD_DESERIALIZE_TIME(); SchemaInfo schemaInfo = getSchemaInfo(pulsarSplit); @@ -714,12 +734,12 @@ public void close() { messageQueue.drain(RawMessage::release); } - if (entryQueue != null) { - entryQueue.drain(Entry::release); - } - if (deserializeEntries != null) { - deserializeEntries.interrupt(); + deserializeEntries.close().whenComplete((r, t) -> { + if (entryQueue != null) { + entryQueue.drain(Entry::release); + } + }); } if (this.cursor != null) { From a5a99911434904f0cdfccaf50ad723a2cb9bbf44 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 25 Feb 2022 16:14:52 +0800 Subject: [PATCH 366/823] [Transaction] Adopt Single_thread to handle TcClient connecting (#13969) ### Motivation The broker will only reconnect the same TC once at the same time, and other connection requests during the reconnection period will be processed together after the connection is completed. There may be concurrency problems in the queue for request addition and the clearing of the queue. ### Modification Use SingleThread to deal TcClient connecting. (cherry picked from commit 29259e1b5c33856cd6bd9413e331f4592fd3007c) --- .../apache/pulsar/broker/PulsarService.java | 5 + .../TransactionMetadataStoreService.java | 152 ++++++++++-------- 2 files changed, 91 insertions(+), 66 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index adeec4c8561c4..7fcf0d02d1555 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -403,6 +403,11 @@ public CompletableFuture closeAsync() { brokerAdditionalServlets = null; } + if (this.transactionMetadataStoreService != null) { + this.transactionMetadataStoreService.close(); + this.transactionMetadataStoreService = null; + } + GracefulExecutorServicesShutdown executorServicesShutdown = GracefulExecutorServicesShutdown .initiate() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 9b60679b938af..b3ced3c7a9641 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -32,7 +33,10 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; @@ -84,9 +88,14 @@ public class TransactionMetadataStoreService { // one connect request open the transactionMetaStore the other request will add to the queue, when the open op // finished the request will be poll and complete the future private final ConcurrentLongHashMap>> pendingConnectRequests; + private final ExecutorService internalPinnedExecutor; private static final long HANDLE_PENDING_CONNECT_TIME_OUT = 30000L; + private final ThreadFactory threadFactory = + new DefaultThreadFactory("transaction-coordinator-thread-factory"); + + public TransactionMetadataStoreService(TransactionMetadataStoreProvider transactionMetadataStoreProvider, PulsarService pulsarService, TransactionBufferClient tbClient, HashedWheelTimer timer) { @@ -98,6 +107,7 @@ public TransactionMetadataStoreService(TransactionMetadataStoreProvider transact this.transactionOpRetryTimer = timer; this.tcLoadSemaphores = new ConcurrentLongHashMap<>(); this.pendingConnectRequests = new ConcurrentLongHashMap<>(); + this.internalPinnedExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); } @Deprecated @@ -152,80 +162,86 @@ public boolean test(NamespaceBundle namespaceBundle) { } public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tcId) { - if (stores.get(tcId) != null) { - return CompletableFuture.completedFuture(null); - } else { - return pulsarService.getBrokerService().checkTopicNsOwnership(TopicName - .TRANSACTION_COORDINATOR_ASSIGN.getPartition((int) tcId.getId()).toString()).thenCompose(v -> { - CompletableFuture completableFuture = new CompletableFuture<>(); - final Semaphore tcLoadSemaphore = this.tcLoadSemaphores - .computeIfAbsent(tcId.getId(), (id) -> new Semaphore(1)); - Deque> deque = pendingConnectRequests - .computeIfAbsent(tcId.getId(), (id) -> new ConcurrentLinkedDeque<>()); - if (tcLoadSemaphore.tryAcquire()) { - // when tcLoadSemaphore.release(), this command will acquire semaphore, so we should jude the store - // exist again. - if (stores.get(tcId) != null) { - return CompletableFuture.completedFuture(null); - } - - openTransactionMetadataStore(tcId).thenAccept((store) -> { - stores.put(tcId, store); - LOG.info("Added new transaction meta store {}", tcId); - long endTime = System.currentTimeMillis() + HANDLE_PENDING_CONNECT_TIME_OUT; - while (true) { - // prevent thread in a busy loop. - if (System.currentTimeMillis() < endTime) { - CompletableFuture future = deque.poll(); - if (future != null) { - // complete queue request future - future.complete(null); - } else { - break; - } - } else { - deque.clear(); - break; - } + CompletableFuture completableFuture = new CompletableFuture<>(); + internalPinnedExecutor.execute(() -> { + if (stores.get(tcId) != null) { + completableFuture.complete(null); + } else { + pulsarService.getBrokerService().checkTopicNsOwnership(TopicName + .TRANSACTION_COORDINATOR_ASSIGN.getPartition((int) tcId.getId()).toString()) + .thenRun(() -> internalPinnedExecutor.execute(() -> { + final Semaphore tcLoadSemaphore = this.tcLoadSemaphores + .computeIfAbsent(tcId.getId(), (id) -> new Semaphore(1)); + Deque> deque = pendingConnectRequests + .computeIfAbsent(tcId.getId(), (id) -> new ConcurrentLinkedDeque<>()); + if (tcLoadSemaphore.tryAcquire()) { + // when tcLoadSemaphore.release(), this command will acquire semaphore, + // so we should jude the store exist again. + if (stores.get(tcId) != null) { + completableFuture.complete(null); } - completableFuture.complete(null); - tcLoadSemaphore.release(); - }).exceptionally(e -> { - completableFuture.completeExceptionally(e.getCause()); - // release before handle request queue, in order to client reconnect infinite loop - tcLoadSemaphore.release(); - long endTime = System.currentTimeMillis() + HANDLE_PENDING_CONNECT_TIME_OUT; - while (true) { - // prevent thread in a busy loop. - if (System.currentTimeMillis() < endTime) { - CompletableFuture future = deque.poll(); - if (future != null) { - // this means that this tc client connection connect fail - future.completeExceptionally(e); + openTransactionMetadataStore(tcId).thenAccept((store) -> internalPinnedExecutor.execute(() -> { + stores.put(tcId, store); + LOG.info("Added new transaction meta store {}", tcId); + long endTime = System.currentTimeMillis() + HANDLE_PENDING_CONNECT_TIME_OUT; + while (true) { + // prevent thread in a busy loop. + if (System.currentTimeMillis() < endTime) { + CompletableFuture future = deque.poll(); + if (future != null) { + // complete queue request future + future.complete(null); + } else { + break; + } } else { + deque.clear(); break; } - } else { - deque.clear(); - break; } + + completableFuture.complete(null); + tcLoadSemaphore.release(); + })).exceptionally(e -> { + internalPinnedExecutor.execute(() -> { + completableFuture.completeExceptionally(e.getCause()); + // release before handle request queue, + //in order to client reconnect infinite loop + tcLoadSemaphore.release(); + long endTime = System.currentTimeMillis() + HANDLE_PENDING_CONNECT_TIME_OUT; + while (true) { + // prevent thread in a busy loop. + if (System.currentTimeMillis() < endTime) { + CompletableFuture future = deque.poll(); + if (future != null) { + // this means that this tc client connection connect fail + future.completeExceptionally(e); + } else { + break; + } + } else { + deque.clear(); + break; + } + } + LOG.error("Add transaction metadata store with id {} error", tcId.getId(), e); + }); + return null; + }); + } else { + // only one command can open transaction metadata store, + // other will be added to the deque, when the op of openTransactionMetadataStore finished + // then handle the requests witch in the queue + deque.add(completableFuture); + if (LOG.isDebugEnabled()) { + LOG.debug("Handle tc client connect added into pending queue! tcId : {}", tcId.toString()); } - LOG.error("Add transaction metadata store with id {} error", tcId.getId(), e); - return null; - }); - } else { - // only one command can open transaction metadata store, - // other will be added to the deque, when the op of openTransactionMetadataStore finished - // then handle the requests witch in the queue - deque.add(completableFuture); - if (LOG.isDebugEnabled()) { - LOG.debug("Handle tc client connect added into pending queue! tcId : {}", tcId.toString()); } - } - return completableFuture; - }); - } + })); + } + }); + return completableFuture; } public CompletableFuture openTransactionMetadataStore(TransactionCoordinatorID tcId) { @@ -537,4 +553,8 @@ private TransactionCoordinatorID getTcIdFromTxnId(TxnID txnId) { public Map getStores() { return Collections.unmodifiableMap(stores); } + + public void close () { + this.internalPinnedExecutor.shutdown(); + } } From 3089aa4040cc6c42c05898fab00c98332a9cb395 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 25 Feb 2022 16:15:53 +0800 Subject: [PATCH 367/823] [Transaction] Fix end transaction at state of timeout (#14370) ### Motivation For concurrency problems, timeout may change the status to timeout before commit/abort changes the status to committing/aborting. ### Modification Cancel timeout when commit or abort and then check the state. (cherry picked from commit 4b480450f32dc6ce5337d0b3d68a35111ddf474e) --- .../pulsar/client/impl/transaction/TransactionImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index 8adc162584469..bba53318695d9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -165,10 +165,10 @@ public synchronized void registerCumulativeAckConsumer(ConsumerImpl consumer) @Override public CompletableFuture commit() { + timeout.cancel(); return checkIfOpenOrCommitting().thenCompose((value) -> { CompletableFuture commitFuture = new CompletableFuture<>(); this.state = State.COMMITTING; - timeout.cancel(); allOpComplete().whenComplete((v, e) -> { if (e != null) { abort().whenComplete((vx, ex) -> commitFuture.completeExceptionally(e)); @@ -194,10 +194,10 @@ public CompletableFuture commit() { @Override public CompletableFuture abort() { + timeout.cancel(); return checkIfOpenOrAborting().thenCompose(value -> { CompletableFuture abortFuture = new CompletableFuture<>(); this.state = State.ABORTING; - timeout.cancel(); allOpComplete().whenComplete((v, e) -> { if (e != null) { log.error(e.getMessage()); From 2bc8f487a71ea812b06d119f24fd7051c82eb6e8 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Fri, 25 Feb 2022 21:38:47 +0800 Subject: [PATCH 368/823] Fix can't read the latest message of the compacted topic (#14449) If the reader enabled read compacted and all the data of topic has been compacted to the compacted ledger, the original topic does not have any data. In this case, the reader is not able to read the latest message of the compacted topic. ```java Reader reader = pulsarClient.newReader() .topic(topic) .startMessageId(MessageId.latest) .startMessageIdInclusive() .readCompacted(true) .create(); ``` The root cause is if the `startMessageIdInclusive` is true and the `startMessageId` is `latest`, the reader will get the last message ID from the broker and then seek to the last message. But, the seek method did not consider if there are messages in the compacted ledger, so not able to seek to last message of the compacted ledger. Add force reset option for the managed cursor, if the seek position < compaction horizon, we should force reset the cursor to the given position, so that the reader able to start read from the compacted ledger. (cherry picked from commit ddc51924e90550ef50fd3cdd099b6aec56aec260) --- .../bookkeeper/mledger/ManagedCursor.java | 6 ++- .../mledger/impl/ManagedCursorImpl.java | 11 ++-- .../impl/ManagedCursorContainerTest.java | 3 +- .../mledger/impl/ManagedCursorTest.java | 6 +-- .../mledger/impl/NonDurableCursorTest.java | 2 +- .../persistent/PersistentSubscription.java | 10 +++- .../broker/service/PersistentTopicTest.java | 3 +- .../pulsar/compaction/CompactedTopicTest.java | 50 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 6 ++- 9 files changed, 82 insertions(+), 15 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index d1fb90a3ca60e..f67cd96ea9f35 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -533,10 +533,14 @@ void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate * * @param position * position to move the cursor to + * @param forceReset + * whether to force reset the position even if the position is no longer in the managed ledger, + * this is used by compacted topic which has data in the compacted ledger, to ensure the cursor can + * read data from the compacted ledger. * @param callback * callback object */ - void asyncResetCursor(final Position position, AsyncCallbacks.ResetCursorCallback callback); + void asyncResetCursor(Position position, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback); /** * Read the specified set of positions from ManagedLedger. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 8c345026d08ab..8c96f0e2859cf 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1135,7 +1135,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { } @Override - public void asyncResetCursor(Position newPos, AsyncCallbacks.ResetCursorCallback callback) { + public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback) { checkArgument(newPos instanceof PositionImpl); final PositionImpl newPosition = (PositionImpl) newPos; @@ -1143,9 +1143,10 @@ public void asyncResetCursor(Position newPos, AsyncCallbacks.ResetCursorCallback ledger.getExecutor().executeOrdered(ledger.getName(), safeRun(() -> { PositionImpl actualPosition = newPosition; - if (!ledger.isValidPosition(actualPosition) && - !actualPosition.equals(PositionImpl.earliest) && - !actualPosition.equals(PositionImpl.latest)) { + if (!ledger.isValidPosition(actualPosition) + && !actualPosition.equals(PositionImpl.earliest) + && !actualPosition.equals(PositionImpl.latest) + && !forceReset) { actualPosition = ledger.getNextValidPosition(actualPosition); if (actualPosition == null) { @@ -1168,7 +1169,7 @@ class Result { final Result result = new Result(); final CountDownLatch counter = new CountDownLatch(1); - asyncResetCursor(newPos, new AsyncCallbacks.ResetCursorCallback() { + asyncResetCursor(newPos, false, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { counter.countDown(); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index af30c9c372136..56de80308e5d1 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -237,7 +237,8 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate } @Override - public void asyncResetCursor(final Position position, AsyncCallbacks.ResetCursorCallback callback) { + public void asyncResetCursor(final Position position, boolean forceReset, + AsyncCallbacks.ResetCursorCallback callback) { } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 676e92fd09d23..892d6b39a9bef 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -687,7 +687,7 @@ void testasyncResetCursor() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(1); PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); - cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() { + cursor.asyncResetCursor(resetPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { moveStatus.set(true); @@ -738,7 +738,7 @@ public AtomicBoolean call() throws Exception { final PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - (5 * idx)); - cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() { + cursor.asyncResetCursor(resetPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { moveStatus.set(true); @@ -787,7 +787,7 @@ void testLastActiveAfterResetCursor() throws Exception { long lastActive = cursor.getLastActive(); - cursor.asyncResetCursor(lastPosition, new AsyncCallbacks.ResetCursorCallback() { + cursor.asyncResetCursor(lastPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { moveStatus.set(true); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java index 4c2944fd79605..5d0271de3d641 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java @@ -371,7 +371,7 @@ void testasyncResetCursor() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(1); PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); - cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() { + cursor.asyncResetCursor(resetPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { moveStatus.set(true); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 8637ecd1e15b8..9e5758034be3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -659,7 +659,15 @@ private void resetCursor(Position finalPosition, CompletableFuture future) topicName, subName); try { - cursor.asyncResetCursor(finalPosition, new AsyncCallbacks.ResetCursorCallback() { + boolean forceReset = false; + if (topic.getCompactedTopic() != null && topic.getCompactedTopic().getCompactionHorizon().isPresent()) { + PositionImpl horizon = (PositionImpl) topic.getCompactedTopic().getCompactionHorizon().get(); + PositionImpl resetTo = (PositionImpl) finalPosition; + if (horizon.compareTo(resetTo) >= 0) { + forceReset = true; + } + } + cursor.asyncResetCursor(finalPosition, forceReset, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { if (log.isDebugEnabled()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index d94320ffc96d6..2314acdc3f51f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.anyString; @@ -2177,7 +2178,7 @@ public void testGetDurableSubscription() throws Exception { doAnswer((Answer) invocationOnMock -> { ((AsyncCallbacks.ResetCursorCallback) invocationOnMock.getArguments()[1]).resetComplete(null); return null; - }).when(mockCursor).asyncResetCursor(any(), any()); + }).when(mockCursor).asyncResetCursor(any(), anyBoolean(), any()); doAnswer((Answer) invocationOnMock -> { ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorComplete(null); return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index accce4ef6f285..4ae699b500f88 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -734,6 +734,7 @@ public void testHasMessageAvailableWithNullValueMessage() throws Exception { Assert.assertNull(reader.readNext(3, TimeUnit.SECONDS)); } + @Test public void testReadCompleteMessagesDuringTopicUnloading() throws Exception { String topic = "persistent://my-property/use/my-ns/testReadCompleteMessagesDuringTopicUnloading-" + UUID.randomUUID(); @@ -789,4 +790,53 @@ public void testReadCompleteMessagesDuringTopicUnloading() throws Exception { Assert.assertEquals(reader.readNext().getValue(), String.format("msg [%d]", i + numMessages)); } } + + @Test + public void testReadCompactedLatestMessageWithInclusive() throws Exception { + String topic = "persistent://my-property/use/my-ns/testLedgerRollover-" + + UUID.randomUUID(); + final int numMessages = 1; + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .blockIfQueueFull(true) + .enableBatching(false) + .create(); + + CompletableFuture lastMessage = null; + for (int i = 0; i < numMessages; ++i) { + lastMessage = producer.newMessage().key(i + "").value(String.format("msg [%d]", i)).sendAsync(); + } + producer.flush(); + lastMessage.join(); + admin.topics().unload(topic); + admin.topics().triggerCompaction(topic); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topic); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1); + Assert.assertEquals(stats.compactedLedger.entries, numMessages); + Assert.assertEquals(admin.topics().getStats(topic) + .getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), 0); + Assert.assertEquals(stats.lastConfirmedEntry, stats.cursors.get(COMPACTION_SUBSCRIPTION).markDeletePosition); + }); + + Awaitility.await() + .pollInterval(3, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + admin.topics().unload(topic); + Assert.assertTrue(admin.topics().getInternalStats(topic).lastConfirmedEntry.endsWith("-1")); + }); + + @Cleanup + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageIdInclusive() + .startMessageId(MessageId.latest) + .readCompacted(true) + .create(); + + Assert.assertTrue(reader.hasMessageAvailable()); + Assert.assertEquals(reader.readNext().getMessageId(), lastMessage.get()); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index e5a5745f82450..856a7758455b9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2012,8 +2012,10 @@ public CompletableFuture hasMessageAvailableAsync() { MessageIdImpl markDeletePosition = MessageIdImpl .convertToMessageIdImpl(response.markDeletePosition); - if (markDeletePosition != null) { - // we only care about comparing ledger ids and entry ids as mark delete position doesn't have other ids such as batch index + if (markDeletePosition != null && !(markDeletePosition.getEntryId() < 0 + && markDeletePosition.getLedgerId() > lastMessageId.getLedgerId())) { + // we only care about comparing ledger ids and entry ids as mark delete position doesn't have + // other ids such as batch index int result = ComparisonChain.start() .compare(markDeletePosition.getLedgerId(), lastMessageId.getLedgerId()) .compare(markDeletePosition.getEntryId(), lastMessageId.getEntryId()) From 4e797416ab9bec5f8b9d93f67ac48555655bcdea Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Sun, 27 Feb 2022 11:53:41 +0800 Subject: [PATCH 369/823] Validate rack name (#14336) If the rack name set in the following case: - Enabled rack aware placement policy, and user set rack name which contains "/" in addition to the head and tail of the string. Such as "/r/a" or "r/a/b" - Enabled region aware placement policy, and user set the rack name which contains multiple "/" in addition to the head and tail of the string. Such as "/region/region/a" or "region/a/rack/b" (cherry picked from commit 25408f5078f29372cc2c2133abc8dbeb0b9a30cf) --- .../pulsar/broker/admin/v2/Bookies.java | 16 +++++ .../pulsar/broker/admin/BookiesApiTest.java | 58 ++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java index 1af839f4c01dc..05eb1dd94f92d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java @@ -43,6 +43,7 @@ import org.apache.bookkeeper.discover.RegistrationClient; import org.apache.bookkeeper.meta.MetadataClientDriver; import org.apache.bookkeeper.net.BookieId; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.common.policies.data.BookieInfo; @@ -55,6 +56,7 @@ @Produces(MediaType.APPLICATION_JSON) @Slf4j public class Bookies extends AdminResource { + private static final String PATH_SEPARATOR = "/"; @GET @Path("/racks-info") @@ -162,6 +164,20 @@ public void updateBookieRackInfo(@Suspended final AsyncResponse asyncResponse, throw new RestException(Status.PRECONDITION_FAILED, "Bookie 'group' parameters is missing"); } + // validate rack name + int separatorCnt = StringUtils.countMatches( + StringUtils.strip(bookieInfo.getRack(), PATH_SEPARATOR), PATH_SEPARATOR); + boolean isRackEnabled = pulsar().getConfiguration().isBookkeeperClientRackawarePolicyEnabled(); + boolean isRegionEnabled = pulsar().getConfiguration().isBookkeeperClientRegionawarePolicyEnabled(); + if (isRackEnabled && ((isRegionEnabled && separatorCnt != 1) || (!isRegionEnabled && separatorCnt != 0))) { + asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, "Bookie 'rack' parameter is invalid, " + + "When `RackawareEnsemblePlacementPolicy` is enabled, the rack name is not allowed to contain " + + "slash (`/`) except for the beginning and end of the rack name string. " + + "When `RegionawareEnsemblePlacementPolicy` is enabled, the rack name can only contain " + + "one slash (`/`) except for the beginning and end of the rack name string.")); + return; + } + getPulsarResources().getBookieResources() .update(optionalBookiesRackConfiguration -> { BookiesRackConfiguration brc = optionalBookiesRackConfiguration diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java index 644d47df7678d..a3bd52b090236 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar.broker.admin; +import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.util.Optional; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.client.PulsarMockBookKeeper; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.BookieInfo; @@ -123,6 +124,61 @@ public void testBasic() throws Exception { .get() .getValue() .size()); + + // test invalid rack name + // use rack aware placement policy + String errorMsg = "Bookie 'rack' parameter is invalid, When `RackawareEnsemblePlacementPolicy` is enabled, " + + "the rack name is not allowed to contain slash (`/`) except for the beginning and end of the rack name " + + "string. When `RegionawareEnsemblePlacementPolicy` is enabled, the rack name can only contain " + + "one slash (`/`) except for the beginning and end of the rack name string."; + + BookieInfo newInfo3 = BookieInfo.builder() + .rack("/rack/a") + .hostname("127.0.0.2") + .build(); + try { + admin.bookies().updateBookieRackInfo(bookie0, "default", newInfo3); + fail(); + } catch (PulsarAdminException e) { + assertEquals(412, e.getStatusCode()); + assertEquals(errorMsg, e.getMessage()); + } + + BookieInfo newInfo4 = BookieInfo.builder() + .rack("/rack") + .hostname("127.0.0.2") + .build(); + try { + admin.bookies().updateBookieRackInfo(bookie0, "default", newInfo4); + } catch (PulsarAdminException e) { + fail(); + } + + // enable region aware placement policy + ServiceConfiguration configuration = new ServiceConfiguration(); + configuration.setBookkeeperClientRegionawarePolicyEnabled(true); + doReturn(configuration).when(pulsar).getConfiguration(); + BookieInfo newInfo5 = BookieInfo.builder() + .rack("/region/rack/a") + .hostname("127.0.0.2") + .build(); + try { + admin.bookies().updateBookieRackInfo(bookie0, "default", newInfo5); + fail(); + } catch (PulsarAdminException e) { + assertEquals(412, e.getStatusCode()); + assertEquals(errorMsg, e.getMessage()); + } + + BookieInfo newInfo6 = BookieInfo.builder() + .rack("/region/rack/") + .hostname("127.0.0.2") + .build(); + try { + admin.bookies().updateBookieRackInfo(bookie0, "default", newInfo6); + } catch (PulsarAdminException e) { + fail(); + } } } From 56f1660b2aa6b8c5eaa9480da448541d7b603cb1 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Sun, 27 Feb 2022 17:36:51 -0800 Subject: [PATCH 370/823] [pulsar-broker] Fix avg-messagePerEntry metrics for consumer (#14330) (cherry picked from commit 7b10bd040e947497113db702ebcb2381d9dfe0fc) --- .../main/java/org/apache/pulsar/broker/service/Consumer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index bfaa6608e774e..bf906a32d1aba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -266,7 +266,7 @@ public Future sendMessages(final List entries, EntryBatchSizes batc // calculate avg message per entry int tmpAvgMessagesPerEntry = AVG_MESSAGES_PER_ENTRY.get(this); - tmpAvgMessagesPerEntry = (int) Math.round(tmpAvgMessagesPerEntry * avgPercent + tmpAvgMessagesPerEntry = (int) Math.floor(tmpAvgMessagesPerEntry * avgPercent + (1 - avgPercent) * totalMessages / entries.size()); AVG_MESSAGES_PER_ENTRY.set(this, tmpAvgMessagesPerEntry); From 5047498c04a2ef72a2b51188ab3eb39fbc3249df Mon Sep 17 00:00:00 2001 From: lin chen <1572139390@qq.com> Date: Mon, 28 Feb 2022 18:10:21 +0800 Subject: [PATCH 371/823] fix npe in ManagedLedgerImpl (#14481) (cherry picked from commit 3da048c8a6df7404df4e6c3301370feb96c6c14b) --- .../org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 87287530a54a8..c07b3ce1ed6e0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3183,9 +3183,11 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, totalEntriesInCurrentLedger = 0; } } else { - totalEntriesInCurrentLedger = ledgers.get(currentLedgerId).getEntries(); + LedgerInfo ledgerInfo = ledgers.get(currentLedgerId); + totalEntriesInCurrentLedger = ledgerInfo != null ? ledgerInfo.getEntries() : 0; } + long unreadEntriesInCurrentLedger = totalEntriesInCurrentLedger - currentEntryId; if (unreadEntriesInCurrentLedger >= entriesToSkip) { From d87a2304fc55828aa1a7a905c2631f594b49b764 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 1 Mar 2022 10:41:07 +0800 Subject: [PATCH 372/823] [Broker] Fix ``Future.join()`` causing deadlock. (#14469) Master issue #14438 ### Motivation Invoking the ``join()`` method in the async method will cause some deadlock. ### Modifications - Refactor ``PersistentTopic#tryToDeletePartitionedMetadata`` to pure async. (cherry picked from commit 65318e83f8d5b4207a9398e100390800425d5433) --- .../service/persistent/PersistentTopic.java | 78 ++++++++++--------- .../service/PersistentTopicE2ETest.java | 22 +++++- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 00934ad13a2d4..e96fdc4dd8fcb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2278,42 +2278,48 @@ private CompletableFuture tryToDeletePartitionedMetadata() { return CompletableFuture.completedFuture(null); } TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName()); - try { - PartitionedTopicResources partitionedTopicResources = getBrokerService().pulsar().getPulsarResources() - .getNamespaceResources() - .getPartitionedTopicResources(); - if (topicName.isPartitioned() && !partitionedTopicResources.partitionedTopicExists(topicName)) { - return CompletableFuture.completedFuture(null); - } - CompletableFuture deleteMetadataFuture = new CompletableFuture<>(); - getBrokerService().fetchPartitionedTopicMetadataAsync(TopicName.get(topicName.getPartitionedTopicName())) - .thenAccept((metadata -> { - // make sure all sub partitions were deleted - for (int i = 0; i < metadata.partitions; i++) { - if (brokerService.getPulsar().getPulsarResources().getTopicResources() - .persistentTopicExists(topicName.getPartition(i)).join()) { - throw new UnsupportedOperationException(); - } - } - })) - .thenAccept((res) -> partitionedTopicResources.deletePartitionedTopicAsync(topicName) - .thenAccept((r) -> { - deleteMetadataFuture.complete(null); - }).exceptionally(ex -> { - deleteMetadataFuture.completeExceptionally(ex.getCause()); - return null; - })) - .exceptionally((e) -> { - if (!(e.getCause() instanceof UnsupportedOperationException)) { - log.error("delete metadata fail", e); - } - deleteMetadataFuture.complete(null); - return null; - }); - return deleteMetadataFuture; - } catch (Exception e) { - return FutureUtil.failedFuture(e); - } + PartitionedTopicResources partitionedTopicResources = getBrokerService().pulsar().getPulsarResources() + .getNamespaceResources() + .getPartitionedTopicResources(); + return partitionedTopicResources.partitionedTopicExistsAsync(topicName) + .thenCompose(partitionedTopicExist -> { + if (!partitionedTopicExist) { + return CompletableFuture.completedFuture(null); + } else { + return getBrokerService() + .fetchPartitionedTopicMetadataAsync(topicName) + .thenCompose((metadata -> { + List> persistentTopicExists = + new ArrayList<>(metadata.partitions); + for (int i = 0; i < metadata.partitions; i++) { + persistentTopicExists.add(brokerService.getPulsar() + .getPulsarResources().getTopicResources() + .persistentTopicExists(topicName.getPartition(i))); + } + List> unmodifiablePersistentTopicExists = + Collections.unmodifiableList(persistentTopicExists); + return FutureUtil.waitForAll(unmodifiablePersistentTopicExists) + .thenCompose(unused -> { + // make sure all sub partitions were deleted after all future complete + Optional anyExistPartition = unmodifiablePersistentTopicExists + .stream() + .map(CompletableFuture::join) + .filter(topicExist -> topicExist) + .findAny(); + if (anyExistPartition.isPresent()) { + log.error("[{}] Delete topic metadata failed because" + + " another partition exist.", topicName); + throw new UnsupportedOperationException( + String.format("Another partition exists for [%s].", + topicName)); + } else { + return partitionedTopicResources + .deletePartitionedTopicAsync(topicName); + } + }); + })); + } + }); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java index 42512b0f38bf8..daf0ed7764d80 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java @@ -96,6 +96,7 @@ public class PersistentTopicE2ETest extends BrokerTestBase { @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + conf.setBrokerDeleteInactivePartitionedTopicMetadataEnabled(true); super.baseSetup(); } @@ -617,8 +618,27 @@ public void testGC() throws Exception { runGC(); assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent()); - } + // write again, the topic will be available + Producer producer2 = pulsarClient.newProducer().topic(topicName).create(); + producer2.close(); + + assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent()); + + // 6. Test for partitioned topic to delete the partitioned metadata + String topicGc = "persistent://prop/ns-abc/topic-gc"; + int partitions = 5; + admin.topics().createPartitionedTopic(topicGc, partitions); + Producer producer3 = pulsarClient.newProducer().topic(topicGc).create(); + producer3.close(); + assertEquals(partitions, pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( + TopicName.get(topicGc)).join().partitions); + runGC(); + Awaitility.await().untilAsserted(()-> { + assertEquals(pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( + TopicName.get(topicGc)).join().partitions, 0); + }); + } @Data @ToString @EqualsAndHashCode From 065ccd4a712f89df0df0c32b5ee8d66feeb9b028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 14 Jan 2022 23:57:34 +0100 Subject: [PATCH 373/823] [owasp] suppress false positive Avro CVE-2021-43045 (#13764) (cherry picked from commit 239133670b927741d673e26ad2810b01c6bab8bd) --- src/owasp-dependency-check-false-positives.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 6cc464e637623..7b945a2bbc9ec 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -47,6 +47,11 @@ org\.apache\.avro:.* CVE-2019-17195 + + CVE-2021-43045 affects only .NET distro, see https://github.com/apache/avro/pull/1357 + org\.apache\.avro:.* + CVE-2021-43045 + Date: Thu, 11 Nov 2021 18:16:49 +0100 Subject: [PATCH 374/823] Flaky Tests: AdminApiSchemaTest#testSchemaInfoApi (#12461) (cherry picked from commit b533fe5825d236415d8bf07fcaaccd8750b63271) --- .../src/test/java/org/apache/pulsar/schema/SchemaTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 45f12aba0b0dd..ff02c74e4dd83 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -658,9 +658,12 @@ public void testNullKeyValueProperty() throws PulsarAdminException, PulsarClient final Map map = new HashMap<>(); map.put("key", null); map.put(null, "value"); // null key is not allowed for JSON, it's only for test here - ((SchemaInfoImpl)Schema.INT32.getSchemaInfo()).setProperties(map); - final Consumer consumer = pulsarClient.newConsumer(Schema.INT32).topic(topic) + // leave INT32 instance unchanged + final Schema integerSchema = Schema.INT32.clone(); + ((SchemaInfoImpl) integerSchema.getSchemaInfo()).setProperties(map); + + final Consumer consumer = pulsarClient.newConsumer(integerSchema).topic(topic) .subscriptionName("sub") .subscribe(); consumer.close(); From 597005bb6626d9980286e2f992cdbba83cf9081a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 7 Mar 2022 18:00:57 +0100 Subject: [PATCH 375/823] [flaky-tests] AdminApiSchemaTest#testSchemaInfoApi (#14508) * [flaky-tests] AdminApiSchemaTest#testSchemaInfoApi * use custom json schema * remove old comment ### Motivation This is the same fix applied here #12461. The problem with the other pull is that the `AbstractSchema#clone()` method does return the same instance, so the fix is not useful at all. I see this test also failing in 2.8 and 2.9 branch, I recommend to cherry-pick it. ### Modifications * Create a new INT schema for the test purpose - [x] `no-need-doc` (cherry picked from commit 32c3cd1009eea884e0701143fe01dadf4556e73b) --- .../src/test/java/org/apache/pulsar/schema/SchemaTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index ff02c74e4dd83..ab78988be94ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -659,8 +659,7 @@ public void testNullKeyValueProperty() throws PulsarAdminException, PulsarClient map.put("key", null); map.put(null, "value"); // null key is not allowed for JSON, it's only for test here - // leave INT32 instance unchanged - final Schema integerSchema = Schema.INT32.clone(); + final Schema integerSchema = Schema.JSON(Integer.class); ((SchemaInfoImpl) integerSchema.getSchemaInfo()).setProperties(map); final Consumer consumer = pulsarClient.newConsumer(integerSchema).topic(topic) From f41ba122279baa467451caa8b4ba2f4bc3dac846 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 2 Mar 2022 16:35:58 +0800 Subject: [PATCH 376/823] [pulsar-io] throw exceptions when kafka offset backing store failed to start (#14491) (cherry picked from commit e6656e1407be80fdf4b6aaf424a57068687840cc) --- .../connect/PulsarOffsetBackingStore.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java index 495c8b9c194c0..86905ad8990e3 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java @@ -30,6 +30,8 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.connect.runtime.WorkerConfig; @@ -49,7 +51,7 @@ @Slf4j public class PulsarOffsetBackingStore implements OffsetBackingStore { - private Map data; + private final Map data = new ConcurrentHashMap<>(); private PulsarClient client; private String topic; private Producer producer; @@ -65,7 +67,6 @@ public PulsarOffsetBackingStore(PulsarClient client) { public void configure(WorkerConfig workerConfig) { this.topic = workerConfig.getString(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG); checkArgument(!isBlank(topic), "Offset storage topic must be specified"); - this.data = new HashMap<>(); log.info("Configure offset backing store on pulsar topic {}", topic); } @@ -126,10 +127,13 @@ private void readNext(CompletableFuture endFuture) { } void processMessage(Message message) { - synchronized (data) { + if (message.getKey() != null) { data.put( ByteBuffer.wrap(message.getKey().getBytes(UTF_8)), ByteBuffer.wrap(message.getValue())); + } else { + log.debug("Got message without key from the offset storage topic, skip it. message value: {}", + message.getValue()); } } @@ -149,10 +153,13 @@ public void start() { CompletableFuture endFuture = new CompletableFuture<>(); readToEnd(endFuture); - endFuture.join(); + endFuture.get(); } catch (PulsarClientException e) { log.error("Failed to setup pulsar producer/reader to cluster", e); throw new RuntimeException("Failed to setup pulsar producer/reader to cluster ", e); + } catch (ExecutionException | InterruptedException e) { + log.error("Failed to start PulsarOffsetBackingStore", e); + throw new RuntimeException("Failed to start PulsarOffsetBackingStore", e); } } @@ -180,6 +187,7 @@ public void stop() { } reader = null; } + data.clear(); // do not close the client, it is provided by the sink context } @@ -191,10 +199,7 @@ public Future> get(Collection keys) { return endFuture.thenApply(ignored -> { Map values = new HashMap<>(); for (ByteBuffer key : keys) { - ByteBuffer value; - synchronized (data) { - value = data.get(key); - } + ByteBuffer value = data.get(key); if (null != value) { values.put(key, value); } From 9ad4ef2d3145346818f8398b5bb4500e13e51dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 10 Mar 2022 07:09:56 +0100 Subject: [PATCH 377/823] [branch-2.9] [tests] Pulsar IO - Elasticsearch - reduce elastic container memory usage and clean up test cases (#14631) --- .../elasticsearch/ElasticSearchBWCTests.java | 2 +- .../ElasticSearchClientSslTests.java | 17 +++----- .../ElasticSearchClientTests.java | 19 +++++---- .../ElasticSearchExtractTests.java | 39 +++++++------------ .../ElasticSearchSinkRawDataTests.java | 13 ++----- .../elasticsearch/ElasticSearchSinkTests.java | 9 ++--- .../elasticsearch/ElasticSearchTestBase.java | 36 +++++++++++++++++ 7 files changed, 72 insertions(+), 63 deletions(-) create mode 100644 pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java index 7b24a7ccb7494..86b9df3229f50 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java @@ -23,7 +23,7 @@ import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; -import org.junit.Test; +import org.testng.annotations.Test; import java.nio.charset.StandardCharsets; diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java index 7cbd24991df8b..325f69c7f73f9 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java @@ -25,17 +25,12 @@ import java.io.IOException; import java.time.Duration; -import java.util.Optional; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; // see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#ssl-tls-settings -public class ElasticSearchClientSslTests { - - public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.10.2-amd64"); +public class ElasticSearchClientSslTests extends ElasticSearchTestBase { final static String INDEX = "myindex"; @@ -44,7 +39,7 @@ public class ElasticSearchClientSslTests { @Test public void testSslBasic() throws IOException { - try(ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE) + try (ElasticsearchContainer container = createElasticsearchContainer() .withCreateContainerCmdModifier(c -> c.withName("elasticsearch")) .withFileSystemBind(sslResourceDir, configDir + "/ssl") .withEnv("ELASTIC_PASSWORD","elastic") // boostrap password @@ -80,7 +75,7 @@ public void testSslBasic() throws IOException { @Test public void testSslWithHostnameVerification() throws IOException { - try(ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE) + try (ElasticsearchContainer container = createElasticsearchContainer() .withCreateContainerCmdModifier(c -> c.withName("elasticsearch")) .withFileSystemBind(sslResourceDir, configDir + "/ssl") .withEnv("ELASTIC_PASSWORD","elastic") // boostrap password @@ -119,7 +114,7 @@ public void testSslWithHostnameVerification() throws IOException { @Test public void testSslWithClientAuth() throws IOException { - try(ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE) + try(ElasticsearchContainer container = createElasticsearchContainer() .withCreateContainerCmdModifier(c -> c.withName("elasticsearch")) .withFileSystemBind(sslResourceDir, configDir + "/ssl") .withEnv("ELASTIC_PASSWORD","elastic") // boostrap password diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java index a831d5cee9cec..1a505dae31af2 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java @@ -26,9 +26,9 @@ import org.awaitility.Awaitility; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; -import org.junit.AfterClass; import org.mockito.Mockito; import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -36,29 +36,28 @@ import java.util.Optional; import java.util.UUID; -import static org.junit.Assert.*; -import static org.mockito.Mockito.verify; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; @Slf4j -public class ElasticSearchClientTests { - - public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2-amd64"); +public class ElasticSearchClientTests extends ElasticSearchTestBase { static ElasticsearchContainer container; @BeforeClass public static final void initBeforeClass() throws IOException { - container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE); + container = createElasticsearchContainer(); container.start(); } - @AfterClass + @AfterClass(alwaysRun = true) public static void closeAfterClass() { container.close(); } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java index 89d686c6edd82..b0d26d92d53b7 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.io.elasticsearch; -import com.google.common.collect.ImmutableList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import com.google.common.collect.ImmutableMap; +import java.util.Optional; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.GenericObject; @@ -31,33 +33,18 @@ import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; -import java.util.Collection; -import java.util.Optional; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; - -@RunWith(Parameterized.class) public class ElasticSearchExtractTests { - SchemaType schemaType; - - @Parameters - public static Collection schemaTypes() { - return ImmutableList.of(SchemaType.JSON, SchemaType.AVRO); - } - - public ElasticSearchExtractTests(SchemaType schemaType) { - this.schemaType = schemaType; + @DataProvider(name = "schemaType") + public Object[] schemaType() { + return new Object[]{SchemaType.JSON, SchemaType.AVRO}; } - @Test - public void testGenericRecord() throws Exception { + @Test(dataProvider = "schemaType") + public void testGenericRecord(SchemaType schemaType) throws Exception { RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value"); valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null); valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null); @@ -154,8 +141,8 @@ public GenericObject getValue() { assertNull(pair4.getRight()); } - @Test - public void testKeyValueGenericRecord() throws Exception { + @Test(dataProvider = "schemaType") + public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception { RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key"); keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null); keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null); @@ -273,4 +260,4 @@ public Object getNativeObject() { assertEquals(pair3.getLeft(), "[\"1\",1]"); assertNull(pair3.getRight()); } -} +} \ No newline at end of file diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java index f195a383d4073..018a11e5aa091 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java @@ -23,11 +23,11 @@ import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.SinkContext; -import org.junit.AfterClass; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -42,13 +42,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; -public class ElasticSearchSinkRawDataTests { - - public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2-amd64"); +public class ElasticSearchSinkRawDataTests extends ElasticSearchTestBase { private static ElasticsearchContainer container; @@ -67,11 +62,11 @@ public class ElasticSearchSinkRawDataTests { @BeforeClass public static final void initBeforeClass() { - container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE); + container = createElasticsearchContainer(); schema = Schema.BYTES; } - @AfterClass + @AfterClass(alwaysRun = true) public static void closeAfterClass() { container.close(); } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java index c7523543beeef..c8c500a3bb377 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java @@ -54,10 +54,7 @@ import java.util.Locale; import static org.testng.Assert.assertNull; -public class ElasticSearchSinkTests { - - public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2-amd64"); +public class ElasticSearchSinkTests extends ElasticSearchTestBase { private static ElasticsearchContainer container; @@ -76,7 +73,7 @@ public class ElasticSearchSinkTests { @BeforeClass public static final void initBeforeClass() { - container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE); + container = createElasticsearchContainer(); valueSchema = Schema.JSON(UserProfile.class); genericSchema = Schema.generic(valueSchema.getSchemaInfo()); @@ -89,7 +86,7 @@ public static final void initBeforeClass() { } - @AfterClass + @AfterClass(alwaysRun = true) public static void closeAfterClass() { container.close(); } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java new file mode 100644 index 0000000000000..34e3fc21abb40 --- /dev/null +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.elasticsearch; + +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import java.util.Optional; + +public class ElasticSearchTestBase { + + private static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE")) + .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64"); + + protected static ElasticsearchContainer createElasticsearchContainer() { + return new ElasticsearchContainer(ELASTICSEARCH_IMAGE) + .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m"); + + } + +} From 70617238125bb5f1bbd0529ae17957909e0a555a Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Mon, 10 Jan 2022 18:27:30 +0800 Subject: [PATCH 378/823] [ Issue 13551 ] FIxed NPE when reset-cursor at a non-existent topic (RestException without cause) (#13573) (cherry picked from commit 78b28991d108f69a632d3e9e097c394ae8eae1ce) --- .../admin/impl/PersistentTopicsBase.java | 2 +- .../pulsar/broker/web/RestException.java | 5 ++--- .../broker/admin/PersistentTopicsTest.java | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index a1481e0c92b6e..b124e404481cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3643,7 +3643,7 @@ private Topic getTopicReference(TopicName topicName) { if (e.getCause() instanceof NotAllowedException) { throw new RestException(Status.BAD_REQUEST, e.getCause()); } - throw new RestException(e.getCause()); + throw new RestException(e.getCause() == null ? e : e.getCause()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java index 0cec819c46a53..2c01a7f0d0166 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java @@ -38,10 +38,9 @@ static String getExceptionData(Throwable t) { writer.append("\n --- An unexpected error occurred in the server ---\n\n"); if (t != null) { writer.append("Message: ").append(t.getMessage()).append("\n\n"); + writer.append("Stacktrace:\n\n"); + t.printStackTrace(new PrintWriter(writer)); } - writer.append("Stacktrace:\n\n"); - - t.printStackTrace(new PrintWriter(writer)); return writer.toString(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 00c4683c68673..78f3d2e7a209f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -43,6 +43,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.WebApplicationException; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Response; @@ -56,6 +57,7 @@ import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TopicResources; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -1146,4 +1148,21 @@ public void testDeleteTopic() throws Exception { Assert.assertEquals(e.getResponse().getStatus(), 404); } } + + @Test + public void testResetCursorReturnTimeoutWhenZKTimeout() { + String topic = "persistent://" + testTenant + "/" + testNamespace + "/" + "topic-2"; + BrokerService brokerService = spy(pulsar.getBrokerService()); + doReturn(brokerService).when(pulsar).getBrokerService(); + CompletableFuture> completableFuture = new CompletableFuture<>(); + doReturn(completableFuture).when(brokerService).getTopicIfExists(topic); + try { + admin.topics().resetCursor(topic, "my-sub", System.currentTimeMillis()); + Assert.fail(); + } catch (PulsarAdminException e) { + String errorMsg = ((InternalServerErrorException) e.getCause()).getResponse().readEntity(String.class); + Assert.assertTrue(errorMsg.contains("TimeoutException")); + } + } + } From 5a03b36e575e311a018606153f2b93fda3dca192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8C=85=E5=AD=90?= Date: Thu, 3 Mar 2022 16:36:54 +0800 Subject: [PATCH 379/823] [Flaky-test]: Fix MLTransactionMetadataStoreTest.testInitTransactionReader fails sporadically (#14532) ### Motivation #14525 When update states is `TxnStatus.COMMITTED`, Not correctly `completableFuture.complete`. ### Modifications - When update states is `TxnStatus.COMMITTED`, Add return to ending. Avoid direct calls `completableFuture.complete` from other logic. ### Documentation - [x ] `no-need-doc` (cherry picked from commit c15d0ef24e37df6b86bc8f4c27d43af27f8cc663) --- .../transaction/coordinator/impl/MLTransactionMetadataStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index f109ec4a3be55..eabb5fbe35ff9 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -391,6 +391,7 @@ public CompletableFuture updateTxnStatus(TxnID txnID, TxnStatus newStatus, txnMetaMap.remove(txnID.getLeastSigBits()); completableFuture.complete(null); }); + return; } completableFuture.complete(null); } catch (InvalidTxnStatusException e) { From e1bcb2b85663d2ecba916bda6c3ccc9846aa01a3 Mon Sep 17 00:00:00 2001 From: Aloys Date: Sat, 5 Mar 2022 18:29:52 +0800 Subject: [PATCH 380/823] [Flaky-test]: AdminApiTest.testNamespaceSplitBundleConcurrent (#14565) (cherry picked from commit 01b55678321b6cf254f5d289f7eb177a6aea9be9) --- .../java/org/apache/pulsar/broker/admin/AdminApiTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index eee39b823e075..348f5cb916d6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -1500,10 +1500,11 @@ public void testNamespaceSplitBundleConcurrent() throws Exception { fail("split bundle shouldn't have thrown exception"); } + Awaitility.await().untilAsserted(() -> + assertEquals(bundleFactory.getBundles(NamespaceName.get(namespace)).getBundles().size(), 4)); String[] splitRange4 = { namespace + "/0x00000000_0x3fffffff", namespace + "/0x3fffffff_0x7fffffff", namespace + "/0x7fffffff_0xbfffffff", namespace + "/0xbfffffff_0xffffffff" }; bundles = bundleFactory.getBundles(NamespaceName.get(namespace)); - assertEquals(bundles.getBundles().size(), 4); for (int i = 0; i < bundles.getBundles().size(); i++) { assertEquals(bundles.getBundles().get(i).toString(), splitRange4[i]); } @@ -1533,13 +1534,13 @@ public void testNamespaceSplitBundleConcurrent() throws Exception { } catch (Exception e) { fail("split bundle shouldn't have thrown exception"); } - + Awaitility.await().untilAsserted(() -> + assertEquals(bundleFactory.getBundles(NamespaceName.get(namespace)).getBundles().size(), 8)); String[] splitRange8 = { namespace + "/0x00000000_0x1fffffff", namespace + "/0x1fffffff_0x3fffffff", namespace + "/0x3fffffff_0x5fffffff", namespace + "/0x5fffffff_0x7fffffff", namespace + "/0x7fffffff_0x9fffffff", namespace + "/0x9fffffff_0xbfffffff", namespace + "/0xbfffffff_0xdfffffff", namespace + "/0xdfffffff_0xffffffff" }; bundles = bundleFactory.getBundles(NamespaceName.get(namespace)); - assertEquals(bundles.getBundles().size(), 8); for (int i = 0; i < bundles.getBundles().size(); i++) { assertEquals(bundles.getBundles().get(i).toString(), splitRange8[i]); } From 3dd810231f964e03509cfd4dc45f6a91f621affd Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 8 Mar 2022 12:53:29 +0800 Subject: [PATCH 381/823] [C++] Fix wrong unit of Access Token Response's `expires_in` field (#14554) The `expires_in` field of Access Token Response is in seconds. See https://datatracker.ietf.org/doc/html/rfc6749#section-4.2.2. However, C++ client treats it as milliseconds currently. It will leads to an earlier expiration of the token. Record the time point via the `std::time_point` class, which supports add operations with a `std::duration` object. Then converts the `expires_in` field via `std::chrono::second` function and calculate the expired time point. It also removes the usage of Boost time functions and makes code more clear. (cherry picked from commit 95c1581d494d59c4d93782eb18547cec5427b503) --- pulsar-client-cpp/lib/auth/AuthOauth2.cc | 15 ++------------- pulsar-client-cpp/lib/auth/AuthOauth2.h | 5 ++++- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.cc b/pulsar-client-cpp/lib/auth/AuthOauth2.cc index 87a217e818f87..c3dfe550a0c19 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.cc +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.cc @@ -23,7 +23,6 @@ #include #include #include -#include #include DECLARE_LOG_OBJECT() @@ -86,22 +85,12 @@ CachedToken::~CachedToken() {} // Oauth2CachedToken -static int64_t currentTimeMillis() { - using namespace boost::posix_time; - using boost::posix_time::milliseconds; - using boost::posix_time::seconds; - static ptime time_t_epoch(boost::gregorian::date(1970, 1, 1)); - - time_duration diff = microsec_clock::universal_time() - time_t_epoch; - return diff.total_milliseconds(); -} - Oauth2CachedToken::Oauth2CachedToken(Oauth2TokenResultPtr token) { latest_ = token; int64_t expiredIn = token->getExpiresIn(); if (expiredIn > 0) { - expiresAt_ = expiredIn + currentTimeMillis(); + expiresAt_ = Clock::now() + std::chrono::seconds(expiredIn); } else { throw std::runtime_error("ExpiresIn in Oauth2TokenResult invalid value: " + std::to_string(expiredIn)); @@ -113,7 +102,7 @@ AuthenticationDataPtr Oauth2CachedToken::getAuthData() { return authData_; } Oauth2CachedToken::~Oauth2CachedToken() {} -bool Oauth2CachedToken::isExpired() { return expiresAt_ < currentTimeMillis(); } +bool Oauth2CachedToken::isExpired() { return expiresAt_ < Clock::now(); } // OauthFlow diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.h b/pulsar-client-cpp/lib/auth/AuthOauth2.h index b1a5ec63a9424..a3658b353eefa 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.h +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include namespace pulsar { @@ -67,13 +68,15 @@ class ClientCredentialFlow : public Oauth2Flow { class Oauth2CachedToken : public CachedToken { public: + using Clock = std::chrono::high_resolution_clock; + Oauth2CachedToken(Oauth2TokenResultPtr token); ~Oauth2CachedToken(); bool isExpired(); AuthenticationDataPtr getAuthData(); private: - int64_t expiresAt_; + std::chrono::time_point expiresAt_; Oauth2TokenResultPtr latest_; AuthenticationDataPtr authData_; }; From 3a7640d3e0888fd2fe8613e62bb692b40ecb8912 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Wed, 9 Mar 2022 09:18:39 +0800 Subject: [PATCH 382/823] [Broker] Fixed wrong behaviour caused by not cleaning up topic policy service state. (#14503) (cherry picked from commit f32154c06c6475fac1cd89d105d3c31d5d8713dc) --- .../SystemTopicBasedTopicPoliciesService.java | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index ddf926dc925ea..964bddea99679 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -22,6 +22,7 @@ import com.google.common.collect.Lists; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -238,8 +239,7 @@ private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture if (ex != null) { log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); result.completeExceptionally(ex); - readerCaches.remove(namespace); - reader.closeAsync(); + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); } else { initPolicesCache(reader, result); result.thenRun(() -> readMorePolicies(reader)); @@ -273,14 +273,7 @@ public CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle n } AtomicInteger bundlesCount = ownedBundlesCountPerNamespace.get(namespace); if (bundlesCount == null || bundlesCount.decrementAndGet() <= 0) { - CompletableFuture> readerCompletableFuture = - readerCaches.remove(namespace); - if (readerCompletableFuture != null) { - readerCompletableFuture.thenAccept(SystemTopicClient.Reader::closeAsync); - ownedBundlesCountPerNamespace.remove(namespace); - policyCacheInitMap.remove(namespace); - policiesCache.entrySet().removeIf(entry -> entry.getKey().getNamespaceObject().equals(namespace)); - } + cleanCacheAndCloseReader(namespace, true); } return CompletableFuture.completedFuture(null); } @@ -314,9 +307,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp log.error("[{}] Failed to check the move events for the system topic", reader.getSystemTopic().getTopicName(), ex); future.completeExceptionally(ex); - readerCaches.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); - policyCacheInitMap.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); - reader.closeAsync(); + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); return; } if (hasMore) { @@ -325,9 +316,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp log.error("[{}] Failed to read event from the system topic.", reader.getSystemTopic().getTopicName(), e); future.completeExceptionally(e); - readerCaches.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); - policyCacheInitMap.remove(reader.getSystemTopic().getTopicName().getNamespaceObject()); - reader.closeAsync(); + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); return; } refreshTopicPoliciesCache(msg); @@ -356,6 +345,18 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp }); } + private void cleanCacheAndCloseReader(NamespaceName namespace, boolean cleanOwnedBundlesCount) { + CompletableFuture> readerFuture = readerCaches.remove(namespace); + policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + if (cleanOwnedBundlesCount) { + ownedBundlesCountPerNamespace.remove(namespace); + } + if (readerFuture != null && !readerFuture.isCompletedExceptionally()) { + readerFuture.thenAccept(SystemTopicClient.Reader::closeAsync); + } + policyCacheInitMap.remove(namespace); + } + private void readMorePolicies(SystemTopicClient.Reader reader) { reader.readNextAsync().whenComplete((msg, ex) -> { if (ex == null) { @@ -365,10 +366,10 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { } else { if (ex instanceof PulsarClientException.AlreadyClosedException) { log.error("Read more topic policies exception, close the read now!", ex); - NamespaceName namespace = reader.getSystemTopic().getTopicName().getNamespaceObject(); - ownedBundlesCountPerNamespace.remove(namespace); - readerCaches.remove(namespace); + cleanCacheAndCloseReader( + reader.getSystemTopic().getTopicName().getNamespaceObject(), false); } else { + log.warn("Read more topic polices exception, read again.", ex); readMorePolicies(reader); } } From a5612df3ea0d597eec16f198b5eab18bf06a0432 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 10 Mar 2022 16:03:02 +0800 Subject: [PATCH 383/823] [C++] Fix thread safety issue for multi topic consumer (#14380) * [C++] Fix thread safety issue for multi topic consumer **Motivation** In C++ client, if a consumer subscribes multiple topics, a `MultiTopicsConsumerImpl` object, which manages a vector of `ConsumerImpl`s (`consumers_` field), will be created. However, `consumers_` could be accessed by multiple threads, while no mutex is locked to protect the access to make it thread safe. **Modifications** - Add a `SynchronizedHashMap` class, which implements some thread safe methods of traverse, remove, find, clear operations. Since the `forEach` methods could call other methods, use the recursive mutex instead of the default mutex. - Add a related test `SynchronizedHashMapTest` to test the methods and the thread safety of `SynchronizedHashMap`. - Use `SynchronizedHashMap` as the type of `MultiTopicsConsumerImpl::consumers_`. * Add findFirstValueIf method * Remove unnecessary return value of forEach * Fix incorrect calls of forEachValue * Add missed header (cherry picked from commit f94eba942b9fb3d2c25b6f7a9e2c0885a194efa0) --- .../lib/MultiTopicsConsumerImpl.cc | 166 ++++++++---------- .../lib/MultiTopicsConsumerImpl.h | 5 +- pulsar-client-cpp/lib/SynchronizedHashMap.h | 127 ++++++++++++++ pulsar-client-cpp/tests/ConsumerTest.cc | 13 +- .../tests/SynchronizedHashMapTest.cc | 125 +++++++++++++ 5 files changed, 335 insertions(+), 101 deletions(-) create mode 100644 pulsar-client-cpp/lib/SynchronizedHashMap.h create mode 100644 pulsar-client-cpp/tests/SynchronizedHashMapTest.cc diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc index 4e31e64d5ee56..0ae86d5879a4f 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc @@ -171,7 +171,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, consumer->getConsumerCreatedFuture().addListener(std::bind( &MultiTopicsConsumerImpl::handleSingleConsumerCreated, shared_from_this(), std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); - consumers_.insert(std::make_pair(topicName->toString(), consumer)); + consumers_.emplace(topicName->toString(), consumer); LOG_DEBUG("Creating Consumer for - " << topicName << " - " << consumerStr_); consumer->start(); @@ -184,7 +184,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, &MultiTopicsConsumerImpl::handleSingleConsumerCreated, shared_from_this(), std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); consumer->setPartitionIndex(i); - consumers_.insert(std::make_pair(topicPartitionName, consumer)); + consumers_.emplace(topicPartitionName, consumer); LOG_DEBUG("Creating Consumer for - " << topicPartitionName << " - " << consumerStr_); consumer->start(); } @@ -232,20 +232,19 @@ void MultiTopicsConsumerImpl::unsubscribeAsync(ResultCallback callback) { state_ = Closing; lock.unlock(); - if (consumers_.empty()) { + std::shared_ptr> consumerUnsubed = std::make_shared>(0); + auto self = shared_from_this(); + int numConsumers = 0; + consumers_.forEachValue( + [&numConsumers, &consumerUnsubed, &self, callback](const ConsumerImplPtr& consumer) { + numConsumers++; + consumer->unsubscribeAsync([self, consumerUnsubed, callback](Result result) { + self->handleUnsubscribedAsync(result, consumerUnsubed, callback); + }); + }); + if (numConsumers == 0) { // No need to unsubscribe, since the list matching the regex was empty callback(ResultOk); - return; - } - - std::shared_ptr> consumerUnsubed = std::make_shared>(0); - - for (ConsumerMap::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - (consumer->second) - ->unsubscribeAsync(std::bind(&MultiTopicsConsumerImpl::handleUnsubscribedAsync, - shared_from_this(), std::placeholders::_1, consumerUnsubed, - callback)); } } @@ -299,17 +298,17 @@ void MultiTopicsConsumerImpl::unsubscribeOneTopicAsync(const std::string& topic, for (int i = 0; i < numberPartitions; i++) { std::string topicPartitionName = topicName->getTopicPartitionName(i); - std::map::iterator iterator = consumers_.find(topicPartitionName); - - if (consumers_.end() == iterator) { + auto optConsumer = consumers_.find(topicPartitionName); + if (optConsumer.is_empty()) { LOG_ERROR("TopicsConsumer not subscribed on topicPartitionName: " << topicPartitionName); callback(ResultUnknownError); + continue; } - (iterator->second) - ->unsubscribeAsync(std::bind(&MultiTopicsConsumerImpl::handleOneTopicUnsubscribedAsync, - shared_from_this(), std::placeholders::_1, consumerUnsubed, - numberPartitions, topicName, topicPartitionName, callback)); + optConsumer.value()->unsubscribeAsync( + std::bind(&MultiTopicsConsumerImpl::handleOneTopicUnsubscribedAsync, shared_from_this(), + std::placeholders::_1, consumerUnsubed, numberPartitions, topicName, topicPartitionName, + callback)); } } @@ -326,10 +325,9 @@ void MultiTopicsConsumerImpl::handleOneTopicUnsubscribedAsync( LOG_DEBUG("Successfully Unsubscribed one Consumer. topicPartitionName - " << topicPartitionName); - std::map::iterator iterator = consumers_.find(topicPartitionName); - if (consumers_.end() != iterator) { - iterator->second->pauseMessageListener(); - consumers_.erase(iterator); + auto optConsumer = consumers_.remove(topicPartitionName); + if (optConsumer.is_present()) { + optConsumer.value()->pauseMessageListener(); } if (consumerUnsubed->load() == numberPartitions) { @@ -363,7 +361,16 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { setState(Closing); - if (consumers_.empty()) { + auto self = shared_from_this(); + int numConsumers = 0; + consumers_.forEach( + [&numConsumers, &self, callback](const std::string& name, const ConsumerImplPtr& consumer) { + numConsumers++; + consumer->closeAsync([self, name, callback](Result result) { + self->handleSingleConsumerClose(result, name, callback); + }); + }); + if (numConsumers == 0) { LOG_DEBUG("TopicsConsumer have no consumers to close " << " topic" << topic_ << " subscription - " << subscriptionName_); setState(Closed); @@ -373,27 +380,13 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { return; } - // close successfully subscribed consumers - for (ConsumerMap::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - std::string topicPartitionName = consumer->first; - ConsumerImplPtr consumerPtr = consumer->second; - - consumerPtr->closeAsync(std::bind(&MultiTopicsConsumerImpl::handleSingleConsumerClose, - shared_from_this(), std::placeholders::_1, topicPartitionName, - callback)); - } - // fail pending recieve failPendingReceiveCallback(); } -void MultiTopicsConsumerImpl::handleSingleConsumerClose(Result result, std::string& topicPartitionName, +void MultiTopicsConsumerImpl::handleSingleConsumerClose(Result result, std::string topicPartitionName, CloseCallback callback) { - std::map::iterator iterator = consumers_.find(topicPartitionName); - if (consumers_.end() != iterator) { - consumers_.erase(iterator); - } + consumers_.remove(topicPartitionName); LOG_DEBUG("Closing the consumer for partition - " << topicPartitionName << " numberTopicPartitions_ - " << numberTopicPartitions_->load()); @@ -543,15 +536,14 @@ void MultiTopicsConsumerImpl::acknowledgeAsync(const MessageId& msgId, ResultCal } const std::string& topicPartitionName = msgId.getTopicName(); - std::map::iterator iterator = consumers_.find(topicPartitionName); + auto optConsumer = consumers_.find(topicPartitionName); - if (consumers_.end() != iterator) { + if (optConsumer.is_present()) { unAckedMessageTrackerPtr_->remove(msgId); - iterator->second->acknowledgeAsync(msgId, callback); + optConsumer.value()->acknowledgeAsync(msgId, callback); } else { LOG_ERROR("Message of topic: " << topicPartitionName << " not in unAckedMessageTracker"); callback(ResultUnknownError); - return; } } @@ -560,11 +552,11 @@ void MultiTopicsConsumerImpl::acknowledgeCumulativeAsync(const MessageId& msgId, } void MultiTopicsConsumerImpl::negativeAcknowledge(const MessageId& msgId) { - auto iterator = consumers_.find(msgId.getTopicName()); + auto optConsumer = consumers_.find(msgId.getTopicName()); - if (consumers_.end() != iterator) { + if (optConsumer.is_present()) { unAckedMessageTrackerPtr_->remove(msgId); - iterator->second->negativeAcknowledge(msgId); + optConsumer.value()->negativeAcknowledge(msgId); } } @@ -605,22 +597,18 @@ bool MultiTopicsConsumerImpl::isOpen() { } void MultiTopicsConsumerImpl::receiveMessages() { - for (ConsumerMap::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - ConsumerImplPtr consumerPtr = consumer->second; - consumerPtr->sendFlowPermitsToBroker(consumerPtr->getCnx().lock(), conf_.getReceiverQueueSize()); - LOG_DEBUG("Sending FLOW command for consumer - " << consumerPtr->getConsumerId()); - } + const auto receiverQueueSize = conf_.getReceiverQueueSize(); + consumers_.forEachValue([receiverQueueSize](const ConsumerImplPtr& consumer) { + consumer->sendFlowPermitsToBroker(consumer->getCnx().lock(), receiverQueueSize); + LOG_DEBUG("Sending FLOW command for consumer - " << consumer->getConsumerId()); + }); } Result MultiTopicsConsumerImpl::pauseMessageListener() { if (!messageListener_) { return ResultInvalidConfiguration; } - for (ConsumerMap::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - (consumer->second)->pauseMessageListener(); - } + consumers_.forEachValue([](const ConsumerImplPtr& consumer) { consumer->pauseMessageListener(); }); return ResultOk; } @@ -628,19 +616,14 @@ Result MultiTopicsConsumerImpl::resumeMessageListener() { if (!messageListener_) { return ResultInvalidConfiguration; } - for (ConsumerMap::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - (consumer->second)->resumeMessageListener(); - } + consumers_.forEachValue([](const ConsumerImplPtr& consumer) { consumer->resumeMessageListener(); }); return ResultOk; } void MultiTopicsConsumerImpl::redeliverUnacknowledgedMessages() { LOG_DEBUG("Sending RedeliverUnacknowledgedMessages command for partitioned consumer."); - for (ConsumerMap::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - (consumer->second)->redeliverUnacknowledgedMessages(); - } + consumers_.forEachValue( + [](const ConsumerImplPtr& consumer) { consumer->redeliverUnacknowledgedMessages(); }); unAckedMessageTrackerPtr_->clear(); } @@ -653,10 +636,9 @@ void MultiTopicsConsumerImpl::redeliverUnacknowledgedMessages(const std::setsecond)->redeliverUnacknowledgedMessages(messageIds); - } + consumers_.forEachValue([&messageIds](const ConsumerImplPtr& consumer) { + consumer->redeliverUnacknowledgedMessages(messageIds); + }); } int MultiTopicsConsumerImpl::getNumOfPrefetchedMessages() const { return messages_.size(); } @@ -671,15 +653,17 @@ void MultiTopicsConsumerImpl::getBrokerConsumerStatsAsync(BrokerConsumerStatsCal MultiTopicsBrokerConsumerStatsPtr statsPtr = std::make_shared(numberTopicPartitions_->load()); LatchPtr latchPtr = std::make_shared(numberTopicPartitions_->load()); - int size = consumers_.size(); lock.unlock(); - ConsumerMap::const_iterator consumer = consumers_.begin(); - for (int i = 0; i < size; i++, consumer++) { - consumer->second->getBrokerConsumerStatsAsync( - std::bind(&MultiTopicsConsumerImpl::handleGetConsumerStats, shared_from_this(), - std::placeholders::_1, std::placeholders::_2, latchPtr, statsPtr, i, callback)); - } + auto self = shared_from_this(); + size_t i = 0; + consumers_.forEachValue([&self, &latchPtr, &statsPtr, &i, callback](const ConsumerImplPtr& consumer) { + size_t index = i++; + consumer->getBrokerConsumerStatsAsync( + [self, latchPtr, statsPtr, index, callback](Result result, BrokerConsumerStats stats) { + self->handleGetConsumerStats(result, stats, latchPtr, statsPtr, index, callback); + }); + }); } void MultiTopicsConsumerImpl::handleGetConsumerStats(Result res, BrokerConsumerStats brokerConsumerStats, @@ -725,10 +709,9 @@ void MultiTopicsConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callb } void MultiTopicsConsumerImpl::setNegativeAcknowledgeEnabledForTesting(bool enabled) { - Lock lock(mutex_); - for (auto&& c : consumers_) { - c.second->setNegativeAcknowledgeEnabledForTesting(enabled); - } + consumers_.forEachValue([enabled](const ConsumerImplPtr& consumer) { + consumer->setNegativeAcknowledgeEnabledForTesting(enabled); + }); } bool MultiTopicsConsumerImpl::isConnected() const { @@ -736,24 +719,19 @@ bool MultiTopicsConsumerImpl::isConnected() const { if (state_ != Ready) { return false; } + lock.unlock(); - for (const auto& topicAndConsumer : consumers_) { - if (!topicAndConsumer.second->isConnected()) { - return false; - } - } - return true; + return consumers_ + .findFirstValueIf([](const ConsumerImplPtr& consumer) { return !consumer->isConnected(); }) + .is_empty(); } uint64_t MultiTopicsConsumerImpl::getNumberOfConnectedConsumer() { - Lock lock(mutex_); uint64_t numberOfConnectedConsumer = 0; - const auto consumers = consumers_; - lock.unlock(); - for (const auto& topicAndConsumer : consumers) { - if (topicAndConsumer.second->isConnected()) { + consumers_.forEachValue([&numberOfConnectedConsumer](const ConsumerImplPtr& consumer) { + if (consumer->isConnected()) { numberOfConnectedConsumer++; } - } + }); return numberOfConnectedConsumer; } diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h index aa6b261a267d8..98b2f318af95a 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h @@ -32,6 +32,7 @@ #include #include #include +#include namespace pulsar { typedef std::shared_ptr> ConsumerSubResultPromisePtr; @@ -93,7 +94,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, std::string consumerStr_; std::string topic_; const ConsumerConfiguration conf_; - typedef std::map ConsumerMap; + typedef SynchronizedHashMap ConsumerMap; ConsumerMap consumers_; std::map topicsPartitions_; mutable std::mutex mutex_; @@ -115,7 +116,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, void handleSinglePartitionConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, unsigned int partitionIndex); - void handleSingleConsumerClose(Result result, std::string& topicPartitionName, CloseCallback callback); + void handleSingleConsumerClose(Result result, std::string topicPartitionName, CloseCallback callback); void notifyResult(CloseCallback closeCallback); void messageReceived(Consumer consumer, const Message& msg); void internalListener(Consumer consumer); diff --git a/pulsar-client-cpp/lib/SynchronizedHashMap.h b/pulsar-client-cpp/lib/SynchronizedHashMap.h new file mode 100644 index 0000000000000..3a784675dd1d7 --- /dev/null +++ b/pulsar-client-cpp/lib/SynchronizedHashMap.h @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include "Utils.h" + +namespace pulsar { + +// V must be default constructible and copyable +template +class SynchronizedHashMap { + using MutexType = std::recursive_mutex; + using Lock = std::lock_guard; + + public: + using OptValue = Optional; + using PairVector = std::vector>; + + SynchronizedHashMap() = default; + + SynchronizedHashMap(const PairVector& pairs) { + for (auto&& kv : pairs) { + data_.emplace(kv.first, kv.second); + } + } + + template + void emplace(Args&&... args) { + Lock lock(mutex_); + data_.emplace(std::forward(args)...); + } + + void forEach(std::function f) const { + Lock lock(mutex_); + for (const auto& kv : data_) { + f(kv.first, kv.second); + } + } + + void forEachValue(std::function f) const { + Lock lock(mutex_); + for (const auto& kv : data_) { + f(kv.second); + } + } + + void clear() { + Lock lock(mutex_); + data_.clear(); + } + + OptValue find(const K& key) const { + Lock lock(mutex_); + auto it = data_.find(key); + if (it != data_.end()) { + return OptValue::of(it->second); + } else { + return OptValue::empty(); + } + } + + OptValue findFirstValueIf(std::function f) const { + Lock lock(mutex_); + for (const auto& kv : data_) { + if (f(kv.second)) { + return OptValue::of(kv.second); + } + } + return OptValue::empty(); + } + + OptValue remove(const K& key) { + Lock lock(mutex_); + auto it = data_.find(key); + if (it != data_.end()) { + auto result = OptValue::of(it->second); + data_.erase(it); + return result; + } else { + return OptValue::empty(); + } + } + + // This method is only used for test + PairVector toPairVector() const { + Lock lock(mutex_); + PairVector pairs; + for (auto&& kv : data_) { + pairs.emplace_back(kv); + } + return pairs; + } + + // This method is only used for test + size_t size() const noexcept { + Lock lock(mutex_); + return data_.size(); + } + + private: + std::unordered_map data_; + // Use recursive_mutex to allow methods being called in `forEach` + mutable MutexType mutex_; +}; + +} // namespace pulsar diff --git a/pulsar-client-cpp/tests/ConsumerTest.cc b/pulsar-client-cpp/tests/ConsumerTest.cc index 100086e2a01c1..b61c15a886630 100644 --- a/pulsar-client-cpp/tests/ConsumerTest.cc +++ b/pulsar-client-cpp/tests/ConsumerTest.cc @@ -530,11 +530,14 @@ TEST(ConsumerTest, testMultiTopicsConsumerUnAckedMessageRedelivery) { multiTopicsConsumerImplPtr->unAckedMessageTrackerPtr_.get()); ASSERT_EQ(numOfMessages * 3, multiTopicsTracker->size()); ASSERT_FALSE(multiTopicsTracker->isEmpty()); - for (auto iter = multiTopicsConsumerImplPtr->consumers_.begin(); - iter != multiTopicsConsumerImplPtr->consumers_.end(); ++iter) { - auto subConsumerPtr = iter->second; - auto tracker = - static_cast(subConsumerPtr->unAckedMessageTrackerPtr_.get()); + + std::vector trackers; + multiTopicsConsumerImplPtr->consumers_.forEach( + [&trackers](const std::string& name, const ConsumerImplPtr& consumer) { + trackers.emplace_back( + static_cast(consumer->unAckedMessageTrackerPtr_.get())); + }); + for (const auto& tracker : trackers) { ASSERT_EQ(0, tracker->size()); ASSERT_TRUE(tracker->isEmpty()); } diff --git a/pulsar-client-cpp/tests/SynchronizedHashMapTest.cc b/pulsar-client-cpp/tests/SynchronizedHashMapTest.cc new file mode 100644 index 0000000000000..62c55c46c8e61 --- /dev/null +++ b/pulsar-client-cpp/tests/SynchronizedHashMapTest.cc @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include +#include +#include +#include "lib/Latch.h" +#include "lib/SynchronizedHashMap.h" + +using namespace pulsar; +using SyncMapType = SynchronizedHashMap; +using OptValue = typename SyncMapType::OptValue; +using PairVector = typename SyncMapType::PairVector; + +inline void sleepMs(long millis) { std::this_thread::sleep_for(std::chrono::milliseconds(millis)); } + +inline PairVector sort(PairVector pairs) { + std::sort(pairs.begin(), pairs.end(), [](const std::pair& lhs, const std::pair& rhs) { + return lhs.first < rhs.first; + }); + return pairs; +} + +TEST(SynchronizedHashMap, testClear) { + SynchronizedHashMap m({{1, 100}, {2, 200}}); + m.clear(); + ASSERT_EQ(m.toPairVector(), PairVector{}); +} + +TEST(SynchronizedHashMap, testRemoveAndFind) { + SyncMapType m({{1, 100}, {2, 200}, {3, 300}}); + + OptValue optValue; + optValue = m.findFirstValueIf([](const int& x) { return x == 200; }); + ASSERT_TRUE(optValue.is_present()); + ASSERT_EQ(optValue.value(), 200); + + optValue = m.findFirstValueIf([](const int& x) { return x >= 301; }); + ASSERT_FALSE(optValue.is_present()); + + optValue = m.find(1); + ASSERT_TRUE(optValue.is_present()); + ASSERT_EQ(optValue.value(), 100); + + ASSERT_FALSE(m.find(0).is_present()); + ASSERT_FALSE(m.remove(0).is_present()); + + optValue = m.remove(1); + ASSERT_TRUE(optValue.is_present()); + ASSERT_EQ(optValue.value(), 100); + + ASSERT_FALSE(m.remove(1).is_present()); + ASSERT_FALSE(m.find(1).is_present()); +} + +TEST(SynchronizedHashMapTest, testForEach) { + SyncMapType m({{1, 100}, {2, 200}, {3, 300}}); + std::vector values; + m.forEachValue([&values](const int& value) { values.emplace_back(value); }); + std::sort(values.begin(), values.end()); + ASSERT_EQ(values, std::vector({100, 200, 300})); + + PairVector pairs; + m.forEach([&pairs](const int& key, const int& value) { pairs.emplace_back(key, value); }); + PairVector expectedPairs({{1, 100}, {2, 200}, {3, 300}}); + ASSERT_EQ(sort(pairs), expectedPairs); +} + +TEST(SynchronizedHashMap, testRecursiveMutex) { + SyncMapType m({{1, 100}}); + OptValue optValue; + m.forEach([&m, &optValue](const int& key, const int& value) { + optValue = m.find(key); // the internal mutex was locked again + }); + ASSERT_TRUE(optValue.is_present()); + ASSERT_EQ(optValue.value(), 100); +} + +TEST(SynchronizedHashMapTest, testThreadSafeForEach) { + SyncMapType m({{1, 100}, {2, 200}, {3, 300}}); + + Latch latch(1); + std::thread t{[&m, &latch] { + latch.wait(); // this thread must start after `m.forEach` started + m.remove(2); + }}; + + std::atomic_bool firstElementDone{false}; + PairVector pairs; + m.forEach([&latch, &firstElementDone, &pairs](const int& key, const int& value) { + pairs.emplace_back(key, value); + if (!firstElementDone) { + latch.countdown(); + firstElementDone = true; + } + sleepMs(200); + }); + { + PairVector expectedPairs({{1, 100}, {2, 200}, {3, 300}}); + ASSERT_EQ(sort(pairs), expectedPairs); + } + t.join(); + { + PairVector expectedPairs({{1, 100}, {3, 300}}); + ASSERT_EQ(sort(m.toPairVector()), expectedPairs); + } +} From c3e65ea55af813571722d082126ecb295cac4355 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 2 Mar 2022 14:21:41 +0200 Subject: [PATCH 384/823] [Tests] Fix thread leak in MLTransactionMetadataStore (#14524) - MLTransactionMetadataStore.internalPinnedExecutor wasn't closed when MLTransactionMetadataStore.closeAsync was called - problem was introduced by #14238 changes - this issue causes tests to fail with OOME. Most likely this also impacts production code. * Close TransactionMetadataStoreService after the broker service has been closed (cherry picked from commit 0ddec86444bbdd94221032a3128f7dbe79934b93) --- .../apache/pulsar/broker/PulsarService.java | 16 ++++++---- .../TransactionMetadataStoreService.java | 12 +++++++- .../TransactionMetadataStoreState.java | 13 ++++++--- .../impl/MLTransactionMetadataStore.java | 29 +++++++++++++------ 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 7fcf0d02d1555..ba4178475f132 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -403,11 +403,6 @@ public CompletableFuture closeAsync() { brokerAdditionalServlets = null; } - if (this.transactionMetadataStoreService != null) { - this.transactionMetadataStoreService.close(); - this.transactionMetadataStoreService = null; - } - GracefulExecutorServicesShutdown executorServicesShutdown = GracefulExecutorServicesShutdown .initiate() @@ -426,7 +421,16 @@ public CompletableFuture closeAsync() { List> asyncCloseFutures = new ArrayList<>(); if (this.brokerService != null) { - asyncCloseFutures.add(this.brokerService.closeAsync()); + CompletableFuture brokerCloseFuture = this.brokerService.closeAsync(); + if (this.transactionMetadataStoreService != null) { + asyncCloseFutures.add(brokerCloseFuture.whenComplete((__, ___) -> { + // close transactionMetadataStoreService after the broker has been closed + this.transactionMetadataStoreService.close(); + this.transactionMetadataStoreService = null; + })); + } else { + asyncCloseFutures.add(brokerCloseFuture); + } this.brokerService = null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index b3ced3c7a9641..28bef1f830d5e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -554,7 +554,17 @@ public Map getStores() { return Collections.unmodifiableMap(stores); } - public void close () { + public synchronized void close () { this.internalPinnedExecutor.shutdown(); + stores.forEach((tcId, metadataStore) -> { + metadataStore.closeAsync().whenComplete((v, ex) -> { + if (ex != null) { + LOG.error("Close transaction metadata store with id " + tcId, ex); + } else { + LOG.info("Removed and closed transaction meta store {}", tcId); + } + }); + }); + stores.clear(); } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreState.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreState.java index 5dbd9ba6dcb9d..8947413191cf6 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreState.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreState.java @@ -33,6 +33,7 @@ public enum State { None, Initializing, Ready, + Closing, Close } @@ -55,10 +56,14 @@ protected boolean changeToInitializingState() { return STATE_UPDATER.compareAndSet(this, State.None, State.Initializing); } + protected boolean changeToClosingState() { + return (STATE_UPDATER.compareAndSet(this, State.Ready, State.Closing) + || STATE_UPDATER.compareAndSet(this, State.None, State.Closing) + || STATE_UPDATER.compareAndSet(this, State.Initializing, State.Closing)); + } + protected boolean changeToCloseState() { - return (STATE_UPDATER.compareAndSet(this, State.Ready, State.Close) - || STATE_UPDATER.compareAndSet(this, State.None, State.Close) - || STATE_UPDATER.compareAndSet(this, State.Initializing, State.Close)); + return STATE_UPDATER.compareAndSet(this, State.Closing, State.Close); } protected boolean checkIfReady() { @@ -68,4 +73,4 @@ protected boolean checkIfReady() { public State getState() { return STATE_UPDATER.get(this); } -} \ No newline at end of file +} diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index eabb5fbe35ff9..19d651c96d7ea 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.transaction.coordinator.impl; +import com.google.common.util.concurrent.MoreExecutors; import io.netty.util.concurrent.DefaultThreadFactory; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -443,15 +445,24 @@ private CompletableFuture>> getTxnPositionPair(TxnI @Override public CompletableFuture closeAsync() { - return transactionLog.closeAsync().thenCompose(v -> { - txnMetaMap.clear(); - this.timeoutTracker.close(); - if (!this.changeToCloseState()) { - return FutureUtil.failedFuture( - new IllegalStateException("Managed ledger transaction metadata store state to close error!")); - } + if (changeToClosingState()) { + // Disable new tasks from being submitted + internalPinnedExecutor.shutdown(); + return transactionLog.closeAsync().thenCompose(v -> { + txnMetaMap.clear(); + this.timeoutTracker.close(); + if (!this.changeToCloseState()) { + return FutureUtil.failedFuture( + new IllegalStateException( + "Managed ledger transaction metadata store state to close error!")); + } + // Shutdown the ExecutorService + MoreExecutors.shutdownAndAwaitTermination(internalPinnedExecutor, Duration.ofSeconds(5L)); + return CompletableFuture.completedFuture(null); + }); + } else { return CompletableFuture.completedFuture(null); - }); + } } @Override @@ -505,4 +516,4 @@ public static List subscriptionToTxnSubscription( public ManagedLedger getManagedLedger() { return this.transactionLog.getManagedLedger(); } -} \ No newline at end of file +} From c4e46a9c1e4362bbc3c039e20f0f89459a2c4ee8 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 11 Mar 2022 16:17:27 +0800 Subject: [PATCH 385/823] Fix delete namespace issue. (#14657) Cherry-pick https://github.com/apache/pulsar/pull/14215 ### Motivation When we delete namespace with `force=false`, sometimes there may occur `Cannot delete non empty bundle`. Because delete system topic and delete namespace bundles are put into futures, can't keep their order. We should delete `system topic` first and then delete `namespace bundles`. ### Modifications - Use admin cli to delete system topic. - Delete system topic first and then delete namespace bundles - Make internalClearZkSources async and the code more readable. --- .../pulsar/broker/admin/AdminResource.java | 3 + .../broker/admin/impl/NamespacesBase.java | 90 +++++++++---------- .../pulsar/broker/admin/NamespacesTest.java | 21 +++-- 3 files changed, 55 insertions(+), 59 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 71c88834d3f69..8d7de948d0820 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -42,6 +42,7 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.internal.TopicsImpl; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.naming.Constants; @@ -750,6 +751,8 @@ protected void resumeAsyncResponseExceptionally(AsyncResponse asyncResponse, Thr asyncResponse.resume(throwable); } else if (throwable instanceof BrokerServiceException.NotAllowedException) { asyncResponse.resume(new RestException(Status.CONFLICT, throwable)); + } else if (throwable instanceof PulsarAdminException) { + asyncResponse.resume(new RestException(((PulsarAdminException) throwable))); } else { asyncResponse.resume(new RestException(throwable)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 30941df5351f8..a9f369d42d4b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -262,52 +262,56 @@ protected void internalDeleteNamespace(AsyncResponse asyncResponse, boolean auth // remove from owned namespace map and ephemeral node from ZK final List> futures = Lists.newArrayList(); - try { - // remove system topics first. - if (!topics.isEmpty()) { - for (String topic : topics) { - pulsar().getBrokerService().getTopicIfExists(topic).whenComplete((topicOptional, ex) -> { - topicOptional.ifPresent(systemTopic -> futures.add(systemTopic.deleteForcefully())); - }); + // remove system topics first. + if (!topics.isEmpty()) { + for (String topic : topics) { + try { + futures.add(pulsar().getAdminClient().topics().deleteAsync(topic, true, true)); + } catch (Exception ex) { + log.error("[{}] Failed to delete system topic {}", clientAppId(), topic, ex); + asyncResponse.resume(new RestException(Status.INTERNAL_SERVER_ERROR, ex)); + return; } } + } + FutureUtil.waitForAll(futures).thenCompose(__ -> { + List> deleteBundleFutures = Lists.newArrayList(); NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() - .getBundles(namespaceName); + .getBundles(namespaceName); for (NamespaceBundle bundle : bundles.getBundles()) { // check if the bundle is owned by any broker, if not then we do not need to delete the bundle - if (pulsar().getNamespaceService().getOwner(bundle).isPresent()) { - futures.add(pulsar().getAdminClient().namespaces() - .deleteNamespaceBundleAsync(namespaceName.toString(), bundle.getBundleRange())); - } - } - } catch (Exception e) { - log.error("[{}] Failed to remove owned namespace {}", clientAppId(), namespaceName, e); - asyncResponse.resume(new RestException(e)); - return; - } - - FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - if (exception.getCause() instanceof PulsarAdminException) { - asyncResponse.resume(new RestException((PulsarAdminException) exception.getCause())); - return null; - } else { - log.error("[{}] Failed to remove owned namespace {}", clientAppId(), namespaceName, exception); - asyncResponse.resume(new RestException(exception.getCause())); - return null; - } + deleteBundleFutures.add(pulsar().getNamespaceService().getOwnerAsync(bundle).thenCompose(ownership -> { + if (ownership.isPresent()) { + try { + return pulsar().getAdminClient().namespaces() + .deleteNamespaceBundleAsync(namespaceName.toString(), + bundle.getBundleRange()); + } catch (PulsarServerException e) { + throw new RestException(e); + } + } else { + return CompletableFuture.completedFuture(null); + } + })); } - - internalClearZkSources(asyncResponse); - + return FutureUtil.waitForAll(deleteBundleFutures); + }) + .thenCompose(__ -> internalClearZkSources()) + .thenAccept(__ -> { + log.info("[{}] Remove namespace successfully {}", clientAppId(), namespaceName); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + log.error("[{}] Failed to remove namespace {}", clientAppId(), namespaceName, ex.getCause()); + resumeAsyncResponseExceptionally(asyncResponse, ex.getCause()); return null; }); } // clear zk-node resources for deleting namespace - protected void internalClearZkSources(AsyncResponse asyncResponse) { + protected CompletableFuture internalClearZkSources() { // clear resource of `/namespace/{namespaceName}` for zk-node - namespaceResources().deleteNamespaceAsync(namespaceName) + return namespaceResources().deleteNamespaceAsync(namespaceName) .thenCompose(ignore -> namespaceResources().getPartitionedTopicResources() .clearPartitionedTopicMetadataAsync(namespaceName)) // clear resource for manager-ledger z-node @@ -319,18 +323,8 @@ protected void internalClearZkSources(AsyncResponse asyncResponse) { // z-node can be deleted now .thenCompose(ignore -> namespaceResources().deletePoliciesAsync(namespaceName)) // clear z-node of local policies - .thenCompose(ignore -> getLocalPolicies().deleteLocalPoliciesAsync(namespaceName)) - .whenComplete((ignore, ex) -> { - if (ex != null) { - log.warn("[{}] Failed to remove namespace or managed-ledger for {}", - clientAppId(), namespaceName, ex); - asyncResponse.resume(new RestException(ex)); - } else { - log.info("[{}] Remove namespace or managed-ledger successfully {}", - clientAppId(), namespaceName); - asyncResponse.resume(Response.noContent().build()); - } - }); + .thenCompose(ignore -> getLocalPolicies().deleteLocalPoliciesAsync(namespaceName)); + } @SuppressWarnings("deprecation") @@ -474,7 +468,7 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo return; } - FutureUtil.waitForAll(futures).handle((result, exception) -> { + FutureUtil.waitForAll(futures).thenCompose(__ -> internalClearZkSources()).handle((result, exception) -> { if (exception != null) { if (exception.getCause() instanceof PulsarAdminException) { asyncResponse.resume(new RestException((PulsarAdminException) exception.getCause())); @@ -487,7 +481,7 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo } } - internalClearZkSources(asyncResponse); + asyncResponse.resume(Response.noContent().build()); return null; }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 9a366a2cc7c1a..abefe449f6dcf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -771,7 +771,7 @@ public void testDeleteNamespaces() throws Exception { @Test public void testDeleteNamespaceWithBundles() throws Exception { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); - String bundledNsLocal = "test-bundled-namespace-1"; + String bundledNsLocal = "test-delete-namespace-with-bundles"; List boundaries = Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) @@ -796,8 +796,8 @@ public boolean matches(NamespaceBundle bundle) { return bundle.getNamespaceObject().equals(testNs); } })); - doReturn(Optional.of(mock(NamespaceEphemeralData.class))).when(nsSvc) - .getOwner(Mockito.argThat(new ArgumentMatcher() { + doReturn(CompletableFuture.completedFuture(Optional.of(mock(NamespaceEphemeralData.class)))).when(nsSvc) + .getOwnerAsync(Mockito.argThat(new ArgumentMatcher() { @Override public boolean matches(NamespaceBundle bundle) { return bundle.getNamespaceObject().equals(testNs); @@ -820,14 +820,14 @@ public boolean matches(NamespaceBundle bundle) { } catch (RestException re) { assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); } - + NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData); + doReturn(Optional.empty()).when(nsSvc).getWebServiceUrl(any(NamespaceBundle.class), any(LookupOptions.class)); AsyncResponse response = mock(AsyncResponse.class); namespaces.deleteNamespace(response, testTenant, testLocalCluster, bundledNsLocal, false, false); ArgumentCaptor captor = ArgumentCaptor.forClass(RestException.class); verify(response, timeout(5000).times(1)).resume(captor.capture()); assertEquals(captor.getValue().getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); - NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData); // make one bundle owned LookupOptions optionsHttps = LookupOptions.builder().authoritative(false).requestHttps(true).readOnly(false).build(); doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(nsBundles.getBundles().get(0), optionsHttps); @@ -844,17 +844,16 @@ public boolean matches(NamespaceBundle bundle) { } response = mock(AsyncResponse.class); - namespaces.deleteNamespace(response, testTenant, testLocalCluster, bundledNsLocal, false, false); - captor = ArgumentCaptor.forClass(RestException.class); - verify(response, timeout(5000).times(1)).resume(captor.capture()); - assertEquals(captor.getValue().getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); + doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(any(NamespaceBundle.class), any(LookupOptions.class)); // ensure all three bundles are owned by the local broker for (NamespaceBundle bundle : nsBundles.getBundles()) { - doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(bundle, optionsHttps); doReturn(true).when(nsSvc).isServiceUnitOwned(bundle); } - doNothing().when(namespacesAdmin).deleteNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + namespaces.deleteNamespace(response, testTenant, testLocalCluster, bundledNsLocal, false, false); + ArgumentCaptor captor2 = ArgumentCaptor.forClass(Response.class); + verify(response, timeout(5000).times(1)).resume(captor2.capture()); + assertEquals(captor2.getValue().getStatus(), Status.NO_CONTENT.getStatusCode()); } @Test From 65af6ce2aac7585cbbedb4e40edb52ff0e6c1ad7 Mon Sep 17 00:00:00 2001 From: ran Date: Mon, 14 Mar 2022 08:51:21 +0800 Subject: [PATCH 386/823] [Branch-2.9] Fix Broker HealthCheck Endpoint Exposes Race Conditions. (#14658) Original PR https://github.com/apache/pulsar/pull/14367 Because the PR https://github.com/apache/pulsar/pull/14367 is based on PR https://github.com/apache/pulsar/pull/14091, so I want to cherry-pick these two PRs to branch-2.9, the PR https://github.com/apache/pulsar/pull/13525 is also needed. --- Fix Issue: #14362 ### Motivation According to relative PR #7724, we will force delete all subscriptions when calling ``healthCheck`` REST API. but it has a race condition when two threads call this API. Please consider this case: > Thread A: Clean up all subscriptions, then create a reader. > Thread B: Prepare to force delete all subscriptions. So, in this case, the reader of thread A is deleted and then an NPE or other exception occurs. ### Modifications - Use ``Completable#handle`` to fix problem 1, the reader needs to be closed regardless of whether there is an exception. - Recheck the subscription after closing reading, and force deletion if it still exists after closing reading. - Added multi-threaded tests for health checks. --- .../apache/pulsar/broker/PulsarService.java | 3 + .../pulsar/broker/admin/impl/BrokersBase.java | 223 ++++++++++-------- .../nonpersistent/NonPersistentTopic.java | 5 + .../service/persistent/PersistentTopic.java | 5 + .../pulsar/broker/web/PulsarWebResource.java | 15 ++ .../broker/admin/AdminApiHealthCheckTest.java | 108 +++++++-- .../apache/pulsar/common/util/FutureUtil.java | 14 ++ 7 files changed, 256 insertions(+), 117 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index ba4178475f132..56d8b2b8ad907 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -272,6 +272,7 @@ public enum State { private volatile CompletableFuture closeFuture; // key is listener name , value is pulsar address and pulsar ssl address private Map advertisedListeners; + private NamespaceName heartbeatNamespaceV2; public PulsarService(ServiceConfiguration config) { this(config, Optional.empty(), (exitCode) -> { @@ -683,6 +684,7 @@ config, localMetadataStore, getZkClient(), this.addWebServerHandlers(webService, metricsServlet, this.config); this.webService.start(); + heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(this.advertisedAddress, this.config); // Refresh addresses and update configuration, since the port might have been dynamically assigned if (config.getBrokerServicePort().equals(Optional.of(0))) { @@ -696,6 +698,7 @@ config, localMetadataStore, getZkClient(), this.brokerServiceUrl = brokerUrl(config); this.brokerServiceUrlTls = brokerUrlTls(config); + if (null != this.webSocketService) { ClusterDataImpl clusterData = ClusterDataImpl.builder() .serviceUrl(webServiceAddress) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index bb9453e09bfdd..cb579ecc44c40 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -24,9 +24,11 @@ import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -47,9 +49,10 @@ import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -294,123 +297,139 @@ public void isReady(@Suspended AsyncResponse asyncResponse) { @GET @Path("/health") - @ApiOperation(value = "Run a healthcheck against the broker") + @ApiOperation(value = "Run a healthCheck against the broker") @ApiResponses(value = { @ApiResponse(code = 200, message = "Everything is OK"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Cluster doesn't exist"), @ApiResponse(code = 500, message = "Internal server error")}) @ApiParam(value = "Topic Version") - public void healthcheck(@Suspended AsyncResponse asyncResponse, - @QueryParam("topicVersion") TopicVersion topicVersion) throws Exception { - String topic; - PulsarClient client; - try { - validateSuperUserAccess(); - NamespaceName heartbeatNamespace = (topicVersion == TopicVersion.V2) - ? - NamespaceService.getHeartbeatNamespaceV2( - pulsar().getAdvertisedAddress(), - pulsar().getConfiguration()) - : - NamespaceService.getHeartbeatNamespace( - pulsar().getAdvertisedAddress(), - pulsar().getConfiguration()); - - - topic = String.format("persistent://%s/%s", heartbeatNamespace, HEALTH_CHECK_TOPIC_SUFFIX); - - LOG.info("Running healthCheck with topic={}", topic); - - client = pulsar().getClient(); - } catch (Exception e) { - LOG.error("Error getting heathcheck topic info", e); - throw new PulsarServerException(e); - } + public void healthCheck(@Suspended AsyncResponse asyncResponse, + @QueryParam("topicVersion") TopicVersion topicVersion) { + validateSuperUserAccess(); + internalRunHealthCheck(topicVersion) + .thenAccept(__ -> { + LOG.info("[{}] Successfully run health check.", clientAppId()); + asyncResponse.resume("ok"); + }).exceptionally(ex -> { + LOG.error("[{}] Fail to run health check.", clientAppId(), ex); + return handleCommonRestAsyncException(asyncResponse, ex); + }); + } - String messageStr = UUID.randomUUID().toString(); + private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion) { + NamespaceName namespaceName = (topicVersion == TopicVersion.V2) + ? NamespaceService.getHeartbeatNamespaceV2(pulsar().getAdvertisedAddress(), pulsar().getConfiguration()) + : NamespaceService.getHeartbeatNamespace(pulsar().getAdvertisedAddress(), pulsar().getConfiguration()); + final String topicName = String.format("persistent://%s/%s", namespaceName, HEALTH_CHECK_TOPIC_SUFFIX); + LOG.info("[{}] Running healthCheck with topic={}", clientAppId(), topicName); + final String messageStr = UUID.randomUUID().toString(); + final String subscriptionName = "healthCheck-" + messageStr; // create non-partitioned topic manually and close the previous reader if present. - try { - pulsar().getBrokerService().getTopic(topic, true).get().ifPresent(t -> { - t.getSubscriptions().forEach((__, value) -> { - try { - value.deleteForcefully(); - } catch (Exception e) { - LOG.warn("Failed to delete previous subscription {} for health check", value.getName(), e); - } - }); + return pulsar().getBrokerService().getTopic(topicName, true) + .thenCompose(topicOptional -> { + if (!topicOptional.isPresent()) { + LOG.error("[{}] Fail to run health check while get topic {}. because get null value.", + clientAppId(), topicName); + throw new RestException(Status.NOT_FOUND, + String.format("Topic [%s] not found after create.", topicName)); + } + PulsarClient client; + try { + client = pulsar().getClient(); + } catch (PulsarServerException e) { + LOG.error("[{}] Fail to run health check while get client.", clientAppId()); + throw new RestException(e); + } + CompletableFuture resultFuture = new CompletableFuture<>(); + client.newProducer(Schema.STRING).topic(topicName).createAsync() + .thenCompose(producer -> client.newReader(Schema.STRING).topic(topicName) + .subscriptionName(subscriptionName) + .startMessageId(MessageId.latest) + .createAsync().exceptionally(createException -> { + producer.closeAsync().exceptionally(ex -> { + LOG.error("[{}] Close producer fail while heath check.", clientAppId()); + return null; + }); + throw FutureUtil.wrapToCompletionException(createException); + }).thenCompose(reader -> producer.sendAsync(messageStr) + .thenCompose(__ -> healthCheckRecursiveReadNext(reader, messageStr)) + .whenComplete((__, ex) -> { + closeAndReCheck(producer, reader, topicOptional.get(), subscriptionName) + .whenComplete((unused, innerEx) -> { + if (ex != null) { + resultFuture.completeExceptionally(ex); + } else { + resultFuture.complete(null); + } + }); + } + )) + ).exceptionally(ex -> { + resultFuture.completeExceptionally(ex); + return null; + }); + return resultFuture; }); - } catch (Exception e) { - LOG.warn("Failed to try to delete subscriptions for health check", e); - } - - CompletableFuture> producerFuture = - client.newProducer(Schema.STRING).topic(topic).createAsync(); - CompletableFuture> readerFuture = client.newReader(Schema.STRING) - .topic(topic).startMessageId(MessageId.latest).createAsync(); - - CompletableFuture completePromise = new CompletableFuture<>(); + } - CompletableFuture.allOf(producerFuture, readerFuture).whenComplete( - (ignore, exception) -> { - if (exception != null) { - completePromise.completeExceptionally(exception); + /** + * Close producer and reader and then to re-check if this operation is success. + * + * Re-check + * - Producer: If close fails we will print error log to notify user. + * - Consumer: If close fails we will force delete subscription. + * + * @param producer Producer + * @param reader Reader + * @param topic Topic + * @param subscriptionName Subscription name + */ + private CompletableFuture closeAndReCheck(Producer producer, Reader reader, + Topic topic, String subscriptionName) { + // no matter exception or success, we still need to + // close producer/reader + CompletableFuture producerFuture = producer.closeAsync(); + CompletableFuture readerFuture = reader.closeAsync(); + List> futures = new ArrayList<>(2); + futures.add(producerFuture); + futures.add(readerFuture); + return FutureUtil.waitForAll(Collections.unmodifiableList(futures)) + .exceptionally(closeException -> { + if (readerFuture.isCompletedExceptionally()) { + LOG.error("[{}] Close reader fail while heath check.", clientAppId()); + Subscription subscription = + topic.getSubscription(subscriptionName); + // re-check subscription after reader close + if (subscription != null) { + LOG.warn("[{}] Force delete subscription {} " + + "when it still exists after the" + + " reader is closed.", + clientAppId(), subscription); + subscription.deleteForcefully() + .exceptionally(ex -> { + LOG.error("[{}] Force delete subscription fail" + + " while health check", + clientAppId(), ex); + return null; + }); + } } else { - producerFuture.thenCompose((producer) -> producer.sendAsync(messageStr)) - .whenComplete((ignore2, exception2) -> { - if (exception2 != null) { - completePromise.completeExceptionally(exception2); - } - }); - - healthcheckReadLoop(readerFuture, completePromise, messageStr); - - // timeout read loop after 10 seconds - FutureUtil.addTimeoutHandling(completePromise, - HEALTHCHECK_READ_TIMEOUT, pulsar().getExecutor(), - () -> FutureUtil.createTimeoutException("Timed out reading", getClass(), - "healthcheck(...)")); + // producer future fail. + LOG.error("[{}] Close producer fail while heath check.", clientAppId()); } + return null; }); + } - completePromise.whenComplete((ignore, exception) -> { - producerFuture.thenAccept((producer) -> { - producer.closeAsync().whenComplete((ignore2, exception2) -> { - if (exception2 != null) { - LOG.warn("Error closing producer for healthcheck", exception2); + private CompletableFuture healthCheckRecursiveReadNext(Reader reader, String content) { + return reader.readNextAsync() + .thenCompose(msg -> { + if (!Objects.equals(content, msg.getValue())) { + return healthCheckRecursiveReadNext(reader, content); } + return CompletableFuture.completedFuture(null); }); - }); - readerFuture.thenAccept((reader) -> { - reader.closeAsync().whenComplete((ignore2, exception2) -> { - if (exception2 != null) { - LOG.warn("Error closing reader for healthcheck", exception2); - } - }); - }); - if (exception != null) { - asyncResponse.resume(new RestException(exception)); - } else { - asyncResponse.resume("ok"); - } - }); - } - - private void healthcheckReadLoop(CompletableFuture> readerFuture, - CompletableFuture completablePromise, - String messageStr) { - readerFuture.thenAccept((reader) -> { - CompletableFuture> readFuture = reader.readNextAsync() - .whenComplete((m, exception) -> { - if (exception != null) { - completablePromise.completeExceptionally(exception); - } else if (m.getValue().equals(messageStr)) { - completablePromise.complete(null); - } else { - healthcheckReadLoop(readerFuture, completablePromise, messageStr); - } - }); - }); } private synchronized void deleteDynamicConfigurationOnZk(String configName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 808b403b655ac..4a5aa1f584a01 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -66,6 +66,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.KeySharedMeta; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; @@ -497,6 +498,10 @@ public CompletableFuture checkReplication() { if (!name.isGlobal()) { return CompletableFuture.completedFuture(null); } + NamespaceName heartbeatNamespace = brokerService.pulsar().getHeartbeatNamespaceV2(); + if (name.getNamespaceObject().equals(heartbeatNamespace)) { + return CompletableFuture.completedFuture(null); + } if (log.isDebugEnabled()) { log.debug("[{}] Checking replication status", name); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index e96fdc4dd8fcb..e6cf77fde3466 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -126,6 +126,7 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.events.EventsTopicNames; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; @@ -1354,6 +1355,10 @@ public CompletableFuture checkReplication() { if (!name.isGlobal()) { return CompletableFuture.completedFuture(null); } + NamespaceName heartbeatNamespace = brokerService.pulsar().getHeartbeatNamespaceV2(); + if (name.getNamespaceObject().equals(heartbeatNamespace)) { + return CompletableFuture.completedFuture(null); + } if (log.isDebugEnabled()) { log.debug("[{}] Checking replication status", name); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 8b6f30b464967..e3ae827ed5a13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -38,6 +38,7 @@ import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.WebApplicationException; +import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; @@ -709,6 +710,11 @@ public static CompletableFuture checkLocalOrGetPeerReplicationC if (!namespace.isGlobal()) { return CompletableFuture.completedFuture(null); } + NamespaceName heartbeatNamespace = pulsarService.getHeartbeatNamespaceV2(); + if (namespace.equals(heartbeatNamespace)) { + return CompletableFuture.completedFuture(null); + } + final CompletableFuture validationFuture = new CompletableFuture<>(); final String localCluster = pulsarService.getConfiguration().getClusterName(); @@ -1055,4 +1061,13 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) { } } + protected Void handleCommonRestAsyncException(AsyncResponse asyncResponse, Throwable ex) { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (realCause instanceof WebApplicationException) { + asyncResponse.resume(realCause); + } else { + asyncResponse.resume(new RestException(realCause)); + } + return null; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java index e0c887f3e3dfc..b4b97cad340a5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java @@ -25,11 +25,15 @@ import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.compaction.Compactor; +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.springframework.util.CollectionUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; - -import java.net.URL; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; @Test(groups = "broker") @Slf4j @@ -58,26 +62,100 @@ public void cleanup() throws Exception { @Test public void testHealthCheckup() throws Exception { - admin.brokers().healthcheck(); + final int times = 30; + CompletableFuture future = new CompletableFuture<>(); + pulsar.getExecutor().execute(() -> { + try { + for (int i = 0; i < times; i++) { + admin.brokers().healthcheck(); + } + future.complete(null); + }catch (PulsarAdminException e) { + future.completeExceptionally(e); + } + }); + for (int i = 0; i < times; i++) { + admin.brokers().healthcheck(); + } + // To ensure we don't have any subscription + final String testHealthCheckTopic = String.format("persistent://pulsar/test/localhost:%s/healthcheck", + pulsar.getConfig().getWebServicePort().get()); + Awaitility.await().untilAsserted(() -> { + Assert.assertFalse(future.isCompletedExceptionally()); + }); + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(CollectionUtils.isEmpty(admin.topics() + .getSubscriptions(testHealthCheckTopic).stream() + // All system topics are using compaction, even though is not explicitly set in the policies. + .filter(v -> !v.equals(Compactor.COMPACTION_SUBSCRIPTION)) + .collect(Collectors.toList()) + )) + ); } @Test public void testHealthCheckupV1() throws Exception { - admin.brokers().healthcheck(TopicVersion.V1); - } - - @Test(expectedExceptions = PulsarAdminException.class) - public void testHealthCheckupV2Error() throws Exception { - admin.brokers().healthcheck(TopicVersion.V2); + final int times = 30; + CompletableFuture future = new CompletableFuture<>(); + pulsar.getExecutor().execute(() -> { + try { + for (int i = 0; i < times; i++) { + admin.brokers().healthcheck(TopicVersion.V1); + } + future.complete(null); + }catch (PulsarAdminException e) { + future.completeExceptionally(e); + } + }); + for (int i = 0; i < times; i++) { + admin.brokers().healthcheck(TopicVersion.V1); + } + final String testHealthCheckTopic = String.format("persistent://pulsar/test/localhost:%s/healthcheck", + pulsar.getConfig().getWebServicePort().get()); + Awaitility.await().untilAsserted(() -> { + Assert.assertFalse(future.isCompletedExceptionally()); + }); + // To ensure we don't have any subscription + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(CollectionUtils.isEmpty(admin.topics() + .getSubscriptions(testHealthCheckTopic).stream() + // All system topics are using compaction, even though is not explicitly set in the policies. + .filter(v -> !v.equals(Compactor.COMPACTION_SUBSCRIPTION)) + .collect(Collectors.toList()) + )) + ); } @Test public void testHealthCheckupV2() throws Exception { - final URL pulsarWebAddress = new URL(pulsar.getWebServiceAddress()); - final String targetNameSpace = "pulsar/" + - pulsarWebAddress.getHost() + ":" + pulsarWebAddress.getPort(); - log.info("Target namespace for broker admin healthcheck V2 endpoint is {}", targetNameSpace); - admin.namespaces().createNamespace(targetNameSpace); - admin.brokers().healthcheck(TopicVersion.V2); + final int times = 30; + CompletableFuture future = new CompletableFuture<>(); + pulsar.getExecutor().execute(() -> { + try { + for (int i = 0; i < times; i++) { + admin.brokers().healthcheck(TopicVersion.V2); + } + future.complete(null); + }catch (PulsarAdminException e) { + future.completeExceptionally(e); + } + }); + for (int i = 0; i < times; i++) { + admin.brokers().healthcheck(TopicVersion.V2); + } + final String testHealthCheckTopic = String.format("persistent://pulsar/localhost:%s/healthcheck", + pulsar.getConfig().getWebServicePort().get()); + Awaitility.await().untilAsserted(() -> { + Assert.assertFalse(future.isCompletedExceptionally()); + }); + // To ensure we don't have any subscription + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(CollectionUtils.isEmpty(admin.topics() + .getSubscriptions(testHealthCheckTopic).stream() + // All system topics are using compaction, even though is not explicitly set in the policies. + .filter(v -> !v.equals(Compactor.COMPACTION_SUBSCRIPTION)) + .collect(Collectors.toList()) + )) + ); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index 3d2fcae918f24..2cdd9fce99508 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -187,4 +187,18 @@ public static Optional getException(CompletableFuture future) } return Optional.empty(); } + + /** + * Wrap throwable exception to CompletionException if that exception is not an instance of CompletionException. + * + * @param throwable Exception + * @return CompletionException + */ + public static CompletionException wrapToCompletionException(Throwable throwable) { + if (throwable instanceof CompletionException) { + return (CompletionException) throwable; + } else { + return new CompletionException(throwable); + } + } } From 5258f1255bd0f4511422d01b9742fc7836411552 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Mon, 14 Mar 2022 16:51:54 +0800 Subject: [PATCH 387/823] [Transaction] Fix cursor readPosition is bigger than maxPosition in OpReadEntry (#14667) Fix cursor read op dead loop. 1. in OpReadEntry we can't use cursor readPosition, because it is not the OpReadEntry readPosition, it is cursor readPosition when one ledger is empty the OpReadEntry readPosition is not equals cursor readPosition, we should use OpReadEntry readPosition to judge the OpReadEntry can be finished. 2. when readPosition is bigger than maxPosition in OpReadEntry, we should complete this OpReadEntry add test (cherry picked from commit a242f03f69791fcfa1934b6cab20689393228381) --- .../bookkeeper/mledger/impl/OpReadEntry.java | 5 ++- .../mledger/impl/ManagedCursorTest.java | 43 ++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java index 91a6e26f567d0..b9c82914a76b9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java @@ -134,8 +134,9 @@ void updateReadPosition(Position newReadPosition) { } void checkReadCompletion() { - if (entries.size() < count && cursor.hasMoreEntries() && - ((PositionImpl) cursor.getReadPosition()).compareTo(maxPosition) < 0) { + // op readPosition is smaller or equals maxPosition then can read again + if (entries.size() < count && cursor.hasMoreEntries() + && maxPosition.compareTo(readPosition) > 0) { // We still have more entries to read from the next ledger, schedule a new async operation if (nextReadPosition.getLedgerId() != readPosition.getLedgerId()) { cursor.ledger.startReadOperationOnLedger(nextReadPosition, OpReadEntry.this); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 892d6b39a9bef..09f11f2c16b96 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -30,7 +30,6 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.collect.Range; @@ -59,7 +58,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; - import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; import lombok.Cleanup; @@ -3710,5 +3708,46 @@ public void testCursorNoRolloverIfNoMetadataSession() throws Exception { assertNotEquals(cursor.getCursorLedger(), initialLedgerId); } + @Test + public void testReadEmptyEntryList() throws Exception { + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setMaxEntriesPerLedger(1); + managedLedgerConfig.setMetadataMaxEntriesPerLedger(1); + managedLedgerConfig.setMinimumRolloverTime(0, TimeUnit.MILLISECONDS); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory + .open("testReadEmptyEntryList", managedLedgerConfig); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("test"); + + PositionImpl lastPosition = (PositionImpl) ledger.addEntry("test".getBytes(Encoding)); + ledger.rollCurrentLedgerIfFull(); + + AtomicBoolean flag = new AtomicBoolean(); + flag.set(false); + ReadEntriesCallback callback = new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + if (entries.size() == 0) { + flag.set(true); + } + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + + } + }; + + // op readPosition is bigger than maxReadPosition + OpReadEntry opReadEntry = OpReadEntry.create(cursor, ledger.lastConfirmedEntry, 10, callback, + null, PositionImpl.get(lastPosition.getLedgerId(), -1)); + Field field = ManagedCursorImpl.class.getDeclaredField("readPosition"); + field.setAccessible(true); + field.set(cursor, PositionImpl.earliest); + ledger.asyncReadEntries(opReadEntry); + + // when readPosition is bigger than maxReadPosition, should complete the opReadEntry + Awaitility.await().untilAsserted(() -> assertTrue(flag.get())); + } + private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class); } From a3f52891593e093c27b583094a1fbfd09bbbae1a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 12 Mar 2022 14:24:38 +0800 Subject: [PATCH 388/823] Fix lost message issue due to ledger rollover. (#14664) (cherry picked from commit ad2cc2d38280b7dd0f056ee981ec8d3b157e3526) --- .../mledger/impl/ManagedLedgerImpl.java | 10 ++--- .../mledger/impl/ManagedCursorTest.java | 3 ++ .../mledger/impl/ManagedLedgerTest.java | 37 +++++++++++++++++-- .../CurrentLedgerRolloverIfFullTest.java | 4 ++ .../MLTransactionMetadataStoreTest.java | 5 ++- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index c07b3ce1ed6e0..59b04fec8f81c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -762,8 +762,8 @@ private synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { } } else if (state == State.ClosedLedger) { // No ledger and no pending operations. Create a new ledger - log.info("[{}] Creating a new ledger", name); if (STATE_UPDATER.compareAndSet(this, State.ClosedLedger, State.CreatingLedger)) { + log.info("[{}] Creating a new ledger", name); this.lastLedgerCreationInitiationTimestamp = System.currentTimeMillis(); mbean.startDataLedgerCreateOp(); asyncCreateLedger(bookKeeper, config, digestType, this, Collections.emptyMap()); @@ -1588,8 +1588,8 @@ synchronized void ledgerClosed(final LedgerHandle lh) { } synchronized void createLedgerAfterClosed() { - if(isNeededCreateNewLedgerAfterCloseLedger()) { - log.info("[{}] Creating a new ledger", name); + if (isNeededCreateNewLedgerAfterCloseLedger()) { + log.info("[{}] Creating a new ledger after closed", name); STATE_UPDATER.set(this, State.CreatingLedger); this.lastLedgerCreationInitiationTimestamp = System.currentTimeMillis(); mbean.startDataLedgerCreateOp(); @@ -1612,8 +1612,8 @@ boolean isNeededCreateNewLedgerAfterCloseLedger() { @Override public void rollCurrentLedgerIfFull() { log.info("[{}] Start checking if current ledger is full", name); - if (currentLedgerEntries > 0 && currentLedgerIsFull()) { - STATE_UPDATER.set(this, State.ClosingLedger); + if (currentLedgerEntries > 0 && currentLedgerIsFull() + && STATE_UPDATER.compareAndSet(this, State.LedgerOpened, State.ClosingLedger)) { currentLedger.asyncClose(new AsyncCallback.CloseCallback() { @Override public void closeComplete(int rc, LedgerHandle lh, Object o) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 09f11f2c16b96..074a55db50091 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -2238,6 +2238,9 @@ void testFindNewestMatchingAfterLedgerRollover() throws Exception { // roll a new ledger int numLedgersBefore = ledger.getLedgersInfo().size(); ledger.getConfig().setMaxEntriesPerLedger(1); + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(ledger, ManagedLedgerImpl.State.LedgerOpened); ledger.rollCurrentLedgerIfFull(); Awaitility.await().atMost(20, TimeUnit.SECONDS) .until(() -> ledger.getLedgersInfo().size() > numLedgersBefore); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index d837651f1fac5..648d3e671d862 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -1936,6 +1936,9 @@ public void testDeletionAfterLedgerClosedAndRetention() throws Exception { c1.skipEntries(1, IndividualDeletedEntries.Exclude); c2.skipEntries(1, IndividualDeletedEntries.Exclude); // let current ledger close + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(ml, ManagedLedgerImpl.State.LedgerOpened); ml.rollCurrentLedgerIfFull(); // let retention expire Thread.sleep(1500); @@ -2205,6 +2208,9 @@ public void testGetPositionAfterN() throws Exception { managedCursor.markDelete(positionMarkDelete); //trigger ledger rollover and wait for the new ledger created + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(managedLedger, ManagedLedgerImpl.State.LedgerOpened); managedLedger.rollCurrentLedgerIfFull(); Awaitility.await().untilAsserted(() -> assertEquals(managedLedger.getLedgersInfo().size(), 3)); assertEquals(5, managedLedger.getLedgersInfoAsList().get(0).getEntries()); @@ -3063,7 +3069,7 @@ public void testManagedLedgerRollOverIfFull() throws Exception { ledger.addEntry(new byte[1024 * 1024]); } - Assert.assertEquals(ledger.getLedgersInfoAsList().size(), msgNum / 2); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getLedgersInfoAsList().size(), msgNum / 2)); List entries = cursor.readEntries(msgNum); Assert.assertEquals(msgNum, entries.size()); @@ -3074,9 +3080,12 @@ public void testManagedLedgerRollOverIfFull() throws Exception { // all the messages have benn acknowledged // and all the ledgers have been removed except the last ledger - Thread.sleep(1000); - Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1); - Assert.assertEquals(ledger.getTotalSize(), 0); + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(ledger, ManagedLedgerImpl.State.LedgerOpened); + ledger.rollCurrentLedgerIfFull(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1)); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getTotalSize(), 0)); } @Test @@ -3094,6 +3103,26 @@ public void testLedgerReachMaximumRolloverTime() throws Exception { .until(() -> firstLedgerId != ml.addEntry("test".getBytes()).getLedgerId()); } + @Test + public void testLedgerNotRolloverWithoutOpenState() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(2); + + ManagedLedgerImpl ml = spy((ManagedLedgerImpl)factory.open("ledger-not-rollover-without-open-state", config)); + ml.addEntry("test1".getBytes()).getLedgerId(); + long ledgerId2 = ml.addEntry("test2".getBytes()).getLedgerId(); + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + // Set state to CreatingLedger to avoid rollover + stateUpdater.set(ml, ManagedLedgerImpl.State.CreatingLedger); + ml.rollCurrentLedgerIfFull(); + Field currentLedger = ManagedLedgerImpl.class.getDeclaredField("currentLedger"); + currentLedger.setAccessible(true); + LedgerHandle lh = (LedgerHandle) currentLedger.get(ml); + Awaitility.await() + .until(() -> ledgerId2 == lh.getId()); + } + @Test public void testExpiredLedgerDeletionAfterManagedLedgerRestart() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java index 77ec229862e90..b05abf3be5218 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import java.lang.reflect.Field; import java.time.Duration; import java.util.concurrent.TimeUnit; import lombok.Cleanup; @@ -98,6 +99,9 @@ public void testCurrentLedgerRolloverIfFull() throws Exception { }); // trigger a ledger rollover + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(managedLedger, ManagedLedgerImpl.State.LedgerOpened); managedLedger.rollCurrentLedgerIfFull(); // the last ledger will be closed and removed and we have one ledger for empty diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java index 7fa3c08300162..a06bf9e6deaae 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java @@ -164,8 +164,11 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) field.get(mlTransactionLog); Position position = managedLedger.getLastConfirmedEntry(); if (isUseManagedLedgerProperties) { + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(managedLedger, ManagedLedgerImpl.State.LedgerOpened); + managedLedger.rollCurrentLedgerIfFull(); Awaitility.await().until(() -> { - managedLedger.rollCurrentLedgerIfFull(); return !managedLedger.ledgerExists(position.getLedgerId()); }); } From 131b455adae0225377293c57a20f79cf473a330e Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 17 Mar 2022 16:34:38 -0500 Subject: [PATCH 389/823] [C++ Client] Add braces around initialization of subobject (#14735) (cherry picked from commit 3ff05b2c5084c73016dceeaee1235b4dbb837f6f) --- pulsar-client-cpp/lib/stats/ProducerStatsImpl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-cpp/lib/stats/ProducerStatsImpl.cc b/pulsar-client-cpp/lib/stats/ProducerStatsImpl.cc index af7ae4b9c0440..a3608e5701720 100644 --- a/pulsar-client-cpp/lib/stats/ProducerStatsImpl.cc +++ b/pulsar-client-cpp/lib/stats/ProducerStatsImpl.cc @@ -26,7 +26,7 @@ namespace pulsar { DECLARE_LOG_OBJECT(); -static const std::array probs = {0.5, 0.9, 0.99, 0.999}; +static const std::array probs = {{0.5, 0.9, 0.99, 0.999}}; std::string ProducerStatsImpl::latencyToString(const LatencyAccumulator& obj) { boost::accumulators::detail::extractor_result< From f9b020305a2d058793b351a41085276d4a110434 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 2 Mar 2022 15:00:42 +0200 Subject: [PATCH 390/823] [Functions] Pass configured metricsPort to k8s runtime (#14502) (cherry picked from commit daed6a0b3ac94e77a3d7d4212ee297b9046317a2) --- conf/functions_worker.yml | 1 + .../runtime/kubernetes/KubernetesRuntime.java | 18 +++++-- .../kubernetes/KubernetesRuntimeFactory.java | 7 ++- .../kubernetes/KubernetesRuntimeTest.java | 52 +++++++++++++++++++ .../worker/FunctionsStatsGenerator.java | 5 +- 5 files changed, 76 insertions(+), 7 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 9ca5f7bc923e5..b235d600756f2 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -206,6 +206,7 @@ functionRuntimeFactoryConfigs: # # The port inside the function pod which is used by the worker to communicate with the pod # grpcPort: 9093 # # The port inside the function pod on which prometheus metrics are exposed +# # An empty value disables prometheus metrics. # metricsPort: 9094 # # The directory inside the function pod where nar packages will be extracted # narExtractionDirectory: diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 70839fc7bda06..c443c4b2f5419 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -427,7 +427,11 @@ public void onSuccess(InstanceCommunication.MetricsData t) { @Override public String getPrometheusMetrics() throws IOException { - return RuntimeUtils.getPrometheusMetrics(metricsPort); + if (metricsPort != null) { + return RuntimeUtils.getPrometheusMetrics(metricsPort); + } else { + return null; + } } @Override @@ -975,10 +979,14 @@ V1StatefulSet createStatefulSet() { } private Map getPrometheusAnnotations() { - final Map annotations = new HashMap<>(); - annotations.put("prometheus.io/scrape", "true"); - annotations.put("prometheus.io/port", String.valueOf(metricsPort)); - return annotations; + if (metricsPort != null) { + final Map annotations = new HashMap<>(); + annotations.put("prometheus.io/scrape", "true"); + annotations.put("prometheus.io/port", String.valueOf(metricsPort)); + return annotations; + } else { + return Collections.emptyMap(); + } } private Map getLabels(Function.FunctionDetails functionDetails) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index d98a161c2e02e..4b2c7e723259c 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -290,6 +290,11 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c String overriddenNamespace = manifestCustomizer.map((customizer) -> customizer.customizeNamespace(instanceConfig.getFunctionDetails(), jobNamespace)).orElse(jobNamespace); String overriddenName = manifestCustomizer.map((customizer) -> customizer.customizeName(instanceConfig.getFunctionDetails(), jobName)).orElse(jobName); + // pass metricsPort configured in functionRuntimeFactoryConfigs.metricsPort in functions_worker.yml + if (metricsPort != null) { + instanceConfig.setMetricsPort(metricsPort); + } + return new KubernetesRuntime( appsClient, coreClient, @@ -351,7 +356,7 @@ public void setupClient() throws Exception { if (k8Uri == null) { log.info("k8Uri is null thus going by defaults"); ApiClient cli; - if (submittingInsidePod) { + if (submittingInsidePod != null && submittingInsidePod) { log.info("Looks like we are inside a k8 pod ourselves. Initializing as cluster"); cli = Config.fromCluster(); } else { diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index b295cf8a72a80..be9c52cb33ffb 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -29,12 +29,14 @@ import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1PodSpec; +import io.kubernetes.client.openapi.models.V1PodTemplateSpec; import io.kubernetes.client.openapi.models.V1ResourceRequirements; import io.kubernetes.client.openapi.models.V1Service; import io.kubernetes.client.openapi.models.V1StatefulSet; import io.kubernetes.client.openapi.models.V1Toleration; import org.apache.commons.lang.StringUtils; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.ConsumerSpec; @@ -42,6 +44,7 @@ import org.apache.pulsar.functions.runtime.RuntimeCustomizer; import org.apache.pulsar.functions.runtime.thread.ThreadRuntime; import org.apache.pulsar.functions.secretsprovider.ClearTextSecretsProvider; +import org.apache.pulsar.functions.secretsproviderconfigurator.DefaultSecretsProviderConfigurator; import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.worker.ConnectorsManager; @@ -66,6 +69,7 @@ import static org.powermock.api.mockito.PowerMockito.doNothing; import static org.powermock.api.mockito.PowerMockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -1145,4 +1149,52 @@ public void testCustomKubernetesDownloadCommandsWithDownloadDirectoryDefined() t String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2); assertTrue(containerCommand.contains(expectedDownloadCommand)); } + + @Test + public void shouldUseConfiguredMetricsPort() throws Exception { + assertMetricsPortConfigured(Collections.singletonMap("metricsPort", 12345), 12345); + } + + @Test + public void shouldUseDefaultMetricsPortWhenMetricsPortIsntSet() throws Exception { + assertMetricsPortConfigured(Collections.emptyMap(), 9094); + } + + @Test + public void shouldNotAddPrometheusAnnotationIfMetricsPortIsSetToEmpty() throws Exception { + assertMetricsPortConfigured(Collections.singletonMap("metricsPort", ""), -1); + } + + private void assertMetricsPortConfigured(Map functionRuntimeFactoryConfigs, + int expectedPort) throws Exception { + KubernetesRuntimeFactory kubernetesRuntimeFactory = new KubernetesRuntimeFactory(); + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setFunctionRuntimeFactoryClassName(KubernetesRuntimeFactory.class.getName()); + workerConfig.setFunctionRuntimeFactoryConfigs(functionRuntimeFactoryConfigs); + AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build(); + kubernetesRuntimeFactory.initialize(workerConfig, authenticationConfig, new DefaultSecretsProviderConfigurator(), Mockito.mock(ConnectorsManager.class), Optional.empty(), Optional.empty()); + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true); + KubernetesRuntime container = kubernetesRuntimeFactory.createContainer(config, userJarFile, userJarFile, 30l); + V1PodTemplateSpec template = container.createStatefulSet().getSpec().getTemplate(); + Map annotations = + template.getMetadata().getAnnotations(); + if (expectedPort != -1) { + // metrics port should be passed to k8s annotation for prometheus scraping + assertEquals(annotations.get("prometheus.io/port"), String.valueOf(expectedPort)); + // scraping annotation should exist + assertEquals(annotations.get("prometheus.io/scrape"), "true"); + + // metrics port should be passed to JavaInstanceStarter with --metrics_port argument + assertTrue(container.getProcessArgs().stream().collect(Collectors.joining(" ")) + .contains("--metrics_port " + expectedPort)); + } else { + // No prometheus annotations should exist + assertFalse(annotations.containsKey("prometheus.io/scrape")); + assertFalse(annotations.containsKey("prometheus.io/port")); + // metrics will be started on random port when the port isn't specified + // check that "--metrics_port 0" argument is passed + assertTrue(container.getProcessArgs().stream().collect(Collectors.joining(" ")) + .contains("--metrics_port 0")); + } + } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java index 7dd1e74b8e5bc..d94f54c253e0e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java @@ -68,7 +68,10 @@ public static void generate(PulsarWorkerService workerService, SimpleTextOutputS if (functionRuntime != null) { try { - out.write(functionRuntime.getPrometheusMetrics()); + String prometheusMetrics = functionRuntime.getPrometheusMetrics(); + if (prometheusMetrics != null) { + out.write(prometheusMetrics); + } } catch (IOException e) { log.warn("Failed to collect metrics for function instance {}", From aafba1ac18dc729b10de62eca0e2df91025e5bce Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 12 Mar 2022 11:11:33 +0800 Subject: [PATCH 391/823] Fix inconsistent prompt message when schema version is empty using AVRO. (#14626) (cherry picked from commit 190e5dbccda455a84ea7fdf491dac52cc50fbbdf) --- .../pulsar/broker/service/ServerCnx.java | 5 ++ .../org/apache/pulsar/schema/SchemaTest.java | 76 ++++++++++++++++++- .../client/impl/BinaryProtoLookupService.java | 8 +- .../pulsar/client/impl/HttpLookupService.java | 5 ++ 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 2ba51d2be342c..c18d525c8caf0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1923,6 +1923,11 @@ remoteAddress, new String(commandGetSchema.getSchemaVersion()), long requestId = commandGetSchema.getRequestId(); SchemaVersion schemaVersion = SchemaVersion.Latest; if (commandGetSchema.hasSchemaVersion()) { + if (commandGetSchema.getSchemaVersion().length == 0) { + commandSender.sendGetSchemaErrorResponse(requestId, ServerError.IncompatibleSchema, + "Empty schema version"); + return; + } schemaVersion = schemaService.versionFromBytes(commandGetSchema.getSchemaVersion()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index ab78988be94ea..c45888f3858e0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -29,12 +29,13 @@ import static org.testng.Assert.fail; import static org.testng.internal.junit.ArrayAsserts.assertArrayEquals; +import com.google.common.base.Throwables; +import lombok.EqualsAndHashCode; import org.apache.avro.Schema.Parser; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Sets; - import java.io.ByteArrayInputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; @@ -44,7 +45,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; - import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; @@ -59,7 +59,9 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaDefinition; @@ -1064,4 +1066,72 @@ private void checkSchemaForAutoSchema(Message message) { } } + @Test + public void testAvroSchemaWithHttpLookup() throws Exception { + stopBroker(); + isTcpLookup = false; + setup(); + testEmptySchema(); + } + + @Test + public void testAvroSchemaWithTcpLookup() throws Exception { + stopBroker(); + isTcpLookup = true; + setup(); + testEmptySchema(); + } + + private void testEmptySchema() throws Exception { + final String namespace = "test-namespace-" + randomName(16); + String ns = PUBLIC_TENANT + "/" + namespace; + admin.namespaces().createNamespace(ns, Sets.newHashSet(CLUSTER_NAME)); + + final String autoProducerTopic = getTopicName(ns, "testEmptySchema"); + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.AVRO(User.class)) + .topic(autoProducerTopic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub-1") + .subscribe(); + + @Cleanup + Producer userProducer = pulsarClient + .newProducer(Schema.AVRO(User.class)) + .topic(autoProducerTopic) + .enableBatching(false) + .create(); + + @Cleanup + Producer producer = pulsarClient + .newProducer() + .topic(autoProducerTopic) + .enableBatching(false) + .create(); + + User test = new User("test"); + userProducer.send(test); + producer.send("test".getBytes(StandardCharsets.UTF_8)); + Message message1 = consumer.receive(); + Assert.assertEquals(test, message1.getValue()); + try { + Message message2 = consumer.receive(); + message2.getValue(); + } catch (Throwable ex) { + Assert.assertTrue(Throwables.getRootCause(ex) instanceof SchemaSerializationException); + Assert.assertEquals(Throwables.getRootCause(ex).getMessage(),"Empty schema version"); + } + } + + @EqualsAndHashCode + static class User implements Serializable { + private String name; + public User() {} + public User(String name) { + this.name = name; + } + } + } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java index eb61a6634213c..ba3281c64cc00 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java @@ -36,6 +36,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse; @@ -217,9 +218,12 @@ public CompletableFuture> getSchema(TopicName topicName) { @Override public CompletableFuture> getSchema(TopicName topicName, byte[] version) { - InetSocketAddress socketAddress = serviceNameResolver.resolveHost(); CompletableFuture> schemaFuture = new CompletableFuture<>(); - + if (version != null && version.length == 0) { + schemaFuture.completeExceptionally(new SchemaSerializationException("Empty schema version")); + return schemaFuture; + } + InetSocketAddress socketAddress = serviceNameResolver.resolveHost(); client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> { long requestId = client.newRequestId(); ByteBuf request = Commands.newGetSchema(requestId, topicName.toString(), diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java index f2cc1692eebe6..72326c3db1f20 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java @@ -39,6 +39,7 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; +import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.lookup.data.LookupData; import org.apache.pulsar.common.naming.NamespaceName; @@ -162,6 +163,10 @@ public CompletableFuture> getSchema(TopicName topicName, by String schemaName = topicName.getSchemaName(); String path = String.format("admin/v2/schemas/%s/schema", schemaName); if (version != null) { + if (version.length == 0) { + future.completeExceptionally(new SchemaSerializationException("Empty schema version")); + return future; + } path = String.format("admin/v2/schemas/%s/schema/%s", schemaName, ByteBuffer.wrap(version).getLong()); From c69a376d3d463f1cc6fb20e81c47fb4869421808 Mon Sep 17 00:00:00 2001 From: wuxuanqicn <89442834+wuxuanqicn@users.noreply.github.com> Date: Sat, 12 Mar 2022 13:50:05 +0800 Subject: [PATCH 392/823] Fixed flaky test MemoryLimitTest#testRejectMessages (#14220) (#14628) Co-authored-by: xuanqi.wu (cherry picked from commit 5f8db372ee3926f93eb109ab3b713038c3b523c8) --- .../pulsar/client/api/MemoryLimitTest.java | 86 ++++++++++--------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MemoryLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MemoryLimitTest.java index ec98e7d1dbec9..431991e61aa19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MemoryLimitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MemoryLimitTest.java @@ -18,20 +18,22 @@ */ package org.apache.pulsar.client.api; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.fail; - -import java.util.concurrent.CountDownLatch; - import lombok.Cleanup; - import org.apache.pulsar.client.api.PulsarClientException.MemoryBufferIsFullError; -import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarTestClient; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + @Test(groups = "broker-api") public class MemoryLimitTest extends ProducerConsumerBase { @@ -62,27 +64,30 @@ public void testRejectMessages() throws Exception { String topic = newTopicName(); - @Cleanup - PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder() + ClientBuilder clientBuilder = PulsarClient.builder() .serviceUrl(pulsar.getBrokerServiceUrl()) - .memoryLimit(100, SizeUnit.KILO_BYTES) - .build(); + .memoryLimit(100, SizeUnit.KILO_BYTES); @Cleanup - Producer producer = client.newProducer() + PulsarTestClient client = PulsarTestClient.create(clientBuilder); + + @Cleanup + ProducerImpl producer = (ProducerImpl) client.newProducer() .topic(topic) .blockIfQueueFull(false) + .sendTimeout(5, TimeUnit.SECONDS) .create(); + // make sure all message pending at pendingMessages queue + // connection with broker can not be established, so handleSendReceipt will not be invoked while sending message + client.dropOpSendMessages(); final int n = 101; - CountDownLatch latch = new CountDownLatch(n); - for (int i = 0; i < n; i++) { - producer.sendAsync(new byte[1024]).thenRun(() -> { - latch.countDown(); - }); + producer.sendAsync(new byte[1024]); } - + Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .until(() -> producer.getPendingQueueSize() == n); assertEquals(client.getMemoryLimitController().currentUsage(), n * 1024); try { @@ -92,8 +97,10 @@ public void testRejectMessages() // Expected } - latch.await(); - + client.allowReconnecting(); + Awaitility.await() + .atMost(Duration.ofSeconds(30)) + .until(() -> producer.getPendingQueueSize() == 0); assertEquals(client.getMemoryLimitController().currentUsage(), 0); // We should now be able to send again @@ -105,41 +112,40 @@ public void testRejectMessagesOnMultipleTopics() throws Exception { String t1 = newTopicName(); String t2 = newTopicName(); - @Cleanup - PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder() + ClientBuilder clientBuilder = PulsarClient.builder() .serviceUrl(pulsar.getBrokerServiceUrl()) - .memoryLimit(100, SizeUnit.KILO_BYTES) - .build(); + .memoryLimit(100, SizeUnit.KILO_BYTES); + + @Cleanup + PulsarTestClient client = PulsarTestClient.create(clientBuilder); @Cleanup - Producer p1 = client.newProducer() + ProducerImpl p1 = (ProducerImpl) client.newProducer() .topic(t1) .blockIfQueueFull(false) + .sendTimeout(5, TimeUnit.SECONDS) .create(); @Cleanup - Producer p2 = client.newProducer() + ProducerImpl p2 = (ProducerImpl) client.newProducer() .topic(t2) .blockIfQueueFull(false) + .sendTimeout(5, TimeUnit.SECONDS) .create(); + client.dropOpSendMessages(); final int n = 101; - CountDownLatch latch = new CountDownLatch(n); - for (int i = 0; i < n / 2; i++) { - p1.sendAsync(new byte[1024]).thenRun(() -> { - latch.countDown(); - }); - p2.sendAsync(new byte[1024]).thenRun(() -> { - latch.countDown(); - }); + p1.sendAsync(new byte[1024]); + p2.sendAsync(new byte[1024]); } // Last message in order to reach the limit - p1.sendAsync(new byte[1024]).thenRun(() -> { - latch.countDown(); - }); + p1.sendAsync(new byte[1024]); + Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .until(() -> (p1.getPendingQueueSize() + p2.getPendingQueueSize()) == n); assertEquals(client.getMemoryLimitController().currentUsage(), n * 1024); try { @@ -156,8 +162,10 @@ public void testRejectMessagesOnMultipleTopics() throws Exception { // Expected } - latch.await(); - + client.allowReconnecting(); + Awaitility.await() + .atMost(Duration.ofSeconds(30)) + .until(() -> (p1.getPendingQueueSize() + p2.getPendingQueueSize()) == 0); assertEquals(client.getMemoryLimitController().currentUsage(), 0); // We should now be able to send again From 4aea15bdbe514ccc33a0705ef711360384c7e2ff Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 14 Mar 2022 22:24:23 +0800 Subject: [PATCH 393/823] [Broker] Ignore the print the log that the topic does not exist (#13535) (cherry picked from commit fe7e55d9f353925a559e88f8ceef2b47b59668e0) --- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index b124e404481cd..b28a51014ad4e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -4248,8 +4248,9 @@ protected void internalHandleResult(AsyncResponse asyncResponse, protected void handleTopicPolicyException(String methodName, Throwable thr, AsyncResponse asyncResponse) { Throwable cause = thr.getCause(); - if (!(cause instanceof WebApplicationException) - || !(((WebApplicationException) cause).getResponse().getStatus() == 307)) { + if (!(cause instanceof WebApplicationException) || !( + ((WebApplicationException) cause).getResponse().getStatus() == 307 + || ((WebApplicationException) cause).getResponse().getStatus() == 404)) { log.error("[{}] Failed to perform {} on topic {}", clientAppId(), methodName, topicName, cause); } From c83d04f315ea0b43bf2f81b914f6ffaf407cba5b Mon Sep 17 00:00:00 2001 From: wenbingshen Date: Tue, 15 Mar 2022 18:21:56 +0800 Subject: [PATCH 394/823] Fix PartitionedProducerImpl flushAsync always fail when one partition send TimeOutException (#14602) Fixes #14598 Master Issue: #14598 Detailed issue description can be found at https://github.com/apache/pulsar/issues/14598 After the `lastSendFuture` returned to application from in `org.apache.pulsar.client.impl.ProducerImpl#flushAsync` acquired , it should not continue to be thrown to the application, the `lastSendFuture` whether an exception occurs or completed, and the application is only allowed to acquire it once, otherwise it will cause `org. apache.pulsar.client.impl.PartitionedProducerImpl#flushAsync ` cannot continue to send data until data is sent again to the abnormal `ProducerImpl`. (cherry picked from commit ddca8521ed30aaa719d64077b65c4a24fb82ce5c) --- .../api/SimpleProducerConsumerTest.java | 78 +++++++++++++++++++ .../client/impl/PartitionedProducerImpl.java | 6 ++ .../pulsar/client/impl/ProducerImpl.java | 42 +++++++++- 3 files changed, 123 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index fc238d125c66d..ce5ba65fa7588 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -88,6 +88,7 @@ import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; +import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; @@ -607,6 +608,83 @@ public void testSendTimeout(int batchMessageDelayMs) throws Exception { log.info("-- Exiting {} test --", methodName); } + @Test(dataProvider = "batch") + public void testSendTimeoutAndRecover(int batchMessageDelayMs) throws Exception { + log.info("-- Starting {} test --", methodName); + + int numPartitions = 6; + TopicName topicName = TopicName.get("persistent://my-property/my-ns/sendTimeoutAndRecover-1"); + admin.topics().createPartitionedTopic(topicName.toString(), numPartitions); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName.toString()) + .subscriptionName("my-subscriber-name").subscribe(); + ProducerBuilder producerBuilder = pulsarClient.newProducer() + .topic(topicName.toString()).sendTimeout(1, TimeUnit.SECONDS); + + if (batchMessageDelayMs != 0) { + producerBuilder.enableBatching(true); + producerBuilder.batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS); + producerBuilder.batchingMaxMessages(5); + } + + @Cleanup + PartitionedProducerImpl partitionedProducer = + (PartitionedProducerImpl) producerBuilder.create(); + final String message = "my-message"; + // 1. Trigger the send timeout + stopBroker(); + + partitionedProducer.sendAsync(message.getBytes()); + + String exceptionMessage = ""; + try { + // 2. execute flush to get results, + // it should be failed because step 1 + partitionedProducer.flush(); + Assert.fail("Send operation should have failed"); + } catch (PulsarClientException e) { + exceptionMessage = e.getMessage(); + } + + // 3. execute flush to get results, + // it shouldn't fail because we already handled the exception in the step 2, unless we keep sending data. + partitionedProducer.flush(); + // 4. execute flushAsync, we only catch the exception once, + // but by getting the original lastSendFuture twice below, + // the same exception information must be caught twice to verify that our handleOnce works as expected. + try { + partitionedProducer.getOriginalLastSendFuture().get(); + Assert.fail("Send operation should have failed"); + } catch (Exception e) { + Assert.assertEquals(PulsarClientException.unwrap(e).getMessage(), exceptionMessage); + } + try { + partitionedProducer.getOriginalLastSendFuture().get(); + Assert.fail("Send operation should have failed"); + } catch (Exception e) { + Assert.assertEquals(PulsarClientException.unwrap(e).getMessage(), exceptionMessage); + } + + startBroker(); + + // 5. We should not have received any message + Message msg = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assert.assertNull(msg); + + // 6. We keep sending data after connection reconnected. + partitionedProducer.sendAsync(message.getBytes()); + // 7. This flush operation must succeed. + partitionedProducer.flush(); + + // 8. We should have received message + msg = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assert.assertNotNull(msg); + Assert.assertEquals(new String(msg.getData()), message); + + log.info("-- Exiting {} test --", methodName); + } + @Test public void testInvalidSequence() throws Exception { log.info("-- Starting {} test --", methodName); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index 216d7755425e9..b120550637920 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -443,4 +443,10 @@ public Timeout getPartitionsAutoUpdateTimeout() { return partitionsAutoUpdateTimeout; } + @VisibleForTesting + public CompletableFuture getOriginalLastSendFuture() { + return CompletableFuture.allOf( + producers.values().stream().map(ProducerImpl::getOriginalLastSendFuture) + .toArray(CompletableFuture[]::new)); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index dc651f36e01ff..2637c4953046f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -54,6 +54,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; @@ -113,6 +114,7 @@ public class ProducerImpl extends ProducerBase implements TimerTask, Conne private final BatchMessageContainerBase batchMessageContainer; private CompletableFuture lastSendFuture = CompletableFuture.completedFuture(null); + private LastSendFutureWrapper lastSendFutureWrapper = LastSendFutureWrapper.create(lastSendFuture); // Globally unique producer name private String producerName; @@ -883,6 +885,31 @@ protected WriteInEventLoopCallback newObject(Handle ha }; } + private static final class LastSendFutureWrapper { + private final CompletableFuture lastSendFuture; + private static final int FALSE = 0; + private static final int TRUE = 1; + private static final AtomicIntegerFieldUpdater THROW_ONCE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(LastSendFutureWrapper.class, "throwOnce"); + private volatile int throwOnce = FALSE; + + private LastSendFutureWrapper(CompletableFuture lastSendFuture) { + this.lastSendFuture = lastSendFuture; + } + static LastSendFutureWrapper create(CompletableFuture lastSendFuture) { + return new LastSendFutureWrapper(lastSendFuture); + } + public CompletableFuture handleOnce() { + return lastSendFuture.handle((ignore, t) -> { + if (t != null && THROW_ONCE_UPDATER.compareAndSet(this, FALSE, TRUE)) { + throw FutureUtil.wrapToCompletionException(t); + } + return null; + }); + } + } + + @Override public CompletableFuture closeAsync() { final State currentState = getAndUpdateState(state -> { @@ -1840,14 +1867,17 @@ private void failPendingBatchMessages(PulsarClientException ex) { @Override public CompletableFuture flushAsync() { - CompletableFuture lastSendFuture; synchronized (ProducerImpl.this) { if (isBatchMessagingEnabled()) { batchMessageAndSend(); } - lastSendFuture = this.lastSendFuture; + CompletableFuture lastSendFuture = this.lastSendFuture; + if (!(lastSendFuture == this.lastSendFutureWrapper.lastSendFuture)) { + this.lastSendFutureWrapper = LastSendFutureWrapper.create(lastSendFuture); + } } - return lastSendFuture.thenApply(ignored -> null); + + return this.lastSendFutureWrapper.handleOnce(); } @Override @@ -2055,5 +2085,11 @@ boolean isErrorStat() { return errorState; } + @VisibleForTesting + CompletableFuture getOriginalLastSendFuture() { + CompletableFuture lastSendFuture = this.lastSendFuture; + return lastSendFuture.thenApply(ignore -> null); + } + private static final Logger log = LoggerFactory.getLogger(ProducerImpl.class); } From 1bea336e7dbea0069d528226b78e0fe29258b780 Mon Sep 17 00:00:00 2001 From: Bharani Chadalavada Date: Tue, 15 Mar 2022 07:12:45 -0700 Subject: [PATCH 395/823] [ Issue 14633] [pulsar-broker] Fix metadata store deadlock when checking BacklogQuota (#14634) (cherry picked from commit 06ed9445bf29ea30b1f094b2d0ff608cb76aa3f6) --- .../org/apache/pulsar/broker/admin/impl/BrokersBase.java | 5 +++-- .../java/org/apache/pulsar/broker/service/BrokerService.java | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index cb579ecc44c40..973efa3df49f9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.collect.Maps; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -270,7 +271,7 @@ public InternalConfigurationData getInternalConfigurationData() { @ApiResponse(code = 500, message = "Internal server error")}) public void backlogQuotaCheck(@Suspended AsyncResponse asyncResponse) { validateSuperUserAccess(); - pulsar().getBrokerService().executor().execute(()->{ + pulsar().getBrokerService().getBacklogQuotaChecker().execute(safeRun(()->{ try { pulsar().getBrokerService().monitorBacklogQuota(); asyncResponse.resume(Response.noContent().build()); @@ -278,7 +279,7 @@ public void backlogQuotaCheck(@Suspended AsyncResponse asyncResponse) { LOG.error("trigger backlogQuotaCheck fail", e); asyncResponse.resume(new RestException(e)); } - }); + })); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d37dfb6a08105..1c452027e76a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -215,6 +215,7 @@ public class BrokerService implements Closeable { private AuthorizationService authorizationService = null; private final ScheduledExecutorService statsUpdater; + @Getter private final ScheduledExecutorService backlogQuotaChecker; protected final AtomicReference lookupRequestSemaphore; @@ -1687,7 +1688,7 @@ public BacklogQuotaManager getBacklogQuotaManager() { return this.backlogQuotaManager; } - public synchronized void monitorBacklogQuota() { + public void monitorBacklogQuota() { forEachTopic(topic -> { if (topic instanceof PersistentTopic) { PersistentTopic persistentTopic = (PersistentTopic) topic; From 08eceb74175f1c1deb78c6cc0bfbf77646bff635 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 15 Mar 2022 22:15:44 +0800 Subject: [PATCH 396/823] Fixed 404 error msg not being returned correctly using http lookup. (#14677) (cherry picked from commit 9a88508426cd2f6baf8d25a225f5f27de5bc1a7a) --- .../org/apache/pulsar/broker/web/RestException.java | 6 +++++- .../broker/service/PersistentTopicE2ETest.java | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java index 2c01a7f0d0166..8ff016c78cda1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java @@ -49,7 +49,11 @@ public RestException(Response.Status status, String message) { } public RestException(int code, String message) { - super(message, Response.status(code).entity(new ErrorData(message)).type(MediaType.APPLICATION_JSON).build()); + super(message, Response + .status(code, message) + .entity(new ErrorData(message)) + .type(MediaType.APPLICATION_JSON) + .build()); } public RestException(Throwable t) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java index daf0ed7764d80..9de664f98ec9d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java @@ -1879,4 +1879,17 @@ public void testProducerBusy() throws Exception { assertEquals(admin.topics().getStats(topicName).getPublishers().size(), 1); } + + @Test + public void testHttpLookupWithNotFoundError() throws Exception { + stopBroker(); + isTcpLookup = false; + setup(); + try { + pulsarClient.newProducer().topic("unknownTenant/unknownNamespace/testNamespaceNotFound").create(); + } catch (Exception ex) { + assertTrue(ex instanceof PulsarClientException.NotFoundException); + assertTrue(ex.getMessage().contains("Namespace not found")); + } + } } From a61aa5c5e925492321583f0b1416b6ac0772ec3a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 15 Mar 2022 16:00:37 +0800 Subject: [PATCH 397/823] Set splitNamespaceBundle with `readonly=false`. (#14680) Master Issue: #14668 Fixes: #14668 ### Motivation When we split a not loaded namespace bundle, we will meet the below error: ``` Failed to find ownership for ServiceUnit:tenant/namespace/0x00000000_0x10000000 ``` Because when validating namespace bundle ownership with `readonly=true` : https://github.com/apache/pulsar/blob/fe7e55d9f353925a559e88f8ceef2b47b59668e0/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java#L1145-L1151 and if the bundle is not owned by any broker, it will return empty(line-392): https://github.com/apache/pulsar/blob/fe7e55d9f353925a559e88f8ceef2b47b59668e0/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java#L388-L400 so throw the below exception : https://github.com/apache/pulsar/blob/fe7e55d9f353925a559e88f8ceef2b47b59668e0/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java#L576-L582 ### Modification - Change readonly from `true` to `false` when validating namespace bundle ownership. (cherry picked from commit 4ffef1adbda405caa3359d108f40eed46fd1a8e8) --- .../pulsar/broker/admin/impl/NamespacesBase.java | 2 +- .../apache/pulsar/broker/admin/NamespacesTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index a9f369d42d4b2..310c0d9d7d584 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1116,7 +1116,7 @@ protected void internalSplitNamespaceBundle(AsyncResponse asyncResponse, String try { nsBundle = validateNamespaceBundleOwnership(namespaceName, policies.bundles, bundleRange, - authoritative, true); + authoritative, false); } catch (Exception e) { asyncResponse.resume(e); return; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index abefe449f6dcf..16ae98f0b43cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -1737,4 +1737,18 @@ private void assertInvalidRetentionPolicyAsPartOfAllPolicies(Policies policies, assertTrue(e.getMessage().startsWith("Invalid retention policy")); } } + + @Test + public void testSplitBundleForMultiTimes() throws Exception{ + String namespace = BrokerTestUtil.newUniqueName(this.testTenant + "/namespace"); + BundlesData data = BundlesData.builder().numBundles(4).build(); + admin.namespaces().createNamespace(namespace, data); + for (int i = 0; i < 10; i ++) { + final BundlesData bundles = admin.namespaces().getBundles(namespace); + final String bundle = bundles.getBoundaries().get(0) + "_" + bundles.getBoundaries().get(1); + admin.namespaces().splitNamespaceBundle(namespace, bundle, true, null); + } + BundlesData bundles = admin.namespaces().getBundles(namespace); + assertEquals(bundles.getNumBundles(), 14); + } } From 091ae73dcf555f294d60ff697b11674b3de6e74c Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Tue, 15 Mar 2022 20:39:43 +0800 Subject: [PATCH 398/823] fix incorrect warn log (#14685) ### Motivation When read offload data failed, it prints a warn log. However, the actual read entryId doesn't print correctly, which will makes hard to debug. ``` WARN org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreBackedReadHandleImpl - Read an unexpected entry id [] which is smaller than the next expected entry id 49648, seeking to the right position ``` ### Modification 1. Print correct actual entryId (cherry picked from commit bcba273b854536b7f35872af3f60ac197229444e) --- .../offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java index f4dc1b8b4e421..73a4dd76e5317 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java @@ -152,7 +152,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } else if (entryId < nextExpectedId && !index.getIndexEntryForEntry(nextExpectedId).equals(index.getIndexEntryForEntry(entryId))) { log.warn("Read an unexpected entry id {} which is smaller than the next expected entry id {}" - + ", seeking to the right position", entries, nextExpectedId); + + ", seeking to the right position", entryId, nextExpectedId); inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); } else if (entryId > lastEntry) { // in the normal case, the entry id should increment in order. But if there has random access in From 878a5527f74cf3cca29a371c6bfb5cb88169cabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Tue, 15 Mar 2022 17:15:56 +0800 Subject: [PATCH 399/823] Fix race condition in consumer redelivery (#14687) (cherry picked from commit dd9bcbe3c00d20e6c3aa63ef5c5235f88659a88c) --- .../pulsar/client/impl/ConsumerImpl.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 856a7758455b9..d19b980a24a9d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1770,20 +1770,20 @@ private CompletableFuture> getRedeliveryMessageIdData(List data = new ArrayList<>(messageIds.size()); - List> futures = new ArrayList<>(messageIds.size()); - messageIds.forEach(messageId -> { + List> futures = messageIds.stream().map(messageId -> { CompletableFuture future = processPossibleToDLQ(messageId); - futures.add(future.thenAccept(sendToDLQ -> { + return future.thenApply(sendToDLQ -> { if (!sendToDLQ) { - data.add(new MessageIdData() + return new MessageIdData() .setPartition(messageId.getPartitionIndex()) .setLedgerId(messageId.getLedgerId()) - .setEntryId(messageId.getEntryId())); + .setEntryId(messageId.getEntryId()); } - })); - }); - return FutureUtil.waitForAll(futures).thenCompose(v -> CompletableFuture.completedFuture(data)); + return null; + }); + }).collect(Collectors.toList()); + return FutureUtil.waitForAll(futures).thenApply(v -> + futures.stream().map(CompletableFuture::join).filter(Objects::nonNull).collect(Collectors.toList())); } private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId) { From 2c36f8b4906854592a5d26224bb93583d68d4764 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 16 Mar 2022 19:01:27 +0800 Subject: [PATCH 400/823] [pulsar-functions] fix some IOExceptions when create functions from package URL (#14553) (cherry picked from commit f0d166f36e1fbd4df1e20ae2ccc7fcae822c17b4) --- .../org/apache/pulsar/client/admin/internal/PackagesImpl.java | 3 ++- .../apache/pulsar/functions/worker/rest/api/FunctionsImpl.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java index 4c7fc4cf29905..77749e6421b9a 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -182,7 +183,7 @@ public void completed(Response response) { if (destinyPath.getParent() != null) { Files.createDirectories(destinyPath.getParent()); } - Files.copy(inputStream, destinyPath); + Files.copy(inputStream, destinyPath, StandardCopyOption.REPLACE_EXISTING); future.complete(null); } catch (IOException e) { future.completeExceptionally(e); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index b00846de3f027..0e3a28a07576c 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -769,6 +769,7 @@ static File downloadPackageFile(PulsarWorkerService worker, String packageName) // use the Nar extraction directory as a temporary directory for downloaded files tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory()); } + Files.createDirectories(tempDirectory); File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); worker.getBrokerAdmin().packages().download(packageName, file.toString()); return file; From 2cb1681ba3e9fcacbe4214bf4c0dc6d08054a545 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 16 Mar 2022 11:37:36 +0800 Subject: [PATCH 401/823] [Broker] Fix precision issue and initial value for Consumer#avgMessagesPerEntry (#14666) ### Motivation 1. Precision issue There is precision issue to use int type for `Consumer#avgMessagesPerEntry`. ``` tmpAvgMessagesPerEntry = (int) Math.floor(tmpAvgMessagesPerEntry * avgPercent + (1 - avgPercent) * totalMessages / entries.size()); ``` For example, if `tmpAvgMessagesPerEntry` = 1 and new value of `totalMessages / entries.size()` is always 5, then the `tmpAvgMessagesPerEntry` is always 1 and never increase. 2. Initial value issue. And the init value of 1000 seems confusing in consumerStats for users, and it need quite a long time to decrease if message rate is very slow. ### Modifications 1. Change type of avgMessagesPerEntry to double. 2. Change init value from 1000 to first `totalMessages / entries.size()`. (cherry picked from commit de2e6c8ce5821b945cbdd0cb2969234667b017ec) --- .../pulsar/broker/service/Consumer.java | 28 +++++++++++-------- ...PersistentDispatcherMultipleConsumers.java | 3 +- ...sistentDispatcherSingleActiveConsumer.java | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index bf906a32d1aba..591b71c71f6b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.AtomicDouble; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.Promise; import java.util.ArrayList; @@ -116,12 +117,10 @@ public class Consumer { /** * It starts keep tracking the average messages per entry. - * The initial value is 1000, when new value comes, it will update with + * The initial value is 0, when new value comes, it will update with * avgMessagesPerEntry = avgMessagePerEntry * avgPercent + (1 - avgPercent) * new Value. */ - private static final AtomicIntegerFieldUpdater AVG_MESSAGES_PER_ENTRY = - AtomicIntegerFieldUpdater.newUpdater(Consumer.class, "avgMessagesPerEntry"); - private volatile int avgMessagesPerEntry = 1000; + private final AtomicDouble avgMessagesPerEntry = new AtomicDouble(0); private static final long [] EMPTY_ACK_SET = new long[0]; private static final double avgPercent = 0.9; @@ -163,7 +162,6 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.set(this, 0); MESSAGE_PERMITS_UPDATER.set(this, 0); UNACKED_MESSAGES_UPDATER.set(this, 0); - AVG_MESSAGES_PER_ENTRY.set(this, 1000); this.metadata = metadata != null ? metadata : Collections.emptyMap(); @@ -265,10 +263,13 @@ public Future sendMessages(final List entries, EntryBatchSizes batc } // calculate avg message per entry - int tmpAvgMessagesPerEntry = AVG_MESSAGES_PER_ENTRY.get(this); - tmpAvgMessagesPerEntry = (int) Math.floor(tmpAvgMessagesPerEntry * avgPercent - + (1 - avgPercent) * totalMessages / entries.size()); - AVG_MESSAGES_PER_ENTRY.set(this, tmpAvgMessagesPerEntry); + if (avgMessagesPerEntry.get() < 1) { //valid avgMessagesPerEntry should always >= 1 + // set init value. + avgMessagesPerEntry.set(1.0 * totalMessages / entries.size()); + } else { + avgMessagesPerEntry.set(avgMessagesPerEntry.get() * avgPercent + + (1 - avgPercent) * totalMessages / entries.size()); + } // reduce permit and increment unackedMsg count with total number of messages in batch-msgs int ackedCount = batchIndexesAcks == null ? 0 : batchIndexesAcks.getTotalAckedIndexCount(); @@ -276,7 +277,7 @@ public Future sendMessages(final List entries, EntryBatchSizes batc if (log.isDebugEnabled()){ log.debug("[{}-{}] Added {} minus {} messages to MESSAGE_PERMITS_UPDATER in broker.service.Consumer" + " for consumerId: {}; avgMessagesPerEntry is {}", - topicName, subscription, ackedCount, totalMessages, consumerId, tmpAvgMessagesPerEntry); + topicName, subscription, ackedCount, totalMessages, consumerId, avgMessagesPerEntry.get()); } incrementUnackedMessages(unackedMessages); msgOut.recordMultipleEvents(totalMessages, totalBytes); @@ -683,8 +684,11 @@ public int getAvailablePermits() { return MESSAGE_PERMITS_UPDATER.get(this); } + /** + * return 0 if there is no entry dispatched yet. + */ public int getAvgMessagesPerEntry() { - return AVG_MESSAGES_PER_ENTRY.get(this); + return (int) Math.round(avgMessagesPerEntry.get()); } public boolean isBlocked() { @@ -729,7 +733,7 @@ public void updateStats(ConsumerStatsImpl consumerStats) { } unackedMessages = consumerStats.unackedMessages; blockedConsumerOnUnackedMsgs = consumerStats.blockedConsumerOnUnackedMsgs; - AVG_MESSAGES_PER_ENTRY.set(this, consumerStats.avgMessagesPerEntry); + avgMessagesPerEntry.set(consumerStats.avgMessagesPerEntry); } public ConsumerStatsImpl getStats() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 0d060056ae6ee..a8a36cdc6c340 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -290,8 +290,9 @@ protected Pair calculateToRead(int currentTotalAvailablePermits) Consumer c = getRandomConsumer(); // if turn on precise dispatcher flow control, adjust the record to read if (c != null && c.isPreciseDispatcherFlowControl()) { + int avgMessagesPerEntry = Math.max(1, c.getAvgMessagesPerEntry()); messagesToRead = Math.min( - (int) Math.ceil(currentTotalAvailablePermits * 1.0 / c.getAvgMessagesPerEntry()), + (int) Math.ceil(currentTotalAvailablePermits * 1.0 / avgMessagesPerEntry), readBatchSize); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 6161847e42dc0..6900a54e35c2c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -369,7 +369,7 @@ protected Pair calculateToRead(Consumer consumer) { long bytesToRead = serviceConfig.getDispatcherMaxReadSizeBytes(); // if turn of precise dispatcher flow control, adjust the records to read if (consumer.isPreciseDispatcherFlowControl()) { - int avgMessagesPerEntry = consumer.getAvgMessagesPerEntry(); + int avgMessagesPerEntry = Math.max(1, consumer.getAvgMessagesPerEntry()); messagesToRead = Math.min((int) Math.ceil(availablePermits * 1.0 / avgMessagesPerEntry), readBatchSize); } From 13212012c9f70eaf701aa4c0e095c708996fdbdf Mon Sep 17 00:00:00 2001 From: Baozi Date: Wed, 16 Mar 2022 19:00:01 +0800 Subject: [PATCH 402/823] Flaky-test: SubscriptionSeekTest.testShouldCloseAllConsumersForMultipleConsumerDispatcherWhenSeek (#14674) (cherry picked from commit 157b808610ec6b4419da1caf6179aa26e90bfa3b) --- .../apache/pulsar/broker/service/SubscriptionSeekTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java index 8c5e96972354d..76d8c0b60310b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java @@ -560,7 +560,7 @@ public void testShouldCloseAllConsumersForMultipleConsumerDispatcherWhenSeek() t .subscriptionName("my-subscription") .subscribe(); - pulsarClient.newConsumer() + org.apache.pulsar.client.api.Consumer consumer2 = pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Shared) .subscriptionName("my-subscription") @@ -579,12 +579,15 @@ public void testShouldCloseAllConsumersForMultipleConsumerDispatcherWhenSeek() t consumer1.seek(MessageId.earliest); // Wait for consumer to reconnect Awaitility.await().until(consumer1::isConnected); + Awaitility.await().until(consumer2::isConnected); consumers = topicRef.getSubscriptions().get("my-subscription").getConsumers(); assertEquals(consumers.size(), 2); for (Consumer consumer : consumers) { assertFalse(connectedSinceSet.contains(consumer.getStats().getConnectedSince())); } + consumer1.close(); + consumer2.close(); } @Test From b5f3f244cbcd84ec0c0f13eac08ef0395effd36a Mon Sep 17 00:00:00 2001 From: llIlll <10194588+llIlll@users.noreply.github.com> Date: Wed, 16 Mar 2022 12:18:39 +0800 Subject: [PATCH 403/823] [flaky-test] SubscriptionSeekTest.testSeekForBatchMessageAndSpecifiedBatchIndex (#14676) (#14689) (cherry picked from commit 2d3e802293c62cf408ca52c1ff76efa713542f6e) --- .../org/apache/pulsar/broker/service/SubscriptionSeekTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java index 76d8c0b60310b..b59eb95512d91 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java @@ -187,7 +187,7 @@ public void testSeekForBatch() throws Exception { @Test public void testSeekForBatchMessageAndSpecifiedBatchIndex() throws Exception { - final String topicName = "persistent://prop/use/ns-abcd/testSeekForBatch"; + final String topicName = "persistent://prop/use/ns-abcd/testSeekForBatchMessageAndSpecifiedBatchIndex"; String subscriptionName = "my-subscription-batch"; Producer producer = pulsarClient.newProducer(Schema.STRING) From 70a1fe1ed20977c2c503c8533c0de12e8445692b Mon Sep 17 00:00:00 2001 From: wenbingshen Date: Thu, 17 Mar 2022 19:50:19 +0800 Subject: [PATCH 404/823] Fix partitionsAutoUpdateFuture never complete (#14625) (cherry picked from commit b06dac68700dfcdf701dfbccb98126db7a7b7ef3) --- .../api/SimpleProducerConsumerTest.java | 49 +++++++++++++++++++ .../client/impl/PartitionedProducerImpl.java | 13 ++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index ce5ba65fa7588..2815ce36bf0e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -39,6 +39,7 @@ import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.util.Timeout; import java.io.ByteArrayInputStream; import java.io.IOException; import java.lang.reflect.Field; @@ -4358,4 +4359,52 @@ public void testShareConsumerWithMessageListener() throws Exception { assertEquals(resultSet.size(), total); }); } + + @Test + public void testPartitionsAutoUpdate() throws Exception { + log.info("-- Starting {} test --", methodName); + + int numPartitions = 3; + TopicName topicName = TopicName.get("persistent://my-property/my-ns/partitionsAutoUpdate-1"); + admin.topics().createPartitionedTopic(topicName.toString(), numPartitions); + + int operationTimeout = 2000; // MILLISECONDS + @Cleanup final PulsarClient client = PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .operationTimeout(operationTimeout, TimeUnit.MILLISECONDS) + .build(); + + ProducerBuilder producerBuilder = client.newProducer() + .topic(topicName.toString()).sendTimeout(1, TimeUnit.SECONDS); + + @Cleanup + PartitionedProducerImpl partitionedProducer = + (PartitionedProducerImpl) producerBuilder.autoUpdatePartitions(true).create(); + + // Trigger the Connection refused exception + stopBroker(); + + log.info("trigger partitionsAutoUpdateTimerTask run failed for producer"); + Timeout timeout = partitionedProducer.getPartitionsAutoUpdateTimeout(); + timeout.task().run(timeout); + Awaitility.await().untilAsserted(() -> { + assertNotNull(partitionedProducer.getPartitionsAutoUpdateFuture()); + assertTrue(partitionedProducer.getPartitionsAutoUpdateFuture().isCompletedExceptionally()); + assertTrue(FutureUtil.getException(partitionedProducer.getPartitionsAutoUpdateFuture()).get().getMessage() + .contains("Connection refused:")); + }); + + startBroker(); + + log.info("trigger partitionsAutoUpdateTimerTask run successful for producer"); + timeout = partitionedProducer.getPartitionsAutoUpdateTimeout(); + timeout.task().run(timeout); + Awaitility.await().untilAsserted(() -> { + assertNotNull(partitionedProducer.getPartitionsAutoUpdateFuture()); + assertTrue(partitionedProducer.getPartitionsAutoUpdateFuture().isDone()); + assertFalse(partitionedProducer.getPartitionsAutoUpdateFuture().isCompletedExceptionally()); + }); + + log.info("-- Exiting {} test --", methodName); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index b120550637920..037f401634034 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -351,7 +351,7 @@ public CompletableFuture onTopicsExtended(Collection topicsExtende if (log.isDebugEnabled()) { log.debug("[{}] partitions number. old: {}, new: {}", - topic, oldPartitionNumber, currentPartitionNumber); + topic, oldPartitionNumber, currentPartitionNumber); } if (oldPartitionNumber == currentPartitionNumber) { @@ -402,10 +402,14 @@ public CompletableFuture onTopicsExtended(Collection topicsExtende } } else { log.error("[{}] not support shrink topic partitions. old: {}, new: {}", - topic, oldPartitionNumber, currentPartitionNumber); + topic, oldPartitionNumber, currentPartitionNumber); future.completeExceptionally(new NotSupportedException("not support shrink topic partitions")); } return future; + }).exceptionally(throwable -> { + log.error("[{}] Auto getting partitions failed", topic, throwable); + future.completeExceptionally(throwable); + return null; }); return future; @@ -438,6 +442,11 @@ public void run(Timeout timeout) throws Exception { } }; + @VisibleForTesting + public CompletableFuture getPartitionsAutoUpdateFuture() { + return partitionsAutoUpdateFuture; + } + @VisibleForTesting public Timeout getPartitionsAutoUpdateTimeout() { return partitionsAutoUpdateTimeout; From 1fae42d723bcb0515c593ba1cedb0cc9d67578d3 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 18 Mar 2022 11:06:47 +0800 Subject: [PATCH 405/823] [Transaction] Fix transaction buffer recover BrokerMetadataException close topic (#14709) ### Motivation When TopicTransactionBuffer recover fail throw BrokerMetadataException, we should close this topic, if we don't close the topic, we can't send message because TopicTransactionBuffer recover fail ![image](https://user-images.githubusercontent.com/39078850/158532983-993c0303-4051-4e56-90e1-c6ce89fa3775.png) ### Modifications When recover fail by BrokerMetadataException, close topic ### Verifying this change add test for it (cherry picked from commit c4e4ddd1dae2249938c8ce15e5282301b167cd5e) --- .../buffer/impl/TopicTransactionBuffer.java | 5 ++ .../TopicTransactionBufferRecoverTest.java | 64 ++++++++++++++++++- .../broker/transaction/TransactionTest.java | 3 +- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index e0a6695ef2260..3b28966def1eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -179,6 +179,11 @@ public void handleTxnEntry(Entry entry) { @Override public void recoverExceptionally(Exception e) { + if (e instanceof PulsarClientException.BrokerMetadataException) { + log.warn("Closing topic {} due to read transaction buffer snapshot while recovering the " + + "transaction buffer throw exception", topic.getName(), e); + topic.close(); + } transactionBufferFuture.completeExceptionally(e); } }, this.topic, this)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 335cecc44138d..5701d22a99c63 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -33,10 +33,14 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.RandomUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TransactionBufferSnapshotService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.matadata.TransactionBufferSnapshot; import org.apache.pulsar.client.api.Consumer; @@ -44,6 +48,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; @@ -60,7 +65,10 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; - +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -442,4 +450,58 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, reader.close(); } + + @Test(timeOut=30000) + public void testTransactionBufferRecoverThrowBrokerMetadataException() throws Exception { + String topic = NAMESPACE1 + "/testTransactionBufferRecoverThrowBrokerMetadataException"; + @Cleanup + Producer producer = pulsarClient + .newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + + producer.newMessage(txn).value("test".getBytes()).sendAsync(); + producer.newMessage(txn).value("test".getBytes()).sendAsync(); + txn.commit().get(); + + // take snapshot + PersistentTopic originalTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); + TransactionBufferSnapshotService transactionBufferSnapshotService = + mock(TransactionBufferSnapshotService.class); + SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class); + // mock reader can't read snapshot fail + doThrow(new PulsarClientException.BrokerMetadataException("")).when(reader).hasMoreEvents(); + doReturn(CompletableFuture.completedFuture(reader)).when(transactionBufferSnapshotService).createReader(any()); + + Field field = PulsarService.class.getDeclaredField("transactionBufferSnapshotService"); + field.setAccessible(true); + TransactionBufferSnapshotService transactionBufferSnapshotServiceOriginal = + (TransactionBufferSnapshotService) field.get(getPulsarServiceList().get(0)); + field.set(getPulsarServiceList().get(0), transactionBufferSnapshotService); + + // recover again will throw BrokerMetadataException then close topic + new TopicTransactionBuffer(originalTopic); + Awaitility.await().untilAsserted(() -> { + // isFenced means closed + Field close = AbstractTopic.class.getDeclaredField("isFenced"); + close.setAccessible(true); + assertTrue((boolean) close.get(originalTopic)); + }); + field.set(getPulsarServiceList().get(0), transactionBufferSnapshotServiceOriginal); + + // topic recover success + txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + + producer.newMessage(txn).value("test".getBytes()).sendAsync(); + txn.commit().get(); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 28d4fcf94c958..2b1a2f4e4e1cf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -806,7 +807,7 @@ public void testCancelTxnTimeout() throws Exception{ public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() throws Exception { PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService() - .getTopic(NAMESPACE1 + "/test", true) + .getTopic(NAMESPACE1 + "/changeMaxReadPositionAndAddAbortTimes" + UUID.randomUUID(), true) .get().get(); TransactionBuffer buffer = persistentTopic.getTransactionBuffer(); Field field = TopicTransactionBuffer.class.getDeclaredField("changeMaxReadPositionAndAddAbortTimes"); From 815cf976fd7f5d11e6a96f28238a15e1da4afd71 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 18 Mar 2022 21:25:36 +0800 Subject: [PATCH 406/823] [fix][txn]: fix transaction pending ack store managed ledger WriteFail state (#14738) like #10711 ``` java.util.concurrent.CompletionException: org.apache.pulsar.broker.service.BrokerServiceException$PersistenceException: org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerAlreadyClosedException: Waiting to recover from failure at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?] at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?] at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:704) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore$2.addFailed(MLPendingAckStore.java:286) ~[io.streamnative-pulsar-broker-2.9.2.5.jar:2.9.2.5] at org.apache.bookkeeper.mledger.impl.OpAddEntry.failed(OpAddEntry.java:138) ~[io.streamnative-managed-ledger-2.9.2.5.jar:2.9.2.5] at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.internalAsyncAddEntry(ManagedLedgerImpl.java:743) ~[io.streamnative-managed-ledger-2.9.2.5.jar:2.9.2.5] at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.lambda$asyncAddEntry$3(ManagedLedgerImpl.java:708) ~[io.streamnative-managed-ledger-2.9.2.5.jar:2.9.2.5] at org.apache.bookkeeper.mledger.util.SafeRun$1.safeRun(SafeRun.java:32) [io.streamnative-managed-ledger-2.9.2.5.jar:2.9.2.5] at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36) [org.apache.bookkeeper-bookkeeper-common-4.14.4.jar:4.14.4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.74.Final.jar:4.1.74.Final] at java.lang.Thread.run(Thread.java:829) [?:?] Caused by: org.apache.pulsar.broker.service.BrokerServiceException$PersistenceException: org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerAlreadyClosedException: Waiting to recover from failure ... 10 more ``` ## Motivation when transaction pending ack managed ledger state become WriteFailed state, should `readyToCreateNewLedger`. ## implement append fail check the managedLedger state and the exception do `readyToCreateNewLedger` ### Verifying this change Add the tests for it --- .../pendingack/impl/MLPendingAckStore.java | 4 + .../pendingack/PendingAckMetadataTest.java | 98 +++++++++++++++++++ .../impl/MLTransactionLogImpl.java | 6 +- 3 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 1ed0992b2ff2d..a7a46f9ed91bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -282,6 +282,10 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", managedLedger.getName(), ctx, exception, pendingAckMetadataEntry.getPendingAckOp()); + + if (exception instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException) { + managedLedger.readyToCreateNewLedger(); + } buf.release(); completableFuture.completeExceptionally(new PersistenceException(exception)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java new file mode 100644 index 0000000000000..c99eee6246371 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.CommandAck; +import org.testng.annotations.Test; +import java.lang.reflect.Field; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; +import static org.testng.Assert.assertTrue; +import static org.testng.AssertJUnit.fail; + +public class PendingAckMetadataTest extends MockedBookKeeperTestCase { + + public PendingAckMetadataTest() { + super(3); + } + + @Test + public void testPendingAckManageLedgerWriteFailState() throws Exception { + ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + factoryConf.setMaxCacheSize(0); + + String pendingAckTopicName = MLPendingAckStore + .getTransactionPendingAckStoreSuffix("test", "test"); + @Cleanup("shutdown") + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); + + CompletableFuture completableFuture = new CompletableFuture<>(); + factory.asyncOpen(pendingAckTopicName, new AsyncCallbacks.OpenLedgerCallback() { + @Override + public void openLedgerComplete(ManagedLedger ledger, Object ctx) { + completableFuture.complete(ledger); + } + + @Override + public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { + + } + }, null); + + ManagedCursor cursor = completableFuture.get().openCursor("test"); + ManagedCursor subCursor = completableFuture.get().openCursor("test"); + MLPendingAckStore pendingAckStore = + new MLPendingAckStore(completableFuture.get(), cursor, subCursor); + + Field field = MLPendingAckStore.class.getDeclaredField("managedLedger"); + field.setAccessible(true); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) field.get(pendingAckStore); + field = ManagedLedgerImpl.class.getDeclaredField("STATE_UPDATER"); + field.setAccessible(true); + AtomicReferenceFieldUpdater state = + (AtomicReferenceFieldUpdater) field.get(managedLedger); + state.set(managedLedger, WriteFailed); + try { + pendingAckStore.appendAbortMark(new TxnID(1, 1), CommandAck.AckType.Cumulative).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause().getCause() instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException); + } + pendingAckStore.appendAbortMark(new TxnID(1, 1), CommandAck.AckType.Cumulative).get(); + + completableFuture.get().close(); + cursor.close(); + subCursor.close(); + } + +} diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index 8bf2ebf4b911a..9ab4bd30b67c0 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -31,8 +31,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.CommandSubscribe; @@ -163,9 +161,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("Transaction log write transaction operation error", exception); - if (exception instanceof ManagedLedgerAlreadyClosedException - && managedLedger instanceof ManagedLedgerImpl - && State.WriteFailed == ((ManagedLedgerImpl) managedLedger).getState()) { + if (exception instanceof ManagedLedgerAlreadyClosedException) { managedLedger.readyToCreateNewLedger(); } buf.release(); From bf98fa7c276542f64d3216ac4baa7c5069c1dd2f Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 19 Mar 2022 14:46:09 +0800 Subject: [PATCH 407/823] cancel offload tasks when managed ledger closed. (#14748) --- .../mledger/impl/ManagedLedgerImpl.java | 9 ++++- .../mledger/impl/ManagedLedgerTest.java | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 59b04fec8f81c..193ce42338527 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2298,13 +2298,13 @@ private void maybeOffload(CompletableFuture finalPromise) { + ", total size = {}, already offloaded = {}, to offload = {}", name, toOffload.stream().map(LedgerInfo::getLedgerId).collect(Collectors.toList()), sizeSummed, alreadyOffloadedSize, toOffloadSize); + offloadLoop(unlockingPromise, toOffload, PositionImpl.latest, Optional.empty()); } else { // offloadLoop will complete immediately with an empty list to offload log.debug("[{}] Nothing to offload, total size = {}, already offloaded = {}, threshold = {}", name, sizeSummed, alreadyOffloadedSize, threshold); + unlockingPromise.complete(PositionImpl.latest); } - - offloadLoop(unlockingPromise, toOffload, PositionImpl.latest, Optional.empty()); } } } @@ -2862,6 +2862,11 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct private void offloadLoop(CompletableFuture promise, Queue ledgersToOffload, PositionImpl firstUnoffloaded, Optional firstError) { + if (getState() == State.Closed) { + promise.completeExceptionally(new ManagedLedgerAlreadyClosedException( + String.format("managed ledger [%s] has already closed", name))); + return; + } LedgerInfo info = ledgersToOffload.poll(); if (info == null) { if (firstError.isPresent()) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 648d3e671d862..af5f6c807e3ff 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -121,6 +122,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Stat; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -3422,4 +3424,42 @@ public void testCancellationOfScheduledTasks() throws Exception { assertTrue(timeoutTask2.isCancelled()); assertTrue(checkLedgerRollTask2.isCancelled()); } + + @Test + public void testOffloadTaskCancelled() throws Exception { + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(2); + config.setMinimumRolloverTime(0, TimeUnit.SECONDS); + + OffloadPoliciesImpl offloadPolicies = new OffloadPoliciesImpl(); + offloadPolicies.setManagedLedgerOffloadDriver("mock"); + offloadPolicies.setManagedLedgerOffloadThresholdInBytes(0L); + LedgerOffloader ledgerOffloader = Mockito.mock(LedgerOffloader.class); + Mockito.when(ledgerOffloader.getOffloadPolicies()).thenReturn(offloadPolicies); + Mockito.when(ledgerOffloader.getOffloadDriverName()).thenReturn(offloadPolicies.getManagedLedgerOffloadDriver()); + config.setLedgerOffloader(ledgerOffloader); + + CompletableFuture readHandle = new CompletableFuture<>(); + readHandle.complete(mock(ReadHandle.class)); + + CompletableFuture offloadFuture = new CompletableFuture<>(); + offloadFuture.complete(null); + Mockito.when(ledgerOffloader.offload(any(ReadHandle.class), any(UUID.class), any(Map.class))).thenReturn(offloadFuture); + + final ManagedLedgerImpl ledgerInit = (ManagedLedgerImpl) factory.open("test-offload-task-close", config); + final ManagedLedgerImpl ledger = spy(ledgerInit); + long ledgerId = 3L; + doReturn(readHandle).when(ledger).getLedgerHandle(ledgerId); + doReturn(ManagedLedgerImpl.State.Closed).when(ledger).getState(); + ledger.addEntry("dummy-entry-1".getBytes(Encoding)); + ledger.addEntry("dummy-entry-2".getBytes(Encoding)); + ledger.addEntry("dummy-entry-3".getBytes(Encoding)); + ledger.close(); + + Awaitility.await().untilAsserted(() -> { + CompletableFuture ledgerInfo = ledger.getLedgerInfo(ledgerId); + Assert.assertFalse(ledgerInfo.get(100, TimeUnit.MILLISECONDS).getOffloadContext().getComplete()); + }); + } } From 2624a0407fb76a9280034b2c4b99147d70946356 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 21 Mar 2022 09:10:51 -0700 Subject: [PATCH 408/823] [fix][broker] Fixed duplicated delayed messages when all consumers disconnect (#14740) --- ...PersistentDispatcherMultipleConsumers.java | 1 + .../persistent/DelayedDeliveryTest.java | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index a8a36cdc6c340..782c597a81e15 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -147,6 +147,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce shouldRewindBeforeReadingOrReplaying = false; } redeliveryMessages.clear(); + delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::clear); } if (isConsumersExceededOnSubscription()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java index cb870f8a70e81..480da2f5b94a8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java @@ -493,4 +493,50 @@ public void testClearDelayedMessagesWhenClearBacklog() throws PulsarClientExcept admin.topics().skipAllMessages(topic, subName); Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 0)); } + + @Test + public void testDelayedDeliveryWithAllConsumersDisconnecting() throws Exception { + String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testDelays"); + + Consumer c1 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + producer.newMessage() + .value("msg") + .deliverAfter(5, TimeUnit.SECONDS) + .send(); + + Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1)); + + c1.close(); + + // Attach a new consumer. Since there are no consumers connected, this will trigger the cursor rewind + @Cleanup + Consumer c2 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .receiverQueueSize(1) + .subscribe(); + + Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1)); + + Message msg = c2.receive(10, TimeUnit.SECONDS); + assertNotNull(msg); + + // No more messages + msg = c2.receive(1, TimeUnit.SECONDS); + assertNull(msg); + + Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 0)); + } } From 456dce613f4e5cf58f67ff6c92eae050131ea5b6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 23 Mar 2022 14:35:38 +0800 Subject: [PATCH 409/823] [C++] Fix producer is never destructed until client is closed (#14797) Fixes #509 ### Motivation When a C++ producer is created successfully, it will start a send timer. However the callback has captured the shared pointer of `ProducerImpl` itself. It extends the lifetime of `ProducerImpl` so that even after the `Producer` object destructs, the underlying `ProducerImpl` object won't be destructed. It could only be destructed after `Client::close()` is called. ### Modifications - Pass a weak pointer of `ProducerImpl` to the send timer and add a `asyncWaitSendTimeout` method for the combination of `expires_from_now` and `async_wait` calls on the timer. - Add `ClientTest.testReferenceCount` to verify the reference count will become 0 after the producer or consumer destructs. (cherry picked from commit f7cbc1eb83ffd27b784d90d5d2dea8660c590ad2) --- pulsar-client-cpp/lib/ProducerImpl.cc | 25 +++++++++++++++--------- pulsar-client-cpp/lib/ProducerImpl.h | 2 ++ pulsar-client-cpp/tests/ClientTest.cc | 27 ++++++++++++++++++++++++++ pulsar-client-cpp/tests/PulsarFriend.h | 8 ++++++++ 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/pulsar-client-cpp/lib/ProducerImpl.cc b/pulsar-client-cpp/lib/ProducerImpl.cc index f81e205475de4..e9812d46054f7 100644 --- a/pulsar-client-cpp/lib/ProducerImpl.cc +++ b/pulsar-client-cpp/lib/ProducerImpl.cc @@ -698,8 +698,8 @@ void ProducerImpl::handleSendTimeout(const boost::system::error_code& err) { std::shared_ptr pendingCallbacks; if (pendingMessagesQueue_.empty()) { // If there are no pending messages, reset the timeout to the configured value. - sendTimer_->expires_from_now(milliseconds(conf_.getSendTimeout())); LOG_DEBUG(getName() << "Producer timeout triggered on empty pending message queue"); + asyncWaitSendTimeout(milliseconds(conf_.getSendTimeout())); } else { // If there is at least one message, calculate the diff between the message timeout and // the current time. @@ -709,17 +709,14 @@ void ProducerImpl::handleSendTimeout(const boost::system::error_code& err) { LOG_DEBUG(getName() << "Timer expired. Calling timeout callbacks."); pendingCallbacks = getPendingCallbacksWhenFailed(); // Since the pending queue is cleared now, set timer to expire after configured value. - sendTimer_->expires_from_now(milliseconds(conf_.getSendTimeout())); + asyncWaitSendTimeout(milliseconds(conf_.getSendTimeout())); } else { // The diff is greater than zero, set the timeout to the diff value LOG_DEBUG(getName() << "Timer hasn't expired yet, setting new timeout " << diff); - sendTimer_->expires_from_now(diff); + asyncWaitSendTimeout(diff); } } - // Asynchronously wait for the timeout to trigger - sendTimer_->async_wait( - std::bind(&ProducerImpl::handleSendTimeout, shared_from_this(), std::placeholders::_1)); lock.unlock(); if (pendingCallbacks) { pendingCallbacks->complete(ResultTimeout); @@ -885,11 +882,21 @@ void ProducerImpl::startSendTimeoutTimer() { // timeout to happen. if (!sendTimer_ && conf_.getSendTimeout() > 0) { sendTimer_ = executor_->createDeadlineTimer(); - sendTimer_->expires_from_now(milliseconds(conf_.getSendTimeout())); - sendTimer_->async_wait( - std::bind(&ProducerImpl::handleSendTimeout, shared_from_this(), std::placeholders::_1)); + asyncWaitSendTimeout(milliseconds(conf_.getSendTimeout())); } } +void ProducerImpl::asyncWaitSendTimeout(DurationType expiryTime) { + sendTimer_->expires_from_now(expiryTime); + + ProducerImplBaseWeakPtr weakSelf = shared_from_this(); + sendTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + auto self = weakSelf.lock(); + if (self) { + std::static_pointer_cast(self)->handleSendTimeout(err); + } + }); +} + } // namespace pulsar /* namespace pulsar */ diff --git a/pulsar-client-cpp/lib/ProducerImpl.h b/pulsar-client-cpp/lib/ProducerImpl.h index d29efed1a13ae..a9eb12b7e9409 100644 --- a/pulsar-client-cpp/lib/ProducerImpl.h +++ b/pulsar-client-cpp/lib/ProducerImpl.h @@ -155,6 +155,8 @@ class ProducerImpl : public HandlerBase, DeadlineTimerPtr sendTimer_; void handleSendTimeout(const boost::system::error_code& err); + using DurationType = typename boost::asio::deadline_timer::duration_type; + void asyncWaitSendTimeout(DurationType expiryTime); Promise producerCreatedPromise_; diff --git a/pulsar-client-cpp/tests/ClientTest.cc b/pulsar-client-cpp/tests/ClientTest.cc index 8f5e68b84a18d..920430d34cb03 100644 --- a/pulsar-client-cpp/tests/ClientTest.cc +++ b/pulsar-client-cpp/tests/ClientTest.cc @@ -19,6 +19,7 @@ #include #include "HttpHelper.h" +#include "PulsarFriend.h" #include #include @@ -176,3 +177,29 @@ TEST(ClientTest, testGetNumberOfReferences) { client.close(); } + +TEST(ClientTest, testReferenceCount) { + Client client(lookupUrl); + const std::string topic = "client-test-reference-count-" + std::to_string(time(nullptr)); + + auto &producers = PulsarFriend::getProducers(client); + auto &consumers = PulsarFriend::getConsumers(client); + + { + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); + ASSERT_EQ(producers.size(), 1); + ASSERT_EQ(producers[0].use_count(), 1); + + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe(topic, "my-sub", consumer)); + ASSERT_EQ(consumers.size(), 1); + ASSERT_EQ(consumers[0].use_count(), 1); + } + + ASSERT_EQ(producers.size(), 1); + ASSERT_EQ(producers[0].use_count(), 0); + ASSERT_EQ(consumers.size(), 1); + ASSERT_EQ(consumers[0].use_count(), 0); + client.close(); +} diff --git a/pulsar-client-cpp/tests/PulsarFriend.h b/pulsar-client-cpp/tests/PulsarFriend.h index aed7096366ad8..74aa1f74c1bb7 100644 --- a/pulsar-client-cpp/tests/PulsarFriend.h +++ b/pulsar-client-cpp/tests/PulsarFriend.h @@ -89,6 +89,14 @@ class PulsarFriend { static std::shared_ptr getClientImplPtr(Client client) { return client.impl_; } + static ClientImpl::ProducersList& getProducers(const Client& client) { + return getClientImplPtr(client)->producers_; + } + + static ClientImpl::ConsumersList& getConsumers(const Client& client) { + return getClientImplPtr(client)->consumers_; + } + static void setNegativeAckEnabled(Consumer consumer, bool enabled) { consumer.impl_->setNegativeAcknowledgeEnabledForTesting(enabled); } From 199b2d22e5e9b495e7ba2579a7ec3bf3d475e387 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:08:36 +0800 Subject: [PATCH 410/823] [fix][txn]: fix transaction log recover throw cursor already close (#14810) ### Motivation When Transactionlog recover fail throw CursorAlreadyClosedException, we should stop the recover op. the cursor was been closed, the transaction log was been closed, so we should stop the recover op, in order to release thread resources like https://github.com/apache/pulsar/pull/14781 ### Modifications When recover fail by CursorAlreadyClosedException, comeplete recover (cherry picked from commit a14a97e9aea230adf0ce85224d8d47a1c5e0fb22) --- .../buffer/impl/TopicTransactionBuffer.java | 3 ++- .../pulsar/broker/transaction/TransactionTest.java | 13 +++++++++++++ .../coordinator/impl/MLTransactionLogImpl.java | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 3b28966def1eb..f108a1690ee4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -710,9 +710,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { && exception instanceof ManagedLedgerException.NonRecoverableLedgerException || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { isReadable = false; + } else { + outstandingReadsRequests.decrementAndGet(); } recover.callBackException(exception); - outstandingReadsRequests.decrementAndGet(); } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 2b1a2f4e4e1cf..a5a39aba299cf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -682,6 +682,19 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ mlTransactionSequenceIdGenerator); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore2.getCoordinatorStats().state, "Ready")); + + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + MLTransactionMetadataStore metadataStore3 = + new MLTransactionMetadataStore(new TransactionCoordinatorID(1), + mlTransactionLog, timeoutTracker, transactionRecoverTracker, + mlTransactionSequenceIdGenerator); + Awaitility.await().untilAsserted(() -> + assertEquals(metadataStore3.getCoordinatorStats().state, "Ready")); } @Test diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index 9ab4bd30b67c0..2a13f950d3d6f 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -271,11 +271,11 @@ public Entry get() { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { if (managedLedgerConfig.isAutoSkipNonRecoverableData() && exception instanceof ManagedLedgerException.NonRecoverableLedgerException - || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { + || exception instanceof ManagedLedgerException.ManagedLedgerFencedException + || exception instanceof ManagedLedgerException.CursorAlreadyClosedException) { isReadable = false; } log.error("Transaction log init fail error!", exception); - outstandingReadsRequests.decrementAndGet(); } } From af632666e0de48348220192c140533e1b7b56c89 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 8 Mar 2022 21:33:27 -0800 Subject: [PATCH 411/823] [C++] Handle exception in creating socket when fd limit is reached (#14587) (cherry picked from commit babae8e98a172302aee0bb3790b0f4e4128a7c35) --- pulsar-client-cpp/lib/ClientConnection.cc | 28 ++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 24cf11c120548..bcee5ca12f50d 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -161,7 +161,6 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: serverProtocolVersion_(ProtocolVersion_MIN), executor_(executor), resolver_(executor_->createTcpResolver()), - socket_(executor_->createSocket()), #if BOOST_VERSION >= 107000 strand_(boost::asio::make_strand(executor_->getIOService().get_executor())), #elif BOOST_VERSION >= 106600 @@ -173,12 +172,20 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: physicalAddress_(physicalAddress), cnxString_("[ -> " + physicalAddress + "] "), incomingBuffer_(SharedBuffer::allocate(DefaultBufferSize)), - connectTimeoutTask_(std::make_shared(executor_->getIOService(), - clientConfiguration.getConnectionTimeout())), outgoingBuffer_(SharedBuffer::allocate(DefaultBufferSize)), - consumerStatsRequestTimer_(executor_->createDeadlineTimer()), maxPendingLookupRequest_(clientConfiguration.getConcurrentLookupRequest()) { + try { + socket_ = executor_->createSocket(); + connectTimeoutTask_ = std::make_shared(executor_->getIOService(), + clientConfiguration.getConnectionTimeout()); + consumerStatsRequestTimer_ = executor_->createDeadlineTimer(); + } catch (const boost::system::system_error& e) { + LOG_ERROR("Failed to initialize connection: " << e.what()); + close(); + return; + } + LOG_INFO(cnxString_ << "Create ClientConnection, timeout=" << clientConfiguration.getConnectionTimeout()); if (clientConfiguration.isUseTls()) { #if BOOST_VERSION >= 105400 @@ -1503,9 +1510,11 @@ void ClientConnection::close(Result result) { } state_ = Disconnected; boost::system::error_code err; - socket_->close(err); - if (err) { - LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); + if (socket_) { + socket_->close(err); + if (err) { + LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); + } } if (tlsSocket_) { @@ -1540,7 +1549,10 @@ void ClientConnection::close(Result result) { consumerStatsRequestTimer_.reset(); } - connectTimeoutTask_->stop(); + if (connectTimeoutTask_) { + connectTimeoutTask_->stop(); + connectTimeoutTask_.reset(); + } lock.unlock(); LOG_INFO(cnxString_ << "Connection closed"); From 869afe4984eab7eb1b525ad4cc468592d2a8afb2 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 24 Mar 2022 03:59:41 +0800 Subject: [PATCH 412/823] [C++] Fix the race condition of connect timeout task (#14823) Fixes #14665 ### Motivation In C++ client, a connect timeout task is created each time before an asynchronous connect operation is performed, if the connection cannot be established in the configured timeout, the callback of the task will be called to close the connection and then the `createProducer` or `subscribe` methods will return `ResultConnectError`. `ClientConnection::connectTimeoutTask_`, which is a shared pointer, represents the timeout task. However, after `ClientConnection::close` is called, the shared pointer will be reset, and the underlying `PeriodicTask` object will be released. After that, when `stop` method is called on the released `PeriodicTask` object in the callback (`handleTcpConnected`), a segmentation fault will happen. The root cause is that `connectTimeoutTask_` can be accessed in two threads while one of them could release the memory. See #14665 for more explanations. This race condition leads to flaky Python tests as well, because we also have the similar test in Python tests. See https://github.com/apache/pulsar/blob/f7cbc1eb83ffd27b784d90d5d2dea8660c590ad2/pulsar-client-cpp/python/pulsar_test.py#L1207-L1221 So this PR might also fix #14714. ### Modifications Remove `connectTimeoutTask_.reset()` in `ClientConnection::close`. After that, the `connectTimeoutTask_` will always points to the same `PeriodicTask` object, whose methods are thread safe. ### Verifying this change Execute the following command ```bash ./tests/main --gtest_filter='ClientTest.testConnectTimeout' --gtest_repeat=10 ``` to runs the `testConnectTimeout` for 10 times. In my local env, it never failed, while before applying this patch, it's very easy to fail. (cherry picked from commit 0c3aad1e0ba0ee53784b963a1238d3d76b6dd8b2) --- pulsar-client-cpp/lib/ClientConnection.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index bcee5ca12f50d..efc3cd5a2a6f4 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -1549,10 +1549,7 @@ void ClientConnection::close(Result result) { consumerStatsRequestTimer_.reset(); } - if (connectTimeoutTask_) { - connectTimeoutTask_->stop(); - connectTimeoutTask_.reset(); - } + connectTimeoutTask_->stop(); lock.unlock(); LOG_INFO(cnxString_ << "Connection closed"); From a62c371148160027984b4312734f02d4e990b227 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 24 Mar 2022 15:29:16 +0800 Subject: [PATCH 413/823] [C++] Fix segmentation fault when creating socket failed (#14834) ### Motivation https://github.com/apache/pulsar/pull/14823 fixes the flaky `testConnectTimeout` but it's also a regression of https://github.com/apache/pulsar/pull/14587. Because when the fd limit is reached, the `connectionTimeoutTask_` won't be initialized with a non-null value. Calling `stop` method on it directly will cause segmentation fault. See https://github.com/apache/pulsar/blob/0fe921f32cefe7648ca428cd9861f9163c69767d/pulsar-client-cpp/lib/ClientConnection.cc#L178-L185 ### Modifications Add the null check for `connectionTimeoutTask_` in `ClientConnection::close`. (cherry picked from commit 54c368ed3744a40240205c17bdcac5cef48130e4) --- pulsar-client-cpp/lib/ClientConnection.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index efc3cd5a2a6f4..6071d0e6b0d74 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -1549,7 +1549,9 @@ void ClientConnection::close(Result result) { consumerStatsRequestTimer_.reset(); } - connectTimeoutTask_->stop(); + if (connectTimeoutTask_) { + connectTimeoutTask_->stop(); + } lock.unlock(); LOG_INFO(cnxString_ << "Connection closed"); From b472453c02d430feff91b63b7554e044b5530d19 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 17 Mar 2022 03:24:54 +0200 Subject: [PATCH 414/823] [Proxy] Log warning when opening connection to broker fails (#14710) (cherry picked from commit 13c45627f6ba65ae50dfdc0da105d678b382045a) --- .../java/org/apache/pulsar/proxy/server/DirectProxyHandler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 37fc3d5a8a298..44a236a8e31c6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -148,6 +148,8 @@ protected void initChannel(SocketChannel ch) { f.addListener(future -> { if (!future.isSuccess()) { // Close the connection if the connection attempt has failed. + log.warn("[{}] Establishing connection to {} ({}) failed. Closing inbound channel.", inboundChannel, + targetBrokerAddress, targetBrokerUrl, future.cause()); inboundChannel.close(); return; } From da5659254161b63a8123485357ac60cb37b75a41 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 22 Mar 2022 05:58:10 +0200 Subject: [PATCH 415/823] [refactor][proxy] Refactor Proxy code and fix connection stalling by switching to auto read mode (#14713) Refactor Proxy code to make it easier to understand and maintain. In addition, switch to use auto read mode since the proxies connections seem to stall in some cases since the proxied connection doesn't use Netty's auto read mode and the read handling doesn't seem complete. Currently, the proxy calls `.read()` when a message is written to the connection. There might be more messages flowing in the other direction and it could result in a blocked connection with the current solution that doesn't use Netty's auto read mode. Currently auto read is disabled in DirectProxyHandler for the connection between the proxy and the broker: https://github.com/apache/pulsar/blob/a26905371749798ec5288fb07a69978a36aacfaa/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java#L112 - replace broker host parsing with a simple solution - pass remote host name to ProxyBackendHandler in the constructor - rename "targetBrokerUrl" to "brokerHostAndPort" since the "targetBrokerUrl" is really "hostname:port" string - move HA proxy message handling to ProxyBackendHandle and extract the logic to a method - remove the static "inboundOutboundChannelMap" which was used for log level 2 - make it obsolete by passing the peer channel id to ParserProxyHandler - Enable auto read in proxy and remove `ctx.read()` / `channel.read()` calls - prepare for IPv6 support (reported as #14732) by improving the `host:port` parsing (pick last `:` since IPv6 address might contains multiple `:` characters) - Handle backpressure properly by switching auto read off when channel writability changes - change auto read of the proxy-broker connection based on the writability of the client-proxy connection - change auto read of the client-proxy connection based on the writability of the proxy-broker connection - Consistently handle write errors by delegating exception handling to exceptionCaught method by using `.addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE)` (cherry picked from commit a1037c75c9c305369799b71d840fa73cb198b293) --- .../proxy/server/BrokerProxyValidator.java | 2 +- .../proxy/server/DirectProxyHandler.java | 182 ++++++++---------- .../proxy/server/ParserProxyHandler.java | 8 +- .../pulsar/proxy/server/ProxyConnection.java | 70 +++---- .../server/BrokerProxyValidatorTest.java | 20 ++ 5 files changed, 136 insertions(+), 146 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java index debe1f7fcac87..b0529c2a777e1 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/BrokerProxyValidator.java @@ -113,7 +113,7 @@ private static List parseCommaSeparatedConfigValue(String configValue) { } public CompletableFuture resolveAndCheckTargetAddress(String hostAndPort) { - int pos = hostAndPort.indexOf(':'); + int pos = hostAndPort.lastIndexOf(':'); String host = hostAndPort.substring(0, pos); int port = Integer.parseInt(hostAndPort.substring(pos + 1)); if (!isPortAllowed(port)) { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 44a236a8e31c6..e04ac1e4ae83d 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -21,15 +21,14 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; - import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelId; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.socket.SocketChannel; @@ -41,27 +40,16 @@ import io.netty.handler.ssl.SslHandler; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.FutureListener; - import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; - import javax.net.ssl.SSLSession; - import lombok.Getter; - import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.tls.TlsHostnameVerifier; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.api.proto.CommandAuthChallenge; @@ -69,6 +57,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarDecoder; import org.apache.pulsar.common.stats.Rate; +import org.apache.pulsar.common.tls.TlsHostnameVerifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,11 +65,11 @@ public class DirectProxyHandler { @Getter private final Channel inboundChannel; + private final ProxyConnection proxyConnection; @Getter Channel outboundChannel; @Getter private final Rate inboundChannelRequestsRate; - protected static Map inboundOutboundChannelMap = new ConcurrentHashMap<>(); private final String originalPrincipal; private final AuthData clientAuthData; private final String clientAuthMethod; @@ -91,12 +80,13 @@ public class DirectProxyHandler { private final ProxyService service; private final Runnable onHandshakeCompleteAction; - public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, String targetBrokerUrl, + public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, String brokerHostAndPort, InetSocketAddress targetBrokerAddress, int protocolVersion, Supplier sslHandlerSupplier) { this.service = service; this.authentication = proxyConnection.getClientAuthentication(); this.inboundChannel = proxyConnection.ctx().channel(); + this.proxyConnection = proxyConnection; this.inboundChannelRequestsRate = new Rate(); this.originalPrincipal = proxyConnection.clientAuthRole; this.clientAuthData = proxyConnection.clientAuthData; @@ -114,7 +104,18 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, if (brokerProxyConnectTimeoutMs > 0) { b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, brokerProxyConnectTimeoutMs); } - b.group(inboundChannel.eventLoop()).channel(inboundChannel.getClass()).option(ChannelOption.AUTO_READ, false); + b.group(inboundChannel.eventLoop()) + .channel(inboundChannel.getClass()); + + String remoteHost; + try { + remoteHost = parseHost(brokerHostAndPort); + } catch (IllegalArgumentException e) { + log.warn("[{}] Failed to parse broker host '{}'", inboundChannel, brokerHostAndPort, e); + inboundChannel.close(); + return; + } + b.handler(new ChannelInitializer() { @Override protected void initChannel(SocketChannel ch) { @@ -128,65 +129,58 @@ protected void initChannel(SocketChannel ch) { } ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder( Commands.DEFAULT_MAX_MESSAGE_SIZE + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, 4)); - ch.pipeline().addLast("proxyOutboundHandler", new ProxyBackendHandler(config, protocolVersion)); + ch.pipeline().addLast("proxyOutboundHandler", + new ProxyBackendHandler(config, protocolVersion, remoteHost)); } }); - URI targetBroker; - try { - // targetBrokerUrl is coming in the "hostname:6650" form, so we need - // to extract host and port - targetBroker = new URI("pulsar://" + targetBrokerUrl); - } catch (URISyntaxException e) { - log.warn("[{}] Failed to parse broker url '{}'", inboundChannel, targetBrokerUrl, e); - inboundChannel.close(); - return; - } - ChannelFuture f = b.connect(targetBrokerAddress); outboundChannel = f.channel(); f.addListener(future -> { if (!future.isSuccess()) { // Close the connection if the connection attempt has failed. log.warn("[{}] Establishing connection to {} ({}) failed. Closing inbound channel.", inboundChannel, - targetBrokerAddress, targetBrokerUrl, future.cause()); + targetBrokerAddress, brokerHostAndPort, future.cause()); inboundChannel.close(); return; } - final ProxyBackendHandler cnx = (ProxyBackendHandler) outboundChannel.pipeline() - .get("proxyOutboundHandler"); - cnx.setRemoteHostName(targetBroker.getHost()); - - // if enable full parsing feature - if (service.getProxyLogLevel() == 2) { - //Set a map between inbound and outbound, - //so can find inbound by outbound or find outbound by inbound - inboundOutboundChannelMap.put(outboundChannel.id() , inboundChannel.id()); - } + }); + } - if (config.isHaProxyProtocolEnabled()) { - if (proxyConnection.hasHAProxyMessage()) { - outboundChannel.writeAndFlush(encodeProxyProtocolMessage(proxyConnection.getHAProxyMessage())); - } else { - if (inboundChannel.remoteAddress() instanceof InetSocketAddress) { - InetSocketAddress clientAddress = (InetSocketAddress) inboundChannel.remoteAddress(); - String sourceAddress = clientAddress.getAddress().getHostAddress(); - int sourcePort = clientAddress.getPort(); - if (outboundChannel.localAddress() instanceof InetSocketAddress) { - InetSocketAddress proxyAddress = (InetSocketAddress) inboundChannel.remoteAddress(); - String destinationAddress = proxyAddress.getAddress().getHostAddress(); - int destinationPort = proxyAddress.getPort(); - HAProxyMessage msg = new HAProxyMessage(HAProxyProtocolVersion.V1, HAProxyCommand.PROXY, - HAProxyProxiedProtocol.TCP4, sourceAddress, destinationAddress, sourcePort, destinationPort); - outboundChannel.writeAndFlush(encodeProxyProtocolMessage(msg)); - msg.release(); - } - } - } + private static String parseHost(String brokerPortAndHost) { + int pos = brokerPortAndHost.lastIndexOf(':'); + if (pos > 0) { + return brokerPortAndHost.substring(0, pos); + } else { + throw new IllegalArgumentException("Illegal broker host:port '" + brokerPortAndHost + "'"); + } + } + + private void writeHAProxyMessage() { + if (proxyConnection.hasHAProxyMessage()) { + outboundChannel.writeAndFlush(encodeProxyProtocolMessage(proxyConnection.getHAProxyMessage())) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); + } else { + if (inboundChannel.remoteAddress() instanceof InetSocketAddress + && outboundChannel.localAddress() instanceof InetSocketAddress) { + InetSocketAddress clientAddress = (InetSocketAddress) inboundChannel.remoteAddress(); + String sourceAddress = clientAddress.getAddress().getHostAddress(); + int sourcePort = clientAddress.getPort(); + InetSocketAddress proxyAddress = (InetSocketAddress) inboundChannel.remoteAddress(); + String destinationAddress = proxyAddress.getAddress().getHostAddress(); + int destinationPort = proxyAddress.getPort(); + HAProxyMessage msg = new HAProxyMessage(HAProxyProtocolVersion.V1, HAProxyCommand.PROXY, + HAProxyProxiedProtocol.TCP4, sourceAddress, destinationAddress, sourcePort, + destinationPort); + outboundChannel.writeAndFlush(encodeProxyProtocolMessage(msg)) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); + msg.release(); } - }); + } } + + private ByteBuf encodeProxyProtocolMessage(HAProxyMessage msg) { // Max length of v1 version proxy protocol message is 108 ByteBuf out = Unpooled.buffer(108); @@ -218,30 +212,45 @@ enum BackendState { Init, HandshakeCompleted } - public class ProxyBackendHandler extends PulsarDecoder implements FutureListener { + public class ProxyBackendHandler extends PulsarDecoder { private BackendState state = BackendState.Init; - private String remoteHostName; + private final String remoteHostName; protected ChannelHandlerContext ctx; private final ProxyConfiguration config; private final int protocolVersion; - public ProxyBackendHandler(ProxyConfiguration config, int protocolVersion) { + public ProxyBackendHandler(ProxyConfiguration config, int protocolVersion, String remoteHostName) { this.config = config; this.protocolVersion = protocolVersion; + this.remoteHostName = remoteHostName; } @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { this.ctx = ctx; + + if (config.isHaProxyProtocolEnabled()) { + writeHAProxyMessage(); + } + // Send the Connect command to broker authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); ByteBuf command; command = Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion, "Pulsar proxy", null /* target broker */, originalPrincipal, clientAuthData, clientAuthMethod); - outboundChannel.writeAndFlush(command); - outboundChannel.read(); + outboundChannel.writeAndFlush(command) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); + } + + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { + // handle backpressure + // stop/resume reading input from connection between the client and the proxy + // when the writability of the connection between the proxy and the broker changes + inboundChannel.config().setAutoRead(ctx.channel().isWritable()); + super.channelWritabilityChanged(ctx); } @Override @@ -262,7 +271,8 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce if (msg instanceof ByteBuf) { ProxyService.bytesCounter.inc(((ByteBuf) msg).readableBytes()); } - inboundChannel.writeAndFlush(msg).addListener(this); + inboundChannel.writeAndFlush(msg) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; default: @@ -301,26 +311,13 @@ protected void handleAuthChallenge(CommandAuthChallenge authChallenge) { log.debug("{} Mutual auth {}", ctx.channel(), authentication.getAuthMethodName()); } - outboundChannel.writeAndFlush(request); - outboundChannel.read(); + outboundChannel.writeAndFlush(request) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } catch (Exception e) { log.error("Error mutual verify", e); } } - @Override - public void operationComplete(Future future) { - // This is invoked when the write operation on the paired connection - // is completed - if (future.isSuccess()) { - outboundChannel.read(); - } else { - log.warn("[{}] [{}] Failed to write on proxy connection. Closing both connections.", inboundChannel, - outboundChannel, future.cause()); - inboundChannel.close(); - } - } - @Override protected void messageReceived() { // no-op @@ -350,18 +347,7 @@ protected void handleConnected(CommandConnected connected) { int maxMessageSize = connected.hasMaxMessageSize() ? connected.getMaxMessageSize() : Commands.INVALID_MAX_MESSAGE_SIZE; inboundChannel.writeAndFlush(Commands.newConnected(connected.getProtocolVersion(), maxMessageSize)) - .addListener(future -> { - if (future.isSuccess()) { - // Start reading from both connections - inboundChannel.read(); - outboundChannel.read(); - } else { - log.warn("[{}] [{}] Failed to write to inbound connection. Closing both connections.", - inboundChannel, - outboundChannel, future.cause()); - inboundChannel.close(); - } - }); + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } private void startDirectProxying(CommandConnected connected) { @@ -390,20 +376,20 @@ private void startDirectProxying(CommandConnected connected) { inboundChannel.pipeline().addBefore("handler", "inboundParser", new ParserProxyHandler(service, inboundChannel, ParserProxyHandler.FRONTEND_CONN, - connected.getMaxMessageSize())); + connected.getMaxMessageSize(), outboundChannel.id())); outboundChannel.pipeline().addBefore("proxyOutboundHandler", "outboundParser", new ParserProxyHandler(service, outboundChannel, ParserProxyHandler.BACKEND_CONN, - connected.getMaxMessageSize())); + connected.getMaxMessageSize(), inboundChannel.id())); } else { inboundChannel.pipeline().addBefore("handler", "inboundParser", new ParserProxyHandler(service, inboundChannel, ParserProxyHandler.FRONTEND_CONN, - Commands.DEFAULT_MAX_MESSAGE_SIZE)); + Commands.DEFAULT_MAX_MESSAGE_SIZE, outboundChannel.id())); outboundChannel.pipeline().addBefore("proxyOutboundHandler", "outboundParser", new ParserProxyHandler(service, outboundChannel, ParserProxyHandler.BACKEND_CONN, - Commands.DEFAULT_MAX_MESSAGE_SIZE)); + Commands.DEFAULT_MAX_MESSAGE_SIZE, inboundChannel.id())); } } } @@ -419,10 +405,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { ctx.close(); } - public void setRemoteHostName(String remoteHostName) { - this.remoteHostName = remoteHostName; - } - private boolean verifyTlsHostName(String hostname, ChannelHandlerContext ctx) { ChannelHandler sslHandler = ctx.channel().pipeline().get("tls"); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java index 9ce7a26d99c00..b5681c64bc7c9 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java @@ -19,6 +19,7 @@ package org.apache.pulsar.proxy.server; +import io.netty.channel.ChannelId; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -56,6 +57,7 @@ public class ParserProxyHandler extends ChannelInboundHandlerAdapter { private String connType; private int maxMessageSize; + private final ChannelId peerChannelId; private final ProxyService service; @@ -64,11 +66,13 @@ public class ParserProxyHandler extends ChannelInboundHandlerAdapter { private static Map producerHashMap = new ConcurrentHashMap<>(); private static Map consumerHashMap = new ConcurrentHashMap<>(); - public ParserProxyHandler(ProxyService service, Channel channel, String type, int maxMessageSize) { + public ParserProxyHandler(ProxyService service, Channel channel, String type, int maxMessageSize, + ChannelId peerChannelId) { this.service = service; this.channel = channel; this.connType = type; this.maxMessageSize = maxMessageSize; + this.peerChannelId = peerChannelId; } private void logging(Channel conn, BaseCommand.Type cmdtype, String info, List messages) throws Exception{ @@ -143,7 +147,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { logging(ctx.channel() , cmd.getType() , "" , null); break; } - topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(String.valueOf(cmd.getMessage().getConsumerId()) + "," + DirectProxyHandler.inboundOutboundChannelMap.get(ctx.channel().id()))); + topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId)); msgBytes = new MutableLong(0); MessageParser.parseMessage(topicName, -1L, -1L,buffer,(message) -> { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index df99acacbef5d..29060faeff25a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import io.netty.channel.ChannelFutureListener; import io.netty.handler.codec.haproxy.HAProxyMessage; import java.net.SocketAddress; import java.util.Collections; @@ -69,7 +70,7 @@ * Handles incoming discovery request from client and sends appropriate response back to client * */ -public class ProxyConnection extends PulsarHandler implements FutureListener { +public class ProxyConnection extends PulsarHandler { private static final Logger LOG = LoggerFactory.getLogger(ProxyConnection.class); // ConnectionPool is used by the proxy to issue lookup requests private ConnectionPool connectionPool; @@ -190,6 +191,17 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E ctx.close(); } + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { + if (directProxyHandler != null && directProxyHandler.outboundChannel != null) { + // handle backpressure + // stop/resume reading input from connection between the proxy and the broker + // when the writability of the connection between the client and the proxy changes + directProxyHandler.outboundChannel.config().setAutoRead(ctx.channel().isWritable()); + } + super.channelWritabilityChanged(ctx); + } + @Override public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exception { if (msg instanceof HAProxyMessage) { @@ -213,7 +225,8 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce directProxyHandler.getInboundChannelRequestsRate().recordEvent(bytes); ProxyService.bytesCounter.inc(bytes); } - directProxyHandler.outboundChannel.writeAndFlush(msg).addListener(this); + directProxyHandler.outboundChannel.writeAndFlush(msg) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; default: @@ -221,18 +234,6 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce } } - @Override - public void operationComplete(Future future) { - // This is invoked when the write operation on the paired connection is - // completed - if (future.isSuccess()) { - ctx.read(); - } else { - LOG.warn("[{}] Error in writing to inbound channel. Closing", remoteAddress, future.cause()); - directProxyHandler.outboundChannel.close(); - } - } - private synchronized void completeConnect(AuthData clientData) throws PulsarClientException { if (service.getConfiguration().isAuthenticationEnabled()) { if (service.getConfiguration().isForwardAuthorizationCredentials()) { @@ -270,18 +271,18 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie ctx() .writeAndFlush( Commands.newError(-1, ServerError.ServiceNotReady, "Target broker isn't available.")) - .addListener(future -> ctx().close()); + .addListener(ChannelFutureListener.CLOSE); return; } brokerProxyValidator.resolveAndCheckTargetAddress(proxyToBrokerUrl) - .thenAccept(address -> ctx().executor().submit(() -> { + .thenAcceptAsync(address -> { // Client already knows which broker to connect. Let's open a // connection there and just pass bytes in both directions state = State.ProxyConnectionToBroker; directProxyHandler = new DirectProxyHandler(service, this, proxyToBrokerUrl, address, protocolVersionToAdvertise, sslHandlerSupplier); - })) + }, ctx.executor()) .exceptionally(throwable -> { if (throwable instanceof TargetAddressDeniedException || throwable.getCause() instanceof TargetAddressDeniedException) { @@ -300,7 +301,7 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie .writeAndFlush( Commands.newError(-1, ServerError.ServiceNotReady, "Target broker cannot be validated.")) - .addListener(future -> ctx().close()); + .addListener(ChannelFutureListener.CLOSE); return null; }); } else { @@ -309,7 +310,8 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie // partitions metadata lookups state = State.ProxyLookupRequests; lookupProxyHandler = new LookupProxyHandler(service, this); - ctx.writeAndFlush(Commands.newConnected(protocolVersionToAdvertise)); + ctx.writeAndFlush(Commands.newConnected(protocolVersionToAdvertise)) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } } @@ -328,7 +330,8 @@ private void doAuthentication(AuthData clientData) throws Exception { } // auth not complete, continue auth with client side. - ctx.writeAndFlush(Commands.newAuthChallenge(authMethod, brokerData, protocolVersionToAdvertise)); + ctx.writeAndFlush(Commands.newAuthChallenge(authMethod, brokerData, protocolVersionToAdvertise)) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); if (LOG.isDebugEnabled()) { LOG.debug("[{}] Authentication in progress client by method {}.", remoteAddress, authMethod); @@ -406,8 +409,8 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), doAuthentication(clientData); } catch (Exception e) { LOG.warn("[{}] Unable to authenticate: ", remoteAddress, e); - ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, "Failed to authenticate")); - close(); + ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, "Failed to authenticate")) + .addListener(ChannelFutureListener.CLOSE); } } @@ -428,8 +431,8 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) { } catch (Exception e) { String msg = "Unable to handleAuthResponse"; LOG.warn("[{}] {} ", remoteAddress, msg, e); - ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, msg)); - close(); + ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, msg)) + .addListener(ChannelFutureListener.CLOSE); } } @@ -462,25 +465,6 @@ protected void handleLookup(CommandLookupTopic lookup) { lookupProxyHandler.handleLookup(lookup); } - private synchronized void close() { - if (state != State.Closed) { - state = State.Closed; - if (directProxyHandler != null && directProxyHandler.outboundChannel != null) { - directProxyHandler.outboundChannel.close(); - directProxyHandler = null; - } - if (connectionPool != null) { - try { - connectionPool.close(); - connectionPool = null; - } catch (Exception e) { - LOG.error("Error closing connection pool", e); - } - } - ctx.close(); - } - } - ClientConfigurationData createClientConfiguration() { ClientConfigurationData clientConf = new ClientConfigurationData(); clientConf.setServiceUrl(service.getServiceUrl()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java index 8e457554cf5ad..fba3c36e26616 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/BrokerProxyValidatorTest.java @@ -90,6 +90,26 @@ public void shouldAllowAllWithWildcard() throws Exception { brokerProxyValidator.resolveAndCheckTargetAddress("myhost.mydomain:6650").get(); } + @Test + public void shouldAllowIPv6Address() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("fd4d:801b:73fa:abcd:0000:0000:0000:0001"), + "*" + , "fd4d:801b:73fa:abcd::/64" + , "6650"); + brokerProxyValidator.resolveAndCheckTargetAddress("myhost.mydomain:6650").get(); + } + + @Test + public void shouldAllowIPv6AddressNumeric() throws Exception { + BrokerProxyValidator brokerProxyValidator = new BrokerProxyValidator( + createMockedAddressResolver("fd4d:801b:73fa:abcd:0000:0000:0000:0001"), + "*" + , "fd4d:801b:73fa:abcd::/64" + , "6650"); + brokerProxyValidator.resolveAndCheckTargetAddress("fd4d:801b:73fa:abcd:0000:0000:0000:0001:6650").get(); + } + private AddressResolver createMockedAddressResolver(String ipAddressResult) { AddressResolver inetSocketAddressResolver = mock(AddressResolver.class); when(inetSocketAddressResolver.resolve(any())).then(invocationOnMock -> { From cb4b004f21b959f8e4a733e45c5972380738264c Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Wed, 23 Mar 2022 11:08:04 +0800 Subject: [PATCH 416/823] [fix][txn]: fix pending ack is recovering throw CursorAlreadyClosedxception (#14781) ### Motivation When Transaction PendingAck recover fail throw CursorAlreadyClosedException, we should stop the recover op. the cursor was been closed, the pendingAck was been closed, so we should stop the recover op, in order to release thread resources ``` 02:03:00.072 [pulsar-transaction-executor-4-1] ERROR org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore - MLPendingAckStore of topic [public/default/persistent/source-topic-partition-13-test__transaction_pending_ack] stat reply fail! org.apache.bookkeeper.mledger.ManagedLedgerException$CursorAlreadyClosedException: Cursor was already closed ``` ### Modifications When recover fail by CursorAlreadyClosedException, comeplete recover ### Verifying this change add test for it (cherry picked from commit 9f30ee909a683458cb0c0ba7b6cf0c5bd874f4ea) --- .../pendingack/impl/MLPendingAckStore.java | 3 ++- .../pulsar/broker/transaction/TransactionTest.java | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index a7a46f9ed91bd..0e4ed4426b9f7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -398,7 +398,8 @@ public Entry get() { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { if (managedLedger.getConfig().isAutoSkipNonRecoverableData() && exception instanceof ManagedLedgerException.NonRecoverableLedgerException - || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { + || exception instanceof ManagedLedgerException.ManagedLedgerFencedException + || exception instanceof ManagedLedgerException.CursorAlreadyClosedException) { isReadable = false; } log.error("MLPendingAckStore of topic [{}] stat reply fail!", managedLedger.getName(), exception); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index a5a39aba299cf..308fa350c9f15 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -622,6 +622,17 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ PendingAckHandleImpl pendingAckHandle2 = new PendingAckHandleImpl(persistentSubscription); Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle2.getStats().state, "Ready")); + + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + PendingAckHandleImpl pendingAckHandle3 = new PendingAckHandleImpl(persistentSubscription); + + Awaitility.await().untilAsserted(() -> + assertEquals(pendingAckHandle3.getStats().state, "Ready")); } @Test From aa66cb8c129d71fbb610f2a8528c7d81dffbbadb Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 14 Mar 2022 21:31:43 +0800 Subject: [PATCH 417/823] Fix normal topic named ends with `healthcheck` becomes system topic issue. (#14671) (cherry picked from commit 3afd1e673304bb2dfb2c42f74a5895a0aa12cfc1) --- .../pulsar/broker/admin/AdminResource.java | 7 +- .../broker/admin/impl/NamespacesBase.java | 7 +- .../pulsar/broker/service/AbstractTopic.java | 3 +- .../pulsar/broker/service/BrokerService.java | 33 ++++++++-- .../broker/systopic/SystemTopicClient.java | 25 ------- .../broker/service/BrokerServiceTest.java | 34 ++++++++++ .../service/InactiveTopicDeleteTest.java | 65 +++++++------------ ...NamespaceEventsSystemTopicServiceTest.java | 7 +- .../systopic/SystemTopicClientTest.java | 47 -------------- 9 files changed, 98 insertions(+), 130 deletions(-) delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/SystemTopicClientTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 8d7de948d0820..3598e03b2ed29 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -39,7 +39,6 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException; -import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -586,12 +585,12 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n } // new create check - if (maxTopicsPerNamespace > 0 && !SystemTopicClient.isSystemTopic(topicName)) { + if (maxTopicsPerNamespace > 0 && !pulsar().getBrokerService().isSystemTopic(topicName)) { List partitionedTopics = getTopicPartitionList(TopicDomain.persistent); // exclude created system topic long topicsCount = - partitionedTopics.stream().filter(t -> !SystemTopicClient.isSystemTopic(TopicName.get(t))) - .count(); + partitionedTopics.stream().filter(t -> + !pulsar().getBrokerService().isSystemTopic(TopicName.get(t))).count(); if (topicsCount + numPartitions > maxTopicsPerNamespace) { log.error("[{}] Failed to create partitioned topic {}, " + "exceed maximum number of topics in namespace", clientAppId(), topicName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 310c0d9d7d584..11d7b39293671 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -58,7 +58,6 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.SubscriptionType; @@ -237,7 +236,7 @@ protected void internalDeleteNamespace(AsyncResponse asyncResponse, boolean auth } boolean hasNonSystemTopic = false; for (String topic : topics) { - if (!SystemTopicClient.isSystemTopic(TopicName.get(topic))) { + if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { hasNonSystemTopic = true; break; } @@ -1892,14 +1891,14 @@ private void clearBacklog(NamespaceName nsName, String bundleRange, String subsc } for (Topic topic : topicList) { if (topic instanceof PersistentTopic - && !SystemTopicClient.isSystemTopic(TopicName.get(topic.getName()))) { + && !pulsar().getBrokerService().isSystemTopic(TopicName.get(topic.getName()))) { futures.add(((PersistentTopic) topic).clearBacklog(subscription)); } } } else { for (Topic topic : topicList) { if (topic instanceof PersistentTopic - && !SystemTopicClient.isSystemTopic(TopicName.get(topic.getName()))) { + && !pulsar().getBrokerService().isSystemTopic(TopicName.get(topic.getName()))) { futures.add(((PersistentTopic) topic).clearBacklog()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 620d35eaf91f1..cc292022432fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -47,7 +47,6 @@ import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; -import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; @@ -540,7 +539,7 @@ public void recordAddLatency(long latency, TimeUnit unit) { } protected void setSchemaCompatibilityStrategy(Policies policies) { - if (SystemTopicClient.isSystemTopic(TopicName.get(this.topic))) { + if (isSystemTopic()) { schemaCompatibilityStrategy = brokerService.pulsar().getConfig().getSystemTopicSchemaCompatibilityStrategy(); return; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 1c452027e76a9..939a389c1eae2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -117,7 +117,7 @@ import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; -import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.validator.BindAddressValidator; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -129,6 +129,7 @@ import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.configuration.BindAddress; import org.apache.pulsar.common.configuration.FieldContext; +import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor; import org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor; import org.apache.pulsar.common.intercept.BrokerEntryMetadataUtils; @@ -2671,8 +2672,30 @@ private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(fin log.debug("No autoSubscriptionCreateOverride policy found for {}", topicName); return null; } - private boolean isSystemTopic(String topic) { - return SystemTopicClient.isSystemTopic(TopicName.get(topic)); + + public boolean isSystemTopic(String topic) { + return isSystemTopic(TopicName.get(topic)); + } + + public boolean isSystemTopic(TopicName topicName) { + if (topicName.getNamespaceObject().equals(NamespaceName.SYSTEM_NAMESPACE) + || topicName.getNamespaceObject().equals(pulsar.getHeartbeatNamespaceV2())) { + return true; + } + + TopicName nonePartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); + + // event topic + if (EventsTopicNames.checkTopicIsEventsNames(nonePartitionedTopicName)) { + return true; + } + + String localName = nonePartitionedTopicName.getLocalName(); + // transaction pending ack topic + if (StringUtils.endsWith(localName, MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { + return true; + } + return false; } /** @@ -2707,13 +2730,13 @@ private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, int maxTopicsPerNamespace = optPolicies.map(p -> p.max_topics_per_namespace) .orElse(pulsar.getConfig().getMaxTopicsPerNamespace()); - if (maxTopicsPerNamespace > 0 && !SystemTopicClient.isSystemTopic(topicName)) { + if (maxTopicsPerNamespace > 0 && !isSystemTopic(topicName)) { return pulsar().getPulsarResources().getTopicResources() .getExistingPartitions(topicName) .thenCompose(topics -> { // exclude created system topic long topicsCount = topics.stream() - .filter(t -> !SystemTopicClient.isSystemTopic(TopicName.get(t))) + .filter(t -> !isSystemTopic(TopicName.get(t))) .count(); if (topicsCount + numPartitions > maxTopicsPerNamespace) { log.error("Failed to create persistent topic {}, " diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java index 122bd9f52b392..33bfc59156768 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClient.java @@ -21,13 +21,9 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.broker.admin.impl.BrokersBase; -import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.TopicName; /** @@ -188,25 +184,4 @@ interface Reader { */ SystemTopicClient getSystemTopic(); } - - static boolean isSystemTopic(TopicName topicName) { - TopicName nonePartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); - - // event topic - if (EventsTopicNames.checkTopicIsEventsNames(nonePartitionedTopicName)) { - return true; - } - - String localName = nonePartitionedTopicName.getLocalName(); - // transaction pending ack topic - if (StringUtils.endsWith(localName, MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { - return true; - } - // health check topic - if (StringUtils.endsWith(localName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX)){ - return true; - } - return false; - } - } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index af8ddb583a58f..c7ffef1a97b7e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -18,10 +18,13 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.common.naming.TopicName.TRANSACTION_COORDINATOR_ASSIGN; +import static org.apache.pulsar.common.naming.TopicName.TRANSACTION_COORDINATOR_LOG; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -66,6 +69,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; @@ -723,6 +727,7 @@ public void testTlsEnabledWithoutNonTlsServicePorts() throws Exception { } finally { pulsarClient.close(); } + resetState(); } @SuppressWarnings("deprecation") @@ -1275,4 +1280,33 @@ public void shouldNotPreventCreatingTopicWhenNonexistingTopicIsCached() throws E getStatsThread.join(); } } + + @Test + public void testIsSystemTopic() { + BrokerService brokerService = pulsar.getBrokerService(); + assertFalse(brokerService.isSystemTopic(TopicName.get("test"))); + assertFalse(brokerService.isSystemTopic(TopicName.get("public/default/test"))); + assertFalse(brokerService.isSystemTopic(TopicName.get("healthcheck"))); + assertFalse(brokerService.isSystemTopic(TopicName.get("public/default/healthcheck"))); + assertFalse(brokerService.isSystemTopic(TopicName.get("persistent://public/default/test"))); + assertFalse(brokerService.isSystemTopic(TopicName.get("non-persistent://public/default/test"))); + + assertTrue(brokerService.isSystemTopic(TopicName.get("__change_events"))); + assertTrue(brokerService.isSystemTopic(TopicName.get("__change_events-partition-0"))); + assertTrue(brokerService.isSystemTopic(TopicName.get("__change_events-partition-1"))); + assertTrue(brokerService.isSystemTopic(TopicName.get("__transaction_buffer_snapshot"))); + assertTrue(brokerService.isSystemTopic(TopicName.get("__transaction_buffer_snapshot-partition-0"))); + assertTrue(brokerService.isSystemTopic(TopicName.get("__transaction_buffer_snapshot-partition-1"))); + assertTrue(brokerService.isSystemTopic(TopicName + .get("topicxxx-partition-0-multiTopicsReader-f433329d68__transaction_pending_ack"))); + assertTrue(brokerService.isSystemTopic( + TopicName.get("topicxxx-multiTopicsReader-f433329d68__transaction_pending_ack"))); + + assertTrue(brokerService.isSystemTopic(TRANSACTION_COORDINATOR_ASSIGN)); + assertTrue(brokerService.isSystemTopic(TRANSACTION_COORDINATOR_LOG)); + NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + assertTrue(brokerService.isSystemTopic("persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck")); + assertTrue(brokerService.isSystemTopic(heartbeatNamespaceV2.toString() + "/healthcheck")); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java index 3b7d9aaec3424..1a2dd423b904e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java @@ -35,11 +35,13 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.apache.pulsar.broker.admin.impl.BrokersBase; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; import org.apache.pulsar.zookeeper.ZooKeeperManagedLedgerCache; @@ -580,49 +582,32 @@ public void testInactiveTopicApplied() throws Exception { } @Test(timeOut = 30000) - public void testInternalTopicInactiveNotClean() throws Exception { + public void testHealthTopicInactiveNotClean() throws Exception { conf.setSystemTopicEnabled(true); conf.setBrokerDeleteInactiveTopicsMode(InactiveTopicDeleteMode.delete_when_no_subscriptions); conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1); super.baseSetup(); // init topic - final String healthCheckTopic = "persistent://prop/ns-abc/"+ BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX; - final String topic = "persistent://prop/ns-abc/testDeleteWhenNoSubscriptions"; - - Producer producer = pulsarClient.newProducer() - .topic(topic) - .create(); - Consumer consumer = pulsarClient.newConsumer() - .topic(topic) - .subscriptionName("sub") - .subscribe(); - - Producer heathCheckProducer = pulsarClient.newProducer() - .topic(healthCheckTopic) - .create(); - Consumer heathCheckConsumer = pulsarClient.newConsumer() - .topic(healthCheckTopic) - .subscriptionName("healthCheck") - .subscribe(); - - consumer.close(); - producer.close(); - heathCheckConsumer.close(); - heathCheckProducer.close(); - - Awaitility.await().untilAsserted(() -> Assert.assertTrue(admin.topics().getList("prop/ns-abc") - .contains(topic))); - Awaitility.await().untilAsserted(() -> { - Assert.assertTrue(admin.topics().getList("prop/ns-abc").contains(healthCheckTopic)); - }); - - admin.topics().deleteSubscription(topic, "sub"); - admin.topics().deleteSubscription(healthCheckTopic, "healthCheck"); - - Awaitility.await().untilAsserted(() -> Assert.assertFalse(admin.topics().getList("prop/ns-abc") - .contains(topic))); - Awaitility.await().pollDelay(2, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertTrue(admin.topics().getList("prop/ns-abc") - .contains(healthCheckTopic))); + NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + final String healthCheckTopicV1 = "persistent://" + heartbeatNamespaceV1 + "/healthcheck"; + + NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + final String healthCheckTopicV2 = "persistent://" + heartbeatNamespaceV2 + "/healthcheck"; + + admin.brokers().healthcheck(TopicVersion.V1); + admin.brokers().healthcheck(TopicVersion.V2); + + List V1Partitions = pulsar + .getPulsarResources() + .getTopicResources() + .getExistingPartitions(TopicName.get(healthCheckTopicV1)) + .get(10, TimeUnit.SECONDS); + List V2Partitions = pulsar + .getPulsarResources() + .getTopicResources() + .getExistingPartitions(TopicName.get(healthCheckTopicV2)) + .get(10, TimeUnit.SECONDS); + Assert.assertTrue(V1Partitions.contains(healthCheckTopicV1)); + Assert.assertTrue(V2Partitions.contains(healthCheckTopicV2)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java index 2daca67520320..d81c89d859f5a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Sets; import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; @@ -143,9 +144,9 @@ public void checkSystemTopic() throws PulsarAdminException { admin.topics().createPartitionedTopic(normalTopic, 3); TopicName systemTopicName = TopicName.get(systemTopic); TopicName normalTopicName = TopicName.get(normalTopic); - - Assert.assertEquals(SystemTopicClient.isSystemTopic(systemTopicName), true); - Assert.assertEquals(SystemTopicClient.isSystemTopic(normalTopicName), false); + BrokerService brokerService = pulsar.getBrokerService(); + Assert.assertEquals(brokerService.isSystemTopic(systemTopicName), true); + Assert.assertEquals(brokerService.isSystemTopic(normalTopicName), false); } private void prepareData() throws PulsarAdminException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/SystemTopicClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/SystemTopicClientTest.java deleted file mode 100644 index d6790b7ec8269..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/SystemTopicClientTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.systopic; - -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; -import org.apache.pulsar.common.naming.TopicName; -import org.testng.annotations.Test; - -public class SystemTopicClientTest { - - @Test - public void testIsSystemTopic() { - assertFalse(SystemTopicClient.isSystemTopic(TopicName.get("test"))); - assertFalse(SystemTopicClient.isSystemTopic(TopicName.get("public/default/test"))); - assertFalse(SystemTopicClient.isSystemTopic(TopicName.get("persistent://public/default/test"))); - assertFalse(SystemTopicClient.isSystemTopic(TopicName.get("non-persistent://public/default/test"))); - - assertTrue(SystemTopicClient.isSystemTopic(TopicName.get("__change_events"))); - assertTrue(SystemTopicClient.isSystemTopic(TopicName.get("__change_events-partition-0"))); - assertTrue(SystemTopicClient.isSystemTopic(TopicName.get("__change_events-partition-1"))); - assertTrue(SystemTopicClient.isSystemTopic(TopicName.get("__transaction_buffer_snapshot"))); - assertTrue(SystemTopicClient.isSystemTopic(TopicName.get("__transaction_buffer_snapshot-partition-0"))); - assertTrue(SystemTopicClient.isSystemTopic(TopicName.get("__transaction_buffer_snapshot-partition-1"))); - assertTrue(SystemTopicClient.isSystemTopic(TopicName - .get("topicxxx-partition-0-multiTopicsReader-f433329d68__transaction_pending_ack"))); - assertTrue(SystemTopicClient.isSystemTopic( - TopicName.get("topicxxx-multiTopicsReader-f433329d68__transaction_pending_ack"))); - - } -} From e0d69a31b684960dbb56f9b124c6f124ca55a034 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 17 Mar 2022 19:47:50 +0800 Subject: [PATCH 418/823] Provide an accurate error message when set ``autoTopicCreation `` (#14684) (cherry picked from commit 50a7e50745e5c26200a4abd2e63e3750e34339fb) --- .../broker/admin/impl/NamespacesBase.java | 7 +++- .../common/policies/data/ValidateResult.java | 40 +++++++++++++++++++ .../impl/AutoTopicCreationOverrideImpl.java | 18 +++++---- .../data/AutoTopicCreationOverrideTest.java | 12 +++--- 4 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ValidateResult.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 11d7b39293671..df8fd1c4e610b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -94,6 +94,7 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.TenantOperation; +import org.apache.pulsar.common.policies.data.ValidateResult; import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; import org.apache.pulsar.common.util.FutureUtil; @@ -803,9 +804,11 @@ protected void internalSetAutoTopicCreation(AsyncResponse asyncResponse, validateNamespacePolicyOperation(namespaceName, PolicyName.AUTO_TOPIC_CREATION, PolicyOperation.WRITE); validatePoliciesReadOnlyAccess(); if (autoTopicCreationOverride != null) { - if (!AutoTopicCreationOverrideImpl.isValidOverride(autoTopicCreationOverride)) { + ValidateResult validateResult = AutoTopicCreationOverrideImpl.validateOverride(autoTopicCreationOverride); + if (!validateResult.isSuccess()) { throw new RestException(Status.PRECONDITION_FAILED, - "Invalid configuration for autoTopicCreationOverride"); + "Invalid configuration for autoTopicCreationOverride. the detail is " + + validateResult.getErrorInfo()); } if (maxPartitions > 0 && autoTopicCreationOverride.getDefaultNumPartitions() > maxPartitions) { throw new RestException(Status.NOT_ACCEPTABLE, diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ValidateResult.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ValidateResult.java new file mode 100644 index 0000000000000..13821947cf217 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ValidateResult.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +import lombok.Getter; + +@Getter +public class ValidateResult { + private final boolean success; + private final String errorInfo; + + private ValidateResult(boolean success, String errorInfo) { + this.success = success; + this.errorInfo = errorInfo; + } + + public static ValidateResult fail(String errorInfo) { + return new ValidateResult(false, errorInfo); + } + + public static ValidateResult success() { + return new ValidateResult(true, null); + } +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/AutoTopicCreationOverrideImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/AutoTopicCreationOverrideImpl.java index 1ce60d1c78b30..ba6bc07780a08 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/AutoTopicCreationOverrideImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/AutoTopicCreationOverrideImpl.java @@ -23,6 +23,7 @@ import lombok.NoArgsConstructor; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.ValidateResult; /** * Override of autoTopicCreation settings on a namespace level. @@ -35,28 +36,29 @@ public final class AutoTopicCreationOverrideImpl implements AutoTopicCreationOve private String topicType; private Integer defaultNumPartitions; - public static boolean isValidOverride(AutoTopicCreationOverride override) { + public static ValidateResult validateOverride(AutoTopicCreationOverride override) { if (override == null) { - return false; + return ValidateResult.fail("[AutoTopicCreationOverride] can not be null"); } if (override.isAllowAutoTopicCreation()) { if (!TopicType.isValidTopicType(override.getTopicType())) { - return false; + return ValidateResult.fail(String.format("Unknown topic type [%s]", override.getTopicType())); } if (TopicType.PARTITIONED.toString().equals(override.getTopicType())) { if (override.getDefaultNumPartitions() == null) { - return false; + return ValidateResult.fail("[defaultNumPartitions] cannot be null when the type is partitioned."); } - if (!(override.getDefaultNumPartitions() > 0)) { - return false; + if (override.getDefaultNumPartitions() <= 0) { + return ValidateResult.fail("[defaultNumPartitions] cannot be less than 1 for partition type."); } } else if (TopicType.NON_PARTITIONED.toString().equals(override.getTopicType())) { if (override.getDefaultNumPartitions() != null) { - return false; + return ValidateResult.fail("[defaultNumPartitions] is not allowed to be" + + " set when the type is non-partition."); } } } - return true; + return ValidateResult.success(); } public static AutoTopicCreationOverrideImplBuilder builder() { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/AutoTopicCreationOverrideTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/AutoTopicCreationOverrideTest.java index 5092d433d0db7..66769f0bfbc18 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/AutoTopicCreationOverrideTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/AutoTopicCreationOverrideTest.java @@ -32,7 +32,7 @@ public void testValidOverrideNonPartitioned() { .allowAutoTopicCreation(true) .topicType(TopicType.NON_PARTITIONED.toString()) .build(); - assertTrue(AutoTopicCreationOverrideImpl.isValidOverride(override)); + assertTrue(AutoTopicCreationOverrideImpl.validateOverride(override).isSuccess()); } @Test @@ -42,7 +42,7 @@ public void testValidOverridePartitioned() { .topicType(TopicType.PARTITIONED.toString()) .defaultNumPartitions(2) .build(); - assertTrue(AutoTopicCreationOverrideImpl.isValidOverride(override)); + assertTrue(AutoTopicCreationOverrideImpl.validateOverride(override).isSuccess()); } @Test @@ -51,7 +51,7 @@ public void testInvalidTopicType() { .allowAutoTopicCreation(true) .topicType("aaa") .build(); - assertFalse(AutoTopicCreationOverrideImpl.isValidOverride(override)); + assertFalse(AutoTopicCreationOverrideImpl.validateOverride(override).isSuccess()); } @Test @@ -61,7 +61,7 @@ public void testNumPartitionsTooLow() { .topicType(TopicType.PARTITIONED.toString()) .defaultNumPartitions(0) .build(); - assertFalse(AutoTopicCreationOverrideImpl.isValidOverride(override)); + assertFalse(AutoTopicCreationOverrideImpl.validateOverride(override).isSuccess()); } @Test @@ -70,7 +70,7 @@ public void testNumPartitionsNotSet() { .allowAutoTopicCreation(true) .topicType(TopicType.PARTITIONED.toString()) .build(); - assertFalse(AutoTopicCreationOverrideImpl.isValidOverride(override)); + assertFalse(AutoTopicCreationOverrideImpl.validateOverride(override).isSuccess()); } @Test @@ -80,6 +80,6 @@ public void testNumPartitionsOnNonPartitioned() { .topicType(TopicType.NON_PARTITIONED.toString()) .defaultNumPartitions(2) .build(); - assertFalse(AutoTopicCreationOverrideImpl.isValidOverride(override)); + assertFalse(AutoTopicCreationOverrideImpl.validateOverride(override).isSuccess()); } } From e0eebaf1d00ea4a0cddd8464c3be26ef531387fa Mon Sep 17 00:00:00 2001 From: xiaolong ran Date: Fri, 18 Mar 2022 12:05:26 +0800 Subject: [PATCH 419/823] Process maxRedeliverCount is 0 of DeadLeddterPolicy (#14706) Signed-off-by: xiaolongran Fixes #14704 ### Motivation When the user uses the function of DeadLetterPolicy, it is better than misoperation. MaxRedeliverCount may be set to 0. When it is set to 0, according to the current processing logic of the Java Client, the message will be pushed to the DeadLetter Topic every time. ### Modifications - When MaxRedeliverCount <= 0 in DeadLetterPolicy, we reset MaxRedeliverCount to default value (cherry picked from commit 601fbdd40eabfcd5d5519e0c5bcc20ae280a8e18) --- .../apache/pulsar/client/impl/ConsumerBuilderImpl.java | 2 ++ .../pulsar/client/impl/ConsumerBuilderImplTest.java | 10 ++++++++++ .../pulsar/websocket/AbstractWebSocketHandlerTest.java | 3 ++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index 471d4bac007b2..dea946b46c022 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -428,6 +428,8 @@ public ConsumerBuilder deadLetterPolicy(DeadLetterPolicy deadLetterPolicy) { if (conf.getAckTimeoutMillis() == 0) { conf.setAckTimeoutMillis(DEFAULT_ACK_TIMEOUT_MILLIS_FOR_DEAD_LETTER); } + + checkArgument(deadLetterPolicy.getMaxRedeliverCount() > 0, "MaxRedeliverCount must be > 0."); conf.setDeadLetterPolicy(deadLetterPolicy); } return this; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java index 13d63baee62ee..36ea53f31ef77 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java @@ -20,6 +20,7 @@ import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.DeadLetterPolicy; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; @@ -288,6 +289,15 @@ public void testConsumerBuilderImplWhenBatchReceivePolicyIsNotValid() { .build()); } + @Test(expectedExceptions = IllegalArgumentException.class) + public void testRedeliverCountOfDeadLetterPolicy() { + consumerBuilderImpl.deadLetterPolicy(DeadLetterPolicy.builder() + .maxRedeliverCount(0) + .deadLetterTopic("test-dead-letter-topic") + .retryLetterTopic("test-retry-letter-topic") + .build()); + } + @Test public void testConsumerBuilderImplWhenNumericPropertiesAreValid() { consumerBuilderImpl.negativeAckRedeliveryDelay(1, TimeUnit.MILLISECONDS); diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java index 9bd9907677996..782e05ea62577 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java @@ -369,11 +369,12 @@ public void consumerBuilderTest() throws IOException { consumerHandler.clearQueryParams(); consumerHandler.putQueryParam("receiverQueueSize", "1001"); consumerHandler.putQueryParam("deadLetterTopic", "dead-letter-topic"); + consumerHandler.putQueryParam("maxRedeliverCount", "3"); conf = consumerHandler.getConf(); // receive queue size is the minimum value of default value (1000) and user defined value(1001) assertEquals(conf.getReceiverQueueSize(), 1000); assertEquals(conf.getDeadLetterPolicy().getDeadLetterTopic(), "dead-letter-topic"); - assertEquals(conf.getDeadLetterPolicy().getMaxRedeliverCount(), 0); + assertEquals(conf.getDeadLetterPolicy().getMaxRedeliverCount(), 3); } } From 973eadd084b4d99a5098f02596bc762b947a2c00 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 22 Mar 2022 00:13:54 +0800 Subject: [PATCH 420/823] [fix][test] Fix wrong retry behavior in MetadataCacheTest (#14778) (cherry picked from commit f89001743b43ade0d1c0fa0440e2d03676e9b3ed) --- .../pulsar/metadata/MetadataCacheTest.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java index 70ed621abd582..322c9bb779bde 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java @@ -36,6 +36,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.function.Supplier; import lombok.AllArgsConstructor; import lombok.Cleanup; @@ -555,4 +556,35 @@ public CustomClass deserialize(String path, byte[] content, Stat stat) throws IO assertEquals(res.getValue().b, 2); assertEquals(res.getValue().path, key1); } + + public static void assertEqualsAndRetry(Supplier actual, + Object expected, + Object expectedAndRetry) throws Exception { + assertEqualsAndRetry(actual, expected, expectedAndRetry, 5, 100); + } + + public static void assertEqualsAndRetry(Supplier actual, + Object expected, + Object expectedAndRetry, + int retryCount, + long intSleepTimeInMillis) throws Exception { + assertTrue(retryStrategically((__) -> { + if (actual.get().equals(expectedAndRetry)) { + return false; + } + assertEquals(actual.get(), expected); + return true; + }, retryCount, intSleepTimeInMillis)); + } + + public static boolean retryStrategically(Predicate predicate, int retryCount, long intSleepTimeInMillis) + throws Exception { + for (int i = 0; i < retryCount; i++) { + if (predicate.test(null)) { + return true; + } + Thread.sleep(intSleepTimeInMillis + (intSleepTimeInMillis * i)); + } + return false; + } } From d71b00304ff6073da7423c452e9afa574ce7769f Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 22 Mar 2022 19:26:18 +0800 Subject: [PATCH 421/823] [fix][broker] Fix cannot delete namespace with system topic (#14730) (cherry picked from commit 7556c4e0165660e8dbd141c4e93bb9e31a67e6f9) --- .../pulsar/broker/web/PulsarWebResource.java | 7 ++- .../pulsar/broker/admin/AdminApiTest2.java | 45 ++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index e3ae827ed5a13..7b5f455ac970b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -722,7 +722,12 @@ public static CompletableFuture checkLocalOrGetPeerReplicationC .getPoliciesAsync(namespace).thenAccept(policiesResult -> { if (policiesResult.isPresent()) { Policies policies = policiesResult.get(); - if (policies.replication_clusters.isEmpty()) { + if (policies.deleted) { + String msg = String.format("Namespace %s is deleted", namespace.toString()); + log.warn(msg); + validationFuture.completeExceptionally(new RestException(Status.PRECONDITION_FAILED, + "Namespace is deleted")); + } else if (policies.replication_clusters.isEmpty()) { String msg = String.format( "Namespace does not have any clusters configured : local_cluster=%s ns=%s", localCluster, namespace.toString()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java index bc1ab256cfb3d..0bf34a16fe020 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java @@ -45,7 +45,6 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; import javax.ws.rs.core.Response.Status; import lombok.Cleanup; @@ -82,6 +81,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; +import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationData; import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationDataImpl; import org.apache.pulsar.common.policies.data.BundlesData; @@ -97,6 +97,7 @@ import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -1356,6 +1357,48 @@ public void testDeleteNamespace() throws Exception { assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); } + @Test + public void testDeleteNamespaceWithTopicPolicies() throws Exception { + stopBroker(); + conf.setSystemTopicEnabled(true); + conf.setTopicLevelPoliciesEnabled(true); + setup(); + + String tenant = "test-tenant"; + assertFalse(admin.tenants().getTenants().contains(tenant)); + + // create tenant + admin.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"))); + assertTrue(admin.tenants().getTenants().contains(tenant)); + + // create namespace2 + String namespace = tenant + "/test-ns2"; + admin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); + // create topic + String topic = namespace + "/test-topic2"; + Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.send("test".getBytes(StandardCharsets.UTF_8)); + BacklogQuota backlogQuota = BacklogQuotaImpl + .builder() + .limitTime(1000) + .limitSize(1000) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(); + admin.topicPolicies().setBacklogQuota(topic, backlogQuota); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topicPolicies() + .getBacklogQuotaMap(topic) + .get(BacklogQuota.BacklogQuotaType.destination_storage), backlogQuota); + }); + producer.close(); + admin.topics().delete(topic); + admin.namespaces().deleteNamespace(namespace); + Awaitility.await().untilAsserted(() -> { + assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty()); + }); + } + @Test(timeOut = 30000) public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminException, InterruptedException { From 862ebc5724f9cccfa88c34a1b405f2ccb46d6a2e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 22 Mar 2022 00:07:52 +0800 Subject: [PATCH 422/823] [fix][admin-cli]: Remove the trust certs check (#14764) (cherry picked from commit 44f92a8118e3ce0461cd4dc1736860fab77f0cae) --- .../main/java/org/apache/pulsar/admin/cli/CmdClusters.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java index eb1ca4f13607e..d43c849647235 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java @@ -76,11 +76,6 @@ protected void validateClusterData(ClusterData clusterData) { "You must specify tls-trust-store-type, tls-trust-store and tls-trust-store-pwd" + " when enable tls-enable-keystore"); } - } else { - if (StringUtils.isBlank(clusterData.getBrokerClientTrustCertsFilePath())) { - throw new RuntimeException("You must specify tls-trust-certs-filepath" - + " when tls-enable-keystore is not enable"); - } } } } From 8a3eccbffaa29c5cca3667e1bc3273483f6adcf1 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Tue, 22 Mar 2022 18:29:23 +0800 Subject: [PATCH 423/823] [fix][admin] Fix NPE in PulsarAdminBuilder when the service is not set (#14769) (cherry picked from commit 0b6b1e2079a9afe92527df2cdb0e843a0ae391dd) --- .../admin/internal/PulsarAdminImpl.java | 3 ++ .../internal/PulsarAdminBuilderImplTest.java | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java index 80cd978814644..427ab6d1aff6d 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.admin.internal; +import static com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.net.URL; import java.util.Map; @@ -148,6 +149,8 @@ public PulsarAdminImpl(String serviceUrl, int autoCertRefreshTime, TimeUnit autoCertRefreshTimeUnit, ClassLoader clientBuilderClassLoader) throws PulsarClientException { + checkArgument(StringUtils.isNotBlank(serviceUrl), "Service URL needs to be specified"); + this.connectTimeout = connectTimeout; this.connectTimeoutUnit = connectTimeoutUnit; this.readTimeout = readTimeout; diff --git a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java new file mode 100644 index 0000000000000..1ea45401eec76 --- /dev/null +++ b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.admin.internal; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClientException; +import org.testng.annotations.Test; + +public class PulsarAdminBuilderImplTest { + + @Test + public void testAdminBuilderWithServiceUrlNotSet() throws PulsarClientException { + try{ + PulsarAdmin.builder().build(); + fail(); + } catch (IllegalArgumentException exception) { + assertEquals("Service URL needs to be specified", exception.getMessage()); + } + } +} From 36bbc18956afe76c025ea626466599f5377106ce Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 22 Mar 2022 12:09:00 +0800 Subject: [PATCH 424/823] Fix flaky test LeaderElectionTest (#14776) (cherry picked from commit 46886b03d0be904ac635d4ea8fa6d74f82b896e5) --- .../apache/pulsar/metadata/LeaderElectionTest.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java index 8412c7129873e..c8a6fed64a61d 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.metadata; +import static org.apache.pulsar.metadata.MetadataCacheTest.assertEqualsAndRetry; import static org.testng.Assert.assertEquals; import java.util.EnumSet; import java.util.Optional; @@ -111,21 +112,21 @@ public void multipleMembers(String provider, Supplier urlSupplier) throw LeaderElectionState les1 = le1.elect("test-1").join(); assertEquals(les1, LeaderElectionState.Leading); - assertEquals(le1.getLeaderValueIfPresent(), Optional.of("test-1")); + assertEqualsAndRetry(() -> le1.getLeaderValueIfPresent(), Optional.of("test-1"), Optional.empty()); assertEquals(le1.getLeaderValue().join(), Optional.of("test-1")); assertEquals(n1.poll(3, TimeUnit.SECONDS), LeaderElectionState.Leading); LeaderElectionState les2 = le2.elect("test-2").join(); assertEquals(les2, LeaderElectionState.Following); assertEquals(le2.getLeaderValue().join(), Optional.of("test-1")); - assertEquals(le2.getLeaderValueIfPresent(), Optional.of("test-1")); + assertEqualsAndRetry(() -> le2.getLeaderValueIfPresent(), Optional.of("test-1"), Optional.empty()); assertEquals(n2.poll(3, TimeUnit.SECONDS), LeaderElectionState.Following); le1.close(); assertEquals(n2.poll(3, TimeUnit.SECONDS), LeaderElectionState.Leading); assertEquals(le2.getState(), LeaderElectionState.Leading); - assertEquals(le2.getLeaderValueIfPresent(), Optional.of("test-2")); + assertEqualsAndRetry(() -> le2.getLeaderValueIfPresent(), Optional.of("test-2"), Optional.empty()); assertEquals(le2.getLeaderValue().join(), Optional.of("test-2")); } @@ -209,7 +210,7 @@ public void revalidateLeaderWithinSameSession(String provider, Supplier LeaderElectionState les = le.elect("test-2").join(); assertEquals(les, LeaderElectionState.Leading); assertEquals(le.getLeaderValue().join(), Optional.of("test-2")); - assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-2")); + assertEqualsAndRetry(() -> le.getLeaderValueIfPresent(), Optional.of("test-2"), Optional.empty()); } @Test(dataProvider = "impl") @@ -239,7 +240,7 @@ public void revalidateLeaderWithDifferentSessionsSameValue(String provider, Supp LeaderElectionState les = le.elect("test-1").join(); assertEquals(les, LeaderElectionState.Leading); assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); - assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-1")); + assertEqualsAndRetry(() -> le.getLeaderValueIfPresent(), Optional.of("test-1"), Optional.empty()); } @@ -275,6 +276,6 @@ public void revalidateLeaderWithDifferentSessionsDifferentValue(String provider, LeaderElectionState les = le.elect("test-2").join(); assertEquals(les, LeaderElectionState.Following); assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); - assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-1")); + assertEqualsAndRetry(() -> le.getLeaderValueIfPresent(), Optional.of("test-1"), Optional.empty()); } } From 9ae8faed33e9dd3c46ad36f3e7872a6b8a2a62b3 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Tue, 22 Mar 2022 00:51:39 -0700 Subject: [PATCH 425/823] Handle kafka sinks that return immutable maps as configs (#14780) (cherry picked from commit b56d7318e73fb6915208dbe1223446e759c2ed0b) --- .../io/kafka/connect/KafkaConnectSink.java | 28 +++++++++++-------- .../SchemaedFileStreamSinkConnector.java | 14 ++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index 268105ce8187a..e8165f8545985 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -24,6 +24,18 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; @@ -44,17 +56,6 @@ import org.apache.pulsar.io.kafka.connect.schema.KafkaConnectData; import org.apache.pulsar.io.kafka.connect.schema.PulsarSchemaToKafkaSchema; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - import static org.apache.pulsar.io.kafka.connect.PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG; @Slf4j @@ -154,6 +155,11 @@ public void open(Map config, SinkContext ctx) throws Exception { Preconditions.checkNotNull(configs); Preconditions.checkArgument(configs.size() == 1); + // configs may contain immutable/unmodifiable maps + configs = configs.stream() + .map(HashMap::new) + .collect(Collectors.toList()); + configs.forEach(x -> { x.put(OFFSET_STORAGE_TOPIC_CONFIG, kafkaSinkConfig.getOffsetStorageTopic()); }); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SchemaedFileStreamSinkConnector.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SchemaedFileStreamSinkConnector.java index a3cce924d1ac5..4a786617f754b 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SchemaedFileStreamSinkConnector.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SchemaedFileStreamSinkConnector.java @@ -22,6 +22,11 @@ import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.file.FileStreamSinkConnector; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + /** * A FileStreamSinkConnector for testing that writes data other than just a value, i.e.: * key, value, key and value schemas. @@ -31,4 +36,13 @@ public class SchemaedFileStreamSinkConnector extends FileStreamSinkConnector { public Class taskClass() { return SchemaedFileStreamSinkTask.class; } + + @Override + public List> taskConfigs(int maxTasks) { + // to test cases when task return immutable maps as configs + return super.taskConfigs(maxTasks) + .stream() + .map(Collections::unmodifiableMap) + .collect(Collectors.toList()); + } } From 60018ba26de64e35343b4f617c625659db312a3e Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 24 Mar 2022 14:10:20 +0800 Subject: [PATCH 426/823] [fix][txn]: fix transaction buffer recover reader and writer fail (#14801) ### Motivation When Transaction buffer recover create reader or create writer fail or read snapshot fail throw PulsarClientException, we should rerecover this topic so close this topic to reinit. (cherry picked from commit bf568633d843f0e6d66b844a72d9d2178171626a) --- .../TransactionBufferSystemTopicClient.java | 22 ++- .../buffer/impl/TopicTransactionBuffer.java | 184 ++++++++++-------- ...TopicTransactionBufferRecoverCallBack.java | 2 +- .../TopicTransactionBufferRecoverTest.java | 48 ++++- 4 files changed, 164 insertions(+), 92 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSystemTopicClient.java index 807bb9d174bfe..aaab858ab1ee2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSystemTopicClient.java @@ -128,10 +128,17 @@ public void close() throws IOException { @Override public CompletableFuture closeAsync() { - return producer.closeAsync().thenCompose(v -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + producer.closeAsync().whenComplete((v, e) -> { + // if close fail, also need remove the producer transactionBufferSystemTopicClient.removeWriter(this); - return CompletableFuture.completedFuture(null); + if (e != null) { + completableFuture.completeExceptionally(e); + return; + } + completableFuture.complete(null); }); + return completableFuture; } @Override @@ -179,10 +186,17 @@ public void close() throws IOException { @Override public CompletableFuture closeAsync() { - return reader.closeAsync().thenCompose(v -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + reader.closeAsync().whenComplete((v, e) -> { + // if close fail, also need remove the reader transactionBufferSystemTopicClient.removeReader(this); - return CompletableFuture.completedFuture(null); + if (e != null) { + completableFuture.completeExceptionally(e); + return; + } + completableFuture.complete(null); }); + return completableFuture; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index f108a1690ee4a..3d6e7da464948 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -37,6 +37,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; @@ -178,15 +179,24 @@ public void handleTxnEntry(Entry entry) { } @Override - public void recoverExceptionally(Exception e) { - if (e instanceof PulsarClientException.BrokerMetadataException) { + public void recoverExceptionally(Throwable e) { + + // when create reader or writer fail throw PulsarClientException, + // should close this topic and then reinit this topic + if (e instanceof PulsarClientException) { + // if transaction buffer recover fail throw PulsarClientException, + // we need to change the PulsarClientException to ServiceUnitNotReadyException, + // the tc do op will retry + transactionBufferFuture.completeExceptionally + (new BrokerServiceException.ServiceUnitNotReadyException(e.getMessage(), e)); log.warn("Closing topic {} due to read transaction buffer snapshot while recovering the " + "transaction buffer throw exception", topic.getName(), e); - topic.close(); + } else { + transactionBufferFuture.completeExceptionally(e); } - transactionBufferFuture.completeExceptionally(e); + topic.close(true); } - }, this.topic, this)); + }, this.topic, this, takeSnapshotWriter)); } @Override @@ -541,98 +551,112 @@ static class TopicTransactionBufferRecover implements Runnable { private final TopicTransactionBuffer topicTransactionBuffer; + private final CompletableFuture> takeSnapshotWriter; + private TopicTransactionBufferRecover(TopicTransactionBufferRecoverCallBack callBack, PersistentTopic topic, - TopicTransactionBuffer transactionBuffer) { + TopicTransactionBuffer transactionBuffer, CompletableFuture< + SystemTopicClient.Writer> takeSnapshotWriter) { this.topic = topic; this.callBack = callBack; this.entryQueue = new SpscArrayQueue<>(2000); this.topicTransactionBuffer = transactionBuffer; + this.takeSnapshotWriter = takeSnapshotWriter; } @SneakyThrows @Override public void run() { - if (!this.topicTransactionBuffer.changeToInitializingState()) { - log.warn("TransactionBuffer {} of topic {} can not change state to Initializing", - this, topic.getName()); - return; - } - topic.getBrokerService().getPulsar().getTransactionBufferSnapshotService() - .createReader(TopicName.get(topic.getName())).thenAcceptAsync(reader -> { - try { - boolean hasSnapshot = false; - while (reader.hasMoreEvents()) { - Message message = reader.readNext(); - if (topic.getName().equals(message.getKey())) { - TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); - if (transactionBufferSnapshot != null) { - hasSnapshot = true; - callBack.handleSnapshot(transactionBufferSnapshot); - this.startReadCursorPosition = PositionImpl.get( - transactionBufferSnapshot.getMaxReadPositionLedgerId(), - transactionBufferSnapshot.getMaxReadPositionEntryId()); - } - } - } - if (!hasSnapshot) { - callBack.noNeedToRecover(); - return; - } - } catch (PulsarClientException pulsarClientException) { - log.error("[{}]Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), pulsarClientException); - callBack.recoverExceptionally(pulsarClientException); - reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); - return null; - }); + this.takeSnapshotWriter.thenRunAsync(() -> { + if (!this.topicTransactionBuffer.changeToInitializingState()) { + log.warn("TransactionBuffer {} of topic {} can not change state to Initializing", + this, topic.getName()); return; } - reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); - return null; - }); - - ManagedCursor managedCursor; - try { - managedCursor = topic.getManagedLedger() - .newNonDurableCursor(this.startReadCursorPosition, SUBSCRIPTION_NAME); - } catch (ManagedLedgerException e) { - callBack.recoverExceptionally(e); - log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); - return; - } - PositionImpl lastConfirmedEntry = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; - FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, - managedCursor, TopicTransactionBufferRecover.this); - if (lastConfirmedEntry.getEntryId() != -1) { - while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 - && fillEntryQueueCallback.fillQueue()) { - Entry entry = entryQueue.poll(); - if (entry != null) { + topic.getBrokerService().getPulsar().getTransactionBufferSnapshotService() + .createReader(TopicName.get(topic.getName())).thenAcceptAsync(reader -> { try { - currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); - callBack.handleTxnEntry(entry); - } finally { - entry.release(); + boolean hasSnapshot = false; + while (reader.hasMoreEvents()) { + Message message = reader.readNext(); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); + if (transactionBufferSnapshot != null) { + hasSnapshot = true; + callBack.handleSnapshot(transactionBufferSnapshot); + this.startReadCursorPosition = PositionImpl.get( + transactionBufferSnapshot.getMaxReadPositionLedgerId(), + transactionBufferSnapshot.getMaxReadPositionEntryId()); + } + } + } + if (!hasSnapshot) { + callBack.noNeedToRecover(); + return; + } + } catch (PulsarClientException pulsarClientException) { + log.error("[{}]Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), pulsarClientException); + callBack.recoverExceptionally(pulsarClientException); + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); + return null; + }); + return; } - } else { + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); + return null; + }); + + ManagedCursor managedCursor; try { - Thread.sleep(1); - } catch (InterruptedException e) { - //no-op + managedCursor = topic.getManagedLedger() + .newNonDurableCursor(this.startReadCursorPosition, SUBSCRIPTION_NAME); + } catch (ManagedLedgerException e) { + callBack.recoverExceptionally(e); + log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); + return; + } + PositionImpl lastConfirmedEntry = + (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; + FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, + managedCursor, TopicTransactionBufferRecover.this); + if (lastConfirmedEntry.getEntryId() != -1) { + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 + && fillEntryQueueCallback.fillQueue()) { + Entry entry = entryQueue.poll(); + if (entry != null) { + try { + currentLoadPosition = PositionImpl.get(entry.getLedgerId(), + entry.getEntryId()); + callBack.handleTxnEntry(entry); + } finally { + entry.release(); + } + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //no-op + } + } + } } - } - } - } - closeCursor(managedCursor); - callBack.recoverComplete(); - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider().getExecutor(this)) - .exceptionally(e -> { - callBack.recoverExceptionally(new Exception(e)); - log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); + closeCursor(managedCursor); + callBack.recoverComplete(); + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)).exceptionally(e -> { + callBack.recoverExceptionally(e.getCause()); + log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); + return null; + }); + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)).exceptionally(e -> { + callBack.recoverExceptionally(e.getCause()); + log.error("[{}]Transaction buffer create snapshot writer fail!", + topic.getName(), e); return null; }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java index 1640459027255..87b8e930a2792 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java @@ -51,5 +51,5 @@ public interface TopicTransactionBufferRecoverCallBack { /** * Topic transaction buffer recover exceptionally. */ - void recoverExceptionally(Exception e); + void recoverExceptionally(Throwable e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 5701d22a99c63..01e03a4066f15 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -59,6 +59,7 @@ import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; @@ -452,7 +453,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, @Test(timeOut=30000) - public void testTransactionBufferRecoverThrowBrokerMetadataException() throws Exception { + public void testTransactionBufferRecoverThrowPulsarClientException() throws Exception { String topic = NAMESPACE1 + "/testTransactionBufferRecoverThrowBrokerMetadataException"; @Cleanup Producer producer = pulsarClient @@ -469,23 +470,55 @@ public void testTransactionBufferRecoverThrowBrokerMetadataException() throws Ex producer.newMessage(txn).value("test".getBytes()).sendAsync(); txn.commit().get(); - // take snapshot PersistentTopic originalTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); TransactionBufferSnapshotService transactionBufferSnapshotService = mock(TransactionBufferSnapshotService.class); SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class); - // mock reader can't read snapshot fail - doThrow(new PulsarClientException.BrokerMetadataException("")).when(reader).hasMoreEvents(); - doReturn(CompletableFuture.completedFuture(reader)).when(transactionBufferSnapshotService).createReader(any()); + SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class); + doReturn(CompletableFuture.completedFuture(reader)).when(transactionBufferSnapshotService).createReader(any()); + doReturn(CompletableFuture.completedFuture(writer)).when(transactionBufferSnapshotService).createWriter(any()); Field field = PulsarService.class.getDeclaredField("transactionBufferSnapshotService"); field.setAccessible(true); TransactionBufferSnapshotService transactionBufferSnapshotServiceOriginal = (TransactionBufferSnapshotService) field.get(getPulsarServiceList().get(0)); + // mock reader can't read snapshot fail + doThrow(new PulsarClientException("test")).when(reader).hasMoreEvents(); + // check reader close topic + checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, + transactionBufferSnapshotService, originalTopic, field, producer); + doReturn(true).when(reader).hasMoreEvents(); + + // mock create reader fail + doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))) + .when(transactionBufferSnapshotService).createReader(any()); + // check create reader fail close topic + originalTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); + checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, + transactionBufferSnapshotService, originalTopic, field, producer); + doReturn(CompletableFuture.completedFuture(reader)).when(transactionBufferSnapshotService).createReader(any()); + + // check create writer fail close topic + originalTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); + // mock create writer fail + doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))) + .when(transactionBufferSnapshotService).createWriter(any()); + checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, + transactionBufferSnapshotService, originalTopic, field, producer); + + } + + private void checkCloseTopic(PulsarClient pulsarClient, + TransactionBufferSnapshotService transactionBufferSnapshotServiceOriginal, + TransactionBufferSnapshotService transactionBufferSnapshotService, + PersistentTopic originalTopic, + Field field, Producer producer) throws Exception { field.set(getPulsarServiceList().get(0), transactionBufferSnapshotService); - // recover again will throw BrokerMetadataException then close topic + // recover again will throw then close topic new TopicTransactionBuffer(originalTopic); Awaitility.await().untilAsserted(() -> { // isFenced means closed @@ -493,10 +526,11 @@ public void testTransactionBufferRecoverThrowBrokerMetadataException() throws Ex close.setAccessible(true); assertTrue((boolean) close.get(originalTopic)); }); + field.set(getPulsarServiceList().get(0), transactionBufferSnapshotServiceOriginal); // topic recover success - txn = pulsarClient.newTransaction() + Transaction txn = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.SECONDS) .build().get(); From 511644b3894da3ba4e984a60b11ffd0f032df729 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 24 Mar 2022 15:46:14 +0800 Subject: [PATCH 427/823] [fix][txn]: fix some exception handle in transaction buffer (#14808) (cherry picked from commit 4824912615bf85f836a72051117462471b0ab306) --- .../buffer/impl/TopicTransactionBuffer.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 3d6e7da464948..7ed656fd1b473 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -227,8 +227,8 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) completableFuture.complete(null); } }).exceptionally(exception -> { - log.error("Topic {}: TransactionBuffer recover failed", this.topic.getName(), exception); - completableFuture.completeExceptionally(exception); + log.error("Topic {}: TransactionBuffer recover failed", this.topic.getName(), exception.getCause()); + completableFuture.completeExceptionally(exception.getCause()); return null; }); return completableFuture; @@ -305,8 +305,8 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { commitMarker.release(); } }).exceptionally(exception -> { - log.error("Transaction {} commit on topic {}.", txnID.toString(), topic.getName(), exception); - completableFuture.completeExceptionally(exception); + log.error("Transaction {} commit on topic {}.", txnID.toString(), topic.getName(), exception.getCause()); + completableFuture.completeExceptionally(exception.getCause()); return null; }); return completableFuture; @@ -351,8 +351,8 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { abortMarker.release(); } }).exceptionally(exception -> { - log.error("Transaction {} abort on topic {}.", txnID.toString(), topic.getName()); - completableFuture.completeExceptionally(exception); + log.error("Transaction {} abort on topic {}.", txnID.toString(), topic.getName(), exception.getCause()); + completableFuture.completeExceptionally(exception.getCause()); return null; }); return completableFuture; From e397e1e4c4b36701b8b11b7164c4481a16f359fb Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 24 Mar 2022 15:49:01 +0800 Subject: [PATCH 428/823] [fix][txn]: fix cannot enable transaction when is allow auto update schema disabled (#14809) (cherry picked from commit b5b0967f12174ba35baaf25092ac521f281e6b7d) --- .../pulsar/broker/service/AbstractTopic.java | 3 ++ .../broker/transaction/TransactionTest.java | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index cc292022432fb..caa0ea29cd3c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -356,6 +356,9 @@ public CompletableFuture addSchema(SchemaData schema) { } private boolean allowAutoUpdateSchema() { + if (brokerService.isSystemTopic(topic)) { + return true; + } if (isAllowAutoUpdateSchema == null) { return brokerService.pulsar().getConfig().isAllowAutoUpdateSchemaEnabled(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 308fa350c9f15..6fd1e92e5a164 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -64,6 +64,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; @@ -89,12 +90,14 @@ import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.client.impl.transaction.TransactionImpl; +import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; @@ -850,4 +853,32 @@ public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() Assert.assertEquals(changeMaxReadPositionAndAddAbortTimes.get(), 0L); } + + @Test + public void testAutoCreateSchemaForTransactionSnapshot() throws Exception { + String namespace = TENANT + "/ns2"; + String topic = namespace + "/test"; + pulsarServiceList.forEach((pulsarService -> + pulsarService.getConfiguration().setAllowAutoUpdateSchemaEnabled(false))); + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + TopicName transactionBufferTopicName = + NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT); + TopicName transactionBufferTopicName1 = + NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(topic).getNamespaceObject(), EventType.TOPIC_POLICY); + Awaitility.await().untilAsserted(() -> { + SchemaInfo schemaInfo = admin + .schemas() + .getSchemaInfo(transactionBufferTopicName.toString()); + Assert.assertNotNull(schemaInfo); + SchemaInfo schemaInfo1 = admin + .schemas() + .getSchemaInfo(transactionBufferTopicName1.toString()); + Assert.assertNotNull(schemaInfo1); + }); + pulsarServiceList.forEach((pulsarService -> + pulsarService.getConfiguration().setAllowAutoUpdateSchemaEnabled(true))); + } } \ No newline at end of file From b75ac4bdfb312dc6b028f5341615ac9de1e1d8ff Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 24 Mar 2022 23:07:19 +0800 Subject: [PATCH 429/823] [improve][tool] Improve transaction perf logs (#14816) (cherry picked from commit f74a58686865edd1a7a6dbba20423a80eaa04d4c) --- .../pulsar/testclient/PerformanceConsumer.java | 9 +++++++-- .../pulsar/testclient/PerformanceProducer.java | 12 ++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 3c7bed9b743c7..77bef8c52b380 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -439,6 +439,9 @@ public static void main(String[] args) throws Exception { if (!arguments.isAbortTransaction) { transaction.commit() .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Commit transaction {}", transaction.getTxnID()); + } totalEndTxnOpSuccessNum.increment(); numTxnOpSuccess.increment(); }) @@ -449,11 +452,13 @@ public static void main(String[] args) throws Exception { }); } else { transaction.abort().thenRun(() -> { - log.info("Abort transaction {}", transaction.getTxnID().toString()); + if (log.isDebugEnabled()) { + log.debug("Abort transaction {}", transaction.getTxnID()); + } totalEndTxnOpSuccessNum.increment(); numTxnOpSuccess.increment(); }).exceptionally(exception -> { - log.error("Commit transaction {} failed with exception", + log.error("Abort transaction {} failed with exception", transaction.getTxnID().toString(), exception); totalEndTxnOpFailNum.increment(); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 0e3d5503734ce..c18cb6cfe3242 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -761,8 +761,10 @@ private static void runProducer(int producerId, if (!arguments.isAbortTransaction) { transaction.commit() .thenRun(() -> { - log.info("Committed transaction {}", - transaction.getTxnID().toString()); + if (log.isDebugEnabled()) { + log.debug("Committed transaction {}", + transaction.getTxnID().toString()); + } totalEndTxnOpSuccessNum.increment(); numTxnOpSuccess.increment(); }) @@ -774,11 +776,13 @@ private static void runProducer(int producerId, }); } else { transaction.abort().thenRun(() -> { - log.info("Abort transaction {}", transaction.getTxnID().toString()); + if (log.isDebugEnabled()) { + log.debug("Abort transaction {}", transaction.getTxnID().toString()); + } totalEndTxnOpSuccessNum.increment(); numTxnOpSuccess.increment(); }).exceptionally(exception -> { - log.error("Commit transaction {} failed with exception", + log.error("Abort transaction {} failed with exception", transaction.getTxnID().toString(), exception); totalEndTxnOpFailNum.increment(); From 06a467825aa1c1ec10518a3300088a1dfc96c594 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 24 Mar 2022 22:44:40 +0800 Subject: [PATCH 430/823] [fix][test]: fix flaky test testTransactionBufferRecoverThrowPulsarClientException (#14846) Co-authored-by: congbo (cherry picked from commit dca5a901528e77f218afb9870b223f06143b055f) --- .../TopicTransactionBufferRecoverTest.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 01e03a4066f15..392a21f86b76f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -454,7 +454,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, @Test(timeOut=30000) public void testTransactionBufferRecoverThrowPulsarClientException() throws Exception { - String topic = NAMESPACE1 + "/testTransactionBufferRecoverThrowBrokerMetadataException"; + String topic = NAMESPACE1 + "/testTransactionBufferRecoverThrowPulsarClientException"; @Cleanup Producer producer = pulsarClient .newProducer() @@ -470,6 +470,8 @@ public void testTransactionBufferRecoverThrowPulsarClientException() throws Exce producer.newMessage(txn).value("test".getBytes()).sendAsync(); txn.commit().get(); + producer.close(); + PersistentTopic originalTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); TransactionBufferSnapshotService transactionBufferSnapshotService = @@ -479,6 +481,8 @@ public void testTransactionBufferRecoverThrowPulsarClientException() throws Exce doReturn(CompletableFuture.completedFuture(reader)).when(transactionBufferSnapshotService).createReader(any()); doReturn(CompletableFuture.completedFuture(writer)).when(transactionBufferSnapshotService).createWriter(any()); + doReturn(CompletableFuture.completedFuture(null)).when(reader).closeAsync(); + doReturn(CompletableFuture.completedFuture(null)).when(writer).closeAsync(); Field field = PulsarService.class.getDeclaredField("transactionBufferSnapshotService"); field.setAccessible(true); TransactionBufferSnapshotService transactionBufferSnapshotServiceOriginal = @@ -487,7 +491,7 @@ public void testTransactionBufferRecoverThrowPulsarClientException() throws Exce doThrow(new PulsarClientException("test")).when(reader).hasMoreEvents(); // check reader close topic checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, - transactionBufferSnapshotService, originalTopic, field, producer); + transactionBufferSnapshotService, originalTopic, field); doReturn(true).when(reader).hasMoreEvents(); // mock create reader fail @@ -497,7 +501,7 @@ public void testTransactionBufferRecoverThrowPulsarClientException() throws Exce originalTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, - transactionBufferSnapshotService, originalTopic, field, producer); + transactionBufferSnapshotService, originalTopic, field); doReturn(CompletableFuture.completedFuture(reader)).when(transactionBufferSnapshotService).createReader(any()); // check create writer fail close topic @@ -507,7 +511,7 @@ public void testTransactionBufferRecoverThrowPulsarClientException() throws Exce doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))) .when(transactionBufferSnapshotService).createWriter(any()); checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, - transactionBufferSnapshotService, originalTopic, field, producer); + transactionBufferSnapshotService, originalTopic, field); } @@ -515,7 +519,7 @@ private void checkCloseTopic(PulsarClient pulsarClient, TransactionBufferSnapshotService transactionBufferSnapshotServiceOriginal, TransactionBufferSnapshotService transactionBufferSnapshotService, PersistentTopic originalTopic, - Field field, Producer producer) throws Exception { + Field field) throws Exception { field.set(getPulsarServiceList().get(0), transactionBufferSnapshotService); // recover again will throw then close topic @@ -529,13 +533,19 @@ private void checkCloseTopic(PulsarClient pulsarClient, field.set(getPulsarServiceList().get(0), transactionBufferSnapshotServiceOriginal); - // topic recover success Transaction txn = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.SECONDS) .build().get(); + @Cleanup + Producer producer = pulsarClient + .newProducer() + .topic(originalTopic.getName()) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); producer.newMessage(txn).value("test".getBytes()).sendAsync(); txn.commit().get(); + producer.close(); } } From c6f17d3ce41c2b24a8a1d4f891d59b4ffc403e02 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:52:40 +0800 Subject: [PATCH 431/823] [fix][txn]: fix transaction buffer no snapshot close recover reader (#14830) ### Motivation now transaction buffer recover no snapshot, we don't close the reader, it will produce the problem of OOM (cherry picked from commit 7a78b505f3f2d9febbcd6a161102b5a584f475c8) --- .../buffer/impl/TopicTransactionBuffer.java | 22 ++++++----- .../TopicTransactionBufferRecoverTest.java | 39 ++++++++++++++----- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 7ed656fd1b473..9ea3b42126792 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -181,6 +181,8 @@ public void handleTxnEntry(Entry entry) { @Override public void recoverExceptionally(Throwable e) { + log.warn("Closing topic {} due to read transaction buffer snapshot while recovering the " + + "transaction buffer throw exception", topic.getName(), e); // when create reader or writer fail throw PulsarClientException, // should close this topic and then reinit this topic if (e instanceof PulsarClientException) { @@ -189,8 +191,6 @@ public void recoverExceptionally(Throwable e) { // the tc do op will retry transactionBufferFuture.completeExceptionally (new BrokerServiceException.ServiceUnitNotReadyException(e.getMessage(), e)); - log.warn("Closing topic {} due to read transaction buffer snapshot while recovering the " - + "transaction buffer throw exception", topic.getName(), e); } else { transactionBufferFuture.completeExceptionally(e); } @@ -590,6 +590,7 @@ public void run() { } } if (!hasSnapshot) { + closeReader(reader); callBack.noNeedToRecover(); return; } @@ -597,16 +598,10 @@ public void run() { log.error("[{}]Transaction buffer recover fail when read " + "transactionBufferSnapshot!", topic.getName(), pulsarClientException); callBack.recoverExceptionally(pulsarClientException); - reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); - return null; - }); + closeReader(reader); return; } - reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); - return null; - }); + closeReader(reader); ManagedCursor managedCursor; try { @@ -679,6 +674,13 @@ private void callBackException(ManagedLedgerException e) { log.error("Transaction buffer recover fail when recover transaction entry!", e); this.exceptionNumber.getAndIncrement(); } + + private void closeReader(SystemTopicClient.Reader reader) { + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); + return null; + }); + } } static class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 392a21f86b76f..c349173898550 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -18,6 +18,16 @@ */ package org.apache.pulsar.broker.transaction; +import static org.apache.pulsar.common.events.EventsTopicNames.TRANSACTION_BUFFER_SNAPSHOT; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -59,6 +69,7 @@ import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; @@ -66,15 +77,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; @Slf4j public class TopicTransactionBufferRecoverTest extends TransactionTestBase { @@ -548,4 +550,23 @@ private void checkCloseTopic(PulsarClient pulsarClient, producer.close(); } + + @Test + public void testTransactionBufferNoSnapshotCloseReader() throws Exception{ + String topic = NAMESPACE1 + "/test"; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).producerName("testTxnTimeOut_producer") + .topic(topic).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create(); + + admin.topics().unload(topic); + + // unload success, all readers have been closed except for the compaction sub + producer.send("test"); + TopicStats stats = admin.topics().getStats(NAMESPACE1 + "/" + TRANSACTION_BUFFER_SNAPSHOT); + + // except for the compaction sub + assertEquals(stats.getSubscriptions().size(), 1); + assertTrue(stats.getSubscriptions().keySet().contains("__compaction")); + } + } From f0a2171cbad894cec5bfb2d4de31cb8de32a3183 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:16:29 +0800 Subject: [PATCH 432/823] [fix][txn]: fix transaction buffer recover throw cursor already close (#14807) ### Motivation When Transaction buffer recover fail throw CursorAlreadyClosedException, we should stop the recover op. the cursor was been closed, the transaction buffer was been closed, so we should stop the recover op, in order to release thread resources like https://github.com/apache/pulsar/pull/14781 (cherry picked from commit aef5f6d5e2d44f84c9c358f1d9dd9db1108a9d99) --- .../buffer/impl/TopicTransactionBuffer.java | 3 ++- .../pulsar/broker/transaction/TransactionTest.java | 12 ++++++++++++ .../coordinator/impl/MLTransactionLogImpl.java | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 9ea3b42126792..66ce8f517673c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -734,7 +734,8 @@ public Entry get() { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { if (recover.topic.getManagedLedger().getConfig().isAutoSkipNonRecoverableData() && exception instanceof ManagedLedgerException.NonRecoverableLedgerException - || exception instanceof ManagedLedgerException.ManagedLedgerFencedException) { + || exception instanceof ManagedLedgerException.ManagedLedgerFencedException + || exception instanceof ManagedLedgerException.CursorAlreadyClosedException) { isReadable = false; } else { outstandingReadsRequests.decrementAndGet(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 6fd1e92e5a164..231c183a59af1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -568,6 +568,18 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer2.getStats().state, "Ready")); managedCursors.removeCursor("transaction-buffer-sub"); + + doAnswer(invocation -> { + AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); + callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null); + return null; + }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + + managedCursors.add(managedCursor); + TransactionBuffer buffer3 = new TopicTransactionBuffer(persistentTopic); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(buffer3.getStats().state, "Ready")); + managedCursors.removeCursor("transaction-buffer-sub"); } @Test diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index 2a13f950d3d6f..f14784e70bbd6 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -274,6 +274,8 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { || exception instanceof ManagedLedgerException.ManagedLedgerFencedException || exception instanceof ManagedLedgerException.CursorAlreadyClosedException) { isReadable = false; + } else { + outstandingReadsRequests.decrementAndGet(); } log.error("Transaction log init fail error!", exception); } From 9fc376bb75a4732122cda9b383a5c2e788207a7c Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 25 Mar 2022 12:03:49 -0500 Subject: [PATCH 433/823] [Broker] Fix NPE when subscription is already removed (#14363) * [Broker] Fix NPE when subscription is already removed * Cover same case for NonPersistentTopic Master Issue: #14362 There is current a race condition when we remove a subscription. The race and how to reproduce it is described in the #14362. One of the consequences of the race is that there is a chance we try to remove the subscription from the topic twice. This leads to an NPE, as described in the issue. * Verify that the `sub` is not null before getting its stats. This is a trivial change. (cherry picked from commit aee1e7dbc55099c6b7cdc49e7b5e1c4cd66994ce) --- .../service/nonpersistent/NonPersistentTopic.java | 10 ++++++---- .../broker/service/persistent/PersistentTopic.java | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 4a5aa1f584a01..886140eb25b78 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -1026,10 +1026,12 @@ public CompletableFuture unsubscribe(String subscriptionName) { // That creates deadlock. so, execute remove it in different thread. return CompletableFuture.runAsync(() -> { NonPersistentSubscription sub = subscriptions.remove(subscriptionName); - // preserve accumulative stats form removed subscription - SubscriptionStatsImpl stats = sub.getStats(); - bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); - msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + if (sub != null) { + // preserve accumulative stats form removed subscription + SubscriptionStatsImpl stats = sub.getStats(); + bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); + msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + } }, brokerService.executor()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index e6cf77fde3466..7052fc5c23412 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1054,10 +1054,12 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { void removeSubscription(String subscriptionName) { PersistentSubscription sub = subscriptions.remove(subscriptionName); - // preserve accumulative stats form removed subscription - SubscriptionStatsImpl stats = sub.getStats(false, false); - bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); - msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + if (sub != null) { + // preserve accumulative stats form removed subscription + SubscriptionStatsImpl stats = sub.getStats(false, false); + bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); + msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + } } /** From ff2fc19c03bf949cb8f9b58dc1e52b7c4cf937ef Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 28 Mar 2022 08:56:05 -0700 Subject: [PATCH 434/823] Support advertised listeners for HTTP and HTTPS services (#14839) --- .../broker/ServiceConfigurationUtils.java | 12 +- .../validator/MultipleListenerValidator.java | 26 +++- .../apache/pulsar/broker/PulsarService.java | 14 ++- .../pulsar/compaction/CompactorTool.java | 3 +- .../loadbalance/AdvertisedListenersTest.java | 119 ++++++++++++++++++ .../data/loadbalancer/AdvertisedListener.java | 25 ++++ 6 files changed, 190 insertions(+), 9 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java index c47b8ca79812d..7507181f34bdd 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java @@ -85,10 +85,20 @@ public static String getAppliedAdvertisedAddress(ServiceConfiguration configurat * Gets the internal advertised listener for broker-to-broker communication. * @return a non-null advertised listener */ - public static AdvertisedListener getInternalListener(ServiceConfiguration config) { + public static AdvertisedListener getInternalListener(ServiceConfiguration config, String protocol) { Map result = MultipleListenerValidator .validateAndAnalysisAdvertisedListener(config); AdvertisedListener internal = result.get(config.getInternalListenerName()); + if (internal == null || !internal.hasUriForProtocol(protocol)) { + // Search for an advertised listener for same protocol + for (AdvertisedListener l : result.values()) { + if (l.hasUriForProtocol(protocol)) { + internal = l; + break; + } + } + } + if (internal == null) { // synthesize an advertised listener based on legacy configuration properties String host = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress()); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java index ce02da974d52e..aa5fdd6a15242 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java @@ -79,7 +79,7 @@ public static Map validateAndAnalysisAdvertisedListe if (entry.getValue().size() > 2) { throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`"); } - URI pulsarAddress = null, pulsarSslAddress = null; + URI pulsarAddress = null, pulsarSslAddress = null, pulsarHttpAddress = null, pulsarHttpsAddress = null; for (final String strUri : entry.getValue()) { try { URI uri = URI.create(strUri); @@ -95,7 +95,22 @@ public static Map validateAndAnalysisAdvertisedListe } else { throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`"); } + } else if (StringUtils.equalsIgnoreCase(uri.getScheme(), "http")) { + if (pulsarHttpAddress == null) { + pulsarHttpAddress = uri; + } else { + throw new IllegalArgumentException("there are redundant configure for listener `" + + entry.getKey() + "`"); + } + } else if (StringUtils.equalsIgnoreCase(uri.getScheme(), "https")) { + if (pulsarHttpsAddress == null) { + pulsarHttpsAddress = uri; + } else { + throw new IllegalArgumentException("there are redundant configure for listener `" + + entry.getKey() + "`"); + } } + String hostPort = String.format("%s:%d", uri.getHost(), uri.getPort()); Set sets = reverseMappings.computeIfAbsent(hostPort, k -> Sets.newTreeSet()); sets.add(entry.getKey()); @@ -103,10 +118,15 @@ public static Map validateAndAnalysisAdvertisedListe throw new IllegalArgumentException("must not specify `" + hostPort + "` to different listener."); } } catch (Throwable cause) { - throw new IllegalArgumentException("the value " + strUri + " in the `advertisedListeners` configure is invalid"); + throw new IllegalArgumentException("the value " + strUri + " in the `advertisedListeners` configure is invalid", cause); } } - result.put(entry.getKey(), AdvertisedListener.builder().brokerServiceUrl(pulsarAddress).brokerServiceUrlTls(pulsarSslAddress).build()); + result.put(entry.getKey(), AdvertisedListener.builder() + .brokerServiceUrl(pulsarAddress) + .brokerServiceUrlTls(pulsarSslAddress) + .brokerHttpUrl(pulsarHttpAddress) + .brokerHttpsUrl(pulsarHttpsAddress) + .build()); } return result; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 56d8b2b8ad907..46884d7cd3e21 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1423,7 +1423,7 @@ public TransactionBufferClient getTransactionBufferClient() { * Gets the broker service URL (non-TLS) associated with the internal listener. */ protected String brokerUrl(ServiceConfiguration config) { - AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config); + AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "pulsar"); return internalListener.getBrokerServiceUrl() != null ? internalListener.getBrokerServiceUrl().toString() : null; } @@ -1436,7 +1436,7 @@ public static String brokerUrl(String host, int port) { * Gets the broker service URL (TLS) associated with the internal listener. */ public String brokerUrlTls(ServiceConfiguration config) { - AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config); + AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "pulsar+ssl"); return internalListener.getBrokerServiceUrlTls() != null ? internalListener.getBrokerServiceUrlTls().toString() : null; } @@ -1447,7 +1447,10 @@ public static String brokerUrlTls(String host, int port) { public String webAddress(ServiceConfiguration config) { if (config.getWebServicePort().isPresent()) { - return webAddress(ServiceConfigurationUtils.getWebServiceAddress(config), getListenPortHTTP().get()); + AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "http"); + return internalListener.getBrokerHttpUrl() != null + ? internalListener.getBrokerHttpUrl().toString() + : webAddress(ServiceConfigurationUtils.getWebServiceAddress(config), getListenPortHTTP().get()); } else { return null; } @@ -1459,7 +1462,10 @@ public static String webAddress(String host, int port) { public String webAddressTls(ServiceConfiguration config) { if (config.getWebServicePortTls().isPresent()) { - return webAddressTls(ServiceConfigurationUtils.getWebServiceAddress(config), getListenPortHTTPS().get()); + AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "https"); + return internalListener.getBrokerHttpsUrl() != null + ? internalListener.getBrokerHttpsUrl().toString() + : webAddressTls(ServiceConfigurationUtils.getWebServiceAddress(config), getListenPortHTTPS().get()); } else { return null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index 1140c49db87f1..ac028ef871b79 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -110,7 +110,7 @@ public static void main(String[] args) throws Exception { brokerConfig.getBrokerClientAuthenticationParameters()); } - AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig); + AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar+ssl"); if (internalListener.getBrokerServiceUrlTls() != null) { log.info("Found a TLS-based advertised listener in configuration file. \n" + "Will connect pulsar use TLS."); @@ -120,6 +120,7 @@ public static void main(String[] args) throws Exception { .tlsTrustCertsFilePath(brokerConfig.getTlsCertificateFilePath()); } else { + internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar"); clientBuilder.serviceUrl(internalListener.getBrokerServiceUrl().toString()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java new file mode 100644 index 0000000000000..6ff49674e2e77 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import static org.testng.Assert.assertEquals; +import java.net.URI; +import java.util.Optional; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.util.PortManager; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.lookup.data.LookupData; +import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class AdvertisedListenersTest extends MultiBrokerBaseTest { + @Override + protected int numberOfAdditionalBrokers() { + return 1; + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + + updateConfig(conf, "BROKER-X"); + } + + @Override + protected ServiceConfiguration createConfForAdditionalBroker(int additionalBrokerIndex) { + ServiceConfiguration conf = super.createConfForAdditionalBroker(additionalBrokerIndex); + updateConfig(conf, "BROKER-" + additionalBrokerIndex); + return conf; + } + + private void updateConfig(ServiceConfiguration conf, String advertisedAddress) { + int pulsarPort = PortManager.nextFreePort(); + int httpPort = PortManager.nextFreePort(); + int httpsPort = PortManager.nextFreePort(); + + // Use invalid domain name as identifier and instead make sure the advertised listeners work as intended + this.conf.setAdvertisedAddress(advertisedAddress); + this.conf.setAdvertisedListeners( + "public:pulsar://localhost:" + pulsarPort + + ",public_http:http://localhost:" + httpPort + + ",public_https:https://localhost:" + httpsPort); + this.conf.setBrokerServicePort(Optional.of(pulsarPort)); + this.conf.setWebServicePort(Optional.of(httpPort)); + this.conf.setWebServicePortTls(Optional.of(httpsPort)); + } + + @Test + public void testLookup() throws Exception { + HttpGet request = + new HttpGet(pulsar.getWebServiceAddress() + "/lookup/v2/topic/persistent/public/default/my-topic"); + request.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + request.addHeader(HttpHeaders.ACCEPT, "application/json"); + + @Cleanup + CloseableHttpClient httpClient = HttpClients.createDefault(); + + @Cleanup + CloseableHttpResponse response = httpClient.execute(request); + + HttpEntity entity = response.getEntity(); + LookupData ld = ObjectMapperFactory.getThreadLocal().readValue(EntityUtils.toString(entity), LookupData.class); + System.err.println("Lookup data: " + ld); + + assertEquals(new URI(ld.getBrokerUrl()).getHost(), "localhost"); + assertEquals(new URI(ld.getHttpUrl()).getHost(), "localhost"); + assertEquals(new URI(ld.getHttpUrlTls()).getHost(), "localhost"); + + + // Produce data + @Cleanup + Producer p = pulsarClient.newProducer(Schema.STRING) + .topic("my-topic") + .create(); + + p.send("hello"); + + // Verify we can get the correct HTTP redirect to the advertised listener + for (PulsarAdmin a : getAllAdmins()) { + TopicStats s = a.topics().getStats("my-topic"); + assertEquals(s.getPublishers().size(), 1); + } + } + +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/AdvertisedListener.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/AdvertisedListener.java index a310974ffa358..b73fdab4483be 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/AdvertisedListener.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/AdvertisedListener.java @@ -44,4 +44,29 @@ public class AdvertisedListener { @Setter // the broker service uri with ssl private URI brokerServiceUrlTls; + + // + @Getter + @Setter + // the broker service uri without ssl + private URI brokerHttpUrl; + // + @Getter + @Setter + // the broker service uri with ssl + private URI brokerHttpsUrl; + + public boolean hasUriForProtocol(String protocol) { + if ("pulsar".equals(protocol)) { + return brokerServiceUrl != null; + } else if ("pulsar+ssl".equals(protocol)) { + return brokerServiceUrlTls != null; + } else if ("http".equals(protocol)) { + return brokerHttpUrl != null; + } else if ("https".equals(protocol)) { + return brokerHttpsUrl != null; + } else { + return false; + } + } } From 6cbcb24159f7ff04f7721e4b37ff9e195f7864ce Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 28 Mar 2022 15:16:57 -0700 Subject: [PATCH 435/823] Fixed merge issue for ff2fc19c03 --- .../src/main/java/org/apache/pulsar/PulsarStandalone.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index ded8addd29d56..aaafd65755b38 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -301,7 +301,7 @@ public void start() throws Exception { broker.start(); final String cluster = config.getClusterName(); - final AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config); + final AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "pulsar"); if (!config.isTlsEnabled()) { checkArgument(config.getWebServicePort().isPresent(), "webServicePort must be present"); checkArgument(internalListener.getBrokerServiceUrl() != null, From 73f44bb6b207b18effa8ab7a5004d7165aad007b Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 24 Feb 2022 23:14:31 +0800 Subject: [PATCH 436/823] [Broker] Optimize RawReader#create when using Compactor (#14447) (cherry picked from commit 992dfbd81f3dbdc648c30ff73e37b6c9ae2b276e) --- .../src/main/java/org/apache/pulsar/client/api/RawReader.java | 2 +- .../main/java/org/apache/pulsar/client/impl/RawReaderImpl.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java index f74157a938914..fe068f2f9435f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java @@ -34,7 +34,7 @@ public interface RawReader { static CompletableFuture create(PulsarClient client, String topic, String subscription) { CompletableFuture> future = new CompletableFuture<>(); RawReader r = new RawReaderImpl((PulsarClientImpl) client, topic, subscription, future); - return future.thenCompose(x -> x.seekAsync(MessageId.earliest)).thenApply(__ -> r); + return future.thenApply(__ -> r); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 217dd5ccc8529..6b032370898e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.api.RawReader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.common.api.proto.CommandAck.AckType; @@ -56,6 +57,7 @@ public RawReaderImpl(PulsarClientImpl client, String topic, String subscription, consumerConfiguration.setSubscriptionType(SubscriptionType.Exclusive); consumerConfiguration.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); consumerConfiguration.setReadCompacted(true); + consumerConfiguration.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); consumer = new RawConsumerImpl(client, consumerConfiguration, consumerFuture); From f27701a5ca57eae91ba9e4483272cf8d735fda69 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 29 Mar 2022 09:47:56 +0200 Subject: [PATCH 437/823] TieredStorage: add debug information (#14907) * TieredStorage: add debug information - enable SLF4 logging in JClouds - add Pulsar Cluster name in Objects metadata - log object names during offloading (cherry picked from commit 83dad0ac855ca81983ff4bf8f5676fe98a55d0ff) --- jclouds-shaded/pom.xml | 5 +++++ .../bookkeeper/mledger/LedgerOffloader.java | 1 + .../org/apache/pulsar/broker/PulsarService.java | 3 ++- .../impl/BlobStoreManagedLedgerOffloader.java | 16 ++++++++++++++-- .../jcloud/provider/JCloudBlobStoreProvider.java | 6 ++++++ 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 3e51419304178..8067c21f6939d 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -39,6 +39,11 @@ jclouds-allblobstore ${jclouds.version} + + org.apache.jclouds.driver + jclouds-slf4j + ${jclouds.version} + javax.annotation javax.annotation-api diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java index ead9db73965e4..4718801b475c8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java @@ -90,6 +90,7 @@ default CompletableFuture AsyncClose() { // TODO: improve the user metadata in subsequent changes String METADATA_SOFTWARE_VERSION_KEY = "S3ManagedLedgerOffloaderSoftwareVersion"; String METADATA_SOFTWARE_GITSHA_KEY = "S3ManagedLedgerOffloaderSoftwareGitSha"; + String METADATA_PULSAR_CLUSTER_NAME = "pulsarClusterName"; /** * Get offload driver name. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 46884d7cd3e21..25357ce17e4fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1224,7 +1224,8 @@ public synchronized LedgerOffloader createManagedLedgerOffloader(OffloadPolicies offloadPolicies, ImmutableMap.of( LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), - LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() + LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha(), + LedgerOffloader.METADATA_PULSAR_CLUSTER_NAME.toLowerCase(), config.getClusterName() ), schemaStorage, getOffloaderScheduler(offloadPolicies)); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index f9c86fd2967cd..5fc2bf7d3b79c 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import java.io.IOException; import java.time.Duration; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -173,6 +174,7 @@ public CompletableFuture offload(ReadHandle readHandle, .withDataBlockHeaderLength(BlockAwareSegmentInputStreamImpl.getHeaderSize()); String dataBlockKey = DataBlockUtils.dataBlockOffloadKey(readHandle.getId(), uuid); String indexBlockKey = DataBlockUtils.indexBlockOffloadKey(readHandle.getId(), uuid); + log.info("ledger {} dataBlockKey {} indexBlockKey {}", readHandle.getId(), dataBlockKey, indexBlockKey); MultipartUpload mpu = null; List parts = Lists.newArrayList(); @@ -180,7 +182,12 @@ public CompletableFuture offload(ReadHandle readHandle, // init multi part upload for data block. try { BlobBuilder blobBuilder = writeBlobStore.blobBuilder(dataBlockKey); - DataBlockUtils.addVersionInfo(blobBuilder, userMetadata); + Map objectMetadata = new HashMap<>(userMetadata); + objectMetadata.put("role", "data"); + if (extraMetadata != null) { + objectMetadata.putAll(extraMetadata); + } + DataBlockUtils.addVersionInfo(blobBuilder, objectMetadata); Blob blob = blobBuilder.build(); mpu = writeBlobStore.initiateMultipartUpload(config.getBucket(), blob.getMetadata(), new PutOptions()); } catch (Throwable t) { @@ -243,7 +250,12 @@ public CompletableFuture offload(ReadHandle readHandle, IndexInputStream indexStream = index.toStream()) { // write the index block BlobBuilder blobBuilder = writeBlobStore.blobBuilder(indexBlockKey); - DataBlockUtils.addVersionInfo(blobBuilder, userMetadata); + Map objectMetadata = new HashMap<>(userMetadata); + objectMetadata.put("role", "index"); + if (extraMetadata != null) { + objectMetadata.putAll(extraMetadata); + } + DataBlockUtils.addVersionInfo(blobBuilder, objectMetadata); Payload indexPayload = Payloads.newInputStreamPayload(indexStream); indexPayload.getContentMetadata().setContentLength((long) indexStream.getStreamSize()); indexPayload.getContentMetadata().setContentType("application/octet-stream"); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java index 49dabb261212c..44aa92ce92438 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.io.Serializable; import java.nio.charset.Charset; +import java.util.Arrays; import java.util.Properties; import java.util.UUID; @@ -61,6 +62,7 @@ import org.jclouds.domain.LocationScope; import org.jclouds.googlecloud.GoogleCredentialsFromJson; import org.jclouds.googlecloudstorage.GoogleCloudStorageProviderMetadata; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; import org.jclouds.providers.AnonymousProviderMetadata; import org.jclouds.providers.ProviderMetadata; import org.jclouds.s3.S3ApiMetadata; @@ -140,6 +142,7 @@ public void validate(TieredStorageConfiguration config) throws IllegalArgumentEx @Override public BlobStore getBlobStore(TieredStorageConfiguration config) { ContextBuilder contextBuilder = ContextBuilder.newBuilder(config.getProviderMetadata()); + contextBuilder.modules(Arrays.asList(new SLF4JLoggingModule())); contextBuilder.overrides(config.getOverrides()); if (config.getProviderCredentials() != null) { @@ -205,6 +208,7 @@ public void validate(TieredStorageConfiguration config) throws IllegalArgumentEx public BlobStore getBlobStore(TieredStorageConfiguration config) { ContextBuilder builder = ContextBuilder.newBuilder("transient"); + builder.modules(Arrays.asList(new SLF4JLoggingModule())); BlobStoreContext ctx = builder .buildView(BlobStoreContext.class); @@ -287,6 +291,7 @@ public ProviderMetadata getProviderMetadata() { static final BlobStoreBuilder BLOB_STORE_BUILDER = (TieredStorageConfiguration config) -> { ContextBuilder contextBuilder = ContextBuilder.newBuilder(config.getProviderMetadata()); + contextBuilder.modules(Arrays.asList(new SLF4JLoggingModule())); contextBuilder.overrides(config.getOverrides()); if (StringUtils.isNotEmpty(config.getServiceEndpoint())) { @@ -371,6 +376,7 @@ public String getAWSSecretKey() { static final BlobStoreBuilder ALIYUN_OSS_BLOB_STORE_BUILDER = (TieredStorageConfiguration config) -> { ContextBuilder contextBuilder = ContextBuilder.newBuilder(config.getProviderMetadata()); + contextBuilder.modules(Arrays.asList(new SLF4JLoggingModule())); Properties overrides = config.getOverrides(); // For security reasons, OSS supports only virtual hosted style access. overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true"); From b659cdce4871b1553b134e2e7e668e9dc1513318 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Mon, 28 Mar 2022 16:43:29 +0800 Subject: [PATCH 438/823] [fix][broker] Fix wrong state for non-durable cursor (#14869) ### Motivation The current non-durable cursor does not have the correct state. For example, when the reader is created, I always see the cursor status as ``Uninitialized`` via the ``getInternalStats`` method. ```json { "reader-xxxxx" : { "markDeletePosition" : "19785:18718", "readPosition" : "19807:42735", "waitingReadOp" : false, "pendingReadOps" : 0, "messagesConsumedCounter" : -2257, "cursorLedger" : -1, "cursorLedgerLastEntry" : -1, "individuallyDeletedMessages" : "[]", "lastLedgerSwitchTimestamp" : "2022-03-24T20:03:51.85Z", "state" : "Uninitialized", "numberOfEntriesSinceFirstNotAckedMessage" : 744993, "totalNonContiguousDeletedMessagesRange" : 0, "subscriptionHavePendingRead" : true, "subscriptionHavePendingReplayRead" : false, "properties" : { } } } ``` ### Modifications - Correct the cursor state. (cherry picked from commit b477557a6f6d36ae18f09d4edfa076e9092a17fe) --- .../mledger/impl/ManagedCursorImpl.java | 2 +- .../mledger/impl/NonDurableCursorImpl.java | 4 ++-- .../apache/pulsar/client/impl/ReaderTest.java | 21 ++++++++++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 8c96f0e2859cf..f882444a4a8c9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -236,7 +236,7 @@ enum State { Closed // The managed cursor has been closed } - private static final AtomicReferenceFieldUpdater STATE_UPDATER = + protected static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, State.class, "state"); protected volatile State state = null; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java index 1d545bd6b48c6..4625f5b58006a 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java @@ -61,7 +61,7 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { // read-position recoverCursor(startCursorPosition); } - + STATE_UPDATER.set(this, State.Open); log.info("[{}] Created non-durable cursor read-position={} mark-delete-position={}", ledger.getName(), readPosition, markDeletePosition); } @@ -110,7 +110,7 @@ protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map reader = pulsarClient.newReader() + .topic(readerNotAckTopic) + .startMessageId(MessageId.earliest) + .create(); + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(readerNotAckTopic); + Assert.assertEquals(internalStats.cursors.size(), 1); + String key = new ArrayList<>(internalStats.cursors.keySet()).get(0); + ManagedLedgerInternalStats.CursorStats cursor = internalStats.cursors.get(key); + Assert.assertEquals(cursor.state, "Open"); + reader.close(); + internalStats = admin.topics().getInternalStats(readerNotAckTopic); + Assert.assertEquals(internalStats.cursors.size(), 0); + } + } From 5ea98b46c631db5928c956eae5481a0ea1dab22a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sun, 27 Mar 2022 22:07:03 +0800 Subject: [PATCH 439/823] [fix][broker] Fix topic policy reader close bug. (#14897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation https://github.com/apache/pulsar/issues/14896 is flaky, after diving into the codes, I find it's a bug about closing topic policy reader. We should use `ex.getCause` instead of `ex`. Stacktrace： ``` 2022-03-27T10:42:16,795+0800 [broker-client-shared-internal-executor-58-1] WARN org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService - Read more topic polices exception, read again. java.util.concurrent.CompletionException: org.apache.pulsar.client.api.PulsarClientException$AlreadyClosedException: Consumer already closed at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_291] at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_291] at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:607) ~[?:1.8.0_291] at java.util.concurrent.CompletableFuture.uniApplyStage(CompletableFuture.java:628) ~[?:1.8.0_291] at java.util.concurrent.CompletableFuture.thenApply(CompletableFuture.java:1996) ~[?:1.8.0_291] at org.apache.pulsar.client.impl.MultiTopicsReaderImpl.readNextAsync(MultiTopicsReaderImpl.java:140) ~[classes/:?] ``` (cherry picked from commit 85c1ba50cb155111f969403a092affd536d9dcb9) --- .../broker/service/SystemTopicBasedTopicPoliciesService.java | 4 +++- .../pulsar/broker/systopic/PartitionedSystemTopicTest.java | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 964bddea99679..1c018141150ff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -47,6 +47,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -364,7 +365,8 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { notifyListener(msg); readMorePolicies(reader); } else { - if (ex instanceof PulsarClientException.AlreadyClosedException) { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof PulsarClientException.AlreadyClosedException) { log.error("Read more topic policies exception, close the read now!", ex); cleanCacheAndCloseReader( reader.getSystemTopic().getTopicName().getNamespaceObject(), false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index 7dd02bdeae7ba..bbd3cae711704 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -74,6 +74,7 @@ public void testAutoCreatedPartitionedSystemTopic() throws Exception { Assert.assertEquals(admin.topics().getPartitionedTopicList(ns).size(), 1); Assert.assertEquals(partitions, PARTITIONS); Assert.assertEquals(admin.topics().getList(ns).size(), PARTITIONS); + reader.close(); } @Test(timeOut = 1000 * 60) @@ -97,6 +98,10 @@ public void testConsumerCreationWhenEnablingTopicPolicy() throws Exception { .subscribeAsync()); } FutureUtil.waitForAll(futureList).get(); + // Close all the consumers after check + for (CompletableFuture> consumer : futureList) { + consumer.join().close(); + } } } From a9938558a8a09414d9adadc520f4c797c750c8a7 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 29 Mar 2022 01:00:20 +0800 Subject: [PATCH 440/823] [fix][transaction] Properly close transaction-buffer-sub non durable cursor (#14900) Fixes #14880 ### Motivation Non durable cursor was not closed properly. ### Modifications For non durable cursor, `cursor.asyncClose` did nothing. The proper way is `topic.getManagedLedger().asyncDeleteCursor` (cherry picked from commit 4e62ffc15714cfa49ed441f3ba7ededb866b9062) --- .../buffer/impl/TopicTransactionBuffer.java | 13 ++++++++----- .../pulsar/broker/transaction/TransactionTest.java | 7 +++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 66ce8f517673c..e2888d954f3a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -55,6 +55,7 @@ import org.apache.pulsar.common.policies.data.TransactionInBufferStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Markers; +import org.apache.pulsar.common.util.Codec; import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; @@ -639,7 +640,7 @@ public void run() { } } - closeCursor(managedCursor); + closeCursor(SUBSCRIPTION_NAME); callBack.recoverComplete(); }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this)).exceptionally(e -> { @@ -656,17 +657,19 @@ public void run() { }); } - private void closeCursor(ManagedCursor cursor) { - cursor.asyncClose(new AsyncCallbacks.CloseCallback() { + private void closeCursor(String subscriptionName) { + topic.getManagedLedger().asyncDeleteCursor(Codec.encode(subscriptionName), + new AsyncCallbacks.DeleteCursorCallback() { @Override - public void closeComplete(Object ctx) { + public void deleteCursorComplete(Object ctx) { log.info("[{}]Transaction buffer snapshot recover cursor close complete.", topic.getName()); } @Override - public void closeFailed(ManagedLedgerException exception, Object ctx) { + public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}]Transaction buffer snapshot recover cursor close fail.", topic.getName()); } + }, null); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 231c183a59af1..ae5e8c4757be5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -538,7 +538,7 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ .getTopic("persistent://" + topic, false).get().get(); persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true); - ManagedCursor managedCursor = mock(ManagedCursor.class); + ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class); doReturn("transaction-buffer-sub").when(managedCursor).getName(); doReturn(true).when(managedCursor).hasMoreEntries(); doAnswer(invocation -> { @@ -579,6 +579,9 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ TransactionBuffer buffer3 = new TopicTransactionBuffer(persistentTopic); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer3.getStats().state, "Ready")); + persistentTopic.getInternalStats(false).thenAccept(internalStats -> { + assertTrue(internalStats.cursors.isEmpty()); + }); managedCursors.removeCursor("transaction-buffer-sub"); } @@ -893,4 +896,4 @@ public void testAutoCreateSchemaForTransactionSnapshot() throws Exception { pulsarServiceList.forEach((pulsarService -> pulsarService.getConfiguration().setAllowAutoUpdateSchemaEnabled(true))); } -} \ No newline at end of file +} From 970c11896e48de42f16987036486be49fce132ca Mon Sep 17 00:00:00 2001 From: ran Date: Wed, 30 Mar 2022 06:30:38 +0800 Subject: [PATCH 441/823] # Motivation (#14895) Currently, the transaction buffer don't be closed when deleting topic. # Modification Close the transaction buffer when deleting topic. (cherry picked from commit e8d52fdb14dca62e7f0d7eb933bb7dc7fb3e916e) --- .../service/persistent/PersistentTopic.java | 7 +- .../buffer/TransactionBufferCloseTest.java | 120 ++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 7052fc5c23412..4f1b8cc31d880 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1155,7 +1155,8 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, deleteTopicAuthenticationFuture.thenCompose( __ -> deleteSchema ? deleteSchema() : CompletableFuture.completedFuture(null)) .thenAccept(__ -> deleteTopicPolicies()) - .thenCompose(__ -> transactionBuffer.clearSnapshot()).whenComplete((v, ex) -> { + .thenCompose(__ -> transactionBufferCleanupAndClose()) + .whenComplete((v, ex) -> { if (ex != null) { log.error("[{}] Error deleting topic", topic, ex); unfenceTopicToResume(); @@ -3317,6 +3318,10 @@ public CompletableFuture getPendingAckManagedLedger(String subNam return subscription.getPendingAckManageLedger(); } + private CompletableFuture transactionBufferCleanupAndClose() { + return transactionBuffer.clearSnapshot().thenCompose(__ -> transactionBuffer.closeAsync()); + } + public long getLastDataMessagePublishedTimestamp() { return lastDataMessagePublishedTimestamp; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java new file mode 100644 index 0000000000000..43d31e7f9ffcd --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer; + +import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.events.EventsTopicNames; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.PublisherStats; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Transaction buffer close test. + */ +@Slf4j +@Test(groups = "broker") +public class TransactionBufferCloseTest extends TransactionTestBase { + + @BeforeMethod + protected void setup() throws Exception { + setUpBase(1, 16, null, 0); + Awaitility.await().until(() -> ((PulsarClientImpl) pulsarClient) + .getTcClient().getState() == TransactionCoordinatorClient.State.READY); + admin.tenants().createTenant(TENANT, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "isPartition") + public Object[][] isPartition() { + return new Object[][]{ + { true }, { false } + }; + } + + @Test(timeOut = 10_000, dataProvider = "isPartition") + public void deleteTopicCloseTransactionBufferTest(boolean isPartition) throws Exception { + int expectedCount = isPartition ? 30 : 1; + TopicName topicName = createAndLoadTopic(isPartition, expectedCount); + checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount); + if (isPartition) { + admin.topics().deletePartitionedTopic(topicName.getPartitionedTopicName(), true); + } else { + admin.topics().delete(topicName.getPartitionedTopicName(), true); + } + checkSnapshotPublisherCount(topicName.getNamespace(), 0); + } + + @Test(timeOut = 10_000, dataProvider = "isPartition") + public void unloadTopicCloseTransactionBufferTest(boolean isPartition) throws Exception { + int expectedCount = isPartition ? 30 : 1; + TopicName topicName = createAndLoadTopic(isPartition, expectedCount); + checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount); + admin.topics().unload(topicName.getPartitionedTopicName()); + checkSnapshotPublisherCount(topicName.getNamespace(), 0); + } + + private TopicName createAndLoadTopic(boolean isPartition, int partitionCount) + throws PulsarAdminException, PulsarClientException { + String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5); + admin.namespaces().createNamespace(namespace, 3); + String topic = namespace + "/tb-close-test-"; + if (isPartition) { + admin.topics().createPartitionedTopic(topic, partitionCount); + } + pulsarClient.newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create() + .close(); + return TopicName.get(topic); + } + + private void checkSnapshotPublisherCount(String namespace, int expectCount) throws PulsarAdminException { + TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace), + EventsTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + List publisherStatsList = + (List) admin.topics() + .getStats(snTopicName.getPartitionedTopicName()).getPublishers(); + Assert.assertEquals(publisherStatsList.size(), expectCount); + } + +} From 105885c9cde045e2fc74b90eeab8f99aa1ae4a7a Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Wed, 30 Mar 2022 12:27:48 +0800 Subject: [PATCH 442/823] [ServerCnx] Improve error logging for topic not found (#13950) (#14891) * [ServerCnx] Improve error logging for topic not found * Log topic not found at INFO level (cherry picked from commit 54c898f18bf8c53d64c7f87049bb63764137acb0) Co-authored-by: Michael Marshall --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index c18d525c8caf0..89541a49f5b1a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1266,8 +1266,9 @@ protected void handleProducer(final CommandProducer cmdProducer) { Throwable cause = exception.getCause(); if (cause instanceof NoSuchElementException) { cause = new TopicNotFoundException("Topic Not Found."); - } - if (!Exceptions.areExceptionsPresentInChain(cause, + log.info("[{}] Failed to load topic {}, producerId={}: Topic not found", + remoteAddress, topicName, producerId); + } else if (!Exceptions.areExceptionsPresentInChain(cause, ServiceUnitNotReadyException.class, ManagedLedgerException.class)) { // Do not print stack traces for expected exceptions log.error("[{}] Failed to create topic {}, producerId={}", From 80a521ff8e676d523a2eafa26901c3cab07be644 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Fri, 1 Apr 2022 12:23:42 +0800 Subject: [PATCH 443/823] [fix][transaction] avoid too many ServiceUnitNotReadyException for transaction buffer handler (#14894) 1. Added max concurrent request limitation for transaction buffer client 2. Add the request to the pending request queue after reaching the concurrent request limitation 3. Avoid duplicated lookup cache invalidation (cherry picked from commit 384c528a8db9547e5ef995b5f177a7e72b3b14aa) --- conf/broker.conf | 3 + .../pulsar/broker/ServiceConfiguration.java | 7 + .../apache/pulsar/broker/PulsarService.java | 3 +- .../impl/TransactionBufferClientImpl.java | 16 +- .../impl/TransactionBufferHandlerImpl.java | 257 +++++++++++++----- .../buffer/TransactionBufferClientTest.java | 47 ++-- .../TransactionBufferHandlerImplTest.java | 67 +++++ .../transaction/TransactionBufferClient.java | 4 + .../transaction/TransactionBufferHandler.java | 4 + 9 files changed, 299 insertions(+), 109 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java diff --git a/conf/broker.conf b/conf/broker.conf index 1cb10a06f606a..4fd2834e84590 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1318,6 +1318,9 @@ transactionBufferSnapshotMaxTransactionCount=1000 # Unit : millisecond transactionBufferSnapshotMinTimeInMillis=5000 +# The max concurrent requests for transaction buffer client, default is 1000 +transactionBufferClientMaxConcurrentRequests=1000 + ### --- Packages management service configuration variables (begin) --- ### # Enable the packages management service or not diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 349c46bc9fc80..4b3b2e1783177 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2215,6 +2215,13 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private int transactionBufferSnapshotMinTimeInMillis = 5000; + + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "The max concurrent requests for transaction buffer client." + ) + private int transactionBufferClientMaxConcurrentRequests = 1000; + /**** --- KeyStore TLS config variables --- ****/ @FieldContext( category = CATEGORY_KEYSTORE_TLS, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 25357ce17e4fb..eb239ddf23f70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -730,7 +730,8 @@ config, localMetadataStore, getZkClient(), this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient()); this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); - transactionBufferClient = TransactionBufferClientImpl.create(getClient(), transactionTimer); + transactionBufferClient = TransactionBufferClientImpl.create(getClient(), transactionTimer, + config.getTransactionBufferClientMaxConcurrentRequests()); transactionMetadataStoreService = new TransactionMetadataStoreService(TransactionMetadataStoreProvider .newProvider(config.getTransactionMetadataStoreProviderClassName()), this, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java index e774f1282a8a7..454e9a6d53beb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java @@ -39,8 +39,10 @@ private TransactionBufferClientImpl(TransactionBufferHandler tbHandler) { this.tbHandler = tbHandler; } - public static TransactionBufferClient create(PulsarClient pulsarClient, HashedWheelTimer timer) { - TransactionBufferHandler handler = new TransactionBufferHandlerImpl(pulsarClient, timer); + public static TransactionBufferClient create(PulsarClient pulsarClient, HashedWheelTimer timer, + int maxConcurrentRequests) { + TransactionBufferHandler handler = new TransactionBufferHandlerImpl(pulsarClient, timer, + maxConcurrentRequests); return new TransactionBufferClientImpl(handler); } @@ -74,4 +76,14 @@ public CompletableFuture abortTxnOnSubscription(String topic, String subs public void close() { tbHandler.close(); } + + @Override + public int getAvailableRequestCredits() { + return tbHandler.getAvailableRequestCredits(); + } + + @Override + public int getPendingRequestsCount() { + return tbHandler.getPendingRequestsCount(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 552ee275dd543..6ea53a3edd263 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.PulsarClient; @@ -42,17 +43,23 @@ import org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscriptionResponse; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; @Slf4j public class TransactionBufferHandlerImpl implements TransactionBufferHandler { - private final ConcurrentSkipListMap pendingRequests; + private final ConcurrentSkipListMap outstandingRequests; + private final GrowableArrayBlockingQueue pendingRequests; private final AtomicLong requestIdGenerator = new AtomicLong(); private final long operationTimeoutInMills; private final HashedWheelTimer timer; private final PulsarClient pulsarClient; - private final LoadingCache> cache = CacheBuilder.newBuilder() + private static final AtomicIntegerFieldUpdater REQUEST_CREDITS_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(TransactionBufferHandlerImpl.class, "requestCredits"); + private volatile int requestCredits; + + private final LoadingCache> lookupCache = CacheBuilder.newBuilder() .maximumSize(100000) .expireAfterAccess(30, TimeUnit.MINUTES) .build(new CacheLoader>() { @@ -61,7 +68,7 @@ public CompletableFuture load(String topic) { CompletableFuture siFuture = getClientCnx(topic); siFuture.whenComplete((si, cause) -> { if (null != cause) { - cache.invalidate(topic); + lookupCache.invalidate(topic); } }); return siFuture; @@ -69,11 +76,13 @@ public CompletableFuture load(String topic) { }); public TransactionBufferHandlerImpl(PulsarClient pulsarClient, - HashedWheelTimer timer) { + HashedWheelTimer timer, int maxConcurrentRequests) { this.pulsarClient = pulsarClient; - this.pendingRequests = new ConcurrentSkipListMap<>(); + this.outstandingRequests = new ConcurrentSkipListMap<>(); + this.pendingRequests = new GrowableArrayBlockingQueue<>(); this.operationTimeoutInMills = 3000L; this.timer = timer; + this.requestCredits = Math.max(100, maxConcurrentRequests); } @Override @@ -87,7 +96,18 @@ public CompletableFuture endTxnOnTopic(String topic, long txnIdMostBits, long requestId = requestIdGenerator.getAndIncrement(); ByteBuf cmd = Commands.newEndTxnOnPartition(requestId, txnIdLeastBits, txnIdMostBits, topic, action, lowWaterMark); - return endTxn(requestId, topic, cmd, cb); + + try { + OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb, lookupCache.get(topic)); + if (checkRequestCredits(op)) { + endTxn(op); + } + } catch (ExecutionException e) { + log.error("[{}] failed to get client cnx from lookup cache", topic, e); + lookupCache.invalidate(topic); + cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage())); + } + return cb; } @Override @@ -102,53 +122,68 @@ public CompletableFuture endTxnOnSubscription(String topic, String subscr long requestId = requestIdGenerator.getAndIncrement(); ByteBuf cmd = Commands.newEndTxnOnSubscription(requestId, txnIdLeastBits, txnIdMostBits, topic, subscription, action, lowWaterMark); - return endTxn(requestId, topic, cmd, cb); - } - - private CompletableFuture endTxn(long requestId, String topic, ByteBuf cmd, CompletableFuture cb) { - OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb); try { - cache.get(topic).whenComplete((clientCnx, throwable) -> { - if (throwable == null) { - if (clientCnx.ctx().channel().isActive()) { - clientCnx.registerTransactionBufferHandler(TransactionBufferHandlerImpl.this); - pendingRequests.put(requestId, op); - timer.newTimeout(timeout -> { - OpRequestSend peek = pendingRequests.remove(requestId); - if (peek != null && !peek.cb.isDone() && !peek.cb.isCompletedExceptionally()) { - peek.cb.completeExceptionally(new TransactionBufferClientException - .RequestTimeoutException()); - onResponse(peek); - } - }, operationTimeoutInMills, TimeUnit.MILLISECONDS); - cmd.retain(); - clientCnx.ctx().writeAndFlush(cmd, clientCnx.ctx().voidPromise()); - } else { - cache.invalidate(topic); - cb.completeExceptionally( - new PulsarClientException.LookupException(topic + " endTxn channel is not active")); - op.recycle(); - } - } else { - log.error("endTxn error topic: [{}]", topic, throwable); - cache.invalidate(topic); - cb.completeExceptionally( - new PulsarClientException.LookupException(throwable.getMessage())); - op.recycle(); - } - }); + OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb, lookupCache.get(topic)); + if (checkRequestCredits(op)) { + endTxn(op); + } } catch (ExecutionException e) { - log.error("endTxn channel is not active exception", e); - cache.invalidate(topic); + log.error("[{}] failed to get client cnx from lookup cache", topic, e); + lookupCache.invalidate(topic); cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage())); - op.recycle(); } return cb; } + private boolean checkRequestCredits(OpRequestSend op) { + int currentPermits = REQUEST_CREDITS_UPDATER.get(this); + if (currentPermits > 0 && pendingRequests.peek() == null) { + if (REQUEST_CREDITS_UPDATER.compareAndSet(this, currentPermits, currentPermits - 1)) { + return true; + } else { + return checkRequestCredits(op); + } + } else { + pendingRequests.add(op); + return false; + } + } + + public void endTxn(OpRequestSend op) { + op.cnx.whenComplete((clientCnx, throwable) -> { + if (throwable == null) { + if (clientCnx.ctx().channel().isActive()) { + clientCnx.registerTransactionBufferHandler(TransactionBufferHandlerImpl.this); + outstandingRequests.put(op.requestId, op); + timer.newTimeout(timeout -> { + OpRequestSend peek = outstandingRequests.remove(op.requestId); + if (peek != null && !peek.cb.isDone() && !peek.cb.isCompletedExceptionally()) { + peek.cb.completeExceptionally(new TransactionBufferClientException + .RequestTimeoutException()); + onResponse(peek); + } + }, operationTimeoutInMills, TimeUnit.MILLISECONDS); + op.cmd.retain(); + clientCnx.ctx().writeAndFlush(op.cmd, clientCnx.ctx().voidPromise()); + } else { + invalidateLookupCache(op); + op.cb.completeExceptionally( + new PulsarClientException.LookupException(op.topic + " endTxn channel is not active")); + onResponse(op); + } + } else { + log.error("endTxn error topic: [{}]", op.topic, throwable); + invalidateLookupCache(op); + op.cb.completeExceptionally( + new PulsarClientException.LookupException(throwable.getMessage())); + onResponse(op); + } + }); + } + @Override public void handleEndTxnOnTopicResponse(long requestId, CommandEndTxnOnPartitionResponse response) { - OpRequestSend op = pendingRequests.remove(requestId); + OpRequestSend op = outstandingRequests.remove(requestId); if (op == null) { if (log.isDebugEnabled()) { log.debug("Got end txn on topic response for timeout {} - {}", response.getTxnidMostBits(), @@ -156,25 +191,32 @@ public void handleEndTxnOnTopicResponse(long requestId, CommandEndTxnOnPartition } return; } - - if (!response.hasError()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Got end txn on topic response for for request {}", op.topic, response.getRequestId()); + try { + if (!response.hasError()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Got end txn on topic response for for request {}", op.topic, + response.getRequestId()); + } + op.cb.complete(new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits())); + } else { + log.error("[{}] Got end txn on topic response for request {} error {}", op.topic, + response.getRequestId(), + response.getError()); + invalidateLookupCache(op); + op.cb.completeExceptionally(ClientCnx.getPulsarClientException(response.getError(), + response.getMessage())); } - op.cb.complete(new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits())); - } else { - log.error("[{}] Got end txn on topic response for request {} error {}", op.topic, response.getRequestId(), - response.getError()); - cache.invalidate(op.topic); - op.cb.completeExceptionally(ClientCnx.getPulsarClientException(response.getError(), response.getMessage())); + } catch (Exception e) { + log.error("[{}] Got exception when complete EndTxnOnTopic op for request {}", op.topic, e); + } finally { + onResponse(op); } - onResponse(op); } @Override public void handleEndTxnOnSubscriptionResponse(long requestId, CommandEndTxnOnSubscriptionResponse response) { - OpRequestSend op = pendingRequests.remove(requestId); + OpRequestSend op = outstandingRequests.remove(requestId); if (op == null) { if (log.isDebugEnabled()) { log.debug("Got end txn on subscription response for timeout {} - {}", response.getTxnidMostBits(), @@ -183,41 +225,96 @@ public void handleEndTxnOnSubscriptionResponse(long requestId, return; } - if (!response.hasError()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Got end txn on subscription response for for request {}", - op.topic, response.getRequestId()); + try { + if (!response.hasError()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Got end txn on subscription response for for request {}", + op.topic, response.getRequestId()); + } + op.cb.complete(new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits())); + } else { + log.error("[{}] Got end txn on subscription response for request {} error {}", + op.topic, response.getRequestId(), response.getError()); + invalidateLookupCache(op); + op.cb.completeExceptionally(ClientCnx.getPulsarClientException(response.getError(), + response.getMessage())); + } + } catch (Exception e) { + log.error("[{}] Got exception when complete EndTxnOnSub op for request {}", op.topic, e); + } finally { + onResponse(op); + } + } + + public void onResponse(OpRequestSend op) { + REQUEST_CREDITS_UPDATER.incrementAndGet(this); + if (op != null) { + ReferenceCountUtil.safeRelease(op.cmd); + op.recycle(); + } + checkPendingRequests(); + } + + private void checkPendingRequests() { + while (true) { + int permits = REQUEST_CREDITS_UPDATER.get(this); + if (permits > 0 && pendingRequests.peek() != null) { + if (REQUEST_CREDITS_UPDATER.compareAndSet(this, permits, permits - 1)) { + OpRequestSend polled = pendingRequests.poll(); + if (polled != null) { + try { + if (polled.cnx != lookupCache.get(polled.topic)) { + OpRequestSend invalid = polled; + polled = OpRequestSend.create(invalid.requestId, invalid.topic, invalid.cmd, invalid.cb, + lookupCache.get(invalid.topic)); + invalid.recycle(); + } + endTxn(polled); + } catch (ExecutionException e) { + log.error("[{}] failed to get client cnx from lookup cache", polled.topic, e); + lookupCache.invalidate(polled.topic); + polled.cb.completeExceptionally(new PulsarClientException.LookupException( + e.getCause().getMessage())); + REQUEST_CREDITS_UPDATER.incrementAndGet(this); + } + } else { + REQUEST_CREDITS_UPDATER.incrementAndGet(this); + } + } + } else { + break; } - op.cb.complete(new TxnID(response.getTxnidMostBits(), response.getTxnidLeastBits())); - } else { - log.error("[{}] Got end txn on subscription response for request {} error {}", - op.topic, response.getRequestId(), response.getError()); - cache.invalidate(op.topic); - op.cb.completeExceptionally(ClientCnx.getPulsarClientException(response.getError(), response.getMessage())); } - onResponse(op); } - void onResponse(OpRequestSend op) { - ReferenceCountUtil.safeRelease(op.byteBuf); - op.recycle(); + private void invalidateLookupCache(OpRequestSend op) { + try { + if (lookupCache.get(op.topic) == op.cnx) { + lookupCache.invalidate(op.topic); + } + } catch (ExecutionException e) { + lookupCache.invalidate(op.topic); + } } - private static final class OpRequestSend { + public static final class OpRequestSend { long requestId; String topic; - ByteBuf byteBuf; + ByteBuf cmd; CompletableFuture cb; long createdAt; + CompletableFuture cnx; - static OpRequestSend create(long requestId, String topic, ByteBuf byteBuf, CompletableFuture cb) { + static OpRequestSend create(long requestId, String topic, ByteBuf cmd, CompletableFuture cb, + CompletableFuture cnx) { OpRequestSend op = RECYCLER.get(); op.requestId = requestId; op.topic = topic; - op.byteBuf = byteBuf; + op.cmd = cmd; op.cb = cb; op.createdAt = System.currentTimeMillis(); + op.cnx = cnx; return op; } @@ -239,7 +336,7 @@ protected OpRequestSend newObject(Handle handle) { }; } - private CompletableFuture getClientCnx(String topic) { + public CompletableFuture getClientCnx(String topic) { return ((PulsarClientImpl) pulsarClient).getConnection(topic); } @@ -247,4 +344,14 @@ private CompletableFuture getClientCnx(String topic) { public void close() { this.timer.stop(); } + + @Override + public int getAvailableRequestCredits() { + return REQUEST_CREDITS_UPDATER.get(this); + } + + @Override + public int getPendingRequestsCount() { + return pendingRequests.size(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index c25447bfb0915..fa1b9e7f287e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -27,12 +27,9 @@ import lombok.Cleanup; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import org.apache.pulsar.broker.service.Topic; -import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; @@ -84,7 +81,7 @@ protected void setup() throws Exception { admin.namespaces().createNamespace(namespace, 10); admin.topics().createPartitionedTopic(partitionedTopicName.getPartitionedTopicName(), partitions); tbClient = TransactionBufferClientImpl.create(pulsarClient, - new HashedWheelTimer(new DefaultThreadFactory("transaction-buffer"))); + new HashedWheelTimer(new DefaultThreadFactory("transaction-buffer")), 1000); } @Override @@ -163,19 +160,19 @@ public void testTransactionBufferClientTimeout() throws Exception { @Cleanup("stop") HashedWheelTimer hashedWheelTimer = new HashedWheelTimer(); TransactionBufferHandlerImpl transactionBufferHandler = - new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer); + new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000); CompletableFuture endFuture = transactionBufferHandler.endTxnOnTopic("test", 1, 1, TxnAction.ABORT, 1); - Field field = TransactionBufferHandlerImpl.class.getDeclaredField("pendingRequests"); + Field field = TransactionBufferHandlerImpl.class.getDeclaredField("outstandingRequests"); field.setAccessible(true); - ConcurrentSkipListMap pendingRequests = + ConcurrentSkipListMap outstandingRequests = (ConcurrentSkipListMap) field.get(transactionBufferHandler); - assertEquals(pendingRequests.size(), 1); + assertEquals(outstandingRequests.size(), 1); Awaitility.await().atLeast(2, TimeUnit.SECONDS).until(() -> { - if (pendingRequests.size() == 0) { + if (outstandingRequests.size() == 0) { return true; } return false; @@ -206,7 +203,7 @@ public void testTransactionBufferChannelUnActive() { @Cleanup("stop") HashedWheelTimer hashedWheelTimer = new HashedWheelTimer(); TransactionBufferHandlerImpl transactionBufferHandler = - new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer); + new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000); try { transactionBufferHandler.endTxnOnTopic("test", 1, 1, TxnAction.ABORT, 1).get(); fail(); @@ -246,8 +243,8 @@ public void testTransactionBufferLookUp() throws Exception { } @Test - public void testTransactionBufferHandlerSemaphore() throws Exception { - String topic = "persistent://" + namespace + "/testTransactionBufferHandlerSemaphore"; + public void testTransactionBufferRequestCredits() throws Exception { + String topic = "persistent://" + namespace + "/testTransactionBufferRequestCredits"; String subName = "test"; String abortTopic = topic + "_abort_sub"; @@ -260,11 +257,17 @@ public void testTransactionBufferHandlerSemaphore() throws Exception { admin.topics().createSubscription(commitTopic, subName, MessageId.earliest); tbClient.abortTxnOnSubscription(abortTopic, "test", 1L, 1L, -1L).get(); - tbClient.commitTxnOnSubscription(commitTopic, "test", 1L, 1L, -1L).get(); tbClient.abortTxnOnTopic(abortTopic, 1L, 1L, -1L).get(); tbClient.commitTxnOnTopic(commitTopic, 1L, 1L, -1L).get(); + + assertEquals(tbClient.getAvailableRequestCredits(), 1000); + } + + @Test + public void testTransactionBufferPendingRequests() throws Exception { + } @Test @@ -291,22 +294,4 @@ public void testEndSubNotExist() throws Exception { tbClient.abortTxnOnSubscription(topic + "_abort_topic", sub, 1L, 1L, -1L).get(); tbClient.abortTxnOnSubscription(topic + "_commit_topic", sub, 1L, 1L, -1L).get(); } - - private void waitPendingAckInit(String topic, String sub) throws Exception { - - boolean exist = false; - for (int i = 0; i < getPulsarServiceList().size(); i++) { - CompletableFuture> completableFuture = getPulsarServiceList().get(i) - .getBrokerService().getTopics().get(topic); - if (completableFuture != null) { - PersistentSubscription persistentSubscription = - (PersistentSubscription) completableFuture.get().get().getSubscription(sub); - Awaitility.await().untilAsserted(() -> - assertEquals(persistentSubscription.getTransactionPendingAckStats().state, "Ready")); - exist = true; - } - } - - assertTrue(exist); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java new file mode 100644 index 0000000000000..ef0cf037772b6 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer; + +import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.api.proto.TxnAction; +import org.testng.annotations.Test; + +import java.util.concurrent.CompletableFuture; + +@Test(groups = "broker") +public class TransactionBufferHandlerImplTest { + + @Test + public void testRequestCredits() { + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + when(pulsarClient.getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); + TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarClient, null, 1000)); + doNothing().when(handler).endTxn(any()); + for (int i = 0; i < 500; i++) { + handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); + } + assertEquals(handler.getAvailableRequestCredits(), 500); + for (int i = 0; i < 500; i++) { + handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); + } + assertEquals(handler.getAvailableRequestCredits(), 0); + handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); + assertEquals(handler.getPendingRequestsCount(), 1); + handler.onResponse(null); + assertEquals(handler.getAvailableRequestCredits(), 0); + assertEquals(handler.getPendingRequestsCount(), 0); + } + + @Test + public void testMinRequestCredits() { + TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(null, null, 50)); + assertEquals(handler.getAvailableRequestCredits(), 100); + } +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionBufferClient.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionBufferClient.java index 24c44f15fc48a..d35f8be73ea8c 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionBufferClient.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/transaction/TransactionBufferClient.java @@ -91,4 +91,8 @@ CompletableFuture abortTxnOnSubscription(String topic, long lowWaterMark); void close(); + + int getAvailableRequestCredits(); + + int getPendingRequestsCount(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBufferHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBufferHandler.java index 4843e6ebc41d3..332857c850d3c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBufferHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBufferHandler.java @@ -72,4 +72,8 @@ CompletableFuture endTxnOnSubscription(String topic, String subscription, * Release resources. */ void close(); + + int getAvailableRequestCredits(); + + int getPendingRequestsCount(); } From 7dc9a4961a23d95409e01cd1e1e25ef501167ad0 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 1 Apr 2022 19:09:14 +0800 Subject: [PATCH 444/823] [C++] Fix flaky tests about reference count (#14854) Fixes #14848 Fixes #14719 ### Motivation #7793 introduced a `testReferenceLeak` to avoid cyclic referenece of the reader. However, it adds a unused field `readerImplWeakPtr_` only for tests. The access to this field is not thread safe that the write operation happens in `handleConsumerCreated` while the read operation can happen anywhere via the getter. So there is a little chance that `readerPtr` in `testReferenceLeak` doesn't point to the right object. In addition, we should only guarantee the reference count becomes 0 after the producer, consumer or reader goes out of its scope. #14797 adds a `ClientTest.testReferenceCount` but it's also flaky. It's caused by the shared pointer of `ProducerImpl` is published to another thread via `shared_from_this()` but the test has a strong expectation that the reference count is exactly 1. ### Modifications - Remove `readerImplWeakPtr_` from `ReaderImpl` and get the weak pointer from `Reader` directly by adding a method to `PulsarFriend`. - Add the check of reader's reference count to `testReferenceCount` and remove the redundant `testReferenceLeak`. - Instead of asserting the reference count of producer/consumer/reader is 1, just assume the it's greater than 0. (cherry picked from commit f84ff571df95f99efa4596e65324def1084fc11b) --- pulsar-client-cpp/lib/ReaderImpl.cc | 5 --- pulsar-client-cpp/lib/ReaderImpl.h | 5 +-- pulsar-client-cpp/tests/ClientTest.cc | 26 +++++++++++-- pulsar-client-cpp/tests/PulsarFriend.h | 7 ++++ pulsar-client-cpp/tests/ReaderTest.cc | 51 ++------------------------ pulsar-client-cpp/tests/ReaderTest.h | 32 ---------------- 6 files changed, 35 insertions(+), 91 deletions(-) delete mode 100644 pulsar-client-cpp/tests/ReaderTest.h diff --git a/pulsar-client-cpp/lib/ReaderImpl.cc b/pulsar-client-cpp/lib/ReaderImpl.cc index 0a7b3215437bf..9401c12011983 100644 --- a/pulsar-client-cpp/lib/ReaderImpl.cc +++ b/pulsar-client-cpp/lib/ReaderImpl.cc @@ -90,11 +90,8 @@ const std::string& ReaderImpl::getTopic() const { return consumer_->getTopic(); void ReaderImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumer) { auto self = shared_from_this(); readerCreatedCallback_(result, Reader(self)); - readerImplWeakPtr_ = self; } -ConsumerImplPtr ReaderImpl::getConsumer() { return consumer_; } - Result ReaderImpl::readNext(Message& msg) { Result res = consumer_->receive(msg); acknowledgeIfNecessary(res, msg); @@ -144,8 +141,6 @@ void ReaderImpl::getLastMessageIdAsync(GetLastMessageIdCallback callback) { }); } -ReaderImplWeakPtr ReaderImpl::getReaderImplWeakPtr() { return readerImplWeakPtr_; } - bool ReaderImpl::isConnected() const { return consumer_->isConnected(); } } // namespace pulsar diff --git a/pulsar-client-cpp/lib/ReaderImpl.h b/pulsar-client-cpp/lib/ReaderImpl.h index a546ae87e790d..6de6c02e460ed 100644 --- a/pulsar-client-cpp/lib/ReaderImpl.h +++ b/pulsar-client-cpp/lib/ReaderImpl.h @@ -53,7 +53,7 @@ class PULSAR_PUBLIC ReaderImpl : public std::enable_shared_from_this Future getReaderCreatedFuture(); - ConsumerImplPtr getConsumer(); + ConsumerImplBaseWeakPtr getConsumer() const noexcept { return consumer_; } void hasMessageAvailableAsync(HasMessageAvailableCallback callback); @@ -62,8 +62,6 @@ class PULSAR_PUBLIC ReaderImpl : public std::enable_shared_from_this void getLastMessageIdAsync(GetLastMessageIdCallback callback); - ReaderImplWeakPtr getReaderImplWeakPtr(); - bool isConnected() const; private: @@ -79,7 +77,6 @@ class PULSAR_PUBLIC ReaderImpl : public std::enable_shared_from_this ConsumerImplPtr consumer_; ReaderCallback readerCreatedCallback_; ReaderListener readerListener_; - ReaderImplWeakPtr readerImplWeakPtr_; }; } // namespace pulsar diff --git a/pulsar-client-cpp/tests/ClientTest.cc b/pulsar-client-cpp/tests/ClientTest.cc index 920430d34cb03..1ba0164ad87b7 100644 --- a/pulsar-client-cpp/tests/ClientTest.cc +++ b/pulsar-client-cpp/tests/ClientTest.cc @@ -24,6 +24,9 @@ #include #include #include "../lib/checksum/ChecksumProvider.h" +#include "lib/LogUtils.h" + +DECLARE_LOG_OBJECT() using namespace pulsar; @@ -184,22 +187,39 @@ TEST(ClientTest, testReferenceCount) { auto &producers = PulsarFriend::getProducers(client); auto &consumers = PulsarFriend::getConsumers(client); + ReaderImplWeakPtr readerWeakPtr; { Producer producer; ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); ASSERT_EQ(producers.size(), 1); - ASSERT_EQ(producers[0].use_count(), 1); + ASSERT_TRUE(producers[0].use_count() > 0); + LOG_INFO("Reference count of the producer: " << producers[0].use_count()); Consumer consumer; ASSERT_EQ(ResultOk, client.subscribe(topic, "my-sub", consumer)); ASSERT_EQ(consumers.size(), 1); - ASSERT_EQ(consumers[0].use_count(), 1); + ASSERT_TRUE(consumers[0].use_count() > 0); + LOG_INFO("Reference count of the consumer: " << consumers[0].use_count()); + + ReaderConfiguration readerConf; + Reader reader; + ASSERT_EQ(ResultOk, + client.createReader(topic + "-reader", MessageId::earliest(), readerConf, reader)); + ASSERT_EQ(consumers.size(), 2); + ASSERT_TRUE(consumers[1].use_count() > 0); + LOG_INFO("Reference count of the reader's underlying consumer: " << consumers[1].use_count()); + + readerWeakPtr = PulsarFriend::getReaderImplWeakPtr(reader); + ASSERT_EQ(readerWeakPtr.use_count(), 1); + LOG_INFO("Reference count of the reader: " << readerWeakPtr.use_count()); } ASSERT_EQ(producers.size(), 1); ASSERT_EQ(producers[0].use_count(), 0); - ASSERT_EQ(consumers.size(), 1); + ASSERT_EQ(consumers.size(), 2); ASSERT_EQ(consumers[0].use_count(), 0); + ASSERT_EQ(consumers[1].use_count(), 0); + ASSERT_EQ(readerWeakPtr.use_count(), 0); client.close(); } diff --git a/pulsar-client-cpp/tests/PulsarFriend.h b/pulsar-client-cpp/tests/PulsarFriend.h index 74aa1f74c1bb7..2d9b558e0fa16 100644 --- a/pulsar-client-cpp/tests/PulsarFriend.h +++ b/pulsar-client-cpp/tests/PulsarFriend.h @@ -25,6 +25,7 @@ #include "lib/ConsumerImpl.h" #include "lib/PartitionedConsumerImpl.h" #include "lib/MultiTopicsConsumerImpl.h" +#include "lib/ReaderImpl.h" using std::string; @@ -79,6 +80,12 @@ class PulsarFriend { return std::static_pointer_cast(consumer.impl_); } + static ConsumerImplPtr getConsumer(Reader reader) { + return std::static_pointer_cast(reader.impl_->getConsumer().lock()); + } + + static ReaderImplWeakPtr getReaderImplWeakPtr(Reader reader) { return reader.impl_; } + static std::shared_ptr getPartitionedConsumerImplPtr(Consumer consumer) { return std::static_pointer_cast(consumer.impl_); } diff --git a/pulsar-client-cpp/tests/ReaderTest.cc b/pulsar-client-cpp/tests/ReaderTest.cc index 8cd535caf7f5e..bf156927590ed 100644 --- a/pulsar-client-cpp/tests/ReaderTest.cc +++ b/pulsar-client-cpp/tests/ReaderTest.cc @@ -18,8 +18,8 @@ */ #include #include -#include "ReaderTest.h" #include "HttpHelper.h" +#include "PulsarFriend.h" #include @@ -28,6 +28,7 @@ #include #include +#include DECLARE_LOG_OBJECT() using namespace pulsar; @@ -423,50 +424,6 @@ TEST(ReaderTest, testReaderReachEndOfTopicMessageWithoutBatches) { client.close(); } -TEST(ReaderTest, testReferenceLeak) { - Client client(serviceUrl); - - std::string topicName = "persistent://public/default/testReferenceLeak"; - - Producer producer; - ASSERT_EQ(ResultOk, client.createProducer(topicName, producer)); - - for (int i = 0; i < 10; i++) { - std::string content = "my-message-" + std::to_string(i); - Message msg = MessageBuilder().setContent(content).build(); - ASSERT_EQ(ResultOk, producer.send(msg)); - } - - ReaderConfiguration readerConf; - Reader reader; - ASSERT_EQ(ResultOk, client.createReader(topicName, MessageId::earliest(), readerConf, reader)); - - ConsumerImplBaseWeakPtr consumerPtr = ReaderTest::getConsumer(reader); - ReaderImplWeakPtr readerPtr = ReaderTest::getReaderImplWeakPtr(reader); - - LOG_INFO("1 consumer use count " << consumerPtr.use_count()); - LOG_INFO("1 reader use count " << readerPtr.use_count()); - - for (int i = 0; i < 10; i++) { - Message msg; - ASSERT_EQ(ResultOk, reader.readNext(msg)); - - std::string content = msg.getDataAsString(); - std::string expected = "my-message-" + std::to_string(i); - ASSERT_EQ(expected, content); - } - - producer.close(); - reader.close(); - // will be released after exit this method. - ASSERT_EQ(1, consumerPtr.use_count()); - ASSERT_EQ(1, readerPtr.use_count()); - client.close(); - // will be released after exit this method. - ASSERT_EQ(1, consumerPtr.use_count()); - ASSERT_EQ(1, readerPtr.use_count()); -} - TEST(ReaderTest, testPartitionIndex) { Client client(serviceUrl); @@ -519,7 +476,7 @@ TEST(ReaderTest, testSubscriptionNameSetting) { Reader reader; ASSERT_EQ(ResultOk, client.createReader(topicName, MessageId::earliest(), readerConf, reader)); - ASSERT_EQ(subName, ReaderTest::getConsumer(reader)->getSubscriptionName()); + ASSERT_EQ(subName, PulsarFriend::getConsumer(reader)->getSubscriptionName()); reader.close(); client.close(); @@ -537,7 +494,7 @@ TEST(ReaderTest, testSetSubscriptionNameAndPrefix) { Reader reader; ASSERT_EQ(ResultOk, client.createReader(topicName, MessageId::earliest(), readerConf, reader)); - ASSERT_EQ(subName, ReaderTest::getConsumer(reader)->getSubscriptionName()); + ASSERT_EQ(subName, PulsarFriend::getConsumer(reader)->getSubscriptionName()); reader.close(); client.close(); diff --git a/pulsar-client-cpp/tests/ReaderTest.h b/pulsar-client-cpp/tests/ReaderTest.h deleted file mode 100644 index fd0387f1d0662..0000000000000 --- a/pulsar-client-cpp/tests/ReaderTest.h +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include "lib/ReaderImpl.h" -#include - -using std::string; - -namespace pulsar { -class ReaderTest { - public: - static ConsumerImplPtr getConsumer(const Reader& reader) { return reader.impl_->getConsumer(); } - static ReaderImplWeakPtr getReaderImplWeakPtr(const Reader& reader) { - return reader.impl_->getReaderImplWeakPtr(); - } -}; -} // namespace pulsar From fb9cd9f73100cb5c92520b5a37b0eee311901050 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 2 Apr 2022 16:55:25 +0800 Subject: [PATCH 445/823] [Fix][Transaction] Fix transaction pendingAckStore asyncMarkDelete (#14974) ### Motivation When we only use transaction to ack message, a message per transaction, `cursor.asyncMarkDelete` will never be executed, because firstPosition and deletePosition both are init as metadataPositions.firstEntry().getKey(); And if only the firstkey need to delete, deletePosition also get metadataPositions.firstKey(); irstPosition == deletePosition, never call `cursor.asyncMarkDelete` ### Modification init firstPosition and deletePosition as PositionImpl.EARLIEST (cherry picked from commit 8a6ecd7d4c9399bb7ce5a224ca854e4a71db79b1) --- .../pendingack/impl/MLPendingAckStore.java | 5 +- .../broker/transaction/TransactionTest.java | 74 +++++++++++++++++++ .../pendingack/PendingAckPersistentTest.java | 2 +- 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 0e4ed4426b9f7..a53db3391baaf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -241,8 +241,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { completableFuture.complete(null); if (!metadataPositions.isEmpty()) { - PositionImpl firstPosition = metadataPositions.firstEntry().getKey(); - PositionImpl deletePosition = metadataPositions.firstEntry().getKey(); + PositionImpl deletePosition = null; while (!metadataPositions.isEmpty() && metadataPositions.firstKey() != null && subManagedCursor.getPersistentMarkDeletedPosition() != null @@ -252,7 +251,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { metadataPositions.remove(metadataPositions.firstKey()); } - if (firstPosition != deletePosition) { + if (deletePosition != null) { PositionImpl finalDeletePosition = deletePosition; cursor.asyncMarkDelete(deletePosition, new AsyncCallbacks.MarkDeleteCallback() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index ae5e8c4757be5..296f04177f8cb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -95,6 +95,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.schema.SchemaInfo; @@ -896,4 +897,77 @@ public void testAutoCreateSchemaForTransactionSnapshot() throws Exception { pulsarServiceList.forEach((pulsarService -> pulsarService.getConfiguration().setAllowAutoUpdateSchemaEnabled(true))); } + + @Test + public void testPendingAckMarkDeletePosition() throws Exception { + String topic = NAMESPACE1 + "/test1"; + + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.BYTES) + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer() + .topic(topic) + .subscriptionName("sub") + .subscribe(); + consumer.getSubscription(); + + PersistentSubscription persistentSubscription = (PersistentSubscription) getPulsarServiceList() + .get(0) + .getBrokerService() + .getTopic(topic, false) + .get() + .get() + .getSubscription("sub"); + + ManagedCursor subscriptionCursor = persistentSubscription.getCursor(); + + subscriptionCursor.getMarkDeletedPosition(); + //pendingAck add message1 and commit mark, metadata add message1 + //PersistentMarkDeletedPosition have not updated + producer.newMessage() + .value("test".getBytes(UTF_8)) + .send(); + Transaction transaction = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES) + .build().get(); + + Message message1 = consumer.receive(10, TimeUnit.SECONDS); + + consumer.acknowledgeAsync(message1.getMessageId(), transaction); + transaction.commit().get(); + //PersistentMarkDeletedPosition of subscription have updated to message1, + //check whether delete markDeletedPosition of pendingAck after append entry to pendingAck + transaction = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES) + .build().get(); + + producer.newMessage() + .value("test".getBytes(UTF_8)) + .send(); + Message message2 = consumer.receive(10, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message2.getMessageId(), transaction); + + Awaitility.await().untilAsserted(() -> { + ManagedLedgerInternalStats managedLedgerInternalStats = admin + .transactions() + .getPendingAckInternalStats(topic, "sub", false) + .pendingAckLogStats + .managedLedgerInternalStats; + String [] markDeletePosition = managedLedgerInternalStats.cursors.get("__pending_ack_state") + .markDeletePosition.split(":"); + String [] lastConfirmedEntry = managedLedgerInternalStats.lastConfirmedEntry.split(":"); + Assert.assertEquals(markDeletePosition[0], lastConfirmedEntry[0]); + //don`t contain commit mark and unCommitted message2 + Assert.assertEquals(Integer.parseInt(markDeletePosition[1]), + Integer.parseInt(lastConfirmedEntry[1]) - 2); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index affaf457e1c13..d8da6631499cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -281,7 +281,7 @@ public void cumulativePendingAckReplayTest() throws Exception { // in order to check out the pending ack cursor is clear whether or not. Awaitility.await() .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) - .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); } @Test From 6cd01a22ac4ae8fdccc61f1cade6fae63036befd Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 2 Apr 2022 17:36:44 +0800 Subject: [PATCH 446/823] [fix][transaction] Fix potentially unfinished CompletableFuture. (#14973) ### Motivation In MLTransactionMetadataStore#addProducedPartitionToTxn, the method getTxnPositionPair(txnID) may throw TransactionNotFoundException, but it does not catch it. It may cause the uncompleted future. https://github.com/apache/pulsar/blob/02eb31b372b2bf72350e8f6cbab552e1627e6197/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java#L260-L290 https://github.com/apache/pulsar/blob/02eb31b372b2bf72350e8f6cbab552e1627e6197/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java#L435-L444 (cherry picked from commit 1544375667f8c724ff9098244f47bb07871c8151) --- .../TransactionMetadataStoreService.java | 3 +- .../pulsar/broker/service/ServerCnx.java | 15 +- .../buffer/impl/TopicTransactionBuffer.java | 2 +- .../impl/MLTransactionMetadataStore.java | 138 +++++++++--------- 4 files changed, 77 insertions(+), 81 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 28bef1f830d5e..97fae3be9f669 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -442,8 +442,7 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea // when managedLedger fence will remove this tc and reload public void handleOpFail(Throwable e, TransactionCoordinatorID tcId) { - if (e.getCause() instanceof ManagedLedgerException.ManagedLedgerFencedException - || e instanceof ManagedLedgerException.ManagedLedgerFencedException) { + if (e instanceof ManagedLedgerException.ManagedLedgerFencedException) { removeTransactionMetadataStore(tcId); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 89541a49f5b1a..07d898f488c81 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2032,24 +2032,23 @@ private boolean checkTransactionEnableAndSenError(long requestId) { } } private Throwable handleTxnException(Throwable ex, String op, long requestId) { - if (ex instanceof CoordinatorException.CoordinatorNotFoundException || ex != null - && ex.getCause() instanceof CoordinatorException.CoordinatorNotFoundException) { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof CoordinatorException.CoordinatorNotFoundException) { if (log.isDebugEnabled()) { log.debug("The Coordinator was not found for the request {}", op); } - return ex; + return cause; } - if (ex instanceof ManagedLedgerException.ManagedLedgerFencedException || ex != null - && ex.getCause() instanceof ManagedLedgerException.ManagedLedgerFencedException) { + if (cause instanceof ManagedLedgerException.ManagedLedgerFencedException) { if (log.isDebugEnabled()) { log.debug("Throw a CoordinatorNotFoundException to client " + "with the message got from a ManagedLedgerFencedException for the request {}", op); } - return new CoordinatorException.CoordinatorNotFoundException(ex.getMessage()); + return new CoordinatorException.CoordinatorNotFoundException(cause.getMessage()); } - log.error("Send response error for {} request {}.", op, requestId, ex); - return ex; + log.error("Send response error for {} request {}.", op, requestId, cause); + return cause; } @Override protected void handleNewTxn(CommandNewTxn command) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index e2888d954f3a5..40b327dde0ace 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -202,7 +202,7 @@ public void recoverExceptionally(Throwable e) { @Override public CompletableFuture getTransactionMeta(TxnID txnID) { - return null; + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index 19d651c96d7ea..f93de8b017552 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -249,15 +249,15 @@ public CompletableFuture newTransaction(long timeOut) { @Override public CompletableFuture addProducedPartitionToTxn(TxnID txnID, List partitions) { - CompletableFuture completableFuture = new CompletableFuture<>(); + CompletableFuture promise = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { if (!checkIfReady()) { - completableFuture + promise .completeExceptionally(new CoordinatorException.TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "add produced partition")); return; } - getTxnPositionPair(txnID).thenAccept(txnMetaListPair -> { + getTxnPositionPair(txnID).thenCompose(txnMetaListPair -> { TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() .setTxnidMostBits(txnID.getMostSigBits()) .setTxnidLeastBits(txnID.getLeastSigBits()) @@ -266,43 +266,42 @@ public CompletableFuture addProducedPartitionToTxn(TxnID txnID, List { - if (exception != null) { - completableFuture.completeExceptionally(exception); - return; - } + return transactionLog.append(transactionMetadataEntry) + .thenAccept(position -> { appendLogCount.increment(); try { synchronized (txnMetaListPair.getLeft()) { txnMetaListPair.getLeft().addProducedPartitions(partitions); txnMetaMap.get(txnID.getLeastSigBits()).getRight().add(position); } - completableFuture.complete(null); + promise.complete(null); } catch (InvalidTxnStatusException e) { transactionLog.deletePosition(Collections.singletonList(position)); log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() + " add produced partition error with TxnStatus : " + txnMetaListPair.getLeft().status().name(), e); - completableFuture.completeExceptionally(e); + promise.completeExceptionally(e); } }); + }).exceptionally(ex -> { + promise.completeExceptionally(ex); + return null; }); }); - return completableFuture; + return promise; } @Override public CompletableFuture addAckedPartitionToTxn(TxnID txnID, List txnSubscriptions) { - CompletableFuture completableFuture = new CompletableFuture<>(); + CompletableFuture promise = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { if (!checkIfReady()) { - completableFuture.completeExceptionally(new CoordinatorException + promise.completeExceptionally(new CoordinatorException .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "add acked partition")); return; } - getTxnPositionPair(txnID).thenAccept(txnMetaListPair -> { + getTxnPositionPair(txnID).thenCompose(txnMetaListPair -> { TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() .setTxnidMostBits(txnID.getMostSigBits()) .setTxnidLeastBits(txnID.getLeastSigBits()) @@ -311,47 +310,46 @@ public CompletableFuture addAckedPartitionToTxn(TxnID txnID, .setLastModificationTime(System.currentTimeMillis()) .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - transactionLog.append(transactionMetadataEntry) - .whenComplete((position, exception) -> { - if (exception != null) { - completableFuture.completeExceptionally(exception); - return; - } + return transactionLog.append(transactionMetadataEntry) + .thenAccept(position -> { appendLogCount.increment(); try { synchronized (txnMetaListPair.getLeft()) { txnMetaListPair.getLeft().addAckedPartitions(txnSubscriptions); txnMetaMap.get(txnID.getLeastSigBits()).getRight().add(position); } - completableFuture.complete(null); + promise.complete(null); } catch (InvalidTxnStatusException e) { transactionLog.deletePosition(Collections.singletonList(position)); log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() + " add acked subscription error with TxnStatus : " + txnMetaListPair.getLeft().status().name(), e); - completableFuture.completeExceptionally(e); + promise.completeExceptionally(e); } }); + }).exceptionally(ex -> { + promise.completeExceptionally(ex); + return null; }); }); - return completableFuture; + return promise; } @Override public CompletableFuture updateTxnStatus(TxnID txnID, TxnStatus newStatus, TxnStatus expectedStatus, boolean isTimeout) { - CompletableFuture completableFuture = new CompletableFuture<>(); + CompletableFuture promise = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { if (!checkIfReady()) { - completableFuture.completeExceptionally(new CoordinatorException + promise.completeExceptionally(new CoordinatorException .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "update transaction status")); return; } - getTxnPositionPair(txnID).thenAccept(txnMetaListPair -> { + getTxnPositionPair(txnID).thenCompose(txnMetaListPair -> { if (txnMetaListPair.getLeft().status() == newStatus) { - completableFuture.complete(null); - return; + promise.complete(null); + return promise; } TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() .setTxnidMostBits(txnID.getMostSigBits()) @@ -362,51 +360,51 @@ public CompletableFuture updateTxnStatus(TxnID txnID, TxnStatus newStatus, .setNewStatus(newStatus) .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - transactionLog.append(transactionMetadataEntry).whenComplete((position, throwable) -> { - if (throwable != null) { - completableFuture.completeExceptionally(throwable); - return; - } - appendLogCount.increment(); - try { - synchronized (txnMetaListPair.getLeft()) { - txnMetaListPair.getLeft().updateTxnStatus(newStatus, expectedStatus); - txnMetaListPair.getRight().add(position); - } - if (newStatus == TxnStatus.ABORTING && isTimeout) { - this.transactionTimeoutCount.increment(); - } - if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { - transactionLog.deletePosition(txnMetaListPair.getRight()).whenComplete((v, exception) -> { - if (exception != null) { - completableFuture.completeExceptionally(exception); - return; + return transactionLog.append(transactionMetadataEntry) + .thenAccept(position -> { + appendLogCount.increment(); + try { + synchronized (txnMetaListPair.getLeft()) { + txnMetaListPair.getLeft().updateTxnStatus(newStatus, expectedStatus); + txnMetaListPair.getRight().add(position); } - this.transactionMetadataStoreStats - .addTransactionExecutionLatencySample(System.currentTimeMillis() - - txnMetaListPair.getLeft().getOpenTimestamp()); - if (newStatus == TxnStatus.COMMITTED) { - committedTransactionCount.increment(); - } else { - abortedTransactionCount.increment(); + if (newStatus == TxnStatus.ABORTING && isTimeout) { + this.transactionTimeoutCount.increment(); } - txnMetaMap.remove(txnID.getLeastSigBits()); - completableFuture.complete(null); - }); - return; - } - completableFuture.complete(null); - } catch (InvalidTxnStatusException e) { - transactionLog.deletePosition(Collections.singletonList(position)); - log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() - + " add update txn status error with TxnStatus : " - + txnMetaListPair.getLeft().status().name(), e); - completableFuture.completeExceptionally(e); - } - }); + if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { + transactionLog.deletePosition(txnMetaListPair.getRight()).whenComplete((v, ex) -> { + if (ex != null) { + promise.completeExceptionally(ex); + return; + } + this.transactionMetadataStoreStats + .addTransactionExecutionLatencySample(System.currentTimeMillis() + - txnMetaListPair.getLeft().getOpenTimestamp()); + if (newStatus == TxnStatus.COMMITTED) { + committedTransactionCount.increment(); + } else { + abortedTransactionCount.increment(); + } + txnMetaMap.remove(txnID.getLeastSigBits()); + promise.complete(null); + }); + return; + } + promise.complete(null); + } catch (InvalidTxnStatusException e) { + transactionLog.deletePosition(Collections.singletonList(position)); + log.error("TxnID : " + txnMetaListPair.getLeft().id().toString() + + " add update txn status error with TxnStatus : " + + txnMetaListPair.getLeft().status().name(), e); + promise.completeExceptionally(e); + } + }); + }).exceptionally(ex -> { + promise.completeExceptionally(ex); + return null; }); }); - return completableFuture; + return promise; } @Override From c5f24385b0d57adb97cb53b167dc4f549e4f2201 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Apr 2022 17:52:11 -0700 Subject: [PATCH 447/823] [Python] Added build for Arm64 on Linux and Python 3.10 (#15004) * [Python] Added build for Arm64 on Linux and Python 3.10 * Fixed docker-build-python3.9.sh * Added arch in docker/pulsar/pom.xml * Fixed spacing in docker/pulsar/pom.xml * Fixed build-wheels.sh when taking args * Removed debug statement * Fixed variable expansion * Pass the arch to the script inside docker --- docker/pulsar/pom.xml | 2 +- pulsar-client-cpp/docker-build-python3.9.sh | 8 +- .../docker/build-wheel-file-within-docker.sh | 4 +- pulsar-client-cpp/docker/build-wheels.sh | 52 +++++++----- pulsar-client-cpp/docker/create-images.sh | 27 +++--- .../docker/manylinux2014/Dockerfile | 85 ++++++------------- pulsar-client-cpp/python/CMakeLists.txt | 11 ++- 7 files changed, 93 insertions(+), 96 deletions(-) diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 9ee0334651e1c..a9a0341ca08c4 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -75,7 +75,7 @@ ${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh - 3.8 cp38-cp38 + 3.8 cp38-cp38 x86_64 diff --git a/pulsar-client-cpp/docker-build-python3.9.sh b/pulsar-client-cpp/docker-build-python3.9.sh index 15cc2fdb102ee..db5c9abd82a99 100755 --- a/pulsar-client-cpp/docker-build-python3.9.sh +++ b/pulsar-client-cpp/docker-build-python3.9.sh @@ -29,16 +29,18 @@ cd $ROOT_DIR/pulsar-client-cpp # Build manylinux2014 build image PYTHON_VERSION="3.9" PYTHON_SPEC="cp39-cp39" -IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC +ARCH="x86_64" +IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH docker build -t $IMAGE_NAME ./docker/manylinux2014 \ --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg PYTHON_SPEC=$PYTHON_SPEC + --build-arg PYTHON_SPEC=$PYTHON_SPEC \ + --build-arg ARCH=$ARCH # Build wheel file BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-pulsar-build}" -IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC +IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"} COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh" diff --git a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh index ade3ca02832d5..c04a3cc2c2eff 100755 --- a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh +++ b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh @@ -31,7 +31,7 @@ cmake . -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERS -DBUILD_TESTS=OFF make clean -make _pulsar -j3 VERBOSE=1 +make _pulsar -j3 cd python python setup.py bdist_wheel @@ -42,4 +42,4 @@ python setup.py bdist_wheel # Audit wheel will make sure no external dependencies are needed for # the shared library and that only symbols supported by most linux # distributions are used. -auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_x86_64.whl +auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_${ARCH}.whl diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh index a61f21516eb05..d3549fd018846 100755 --- a/pulsar-client-cpp/docker/build-wheels.sh +++ b/pulsar-client-cpp/docker/build-wheels.sh @@ -28,32 +28,41 @@ ROOT_DIR=`cd $(dirname $0)/../..; pwd` cd $ROOT_DIR PYTHON_VERSIONS=( - '2.7 cp27-cp27mu' - '2.7 cp27-cp27m' - '3.5 cp35-cp35m' - '3.6 cp36-cp36m' - '3.7 cp37-cp37m' - '3.8 cp38-cp38' - '3.9 cp39-cp39' + '2.7 cp27-cp27mu x86_64' + '2.7 cp27-cp27m x86_64' + '3.5 cp35-cp35m x86_64' + '3.6 cp36-cp36m x86_64' + '3.7 cp37-cp37m x86_64' + '3.8 cp38-cp38 x86_64' + '3.9 cp39-cp39 x86_64' + '3.10 cp310-cp310 x86_64' + '3.7 cp37-cp37m aarch64' + '3.8 cp38-cp38 aarch64' + '3.9 cp39-cp39 aarch64' + '3.10 cp310-cp310 aarch64' ) -function contains() { - local n=$# - local value=${!n} - for ((i=1;i < $#;i++)) { - if [ "${!i}" == "${value}" ]; then - echo "y" - return 0 +function contains_build_version { + for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a v <<< "$line" + value="${v[0]} ${v[1]} ${v[2]}" + + if [ "${build_version}" == "${value}" ]; then + # found + res=1 + return fi - } - echo "n" - return 1 + done + + # not found + res=0 } if [ $# -ge 1 ]; then build_version=$@ - if [ $(contains "${PYTHON_VERSIONS[@]}" "${build_version}") == "y" ]; then + contains_build_version + if [ $res == 1 ]; then PYTHON_VERSIONS=( "${build_version}" ) @@ -69,9 +78,10 @@ for line in "${PYTHON_VERSIONS[@]}"; do read -r -a PY <<< "$line" PYTHON_VERSION=${PY[0]} PYTHON_SPEC=${PY[1]} - echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC" + ARCH=${PY[2]} + echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH" - IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC + IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH echo "Using image: $IMAGE" @@ -79,6 +89,6 @@ for line in "${PYTHON_VERSIONS[@]}"; do COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh" DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX ${IMAGE}" - $DOCKER_CMD bash -c "${COMMAND}" + $DOCKER_CMD bash -c "ARCH=$ARCH ${COMMAND}" done diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh index 6aa1d69ef44dd..0a282375f7d56 100755 --- a/pulsar-client-cpp/docker/create-images.sh +++ b/pulsar-client-cpp/docker/create-images.sh @@ -24,13 +24,18 @@ set -e PYTHON_VERSIONS=( - '2.7 cp27-cp27mu manylinux1' - '2.7 cp27-cp27m manylinux1' - '3.5 cp35-cp35m manylinux2014' - '3.6 cp36-cp36m manylinux2014' - '3.7 cp37-cp37m manylinux2014' - '3.8 cp38-cp38 manylinux2014' - '3.9 cp39-cp39 manylinux2014' + '2.7 cp27-cp27mu manylinux1 x86_64' + '2.7 cp27-cp27m manylinux1 x86_64' + '3.5 cp35-cp35m manylinux2014 x86_64' + '3.6 cp36-cp36m manylinux2014 x86_64' + '3.7 cp37-cp37m manylinux2014 x86_64' + '3.8 cp38-cp38 manylinux2014 x86_64' + '3.9 cp39-cp39 manylinux2014 x86_64' + '3.10 cp310-cp310 manylinux2014 x86_64' + '3.7 cp37-cp37m manylinux2014 aarch64' + '3.8 cp38-cp38 manylinux2014 aarch64' + '3.9 cp39-cp39 manylinux2014 aarch64' + '3.10 cp310-cp310 manylinux2014 aarch64' ) for line in "${PYTHON_VERSIONS[@]}"; do @@ -38,13 +43,15 @@ for line in "${PYTHON_VERSIONS[@]}"; do PYTHON_VERSION=${PY[0]} PYTHON_SPEC=${PY[1]} BASE_IMAGE=${PY[2]} - echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC" + ARCH=${PY[3]} + echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH" - IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC + IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH docker build -t $IMAGE_NAME $BASE_IMAGE \ --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg PYTHON_SPEC=$PYTHON_SPEC + --build-arg PYTHON_SPEC=$PYTHON_SPEC \ + --build-arg ARCH=$ARCH echo "==== Successfully built image $IMAGE_NAME" done diff --git a/pulsar-client-cpp/docker/manylinux2014/Dockerfile b/pulsar-client-cpp/docker/manylinux2014/Dockerfile index 20a7247e30a40..5bc1b0315ed61 100644 --- a/pulsar-client-cpp/docker/manylinux2014/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux2014/Dockerfile @@ -17,10 +17,8 @@ # under the License. # - -FROM quay.io/pypa/manylinux2014_x86_64 - -RUN yum install -y gtest-devel +ARG ARCH +FROM quay.io/pypa/manylinux2014_${ARCH} ARG PYTHON_VERSION ARG PYTHON_SPEC @@ -28,6 +26,10 @@ ARG PYTHON_SPEC ENV PYTHON_VERSION=${PYTHON_VERSION} ENV PYTHON_SPEC=${PYTHON_SPEC} +ARG ARCH +ENV ARCH=${ARCH} + + ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION} @@ -46,84 +48,51 @@ RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \ #################################### # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \ - tar xvfz zlib-1.2.11.tar.gz && \ - cd zlib-1.2.11 && \ +RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ + tar xvfz zlib-1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11 + make -j8 && make install && \ + rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 # Compile OpenSSL -RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ - tar xvfz OpenSSL_1_1_0j.tar.gz && \ - cd openssl-OpenSSL_1_1_0j/ && \ - ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-x86_64 && \ - make && make install && \ - rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j +RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ + tar xvfz OpenSSL_1_1_1n.tar.gz && \ + cd openssl-OpenSSL_1_1_1n/ && \ + ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \ + make -j8 && make install && \ + rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n # Download and compile boost -RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.gz && \ - tar xvfz boost_1_68_0.tar.gz && \ - cd /boost_1_68_0 && \ +RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \ + tar xvfz boost_1_78_0.tar.gz && \ + cd /boost_1_78_0 && \ ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \ - ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost_1_68_0.tar.gz /boost_1_68_0 + ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \ + rm -rf /boost_1_78_0.tar.gz /boost_1_78_0 # Download and copile protoubf RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz && \ tar xvfz protobuf-cpp-3.3.0.tar.gz && \ cd protobuf-3.3.0/ && \ CXXFLAGS=-fPIC ./configure && \ - make && make install && ldconfig && \ + make -j8 && make install && ldconfig && \ rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0 -# Compile APR -RUN curl -O -L http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz && \ - tar xvfz apr-1.5.2.tar.gz && \ - cd apr-1.5.2 && \ - CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \ - make && make install && \ - rm -rf /apr-1.5.2.tar.gz /apr-1.5.2 - -# Compile APR-Util -RUN curl -O -L http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz && \ - tar xvfz apr-util-1.5.4.tar.gz && \ - cd apr-util-1.5.4 && \ - CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure -with-apr=/usr/local/apr && \ - make && make install && \ - rm -rf /apr-util-1.5.4.tar.gz /apr-util-1.5.4 - -# Libtool -RUN curl -L -O https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz && \ - tar xvfz libtool-2.4.6.tar.gz && \ - cd libtool-2.4.6 && \ - ./configure && \ - make && make install && \ - rm -rf /libtool-2.4.6.tar.gz /libtool-2.4.6 - -# Compile log4cxx -RUN curl -O -L https://github.com/apache/logging-log4cxx/archive/v0.11.0.tar.gz && \ - tar xvfz v0.11.0.tar.gz && \ - cd logging-log4cxx-0.11.0 && \ - ./autogen.sh && \ - CXXFLAGS=-fPIC ./configure && \ - make && make install && \ - rm -rf /v0.11.0.tar.gz /logging-log4cxx-0.11.0 - # Compile expat RUN curl -O -L https://github.com/libexpat/libexpat/archive/R_2_2_0.tar.gz && \ tar xfvz R_2_2_0.tar.gz && \ cd libexpat-R_2_2_0/expat && \ ./buildconf.sh && \ CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \ - make && make installlib && \ + make -j8 && make installlib && \ rm -rf /R_2_2_0.tar.gz /libexpat-R_2_2_0 RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.12.1.tar.gz && \ tar xvfz v3.12.1.tar.gz && \ cd CMake-3.12.1 && \ ./configure && \ - make && make install && \ + make -j8 && make install && \ rm -rf /v3.12.1.tar.gz /CMake-3.12.1 # Zstandard @@ -139,7 +108,7 @@ RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1 tar xvfz snappy-1.1.3.tar.gz && \ cd snappy-1.1.3 && \ CXXFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ + make -j8 && make install && \ rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz # LibCurl @@ -147,7 +116,7 @@ RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl- tar xvfz curl-7.61.0.tar.gz && \ cd curl-7.61.0 && \ CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ - make && make install && \ + make -j8 && make install && \ rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 RUN pip install twine diff --git a/pulsar-client-cpp/python/CMakeLists.txt b/pulsar-client-cpp/python/CMakeLists.txt index 30631cd8a5047..ee4a6b2b03284 100644 --- a/pulsar-client-cpp/python/CMakeLists.txt +++ b/pulsar-client-cpp/python/CMakeLists.txt @@ -60,6 +60,10 @@ if (NOT DEFINED ${Boost_PYTHON39-MT_LIBRARY}) set(Boost_PYTHON39-MT_LIBRARY ${Boost_PYTHON39_LIBRARY}) endif() +if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY}) + set(Boost_PYTHON310-MT_LIBRARY ${Boost_PYTHON310_LIBRARY}) +endif() + # Try all possible boost-python variable namings set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY} ${Boost_PYTHON3_LIBRARY} @@ -69,7 +73,9 @@ set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY} ${Boost_PYTHON35_LIBRARY} ${Boost_PYTHON36_LIBRARY} ${Boost_PYTHON38_LIBRARY} - ${Boost_PYTHON39_LIBRARY}) + ${Boost_PYTHON39_LIBRARY} + ${Boost_PYTHON310_LIBRARY} + ) if (APPLE) if (Boost_PYTHON27-MT_LIBRARY_RELEASE) @@ -84,6 +90,9 @@ if (APPLE) if (Boost_PYTHON39-MT_LIBRARY_RELEASE) set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON39-MT_LIBRARY_RELEASE}) endif () + if (Boost_PYTHON310-MT_LIBRARY_RELEASE) + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON310-MT_LIBRARY_RELEASE}) + endif () endif() message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}") From 29f0ffd777e1d2ea2fe5fd49f55162d5adb2e0d3 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 4 Apr 2022 10:06:06 -0700 Subject: [PATCH 448/823] [Python] Added build for wheels on Alpine linux (#15016) --- docker/pulsar/pom.xml | 2 +- pulsar-client-cpp/docker/build-wheels.sh | 24 +--- pulsar-client-cpp/docker/create-images.sh | 17 +-- .../docker/manylinux_musl/Dockerfile | 116 ++++++++++++++++++ pulsar-client-cpp/docker/push-images.sh | 18 +-- pulsar-client-cpp/docker/python-versions.sh | 44 +++++++ 6 files changed, 175 insertions(+), 46 deletions(-) create mode 100644 pulsar-client-cpp/docker/manylinux_musl/Dockerfile create mode 100644 pulsar-client-cpp/docker/python-versions.sh diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index a9a0341ca08c4..36ebfa4161072 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -75,7 +75,7 @@ ${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh - 3.8 cp38-cp38 x86_64 + 3.8 cp38-cp38 manylinux2014 x86_64 diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh index d3549fd018846..25ac64f4846ec 100755 --- a/pulsar-client-cpp/docker/build-wheels.sh +++ b/pulsar-client-cpp/docker/build-wheels.sh @@ -27,25 +27,12 @@ BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" ROOT_DIR=`cd $(dirname $0)/../..; pwd` cd $ROOT_DIR -PYTHON_VERSIONS=( - '2.7 cp27-cp27mu x86_64' - '2.7 cp27-cp27m x86_64' - '3.5 cp35-cp35m x86_64' - '3.6 cp36-cp36m x86_64' - '3.7 cp37-cp37m x86_64' - '3.8 cp38-cp38 x86_64' - '3.9 cp39-cp39 x86_64' - '3.10 cp310-cp310 x86_64' - '3.7 cp37-cp37m aarch64' - '3.8 cp38-cp38 aarch64' - '3.9 cp39-cp39 aarch64' - '3.10 cp310-cp310 aarch64' -) +source ./pulsar-client-cpp/docker/python-versions.sh function contains_build_version { for line in "${PYTHON_VERSIONS[@]}"; do read -r -a v <<< "$line" - value="${v[0]} ${v[1]} ${v[2]}" + value="${v[0]} ${v[1]} ${v[2]} ${v[3]}" if [ "${build_version}" == "${value}" ]; then # found @@ -78,10 +65,11 @@ for line in "${PYTHON_VERSIONS[@]}"; do read -r -a PY <<< "$line" PYTHON_VERSION=${PY[0]} PYTHON_SPEC=${PY[1]} - ARCH=${PY[2]} - echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH" + IMAGE=${PY[2]} + ARCH=${PY[3]} + echo "--------- Build Python wheel for $PYTHON_VERSION -- $IMAGE -- $PYTHON_SPEC -- $ARCH" - IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH + IMAGE=$BUILD_IMAGE_NAME:${IMAGE}-$PYTHON_SPEC-$ARCH echo "Using image: $IMAGE" diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh index 0a282375f7d56..14938a40c2a9e 100755 --- a/pulsar-client-cpp/docker/create-images.sh +++ b/pulsar-client-cpp/docker/create-images.sh @@ -23,20 +23,7 @@ set -e -PYTHON_VERSIONS=( - '2.7 cp27-cp27mu manylinux1 x86_64' - '2.7 cp27-cp27m manylinux1 x86_64' - '3.5 cp35-cp35m manylinux2014 x86_64' - '3.6 cp36-cp36m manylinux2014 x86_64' - '3.7 cp37-cp37m manylinux2014 x86_64' - '3.8 cp38-cp38 manylinux2014 x86_64' - '3.9 cp39-cp39 manylinux2014 x86_64' - '3.10 cp310-cp310 manylinux2014 x86_64' - '3.7 cp37-cp37m manylinux2014 aarch64' - '3.8 cp38-cp38 manylinux2014 aarch64' - '3.9 cp39-cp39 manylinux2014 aarch64' - '3.10 cp310-cp310 manylinux2014 aarch64' -) +source python-versions.sh for line in "${PYTHON_VERSIONS[@]}"; do read -r -a PY <<< "$line" @@ -46,7 +33,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do ARCH=${PY[3]} echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH" - IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH + IMAGE_NAME=pulsar-build:$BASE_IMAGE-$PYTHON_SPEC-$ARCH docker build -t $IMAGE_NAME $BASE_IMAGE \ --build-arg PYTHON_VERSION=$PYTHON_VERSION \ diff --git a/pulsar-client-cpp/docker/manylinux_musl/Dockerfile b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile new file mode 100644 index 0000000000000..89dd2d107558c --- /dev/null +++ b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ARG ARCH +FROM quay.io/pypa/musllinux_1_1_${ARCH} + +ARG PYTHON_VERSION +ARG PYTHON_SPEC + +ENV PYTHON_VERSION=${PYTHON_VERSION} +ENV PYTHON_SPEC=${PYTHON_SPEC} + +ARG ARCH +ENV ARCH=${ARCH} + + +ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" + +RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION} + +# Perl (required for building OpenSSL) +RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \ + tar xvfz perl-5.10.0.tar.gz && \ + cd perl-5.10.0 && \ + ./configure.gnu --prefix=/usr/local/ && \ + make && make install && \ + rm -rf /perl-5.10.0.tar.gz /perl-5.10.0 + +#################################### +# These dependencies can be found in Ubuntu but they're not compiled with -fPIC, +# so they cannot be statically linked into a shared library +#################################### + +# ZLib +RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ + tar xvfz zlib-1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ + CFLAGS="-fPIC -O3" ./configure && \ + make -j8 && make install && \ + rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 + +# Compile OpenSSL +RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ + tar xvfz OpenSSL_1_1_1n.tar.gz && \ + cd openssl-OpenSSL_1_1_1n/ && \ + ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \ + make -j8 && make install && \ + rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n + +# Download and compile boost +RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \ + tar xvfz boost_1_78_0.tar.gz && \ + cd /boost_1_78_0 && \ + ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \ + ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \ + rm -rf /boost_1_78_0.tar.gz /boost_1_78_0 + +# Download and copile protoubf +RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \ + tar xvfz protobuf-cpp-3.20.0.tar.gz && \ + cd protobuf-3.20.0/ && \ + CXXFLAGS=-fPIC ./configure && \ + make -j8 && make install && \ + rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0 + +RUN apk add cmake + +# Zstandard +RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ + tar xvfz zstd-1.3.7.tar.gz && \ + cd zstd-1.3.7 && \ + CFLAGS="-fPIC -O3" make -j8 && \ + make install && \ + rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz + +# Snappy +RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \ + tar xvfz snappy-1.1.3.tar.gz && \ + cd snappy-1.1.3 && \ + CXXFLAGS="-fPIC -O3" ./configure && \ + make -j8 && make install && \ + rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz + +# LibCurl +RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ + tar xvfz curl-7.61.0.tar.gz && \ + cd curl-7.61.0 && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ + make -j8 && make install && \ + rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + +RUN pip install twine +RUN pip install fastavro +RUN pip install six +RUN pip install enum34 + + +ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include +ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ diff --git a/pulsar-client-cpp/docker/push-images.sh b/pulsar-client-cpp/docker/push-images.sh index 0501670e52664..a1806fa29df92 100755 --- a/pulsar-client-cpp/docker/push-images.sh +++ b/pulsar-client-cpp/docker/push-images.sh @@ -23,24 +23,18 @@ set -e -DOCKER_ORG=apachepulsar +source python-versions.sh -PYTHON_VERSIONS=( - '2.7 cp27-cp27mu' - '2.7 cp27-cp27m' - '3.5 cp35-cp35m' - '3.6 cp36-cp36m' - '3.7 cp37-cp37m' - '3.8 cp38-cp38' - '3.9 cp39-cp39' -) +DOCKER_ORG=apachepulsar for line in "${PYTHON_VERSIONS[@]}"; do read -r -a PY <<< "$line" PYTHON_VERSION=${PY[0]} PYTHON_SPEC=${PY[1]} - - IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC + BASE_IMAGE=${PY[2]} + ARCH=${PY[3]} + + IMAGE_NAME=pulsar-build:$BASE_IMAGE-$PYTHON_SPEC-$ARCH FULL_NAME=$DOCKER_ORG/$IMAGE_NAME echo "IMAGE_NAME: $IMAGE_NAME" diff --git a/pulsar-client-cpp/docker/python-versions.sh b/pulsar-client-cpp/docker/python-versions.sh new file mode 100644 index 0000000000000..a264ee179c0f2 --- /dev/null +++ b/pulsar-client-cpp/docker/python-versions.sh @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +PYTHON_VERSIONS=( + '2.7 cp27-cp27mu manylinux1 x86_64' + '2.7 cp27-cp27m manylinux1 x86_64' + '3.5 cp35-cp35m manylinux2014 x86_64' + '3.6 cp36-cp36m manylinux2014 x86_64' + '3.7 cp37-cp37m manylinux2014 x86_64' + '3.8 cp38-cp38 manylinux2014 x86_64' + '3.9 cp39-cp39 manylinux2014 x86_64' + '3.10 cp310-cp310 manylinux2014 x86_64' + '3.7 cp37-cp37m manylinux2014 aarch64' + '3.8 cp38-cp38 manylinux2014 aarch64' + '3.9 cp39-cp39 manylinux2014 aarch64' + '3.10 cp310-cp310 manylinux2014 aarch64' + + # Alpine compatible wheels + '3.7 cp37-cp37m manylinux_musl aarch64' + '3.8 cp38-cp38 manylinux_musl aarch64' + '3.9 cp39-cp39 manylinux_musl aarch64' + '3.10 cp310-cp310 manylinux_musl aarch64' + + '3.7 cp37-cp37m manylinux_musl x86_64' + '3.8 cp38-cp38 manylinux_musl x86_64' + '3.9 cp39-cp39 manylinux_musl x86_64' + '3.10 cp310-cp310 manylinux_musl x86_64' +) From cc9bb44be74a6eb31df0b91cc775965b25351997 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 4 Apr 2022 18:47:16 +0800 Subject: [PATCH 449/823] [C++] Fix connection is not closed when broker closes the connection to proxy (#15009) ### Motivation ### Motivation Recently we found C++ client might not refresh the OAuth2 token after the reconnection. It's because when Pulsar proxy is enabled, the `AuthResponse` might be sent by proxy, which leads to a disconnection from broker side. See https://github.com/apache/pulsar/blob/94cc46a66fbe322472dbb657803d21320e59079c/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java#L687 Then, proxy will return a `ServiceNotReady` error to client as the result of topic lookup. See https://github.com/apache/pulsar/blob/d1a302e60a8f61615800679522016f21dcd01295/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java#L150-L154 https://github.com/apache/pulsar/blob/d1a302e60a8f61615800679522016f21dcd01295/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java#L242-L247 However, in this case, C++ client only completes the future of lookup with `ResultConnectError`. The `ClientConnection` object is still cached in the pool and will be used for followed interactions. ### Modifications Like what Java client does, add `checkServerError` method to `ClientConnection`, which closes the socket for `ServiceNotReady` error. Here we don't call `close` directly just not to interrupt the execution of `handleIncomingCommand`. `close` will be called in `handleRead` because the `err` will be `boost::asio::error::bad_descriptor`. ### Verifying this change It's hard to mock the `ServiceNotReady` case in unit test. I have tested this patch in my private env and let the OAuth2 token expire quickly. We can see the following logs (hosts are hidden): ``` 2022-04-03 19:00:54.357 INFO [0x110af6600] HandlerBase:142 | [persistent://public/default/topic-1, test-oauth2-50-7] Schedule reconnection in 0.1 s 2022-04-03 19:00:54.458 INFO [0x7000056de000] HandlerBase:64 | [persistent://public/default/topic-1, test-oauth2-50-7] Getting connection from pool 2022-04-03 19:00:54.953 ERROR [0x7000056de000] ClientConnection:1001 | [LOCAL_IP:50660 -> REMOTE_IP:6651] Failed lookup req_id: 90 error: ServiceUnitNotReady msg: Disconnected from server at test-oauth2-broker.sndev.svc.cluster.local/172.20.116.73:6650 2022-04-03 19:00:54.953 INFO [0x7000056de000] HandlerBase:142 | [persistent://public/default/topic-1, test-oauth2-50-7] Schedule reconnection in 0.182 s 2022-04-03 19:00:54.953 ERROR [0x7000056de000] ClientConnection:597 | [LOCAL_IP:50660 -> REMOTE_IP:6651] Read operation failed: Bad file descriptor 2022-04-03 19:00:54.953 INFO [0x7000056de000] ClientConnection:1560 | [LOCAL_IP:50660 -> REMOTE_IP:6651] Connection closed 2022-04-03 19:00:54.953 INFO [0x7000056de000] ClientConnection:263 | [LOCAL_IP:50660 -> REMOTE_IP:6651] Destroyed connection 2022-04-03 19:00:55.136 INFO [0x7000056de000] HandlerBase:64 | [persistent://public/default/topic-1, test-oauth2-50-7] Getting connection from pool 2022-04-03 19:00:55.136 INFO [0x7000056de000] ConnectionPool:86 | Deleting stale connection from pool for SERVICE_URL:6651 use_count: -1 @ 0x0 2022-04-03 19:00:55.137 INFO [0x7000056de000] ClientConnection:189 | [ -> SERVICE_URL:6651] Create ClientConnection, timeout=10000 2022-04-03 19:00:55.250 INFO [0x7000056de000] ConnectionPool:96 | Created connection for SERVICE_URL:6651 2022-04-03 19:00:55.702 INFO [0x7000056de000] ClientConnection:375 | [LOCAL_IP:51081 -> REMOTE_IP:6651] Connected to broker 2022-04-03 19:00:57.750 INFO [0x7000056de000] ProducerImpl:189 | [persistent://public/default/topic-1, test-oauth2-50-7] Created producer on broker [LOCAL_IP:50675 -> REMOTE_IP:6651] ``` We can see a new connection was established after handling the `ServiceUnitNotReady` error (port from 50660 to 51081). (cherry picked from commit 8c7e1d9e31172fed3ebc32ebf0c725226a0eae8e) --- pulsar-client-cpp/lib/ClientConnection.cc | 48 +++++++++++++++++------ pulsar-client-cpp/lib/ClientConnection.h | 3 ++ pulsar-client-cpp/tests/AuthTokenTest.cc | 4 +- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 6071d0e6b0d74..95d1bf83adcb5 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -910,13 +910,16 @@ void ClientConnection::handleIncomingCommand() { if (partitionMetadataResponse.has_error()) { LOG_ERROR(cnxString_ << "Failed partition-metadata lookup req_id: " << partitionMetadataResponse.request_id() - << " error: " << partitionMetadataResponse.error()); + << " error: " << partitionMetadataResponse.error() + << " msg: " << partitionMetadataResponse.message()); + checkServerError(partitionMetadataResponse.error()); + lookupDataPromise->setFailed(getResult(partitionMetadataResponse.error())); } else { LOG_ERROR(cnxString_ << "Failed partition-metadata lookup req_id: " << partitionMetadataResponse.request_id() << " with empty response: "); + lookupDataPromise->setFailed(ResultConnectError); } - lookupDataPromise->setFailed(ResultConnectError); } else { LookupDataResultPtr lookupResultPtr = std::make_shared(); lookupResultPtr->setPartitions(partitionMetadataResponse.partitions()); @@ -994,13 +997,16 @@ void ClientConnection::handleIncomingCommand() { if (lookupTopicResponse.has_error()) { LOG_ERROR(cnxString_ << "Failed lookup req_id: " << lookupTopicResponse.request_id() - << " error: " << lookupTopicResponse.error()); + << " error: " << lookupTopicResponse.error() + << " msg: " << lookupTopicResponse.message()); + checkServerError(lookupTopicResponse.error()); + lookupDataPromise->setFailed(getResult(lookupTopicResponse.error())); } else { LOG_ERROR(cnxString_ << "Failed lookup req_id: " << lookupTopicResponse.request_id() << " with empty response: "); + lookupDataPromise->setFailed(ResultConnectError); } - lookupDataPromise->setFailed(ResultConnectError); } else { LOG_DEBUG(cnxString_ << "Received lookup response from server. req_id: " @@ -1509,15 +1515,10 @@ void ClientConnection::close(Result result) { return; } state_ = Disconnected; - boost::system::error_code err; - if (socket_) { - socket_->close(err); - if (err) { - LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); - } - } + closeSocket(); if (tlsSocket_) { + boost::system::error_code err; tlsSocket_->lowest_layer().close(err); if (err) { LOG_WARN(cnxString_ << "Failed to close TLS socket: " << err.message()); @@ -1662,4 +1663,29 @@ Future ClientConnection::newGetTopicsOfNamespace(con return promise.getFuture(); } +void ClientConnection::closeSocket() { + boost::system::error_code err; + if (socket_) { + socket_->close(err); + if (err) { + LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); + } + } +} + +void ClientConnection::checkServerError(const proto::ServerError& error) { + switch (error) { + case proto::ServerError::ServiceNotReady: + closeSocket(); + break; + case proto::ServerError::TooManyRequests: + // TODO: Implement maxNumberOfRejectedRequestPerConnection like + // https://github.com/apache/pulsar/pull/274 + closeSocket(); + break; + default: + break; + } +} + } // namespace pulsar diff --git a/pulsar-client-cpp/lib/ClientConnection.h b/pulsar-client-cpp/lib/ClientConnection.h index 7ca5f378f30e5..b615eaab2d08f 100644 --- a/pulsar-client-cpp/lib/ClientConnection.h +++ b/pulsar-client-cpp/lib/ClientConnection.h @@ -340,6 +340,9 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this Date: Tue, 5 Apr 2022 08:37:15 -0700 Subject: [PATCH 450/823] [Python] Added script to generate Wheel files for MacOS (#15024) --- pulsar-client-cpp/CMakeLists.txt | 16 +- pulsar-client-cpp/python/build-mac-wheels.sh | 255 +++++++++++++++++++ 2 files changed, 266 insertions(+), 5 deletions(-) create mode 100755 pulsar-client-cpp/python/build-mac-wheels.sh diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 2d98281a1809d..8b9085307be45 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -18,6 +18,7 @@ # cmake_minimum_required(VERSION 3.4) + project (pulsar-cpp) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") @@ -114,13 +115,13 @@ endif(NOT LOG_CATEGORY_NAME) add_definitions(-DLOG_CATEGORY_NAME=${LOG_CATEGORY_NAME} -DBUILDING_PULSAR -DBOOST_ALL_NO_LIB -DBOOST_ALLOW_DEPRECATED_HEADERS) -set(OPENSSL_ROOT_DIR /usr/lib64/) +set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/) ### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY -### After find the libs, will unset related cache, and will not affact another same call to find_package. +### After find the libs, will unset related cache, and will not affect another same call to find_package. if (APPLE) set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include) - set(OPENSSL_ROOT_DIR /usr/local/opt/openssl/ /opt/homebrew/opt/openssl) + set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl) endif () set(OPENSSL_USE_STATIC_LIBS FALSE) @@ -139,11 +140,15 @@ unset(OPENSSL_VERSION CACHE) if (LINK_STATIC) find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib) + message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}") find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf) + message(STATUS "Protobuf: ${Protobuf_LIBRARIES}") find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a) + message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}") find_library(LIB_ZSTD NAMES libzstd.a) + message(STATUS "ZStd: ${LIB_ZSTD}") find_library(LIB_SNAPPY NAMES libsnappy.a) - message(STATUS "Protobuf_LIBRARIES: ${Protobuf_LIBRARIES}") + message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") set(COMMON_LIBS ${Protobuf_LIBRARIES} ${COMMON_LIBS}) if (USE_LOG4CXX) @@ -268,7 +273,7 @@ if (BUILD_PYTHON_WRAPPER) list(GET PYTHONLIBS_VERSION_NO_LIST 1 PYTHONLIBS_VERSION_MINOR) set(BOOST_PYTHON_NAME_POSTFIX ${PYTHONLIBS_VERSION_MAJOR}${PYTHONLIBS_VERSION_MINOR}) # For python3 the lib name is boost_python3 - set(BOOST_PYTHON_NAME_LIST python36;python37;python38;python39;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX}) + set(BOOST_PYTHON_NAME_LIST python36;python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX}) else () # Regular boost_python set(BOOST_PYTHON_NAME_LIST python;python-mt;python-py27;python27-mt;python27) @@ -286,6 +291,7 @@ if (BUILD_PYTHON_WRAPPER) MESSAGE(FATAL_ERROR "Could not find Boost Python library") endif () + MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND}) find_package(Boost REQUIRED COMPONENTS ${BOOST_PYTHON_NAME_FOUND}) endif (BUILD_PYTHON_WRAPPER) diff --git a/pulsar-client-cpp/python/build-mac-wheels.sh b/pulsar-client-cpp/python/build-mac-wheels.sh new file mode 100755 index 0000000000000..68547b70f6b11 --- /dev/null +++ b/pulsar-client-cpp/python/build-mac-wheels.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +PYTHON_VERSIONS=( + '3.8 3.8.13' + '3.9 3.9.10' + '3.10 3.10.2' +) + +export MACOSX_DEPLOYMENT_TARGET=11.0 +MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*} + +ZLIB_VERSION=1.2.12 +OPENSSL_VERSION=1_1_1n +BOOST_VERSION=1.78.0 +PROTOBUF_VERSION=3.20.0 +ZSTD_VERSION=1.5.2 +SNAPPY_VERSION=1.1.3 +CURL_VERSION=7.61.0 + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd "${ROOT_DIR}/pulsar-client-cpp" + + +# Compile and cache dependencies +CACHE_DIR=~/.pulsar-mac-wheels-cache +mkdir -p $CACHE_DIR + +PREFIX=$CACHE_DIR/install + +cd $CACHE_DIR + +############################################################################### +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_VERSION_LONG=${PY[1]} + + if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then + echo "Building Python $PYTHON_VERSION_LONG" + curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz + tar xfz Python-${PYTHON_VERSION_LONG}.tgz + + PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION + pushd Python-${PYTHON_VERSION_LONG} + CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --prefix=$PY_PREFIX --enable-shared + make -j16 + make install + + curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl + $PY_PREFIX/bin/pip3 install wheel-*.whl + + touch .done + popd + else + echo "Using cached Python $PYTHON_VERSION_LONG" + fi +done + +############################################################################### +if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then + echo "Building ZLib" + curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz + tar xvfz zlib-$ZLIB_VERSION.tar.gz + pushd zlib-$ZLIB_VERSION + CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached ZLib" +fi + +############################################################################### +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}/.done ]; then + echo "Building OpenSSL" + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.tar.gz + tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz + pushd openssl-OpenSSL_${OPENSSL_VERSION} + ./Configure -fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} --prefix=$PREFIX no-shared darwin64-arm64-cc + make -j8 + make install + touch .done + popd +else + echo "Using cached OpenSSL" +fi + +############################################################################### +BOOST_VERSION_=${BOOST_VERSION//./_} +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_VERSION_LONG=${PY[1]} + + DIR=boost-src-${BOOST_VERSION}-python-${PYTHON_VERSION} + if [ ! -f $DIR/.done ]; then + echo "Building Boost for Py $PYTHON_VERSION" + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz + tar xfz boost_${BOOST_VERSION_}.tar.gz + mv boost_${BOOST_VERSION_} $DIR + + PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION + + pushd $DIR + cat < user-config.jam + using python : $PYTHON_VERSION + : python3 + : ${PY_PREFIX}/include/python${PYTHON_VERSION} + : ${PY_PREFIX}/lib + ; +EOF + ./bootstrap.sh --with-libraries=python --with-python=python3 --with-python-root=$PY_PREFIX \ + --prefix=$CACHE_DIR/boost-py-$PYTHON_VERSION + ./b2 address-model=64 cxxflags="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" link=static threading=multi \ + --user-config=./user-config.jam \ + variant=release python=${PYTHON_VERSION} \ + -j16 \ + install + touch .done + popd + else + echo "Using cached Boost for Py $PYTHON_VERSION" + fi + +done + + + +############################################################################### +if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then + echo "Building Protobuf" + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + tar xvfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + pushd protobuf-${PROTOBUF_VERSION} + CXXFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached Protobuf" +fi + +############################################################################### +if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then + echo "Building ZStd" + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz + tar xvfz zstd-${ZSTD_VERSION}.tar.gz + pushd zstd-${ZSTD_VERSION} + CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX make -j16 install + touch .done + popd +else + echo "Using cached ZStd" +fi + +############################################################################### +if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then + echo "Building Snappy" + curl -O -L https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz + tar xvfz snappy-${SNAPPY_VERSION}.tar.gz + pushd snappy-${SNAPPY_VERSION} + CXXFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached Snappy" +fi + +############################################################################### +if [ ! -f curl-${CURL_VERSION}/.done ]; then + echo "Building LibCurl" + CURL_VERSION_=${CURL_VERSION//./_} + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz + tar xfz curl-${CURL_VERSION}.tar.gz + pushd curl-${CURL_VERSION} + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --with-ssl=$PREFIX \ + --without-nghttp2 --without-libidn2 --prefix=$PREFIX + make -j16 install + touch .done + popd +else + echo "Using cached LibCurl" +fi + +############################################################################### +############################################################################### +############################################################################### +############################################################################### + +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_VERSION_LONG=${PY[1]} + echo '----------------------------------------------------------------------------' + echo '----------------------------------------------------------------------------' + echo '----------------------------------------------------------------------------' + echo "Build wheel for Python $PYTHON_VERSION" + + cd "${ROOT_DIR}/pulsar-client-cpp" + + find . -name CMakeCache.txt | xargs -r rm + find . -name CMakeFiles | xargs -r rm -rf + + PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION + PY_EXE=$PY_PREFIX/bin/python3 + + cmake . \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ + -DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET_MAJOR}.sdk \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$PREFIX \ + -DCMAKE_CXX_FLAGS=-I$PREFIX/include \ + -DCMAKE_FIND_FRAMEWORK=$PREFIX \ + -DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \ + -DBoost_LIBRARY_DIRS=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \ + -DPYTHON_INCLUDE_DIR=$PY_PREFIX/include/python$PYTHON_VERSION \ + -DPYTHON_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \ + -DLINK_STATIC=ON \ + -DBUILD_TESTS=OFF \ + -DBUILD_WIRESHARK=OFF \ + -DPROTOC_PATH=$PREFIX/bin/protoc + + make clean + make _pulsar -j16 + + cd python + $PY_EXE setup.py bdist_wheel +done \ No newline at end of file From 4a92407226dc7923dd9c1f5110a2fe38c45540a8 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Wed, 6 Apr 2022 02:26:06 +0800 Subject: [PATCH 451/823] [fix][client] Fix potentially unfinished CompletableFuture in doReconsumeLater (#14947) As the code is shown below, if the future returned by `doAcknowledge` is completed exceptionally, the ``result`` future can't complete. ```java typedMessageBuilderNew.sendAsync() .thenAccept(__ -> doAcknowledge(finalMessageId, ackType, Collections.emptyMap(), null) .thenAccept(v -> result.complete(null))) .exceptionally(ex -> { result.completeExceptionally(ex); return null; }); ``` - Use ``thenCompose`` to instead of ``thenAccept``. - [x] Make sure that the change passes the CI checks. - [x] `no-need-doc` (cherry picked from commit 64f020cf699ecaebfec359b58d4b13491cb5f232) --- .../main/java/org/apache/pulsar/client/impl/ConsumerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index d19b980a24a9d..ff2612beee47c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -624,7 +624,8 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a typedMessageBuilderNew.key(message.getKey()); } typedMessageBuilderNew.sendAsync() - .thenAccept(__ -> doAcknowledge(finalMessageId, ackType, properties, null).thenAccept(v -> result.complete(null))) + .thenCompose(__ -> doAcknowledge(finalMessageId, ackType, properties, null)) + .thenAccept(v -> result.complete(null)) .exceptionally(ex -> { result.completeExceptionally(ex); return null; From adb4958121349aecc3e4e02e9fe6404debe2b4cb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 5 Apr 2022 22:44:25 +0300 Subject: [PATCH 452/823] [ML] Fix race condition in updating lastMarkDeleteEntry field (#15031) - missed updates can lead to the subscription and consuming getting stuck (cherry picked from commit ad2f397fb9bb1d6f90a980e68f0161ed32900309) --- .../mledger/impl/ManagedCursorImpl.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index f882444a4a8c9..b3ba61810c2ff 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1732,7 +1732,7 @@ public void asyncMarkDelete(final Position position, Map propertie // Apply rate limiting to mark-delete operations if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) { isDirty = true; - lastMarkDeleteEntry = new MarkDeleteEntry(newPosition, properties, null, null); + updateLastMarkDeleteEntryToLatest(newPosition, properties); callback.markDeleteComplete(ctx); return; } @@ -1786,7 +1786,14 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { // ledger is postponed to when the counter goes to 0. PENDING_MARK_DELETED_SUBMITTED_COUNT_UPDATER.incrementAndGet(this); - lastMarkDeleteEntry = mdEntry; + LAST_MARK_DELETE_ENTRY_UPDATER.updateAndGet(this, last -> { + if (last != null && last.newPosition.compareTo(mdEntry.newPosition) > 0) { + // keep the current value since it's later then the mdEntry.newPosition + return last; + } else { + return mdEntry; + } + }); persistPositionToLedger(cursorLedger, mdEntry, new VoidCallback() { @Override @@ -2049,9 +2056,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb // Apply rate limiting to mark-delete operations if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) { isDirty = true; - PositionImpl finalNewMarkDeletePosition = newMarkDeletePosition; - LAST_MARK_DELETE_ENTRY_UPDATER.updateAndGet(this, - last -> new MarkDeleteEntry(finalNewMarkDeletePosition, last.properties, null, null)); + updateLastMarkDeleteEntryToLatest(newMarkDeletePosition, null); callback.deleteComplete(ctx); return; } @@ -2083,6 +2088,22 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { } } + // update lastMarkDeleteEntry field if newPosition is later than the current lastMarkDeleteEntry.newPosition + private void updateLastMarkDeleteEntryToLatest(final PositionImpl newPosition, + final Map properties) { + LAST_MARK_DELETE_ENTRY_UPDATER.updateAndGet(this, last -> { + if (last != null && last.newPosition.compareTo(newPosition) > 0) { + // keep current value, don't update + return last; + } else { + // use given properties or when missing, use the properties from the previous field value + Map propertiesToUse = + properties != null ? properties : (last != null ? last.properties : Collections.emptyMap()); + return new MarkDeleteEntry(newPosition, propertiesToUse, null, null); + } + }); + } + /** * Given a list of entries, filter out the entries that have already been individually deleted. * From 218719429bfe864f4c517a6e7d80ff807fdbcd4a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 6 Apr 2022 21:05:53 -0700 Subject: [PATCH 453/823] [Python] Generate universal2 wheel files for MacOS (#15054) --- pulsar-client-cpp/CMakeLists.txt | 2 +- .../lib/checksum/crc32c_sse42.cc | 11 ++-- pulsar-client-cpp/python/build-mac-wheels.sh | 61 ++++++++++++++----- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 8b9085307be45..5068356410455 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -94,7 +94,7 @@ else() # GCC or Clang are mostly compatible: add_compile_options(-Wall -Wformat-security -Wvla -Werror) # Turn off certain warnings that are too much pain for too little gain: add_compile_options(-Wno-sign-compare -Wno-deprecated-declarations -Wno-error=cpp) - if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR APPLE) add_compile_options(-msse4.2 -mpclmul) endif() # Options unique to Clang or GCC: diff --git a/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc b/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc index 1ed88e888e39c..f866480f09ea4 100644 --- a/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc +++ b/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc @@ -31,7 +31,8 @@ #include "lib/checksum/crc32c_sw.h" #include "gf2.hpp" -#if BOOST_ARCH_X86_64 +#if BOOST_ARCH_X86_64 && !defined(__arm64__) +#define PULSAR_X86_64 #include // SSE4.2 #include // PCLMUL #else @@ -44,7 +45,7 @@ #ifdef _MSC_VER #include -#elif BOOST_ARCH_X86_64 +#elif defined(PULSAR_X86_64) #include #endif @@ -79,7 +80,7 @@ bool crc32c_initialize() { __cpuid(CPUInfo, 1); has_sse42 = (CPUInfo[2] & cpuid_ecx_sse42) != 0; has_pclmulqdq = (CPUInfo[2] & cpuid_ecx_pclmulqdq) != 0; -#elif BOOST_ARCH_X86_64 +#elif defined(PULSAR_X86_64) const uint32_t cpuid_ecx_sse42 = (1 << 20); const uint32_t cpuid_ecx_pclmulqdq = (1 << 1); unsigned int eax, ebx, ecx, edx; @@ -116,7 +117,7 @@ void chunk_config::make_shift_table(size_t bytes, uint32_t table[256]) { for (unsigned int i = 0; i < 256; ++i) table[i] = (const bitvector<32>)mul(m, bitvector<32>(i)); } -#if BOOST_ARCH_X86_64 +#ifdef PULSAR_X86_64 static uint32_t crc32c_chunk(uint32_t crc, const void *buf, const chunk_config &config) { DEBUG_PRINTF3(" crc32c_chunk(crc = 0x%08x, buf = %p, config.words = " SIZE_T_FORMAT ")", crc, buf, @@ -259,7 +260,7 @@ uint32_t crc32c(uint32_t init, const void *buf, size_t len, const chunk_config * return crc; } -#else // ! BOOST_ARCH_X86_64 +#else // ! PULSAR_X86_64 uint32_t crc32c(uint32_t init, const void *buf, size_t len, const chunk_config *config) { // SSE 4.2 extension for hw implementation are not present diff --git a/pulsar-client-cpp/python/build-mac-wheels.sh b/pulsar-client-cpp/python/build-mac-wheels.sh index 68547b70f6b11..0236eb44f79c5 100755 --- a/pulsar-client-cpp/python/build-mac-wheels.sh +++ b/pulsar-client-cpp/python/build-mac-wheels.sh @@ -20,6 +20,11 @@ set -e +ARCHS=( + 'x86_64' + 'arm64' +) + PYTHON_VERSIONS=( '3.8 3.8.13' '3.9 3.9.10' @@ -45,10 +50,10 @@ cd "${ROOT_DIR}/pulsar-client-cpp" CACHE_DIR=~/.pulsar-mac-wheels-cache mkdir -p $CACHE_DIR -PREFIX=$CACHE_DIR/install - cd $CACHE_DIR +PREFIX=$CACHE_DIR/install + ############################################################################### for line in "${PYTHON_VERSIONS[@]}"; do read -r -a PY <<< "$line" @@ -63,7 +68,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION pushd Python-${PYTHON_VERSION_LONG} CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --prefix=$PY_PREFIX --enable-shared + ./configure --prefix=$PY_PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 make -j16 make install @@ -83,7 +88,7 @@ if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz tar xvfz zlib-$ZLIB_VERSION.tar.gz pushd zlib-$ZLIB_VERSION - CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX make -j16 make install touch .done @@ -93,16 +98,35 @@ else fi ############################################################################### -if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}/.done ]; then +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}.done ]; then echo "Building OpenSSL" curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.tar.gz + # -arch arm64 -arch x86_64 tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz - pushd openssl-OpenSSL_${OPENSSL_VERSION} - ./Configure -fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} --prefix=$PREFIX no-shared darwin64-arm64-cc + mv openssl-OpenSSL_${OPENSSL_VERSION} openssl-OpenSSL_${OPENSSL_VERSION}-arm64 + pushd openssl-OpenSSL_${OPENSSL_VERSION}-arm64 + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared darwin64-arm64-cc make -j8 make install - touch .done popd + + tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz + mv openssl-OpenSSL_${OPENSSL_VERSION} openssl-OpenSSL_${OPENSSL_VERSION}-x86_64 + pushd openssl-OpenSSL_${OPENSSL_VERSION}-x86_64 + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared darwin64-x86_64-cc + make -j8 + make install + popd + + # Create universal binaries + lipo -create openssl-OpenSSL_${OPENSSL_VERSION}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION}-x86_64/libssl.a \ + -output $PREFIX/lib/libssl.a + lipo -create openssl-OpenSSL_${OPENSSL_VERSION}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION}-x86_64/libcrypto.a \ + -output $PREFIX/lib/libcrypto.a + + touch openssl-OpenSSL_${OPENSSL_VERSION}.done else echo "Using cached OpenSSL" fi @@ -133,7 +157,8 @@ for line in "${PYTHON_VERSIONS[@]}"; do EOF ./bootstrap.sh --with-libraries=python --with-python=python3 --with-python-root=$PY_PREFIX \ --prefix=$CACHE_DIR/boost-py-$PYTHON_VERSION - ./b2 address-model=64 cxxflags="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" link=static threading=multi \ + ./b2 address-model=64 cxxflags="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + link=static threading=multi \ --user-config=./user-config.jam \ variant=release python=${PYTHON_VERSION} \ -j16 \ @@ -154,7 +179,8 @@ if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz tar xvfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz pushd protobuf-${PROTOBUF_VERSION} - CXXFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + CXXFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --prefix=$PREFIX make -j16 make install touch .done @@ -169,7 +195,8 @@ if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz tar xvfz zstd-${ZSTD_VERSION}.tar.gz pushd zstd-${ZSTD_VERSION} - CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX make -j16 install + CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ + make -j16 install touch .done popd else @@ -182,7 +209,8 @@ if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then curl -O -L https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz tar xvfz snappy-${SNAPPY_VERSION}.tar.gz pushd snappy-${SNAPPY_VERSION} - CXXFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --prefix=$PREFIX make -j16 make install touch .done @@ -198,8 +226,10 @@ if [ ! -f curl-${CURL_VERSION}/.done ]; then curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz tar xfz curl-${CURL_VERSION}.tar.gz pushd curl-${CURL_VERSION} - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --with-ssl=$PREFIX \ - --without-nghttp2 --without-libidn2 --prefix=$PREFIX + CFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --with-ssl=$PREFIX \ + --without-nghttp2 --without-libidn2 --disable-ldap \ + --prefix=$PREFIX make -j16 install touch .done popd @@ -230,6 +260,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do PY_EXE=$PY_PREFIX/bin/python3 cmake . \ + -DCMAKE_OSX_ARCHITECTURES='arm64;x86_64' \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ -DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET_MAJOR}.sdk \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ @@ -252,4 +283,4 @@ for line in "${PYTHON_VERSIONS[@]}"; do cd python $PY_EXE setup.py bdist_wheel -done \ No newline at end of file +done From 746cfb5b78959546b111e1a7f148e07cfb3a2baf Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 7 Apr 2022 14:55:16 +0800 Subject: [PATCH 454/823] [Client] Add test to ensure the message order in listener callbacks (#15049) (cherry picked from commit c2c05c49aff1ebc7b2b7a1d5bd547c33211e4479) --- .../api/SimpleProducerConsumerTest.java | 30 +++++++++++++++++++ .../pulsar/client/impl/ConsumerBase.java | 9 ++++-- .../pulsar/client/impl/ConsumerImpl.java | 6 ---- .../client/impl/MultiTopicsConsumerImpl.java | 4 +-- .../client/impl/ZeroQueueConsumerImpl.java | 2 +- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 2815ce36bf0e4..f5677d362737c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -60,6 +60,7 @@ import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; @@ -4407,4 +4408,33 @@ public void testPartitionsAutoUpdate() throws Exception { log.info("-- Exiting {} test --", methodName); } + + @Test(invocationCount = 5) + public void testListenerOrdering() throws Exception { + final String topic = "persistent://my-property/my-ns/test-listener-ordering-" + System.currentTimeMillis(); + final int numMessages = 1000; + final CountDownLatch latch = new CountDownLatch(numMessages); + final List values = new CopyOnWriteArrayList<>(); + final Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .messageListener((MessageListener) (consumer1, msg) -> { + values.add(msg.getValue()); + latch.countDown(); + }) + .subscribe(); + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + for (int i = 0; i < numMessages; i++) { + producer.send("msg-" + i); + } + latch.await(3, TimeUnit.SECONDS); + producer.close(); + consumer.close(); + assertEquals(values.size(), numMessages); + for (int i = 0; i < numMessages; i++) { + assertEquals(values.get(i), "msg-" + i); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 20baf47d9def8..b50a570009398 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -913,8 +913,13 @@ private void doPendingBatchReceiveTask(Timeout timeout) { } } - protected void triggerListener() { - // Use internalPinnedExecutor to maintain message ordering + protected void tryTriggerListener() { + if (listener != null) { + triggerListener(); + } + } + + private void triggerListener() { internalPinnedExecutor.execute(() -> { try { // Listener should only have one pending/running executable to process a message diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index ff2612beee47c..21a3151d04aff 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1236,12 +1236,6 @@ void messageReceived(MessageIdData messageId, int redeliveryCount, List ac } - private void tryTriggerListener() { - if (listener != null) { - triggerListener(); - } - } - private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, MessageIdData messageId, ClientCnx cnx) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 74898a2a62a3e..1d479976950ce 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -301,9 +301,7 @@ private void messageReceived(ConsumerImpl consumer, Message message) { notifyPendingBatchReceivedCallBack(); } - if (listener != null) { - triggerListener(); - } + tryTriggerListener(); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java index 42361b67caf47..5324c33d01127 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java @@ -174,7 +174,7 @@ private void triggerZeroQueueSizeListener(final Message message) { } @Override - protected void triggerListener() { + protected void tryTriggerListener() { // Ignore since it was already triggered in the triggerZeroQueueSizeListener() call } From 20f6f4bce306d3a73bb75180127f1b6a3c737378 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 8 Apr 2022 12:35:27 +0800 Subject: [PATCH 455/823] [C++] Fix single message metadata not set correctly (#15072) ### Motivation Recently I found the messages sent by C++ producer don't have the schema version, which causes Java consumer cannot consume them with `AUTO_CONSUME` schema. After rechecking the code, I found the C++ client doesn't set single message metadata correctly, i.e. when batching is enabled, some messages' metadata could be wrong. - In `initBatchMessageMetadata`, the schema version is not set. - In `serializeSingleMessageInBatchWithPayload`, the ordering key and the sequence id are not set. In addition, when a C++ consumer consumes batched messages from a Java producer, some metadata might be wrong. Because even for batched messages, Java producer also sets the partition key and the ordering key. It's redundant because only keys in `SingleMessageMetadata` should be set. To avoid 2nd and later single messages in the batch reuse the keys in `MessageMetadata`, Java client clears these fields if the `SingleMessageMetadata` doesn't contain them when a `MessageImpl` is constructed. ### Modifications - Set the fields that were not set before when creating a batch. Some fields like `null_value` and `null_partition_key` are not set because they are not supported by C++ client at this moment. - Use a more efficient way to copy the repeated fields of ProtoBuf. - Clear some fields when they are not contained by the `SingleMessageMetadata` object when creating a `MessageImpl` so that the bahavior could be consisitent with Java client. ### Verifying this change Following tests are added: - `BatchMessageTest.testSingleMessageMetadata`: test 3 single messages in batch are consumed successfully, i.e. the correct metadata is received. - `SchemaTest.testHasSchemaVersion`: test when schema is configured, all messages should has the schema version. - The validation for schema version is also added to `ProtobufNativeSchemaTest.testEndToEnd`. (cherry picked from commit 6f41fdebf642b08e2c3f45f3574963ec0936868c) --- pulsar-client-cpp/lib/Commands.cc | 31 +++++--- pulsar-client-cpp/lib/Message.cc | 26 +++++++ pulsar-client-cpp/tests/BatchMessageTest.cc | 78 +++++++++++++++++++ .../tests/ProtobufNativeSchemaTest.cc | 3 + pulsar-client-cpp/tests/SchemaTest.cc | 34 ++++++++ 5 files changed, 161 insertions(+), 11 deletions(-) diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc index 1094efb8eee35..33db6e42a13d2 100644 --- a/pulsar-client-cpp/lib/Commands.cc +++ b/pulsar-client-cpp/lib/Commands.cc @@ -687,26 +687,35 @@ void Commands::initBatchMessageMetadata(const Message& msg, pulsar::proto::Messa batchMetadata.add_replicate_to(metadata.replicate_to(i)); } } - // TODO: set other optional fields + if (metadata.has_schema_version()) { + batchMetadata.set_schema_version(metadata.schema_version()); + } } uint64_t Commands::serializeSingleMessageInBatchWithPayload(const Message& msg, SharedBuffer& batchPayLoad, unsigned long maxMessageSizeInBytes) { + const auto& msgMetadata = msg.impl_->metadata; SingleMessageMetadata metadata; - if (msg.impl_->hasPartitionKey()) { - metadata.set_partition_key(msg.impl_->getPartitionKey()); + if (msgMetadata.has_partition_key()) { + metadata.set_partition_key(msgMetadata.partition_key()); + } + if (msgMetadata.has_ordering_key()) { + metadata.set_ordering_key(msgMetadata.ordering_key()); } - for (MessageBuilder::StringMap::const_iterator it = msg.impl_->properties().begin(); - it != msg.impl_->properties().end(); it++) { - proto::KeyValue* keyValue = proto::KeyValue().New(); - keyValue->set_key(it->first); - keyValue->set_value(it->second); + metadata.mutable_properties()->Reserve(msgMetadata.properties_size()); + for (int i = 0; i < msgMetadata.properties_size(); i++) { + auto keyValue = proto::KeyValue().New(); + *keyValue = msgMetadata.properties(i); metadata.mutable_properties()->AddAllocated(keyValue); } - if (msg.impl_->getEventTimestamp() != 0) { - metadata.set_event_time(msg.impl_->getEventTimestamp()); + if (msgMetadata.has_event_time()) { + metadata.set_event_time(msgMetadata.event_time()); + } + + if (msgMetadata.has_sequence_id()) { + metadata.set_sequence_id(msgMetadata.sequence_id()); } // Format of batch message @@ -736,7 +745,7 @@ uint64_t Commands::serializeSingleMessageInBatchWithPayload(const Message& msg, batchPayLoad.bytesWritten(msgMetadataSize); batchPayLoad.write(msg.impl_->payload.data(), payloadSize); - return msg.impl_->metadata.sequence_id(); + return msgMetadata.sequence_id(); } Message Commands::deSerializeSingleMessageInBatch(Message& batchedMessage, int32_t batchIndex) { diff --git a/pulsar-client-cpp/lib/Message.cc b/pulsar-client-cpp/lib/Message.cc index 76e408ffef428..b928945cfae21 100644 --- a/pulsar-client-cpp/lib/Message.cc +++ b/pulsar-client-cpp/lib/Message.cc @@ -79,12 +79,38 @@ Message::Message(const MessageId& messageID, proto::MessageMetadata& metadata, S impl_->metadata.mutable_properties()->CopyFrom(singleMetadata.properties()); impl_->topicName_ = &topicName; + impl_->metadata.clear_properties(); + if (singleMetadata.properties_size() > 0) { + impl_->metadata.mutable_properties()->Reserve(singleMetadata.properties_size()); + for (int i = 0; i < singleMetadata.properties_size(); i++) { + auto keyValue = proto::KeyValue().New(); + *keyValue = singleMetadata.properties(i); + impl_->metadata.mutable_properties()->AddAllocated(keyValue); + } + } + if (singleMetadata.has_partition_key()) { impl_->metadata.set_partition_key(singleMetadata.partition_key()); + } else { + impl_->metadata.clear_partition_key(); + } + + if (singleMetadata.has_ordering_key()) { + impl_->metadata.set_ordering_key(singleMetadata.ordering_key()); + } else { + impl_->metadata.clear_ordering_key(); } if (singleMetadata.has_event_time()) { impl_->metadata.set_event_time(singleMetadata.event_time()); + } else { + impl_->metadata.clear_event_time(); + } + + if (singleMetadata.has_sequence_id()) { + impl_->metadata.set_sequence_id(singleMetadata.sequence_id()); + } else { + impl_->metadata.clear_sequence_id(); } } diff --git a/pulsar-client-cpp/tests/BatchMessageTest.cc b/pulsar-client-cpp/tests/BatchMessageTest.cc index ebe6bf159ba63..62fd5fff25c2d 100644 --- a/pulsar-client-cpp/tests/BatchMessageTest.cc +++ b/pulsar-client-cpp/tests/BatchMessageTest.cc @@ -1071,3 +1071,81 @@ TEST(BatchMessageTest, testProducerQueueWithBatches) { ASSERT_EQ(rejectedMessges, 10); } + +TEST(BatchMessageTest, testSingleMessageMetadata) { + const auto topic = "BatchMessageTest-SingleMessageMetadata-" + std::to_string(time(nullptr)); + constexpr int numMessages = 3; + + Client client(lookupUrl); + + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe(topic, "sub", consumer)); + + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer( + topic, ProducerConfiguration().setBatchingMaxMessages(numMessages), producer)); + + producer.sendAsync(MessageBuilder() + .setContent("msg-0") + .setPartitionKey("key-0") + .setOrderingKey("ordering-key-0") + .setEventTimestamp(10UL) + .setProperty("k0", "v0") + .setProperty("k1", "v1") + .build(), + nullptr); + producer.sendAsync(MessageBuilder() + .setContent("msg-1") + .setOrderingKey("ordering-key-1") + .setEventTimestamp(11UL) + .setProperty("k2", "v2") + .build(), + nullptr); + producer.sendAsync(MessageBuilder().setContent("msg-2").build(), nullptr); + ASSERT_EQ(ResultOk, producer.flush()); + + Message msgs[numMessages]; + for (int i = 0; i < numMessages; i++) { + Message msg; + ASSERT_EQ(ResultOk, consumer.receive(msg, 3000)); + msgs[i] = msg; + LOG_INFO("message " << i << ": " << msg.getDataAsString() + << ", key: " << (msg.hasPartitionKey() ? msg.getPartitionKey() : "(null)") + << ", ordering key: " << (msg.hasOrderingKey() ? msg.getOrderingKey() : "(null)") + << ", event time: " << (msg.getEventTimestamp()) + << ", properties count: " << msg.getProperties().size() + << ", has schema version: " << msg.hasSchemaVersion()); + } + + ASSERT_EQ(msgs[0].getDataAsString(), "msg-0"); + ASSERT_TRUE(msgs[0].hasPartitionKey()); + ASSERT_EQ(msgs[0].getPartitionKey(), "key-0"); + ASSERT_TRUE(msgs[0].hasOrderingKey()); + ASSERT_EQ(msgs[0].getOrderingKey(), "ordering-key-0"); + ASSERT_EQ(msgs[0].getEventTimestamp(), 10UL); + ASSERT_EQ(msgs[0].getProperties().size(), 2); + ASSERT_TRUE(msgs[0].hasProperty("k0")); + ASSERT_EQ(msgs[0].getProperty("k0"), "v0"); + ASSERT_TRUE(msgs[0].hasProperty("k1")); + ASSERT_EQ(msgs[0].getProperty("k1"), "v1"); + ASSERT_FALSE(msgs[0].hasSchemaVersion()); + + ASSERT_EQ(msgs[1].getDataAsString(), "msg-1"); + ASSERT_FALSE(msgs[1].hasPartitionKey()); + ASSERT_TRUE(msgs[1].hasOrderingKey()); + ASSERT_EQ(msgs[1].getOrderingKey(), "ordering-key-1"); + ASSERT_EQ(msgs[1].getEventTimestamp(), 11UL); + ASSERT_EQ(msgs[1].getProperties().size(), 1); + ASSERT_TRUE(msgs[1].hasProperty("k2")); + ASSERT_EQ(msgs[1].getProperty("k2"), "v2"); + ASSERT_FALSE(msgs[1].hasSchemaVersion()); + + ASSERT_EQ(msgs[2].getDataAsString(), "msg-2"); + ASSERT_FALSE(msgs[2].hasPartitionKey()); + ASSERT_FALSE(msgs[2].hasOrderingKey()); + ASSERT_EQ(msgs[2].getEventTimestamp(), 0UL); + ASSERT_EQ(msgs[2].getProperties().size(), 0); + ASSERT_FALSE(msgs[2].hasSchemaVersion()); + + client.close(); +} diff --git a/pulsar-client-cpp/tests/ProtobufNativeSchemaTest.cc b/pulsar-client-cpp/tests/ProtobufNativeSchemaTest.cc index ab4fedb79e141..df1f9c6055782 100644 --- a/pulsar-client-cpp/tests/ProtobufNativeSchemaTest.cc +++ b/pulsar-client-cpp/tests/ProtobufNativeSchemaTest.cc @@ -121,6 +121,9 @@ TEST(ProtobufNativeSchemaTest, testEndToEnd) { receivedTestMessage.ParseFromArray(msg.getData(), msg.getLength()); ASSERT_EQ(receivedTestMessage.testenum(), ::proto::TestEnum::FAILOVER); + ASSERT_TRUE(msg.hasSchemaVersion()); + ASSERT_EQ(msg.getSchemaVersion(), std::string(8L, '\0')); + client.close(); } diff --git a/pulsar-client-cpp/tests/SchemaTest.cc b/pulsar-client-cpp/tests/SchemaTest.cc index 1776460f895a2..f15365219f748 100644 --- a/pulsar-client-cpp/tests/SchemaTest.cc +++ b/pulsar-client-cpp/tests/SchemaTest.cc @@ -73,3 +73,37 @@ TEST(SchemaTest, testSchema) { client.close(); } + +TEST(SchemaTest, testHasSchemaVersion) { + Client client(lookupUrl); + std::string topic = "SchemaTest-HasSchemaVersion"; + SchemaInfo stringSchema(SchemaType::STRING, "String", ""); + + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe(topic + "1", "sub", ConsumerConfiguration().setSchema(stringSchema), + consumer)); + Producer batchedProducer; + ASSERT_EQ(ResultOk, client.createProducer(topic + "1", ProducerConfiguration().setSchema(stringSchema), + batchedProducer)); + Producer nonBatchedProducer; + ASSERT_EQ(ResultOk, client.createProducer(topic + "1", ProducerConfiguration().setSchema(stringSchema), + nonBatchedProducer)); + + ASSERT_EQ(ResultOk, batchedProducer.send(MessageBuilder().setContent("msg-0").build())); + ASSERT_EQ(ResultOk, nonBatchedProducer.send(MessageBuilder().setContent("msg-1").build())); + + Message msgs[2]; + ASSERT_EQ(ResultOk, consumer.receive(msgs[0], 3000)); + ASSERT_EQ(ResultOk, consumer.receive(msgs[1], 3000)); + + std::string schemaVersion(8, '\0'); + ASSERT_EQ(msgs[0].getDataAsString(), "msg-0"); + ASSERT_TRUE(msgs[0].hasSchemaVersion()); + ASSERT_EQ(msgs[0].getSchemaVersion(), schemaVersion); + + ASSERT_EQ(msgs[1].getDataAsString(), "msg-1"); + ASSERT_TRUE(msgs[1].hasSchemaVersion()); + ASSERT_EQ(msgs[1].getSchemaVersion(), schemaVersion); + + client.close(); +} From 22121152f92a50cbba962a915d276958d260760c Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 8 Apr 2022 18:08:42 +0800 Subject: [PATCH 456/823] [fix][txn]: fix transaction producer stuck problem (#15061) ### Motivation when transaction buffer recover fail, we don't remove the producer future from the producer map. If producer reconnect to this broker, it will throw ``` pulsar-io-5-1] WARN org.apache.pulsar.broker.service.ServerCnx - [/10.124.4.69:49108][persistent://public/default/s_topic-partition-5] Producer with id is already present on the connection, producerId=45",2022-04-02T06:34:17.042900012Z ``` and if client don't restart, we can't recover this producer the producer has been stucked ### Modifications when topic transaction buffer recvoer fail, remove the producer from the map. in master branch, https://github.com/apache/pulsar/pull/14467 has fixed this problem --- .../apache/pulsar/broker/PulsarService.java | 2 + .../pulsar/broker/service/BrokerService.java | 1 + .../pulsar/broker/service/ServerCnx.java | 1 + .../TopicTransactionBufferRecoverTest.java | 48 +++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index eb239ddf23f70..ac4759d79484d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -273,6 +273,7 @@ public enum State { // key is listener name , value is pulsar address and pulsar ssl address private Map advertisedListeners; private NamespaceName heartbeatNamespaceV2; + private NamespaceName heartbeatNamespaceV1; public PulsarService(ServiceConfiguration config) { this(config, Optional.empty(), (exitCode) -> { @@ -684,6 +685,7 @@ config, localMetadataStore, getZkClient(), this.addWebServerHandlers(webService, metricsServlet, this.config); this.webService.start(); + heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(this.advertisedAddress, this.config); heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(this.advertisedAddress, this.config); // Refresh addresses and update configuration, since the port might have been dynamically assigned diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 939a389c1eae2..9f14da56653ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2679,6 +2679,7 @@ public boolean isSystemTopic(String topic) { public boolean isSystemTopic(TopicName topicName) { if (topicName.getNamespaceObject().equals(NamespaceName.SYSTEM_NAMESPACE) + || topicName.getNamespaceObject().equals(pulsar.getHeartbeatNamespaceV1()) || topicName.getNamespaceObject().equals(pulsar.getHeartbeatNamespaceV2())) { return true; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 07d898f488c81..fd287329a7e71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1256,6 +1256,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { Throwable cause = exception.getCause(); log.error("producerId {}, requestId {} : TransactionBuffer recover failed", producerId, requestId, exception); + producers.remove(producerId, producerFuture); commandSender.sendErrorResponse(requestId, ServiceUnitNotReadyException.getClientErrorCode(cause), cause.getMessage()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index c349173898550..fe724dd2be7ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -46,6 +46,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TransactionBufferSnapshotService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -73,6 +74,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; +import org.powermock.reflect.Whitebox; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -569,4 +571,50 @@ public void testTransactionBufferNoSnapshotCloseReader() throws Exception{ assertTrue(stats.getSubscriptions().keySet().contains("__compaction")); } + @Test + public void transactionBufferRecoverFailRemoveProducerFuture() throws Exception { + String topic = NAMESPACE1 + "/transactionBufferRecoverFailRemoveProducerFuture"; + + @Cleanup + Producer producer = pulsarClient + .newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + + // txn buffer init success + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + producer.newMessage(txn).value("test".getBytes()).sendAsync(); + producer.newMessage(txn).value("test".getBytes()).sendAsync(); + txn.commit().get(); + + PersistentTopic originalTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); + TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) originalTopic.getTransactionBuffer(); + + CompletableFuture bufferFuture = new CompletableFuture<>(); + bufferFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException("test")); + + // set fail future to topic transaction buffer + Whitebox.setInternalState(topicTransactionBuffer, "transactionBufferFuture", bufferFuture); + + originalTopic.getProducers().get(originalTopic.getProducers().keySet().toArray()[0]).disconnect().get(); + // client producer has been close and reconnect + Awaitility.await().untilAsserted(() -> assertFalse(producer.isConnected())); + // producer is dis connect + Awaitility.await().until(() -> !producer.isConnected()); + // client producer can't reconnect to broker + Awaitility.await().during(5, TimeUnit.SECONDS).until(() -> !producer.isConnected()); + + // recover the buffer future + bufferFuture = new CompletableFuture<>(); + bufferFuture.complete(null); + Whitebox.setInternalState(topicTransactionBuffer, "transactionBufferFuture", bufferFuture); + + // client producer can't connect to broker + Awaitility.await().until(producer::isConnected); + } } From 1f9dcbf904077bb298096339b851da557113648a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 8 Apr 2022 01:27:12 -0500 Subject: [PATCH 457/823] Ignore case when obfuscating passwords in python configuration scripts (#15077) (cherry picked from commit 8a67a383fb5e53f947d0513fc12656dd37961690) --- docker/pulsar/scripts/apply-config-from-env-with-prefix.py | 4 ++-- docker/pulsar/scripts/apply-config-from-env.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py index a3a4adafcabfb..3f6bc2e4d3b85 100755 --- a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py +++ b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py @@ -60,7 +60,7 @@ v = os.environ[k].strip() # Hide the value in logs if is password. - if "password" in k: + if "password" in k.lower(): displayValue = "********" else: displayValue = v @@ -80,7 +80,7 @@ continue # Hide the value in logs if is password. - if "password" in k: + if "password" in k.lower(): displayValue = "********" else: displayValue = v diff --git a/docker/pulsar/scripts/apply-config-from-env.py b/docker/pulsar/scripts/apply-config-from-env.py index 6e2eb746cd4e9..b9757d77f5ef5 100755 --- a/docker/pulsar/scripts/apply-config-from-env.py +++ b/docker/pulsar/scripts/apply-config-from-env.py @@ -62,7 +62,7 @@ v = os.environ[k].strip() # Hide the value in logs if is password. - if "password" in k: + if "password" in k.lower(): displayValue = "********" else: displayValue = v @@ -82,7 +82,7 @@ continue # Hide the value in logs if is password. - if "password" in k: + if "password" in k.lower(): displayValue = "********" else: displayValue = v From 6c2cc9bd3b841bd970ebf54b820469019e99997a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 14 Apr 2022 13:04:25 +0300 Subject: [PATCH 458/823] [ML] Follow up on race condition fixes in ManagedCursorImpl #15031 (#15067) - follow up on #15031 * [ML] Fix race in persisting mark delete position * [ML] Resetting should reset lastMarkDeleteEntry * [ML] Reset fields in initializeCursorPosition method (cherry picked from commit a19a30a5e89bfd2c2da5460db6120c8e0a48d8f7) --- .../mledger/impl/ManagedCursorImpl.java | 100 +++++++++++++----- 1 file changed, 75 insertions(+), 25 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index b3ba61810c2ff..adb29bb37bb96 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -118,6 +118,11 @@ public class ManagedCursorImpl implements ManagedCursor { // this position is have persistent mark delete position protected volatile PositionImpl persistentMarkDeletePosition; + protected static final AtomicReferenceFieldUpdater + INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, + "inProgressMarkDeletePersistPosition"); + protected volatile PositionImpl inProgressMarkDeletePersistPosition; protected static final AtomicReferenceFieldUpdater READ_POSITION_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, "readPosition"); @@ -214,6 +219,31 @@ public MarkDeleteEntry(PositionImpl newPosition, Map properties, this.callback = callback; this.ctx = ctx; } + + public void triggerComplete() { + // Trigger the final callback after having (eventually) triggered the switchin-ledger operation. This + // will ensure that no race condition will happen between the next mark-delete and the switching + // operation. + if (callbackGroup != null) { + // Trigger the callback for every request in the group + for (MarkDeleteEntry e : callbackGroup) { + e.callback.markDeleteComplete(e.ctx); + } + } else if (callback != null) { + // Only trigger the callback for the current request + callback.markDeleteComplete(ctx); + } + } + + public void triggerFailed(ManagedLedgerException exception) { + if (callbackGroup != null) { + for (MarkDeleteEntry e : callbackGroup) { + e.callback.markDeleteFailed(exception, e.ctx); + } + } else if (callback != null) { + callback.markDeleteFailed(exception, ctx); + } + } } protected final ArrayDeque pendingMarkDeleteOps = new ArrayDeque<>(); @@ -540,6 +570,7 @@ private void recoveredCursor(PositionImpl position, Map properties messagesConsumedCounter = -getNumberOfEntries(Range.openClosed(position, ledger.getLastPosition())); markDeletePosition = position; persistentMarkDeletePosition = position; + inProgressMarkDeletePersistPosition = null; readPosition = ledger.getNextValidPosition(position); lastMarkDeleteEntry = new MarkDeleteEntry(markDeletePosition, properties, null, null); // assign cursor-ledger so, it can be deleted when new ledger will be switched @@ -1121,7 +1152,11 @@ public void operationFailed(ManagedLedgerException exception) { }; - internalAsyncMarkDelete(newPosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), new MarkDeleteCallback() { + persistentMarkDeletePosition = null; + inProgressMarkDeletePersistPosition = null; + lastMarkDeleteEntry = new MarkDeleteEntry(newPosition, getProperties(), null, null); + internalAsyncMarkDelete(newPosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), + new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { finalCallback.operationComplete(); @@ -1567,6 +1602,9 @@ boolean hasMoreEntries(PositionImpl position) { void initializeCursorPosition(Pair lastPositionCounter) { readPosition = ledger.getNextValidPosition(lastPositionCounter.getLeft()); markDeletePosition = lastPositionCounter.getLeft(); + lastMarkDeleteEntry = new MarkDeleteEntry(markDeletePosition, getProperties(), null, null); + persistentMarkDeletePosition = null; + inProgressMarkDeletePersistPosition = null; // Initialize the counter such that the difference between the messages written on the ML and the // messagesConsumed is 0, to ensure the initial backlog count is 0. @@ -1781,6 +1819,34 @@ protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map { + if (current != null && current.compareTo(mdEntry.newPosition) > 0) { + return current; + } else { + return mdEntry.newPosition; + } + }); + + // if there's a newer or equal mark delete update in progress, skip it. + if (inProgressLatest != mdEntry.newPosition) { + if (log.isInfoEnabled()) { + log.info("Skipping updating mark delete position to {}. The mark delete position update " + + "in progress {} is later.", mdEntry.newPosition, inProgressLatest); + } + mdEntry.triggerComplete(); + return; + } + // The counter is used to mark all the pending mark-delete request that were submitted to BK and that are not // yet finished. While we have outstanding requests we cannot close the current ledger, so the switch to new // ledger is postponed to when the counter goes to 0. @@ -1803,6 +1869,9 @@ public void operationComplete() { mdEntry.newPosition); } + INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.compareAndSet(ManagedCursorImpl.this, + mdEntry.newPosition, null); + // Remove from the individual deleted messages all the entries before the new mark delete // point. lock.writeLock().lock(); @@ -1814,11 +1883,7 @@ public void operationComplete() { subMap.values().forEach(BitSetRecyclable::recycle); subMap.clear(); } - if (persistentMarkDeletePosition == null - || mdEntry.newPosition.compareTo(persistentMarkDeletePosition) > 0) { - persistentMarkDeletePosition = mdEntry.newPosition; - } - + persistentMarkDeletePosition = mdEntry.newPosition; } finally { lock.writeLock().unlock(); } @@ -1827,22 +1892,13 @@ public void operationComplete() { decrementPendingMarkDeleteCount(); - // Trigger the final callback after having (eventually) triggered the switchin-ledger operation. This - // will ensure that no race condition will happen between the next mark-delete and the switching - // operation. - if (mdEntry.callbackGroup != null) { - // Trigger the callback for every request in the group - for (MarkDeleteEntry e : mdEntry.callbackGroup) { - e.callback.markDeleteComplete(e.ctx); - } - } else { - // Only trigger the callback for the current request - mdEntry.callback.markDeleteComplete(mdEntry.ctx); - } + mdEntry.triggerComplete(); } @Override public void operationFailed(ManagedLedgerException exception) { + INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.compareAndSet(ManagedCursorImpl.this, + mdEntry.newPosition, null); isDirty = true; log.warn("[{}] Failed to mark delete position for cursor={} position={}", ledger.getName(), ManagedCursorImpl.this, mdEntry.newPosition); @@ -1853,13 +1909,7 @@ public void operationFailed(ManagedLedgerException exception) { decrementPendingMarkDeleteCount(); - if (mdEntry.callbackGroup != null) { - for (MarkDeleteEntry e : mdEntry.callbackGroup) { - e.callback.markDeleteFailed(exception, e.ctx); - } - } else { - mdEntry.callback.markDeleteFailed(exception, mdEntry.ctx); - } + mdEntry.triggerFailed(exception); } }); } From 72543fd19a6a8e1764b8ec6cb06f2509428b79a3 Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Sat, 16 Apr 2022 01:39:13 +0800 Subject: [PATCH 459/823] [CI] Change python cp35-cp35m lib build version from manylinux2014 to manylinux1 (#15180) --- pulsar-client-cpp/docker/python-versions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-cpp/docker/python-versions.sh b/pulsar-client-cpp/docker/python-versions.sh index a264ee179c0f2..246eae914a372 100644 --- a/pulsar-client-cpp/docker/python-versions.sh +++ b/pulsar-client-cpp/docker/python-versions.sh @@ -20,7 +20,7 @@ PYTHON_VERSIONS=( '2.7 cp27-cp27mu manylinux1 x86_64' '2.7 cp27-cp27m manylinux1 x86_64' - '3.5 cp35-cp35m manylinux2014 x86_64' + '3.5 cp35-cp35m manylinux1 x86_64' '3.6 cp36-cp36m manylinux2014 x86_64' '3.7 cp37-cp37m manylinux2014 x86_64' '3.8 cp38-cp38 manylinux2014 x86_64' From 3782beb8804e162f8bf6ab7d939fd270a4978db7 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 3 Nov 2021 21:02:36 +0800 Subject: [PATCH 460/823] Cleanup already deleted namespace topics. (#12597) Cherry pick from #7473. #7473 has fix the `Cleanup already deleted namespace topics` issue, but with #8129 involved, changes have been changed back. ### Motivation We are having frequent issues when user removes cluster from the global namespace where broker from removed-cluster fails to unload topic and namespace bundle still loaded with the broker. It happens when broker from removed-cluster receives below error ``` 17:38:52.199 [pulsar-io-22-28] ERROR org.apache.pulsar.broker.service.persistent.PersistentReplicator - [persistent://prop/global/ns/tp1][east -> west] Failed to close dispatch rate limiter: org.apache.pulsar.client.api.PulsarClientException: Producer was not registered on the connection : 17:38:52.199 [pulsar-io-22-28] WARN org.apache.pulsar.broker.service.AbstractReplicator - [persistent://prop/global/ns/tp1][east -> west]] Exception: 'org.apache.pulsar.client.api.PulsarClientException: Producer was not registered on the connection' occured while trying to close the producer. retrying again in 0.1 s : 17:38:52.351 [pulsar-io-22-37] ERROR org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://prop/global/ns/tp1] Error closing topic java.util.concurrent.CompletionException: org.apache.pulsar.client.api.PulsarClientException: Producer was not registered on the connection at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?] ``` ### Modification Source Broker should return explicit error-code when producer is already closed and dest-broker from removed-cluster should handle this error and clean up the replicator and topic gracefully. (cherry picked from commit 6b3fb4193857c324c748ea53ae5e6028137b2e35) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index fd287329a7e71..87eea84af8c59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1602,9 +1602,8 @@ protected void handleCloseProducer(CommandCloseProducer closeProducer) { CompletableFuture producerFuture = producers.get(producerId); if (producerFuture == null) { - log.warn("[{}] Producer was not registered on the connection. producerId={}", remoteAddress, producerId); - commandSender.sendErrorResponse(requestId, ServerError.UnknownError, - "Producer was not registered on the connection"); + log.info("[{}] Producer {} was not registered on the connection", remoteAddress, producerId); + ctx.writeAndFlush(Commands.newSuccess(requestId)); return; } @@ -1649,8 +1648,8 @@ protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) { CompletableFuture consumerFuture = consumers.get(consumerId); if (consumerFuture == null) { - log.warn("[{}] Consumer was not registered on the connection: consumerId={}", remoteAddress, consumerId); - commandSender.sendErrorResponse(requestId, ServerError.MetadataError, "Consumer not found"); + log.info("[{}] Consumer was not registered on the connection: {}", consumerId, remoteAddress); + ctx.writeAndFlush(Commands.newSuccess(requestId)); return; } From eb6bde2577219819581949e5f89789f54ecc8b40 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Fri, 18 Feb 2022 08:26:16 +0800 Subject: [PATCH 461/823] [LoadBalance] Optimize find nics process. (#14340) (cherry picked from commit 77d60b36422738498b7578132090f560c382e89a) --- .../impl/LinuxBrokerHostUsageImpl.java | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index fa18795000316..53d8d3e379da3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -227,23 +227,18 @@ public int getNicCount() { } private boolean isPhysicalNic(Path path) { - if (!path.toString().contains("/virtual/")) { - try { - Files.readAllBytes(path.resolve("speed")); - return true; - } catch (Exception e) { - // In some cases, VMs in EC2 won't have the speed reported on the NIC and will give a read-error. - // Check the type to make sure it's ethernet (type "1") - try { - String type = new String(Files.readAllBytes(path.resolve("type")), StandardCharsets.UTF_8).trim(); - return Integer.parseInt(type) == 1; - } catch (IOException ioe) { - // wireless nics don't report speed, ignore them. - return false; - } - } + if (path.toString().contains("/virtual/")) { + return false; + } + try { + // Check the type to make sure it's ethernet (type "1") + String type = new String(Files.readAllBytes(path.resolve("type")), StandardCharsets.UTF_8).trim(); + // wireless NICs don't report speed, ignore them. + return Integer.parseInt(type) == 1; + } catch (Exception e) { + // Read type got error. + return false; } - return false; } private Path getNicSpeedPath(String nic) { @@ -254,12 +249,13 @@ private double getTotalNicLimitKbps(List nics) { // Use the override value as configured. Return the total max speed across all available NICs, converted // from Gbps into Kbps return overrideBrokerNicSpeedGbps.map(aDouble -> aDouble * nics.size() * 1024 * 1024) - .orElseGet(() -> nics.stream().mapToDouble(s -> { + .orElseGet(() -> nics.stream().mapToDouble(nicPath -> { // Nic speed is in Mbits/s, return kbits/s try { - return Double.parseDouble(new String(Files.readAllBytes(getNicSpeedPath(s)))); + return Double.parseDouble(new String(Files.readAllBytes(getNicSpeedPath(nicPath)))); } catch (IOException e) { - log.error("Failed to read speed for nic " + s, e); + log.error(String.format("Failed to read speed for nic %s, maybe you can set broker" + + " config [loadBalancerOverrideBrokerNicSpeedGbps] to override it.", nicPath), e); return 0d; } }).sum() * 1024); From 1c2eb897fb49c8ad3e70f88d104979d197564b57 Mon Sep 17 00:00:00 2001 From: grayson <916028390@qq.com> Date: Thu, 24 Mar 2022 23:08:50 +0800 Subject: [PATCH 462/823] [fix][broker] filter the virtual NIC with relative path (#14829) (cherry picked from commit 9e919c71a46a2b936a44ed0ea9f2592b872cee86) --- .../broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 53d8d3e379da3..9bd5f4e65247b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -227,10 +227,10 @@ public int getNicCount() { } private boolean isPhysicalNic(Path path) { - if (path.toString().contains("/virtual/")) { - return false; - } try { + if (path.toRealPath().toString().contains("/virtual/")) { + return false; + } // Check the type to make sure it's ethernet (type "1") String type = new String(Files.readAllBytes(path.resolve("type")), StandardCharsets.UTF_8).trim(); // wireless NICs don't report speed, ignore them. From e5a7da929160733089661f0ef8810ceeae7360d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 28 Mar 2022 18:18:42 +0200 Subject: [PATCH 463/823] [fix][security] Upgrade jackson and jackson-databind (2.13.2.1) to get rid of CVE-2020-36518 (#14871) * [fix][security] Upgrade JacksonXML to get rid of CVE-2020-36518 * force jackson-databind version (cherry picked from commit 6d9ba7b60ab97257aa6e4e032a9764b5c3aefffa) --- .../server/src/assemble/LICENSE.bin.txt | 16 +++++------ pom.xml | 9 ++++-- pulsar-functions/runtime-all/pom.xml | 1 - pulsar-io/elastic-search/pom.xml | 1 - pulsar-sql/presto-distribution/LICENSE | 28 +++++++++---------- pulsar-sql/presto-distribution/pom.xml | 4 +-- 6 files changed, 31 insertions(+), 28 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 358d61c3c0260..deefa4eab91ad 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -312,14 +312,14 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.78.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.7.3.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.12.6.jar - - com.fasterxml.jackson.core-jackson-core-2.12.6.jar - - com.fasterxml.jackson.core-jackson-databind-2.12.6.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.12.6.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.12.6.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.12.6.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.12.6.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.12.6.jar + - com.fasterxml.jackson.core-jackson-annotations-2.13.2.jar + - com.fasterxml.jackson.core-jackson-core-2.13.2.jar + - com.fasterxml.jackson.core-jackson-databind-2.13.2.1.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.2.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.2.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar diff --git a/pom.xml b/pom.xml index 30df8e64aadb3..8b7eb2bf343ce 100644 --- a/pom.xml +++ b/pom.xml @@ -123,8 +123,8 @@ flexible messaging model and an intuitive client API. 2.17.1 1.69 1.0.2 - 2.12.6 - 2.12.6 + 2.13.2 + 2.13.2.1 0.9.11 1.6.2 8.37 @@ -779,6 +779,11 @@ flexible messaging model and an intuitive client API. pom import + + com.fasterxml.jackson.core + jackson-databind + ${jackson.databind.version} + log4j diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index bf01c32367513..6a11a7551fdc1 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -76,7 +76,6 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 180e338099f7f..081620e25ec6d 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -52,7 +52,6 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index ad17a62ce437d..b6f2eefe59521 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.12.6.jar - - jackson-core-2.12.6.jar - - jackson-databind-2.12.6.jar - - jackson-dataformat-smile-2.12.6.jar - - jackson-datatype-guava-2.12.6.jar - - jackson-datatype-jdk8-2.12.6.jar - - jackson-datatype-joda-2.12.6.jar - - jackson-datatype-jsr310-2.12.6.jar - - jackson-dataformat-yaml-2.12.6.jar - - jackson-jaxrs-base-2.12.6.jar - - jackson-jaxrs-json-provider-2.12.6.jar - - jackson-module-jaxb-annotations-2.12.6.jar - - jackson-module-jsonSchema-2.12.6.jar + - jackson-annotations-2.13.2.jar + - jackson-core-2.13.2.jar + - jackson-databind-2.13.2.1.jar + - jackson-dataformat-smile-2.13.2.jar + - jackson-datatype-guava-2.13.2.jar + - jackson-datatype-jdk8-2.13.2.jar + - jackson-datatype-joda-2.13.2.jar + - jackson-datatype-jsr310-2.13.2.jar + - jackson-dataformat-yaml-2.13.2.jar + - jackson-jaxrs-base-2.13.2.jar + - jackson-jaxrs-json-provider-2.13.2.jar + - jackson-module-jaxb-annotations-2.13.2.jar + - jackson-module-jsonSchema-2.13.2.jar * Guava - guava-30.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -440,7 +440,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.7.jar * Jackson - - jackson-module-parameter-names-2.12.6.jar + - jackson-module-parameter-names-2.13.2.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index e3f0ae3b51475..0d9c145dff5bb 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -39,10 +39,10 @@ 2.6 0.0.12 4.2.0 - 2.12.6 + 2.13.2 - 2.12.6 + 2.13.2.1 3.0.5 30.1-jre 2.12.1 From b6f2d6ceaf59198b7e30e8ec261c030269d41fe9 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 1 Apr 2022 18:08:22 +0800 Subject: [PATCH 464/823] [fix][pulsar-admin] Fix pulsar-admin not prompting message when there is a 500 error. (#14856) ### Motivation When pulsar-admin occurs 500 error, it doesn't return the root cause message to the user, only prompt `Internal Server Error`. The root cause is that: the broker side return 'TEXT_PLAIN' type message, https://github.com/apache/pulsar/blob/9644448fa2d85701e47fb8b6bf90ac149f70cf56/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java#L77-L88 But client side tries to parse it as json type (line-222): https://github.com/apache/pulsar/blob/9644448fa2d85701e47fb8b6bf90ac149f70cf56/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java#L219-L224 https://github.com/apache/pulsar/blob/9644448fa2d85701e47fb8b6bf90ac149f70cf56/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java#L272-L288 So if broker return 500, `getReasonFromServer` will throw exception 3 times and then use the original 500 message to the user. So we need to keep the server to return JSON type data to keep consistent. (cherry picked from commit 98260fe3d61f448504b05e755a412ba0e41310f5) --- .../pulsar/broker/web/RestException.java | 6 ++--- .../apache/pulsar/broker/admin/AdminTest.java | 27 +++++++++++++++++++ .../pulsar/broker/web/RestExceptionTest.java | 4 ++- .../client/admin/internal/BaseResource.java | 2 +- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java index 8ff016c78cda1..dcdd69f90c2d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java @@ -80,9 +80,9 @@ private static Response getResponse(Throwable t) { return e.getResponse(); } else { return Response - .status(Status.INTERNAL_SERVER_ERROR) - .entity(getExceptionData(t)) - .type(MediaType.TEXT_PLAIN) + .status(Status.INTERNAL_SERVER_ERROR.getStatusCode(), t.getMessage()) + .entity(new ErrorData(getExceptionData(t))) + .type(MediaType.APPLICATION_JSON) .build(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 63ade238aad56..f9300a7fcac72 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -43,8 +43,10 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -67,8 +69,10 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; import org.apache.pulsar.broker.loadbalance.LeaderBroker; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -79,6 +83,7 @@ import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.ErrorData; import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.ResourceQuota; @@ -830,6 +835,28 @@ public void testUpdatePartitionedTopicCoontainedInOldTopic() throws Exception { false, 10); } + @Test + public void test500Error() throws Exception { + final String property = "prop-xyz"; + final String cluster = "use"; + final String namespace = "ns"; + final String partitionedTopicName = "error-500-topic"; + AsyncResponse response1 = mock(AsyncResponse.class); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(RestException.class); + NamespaceName namespaceName = NamespaceName.get(property, cluster, namespace); + NamespaceService ns = spy(pulsar.getNamespaceService()); + Field namespaceField = pulsar.getClass().getDeclaredField("nsService"); + namespaceField.setAccessible(true); + namespaceField.set(pulsar, ns); + CompletableFuture> future = new CompletableFuture(); + future.completeExceptionally(new RuntimeException("500 error contains error message")); + doReturn(future).when(ns).getListOfTopics(namespaceName, CommandGetTopicsOfNamespace.Mode.ALL); + persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, 5, false); + verify(response1, timeout(5000).times(1)).resume(responseCaptor.capture()); + Assert.assertEquals(responseCaptor.getValue().getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); + Assert.assertTrue(((ErrorData)responseCaptor.getValue().getResponse().getEntity()).reason.contains("500 error contains error message")); + } + static class TestAsyncResponse implements AsyncResponse { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java index 1703f73983b28..c1938adfc5f86 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java @@ -22,6 +22,7 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response.Status; +import org.apache.pulsar.common.policies.data.ErrorData; import org.testng.annotations.Test; /** @@ -54,7 +55,8 @@ public void testOtherException() { RestException testException = new RestException(otherException); assertEquals(Status.INTERNAL_SERVER_ERROR.getStatusCode(), testException.getResponse().getStatus()); - assertEquals(RestException.getExceptionData(otherException), testException.getResponse().getEntity()); + ErrorData errorData = (ErrorData)testException.getResponse().getEntity(); + assertEquals(RestException.getExceptionData(otherException), errorData.reason); } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java index 6838fd8b10bbe..e4ea751672a92 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java @@ -216,7 +216,7 @@ public PulsarAdminException getApiException(Throwable e) { ServerErrorException see = (ServerErrorException) e; int statusCode = see.getResponse().getStatus(); String httpError = getReasonFromServer(see); - return new ServerSideErrorException(see, e.getMessage(), httpError, statusCode); + return new ServerSideErrorException(see, httpError, httpError, statusCode); } else if (e instanceof ClientErrorException) { // Handle 4xx exceptions ClientErrorException cee = (ClientErrorException) e; From cd835c59df13bf89d75b1c047a2647791a11cba5 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Sat, 2 Apr 2022 11:16:02 +0800 Subject: [PATCH 465/823] [fix][client] ConsumerBuilderImpl can not set null to deadLetterPolicy (#14980) `ConsumerBuilderImpl` can not set null to deadLetterPolicy if it's already assigned a non-null value. Calls `conf.setDeadLetterPolicy` if `deadLetterPolicy` is null. (cherry picked from commit 2f530717851d5e852b4489ff70184d14634ef2b9) --- .../client/impl/ConsumerBuilderImpl.java | 3 +- .../client/impl/ConsumerBuilderImplTest.java | 29 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index dea946b46c022..a3291db70950d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -428,10 +428,9 @@ public ConsumerBuilder deadLetterPolicy(DeadLetterPolicy deadLetterPolicy) { if (conf.getAckTimeoutMillis() == 0) { conf.setAckTimeoutMillis(DEFAULT_ACK_TIMEOUT_MILLIS_FOR_DEAD_LETTER); } - checkArgument(deadLetterPolicy.getMaxRedeliverCount() > 0, "MaxRedeliverCount must be > 0."); - conf.setDeadLetterPolicy(deadLetterPolicy); } + conf.setDeadLetterPolicy(deadLetterPolicy); return this; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java index 36ea53f31ef77..d648ee75af78c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java @@ -18,6 +18,17 @@ */ package org.apache.pulsar.client.impl; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertNotNull; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.DeadLetterPolicy; @@ -29,18 +40,6 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertNotNull; - /** * Unit tests of {@link ConsumerBuilderImpl}. */ @@ -298,6 +297,12 @@ public void testRedeliverCountOfDeadLetterPolicy() { .build()); } + @Test + public void testNullDeadLetterPolicy() { + consumerBuilderImpl.deadLetterPolicy(null); + verify(consumerBuilderImpl.getConf()).setDeadLetterPolicy(null); + } + @Test public void testConsumerBuilderImplWhenNumericPropertiesAreValid() { consumerBuilderImpl.negativeAckRedeliveryDelay(1, TimeUnit.MILLISECONDS); From c16f500939fa80e48e97101a48612c72857108f5 Mon Sep 17 00:00:00 2001 From: Qiang Huang Date: Sat, 2 Apr 2022 16:54:58 +0800 Subject: [PATCH 466/823] [fix][broker] Fix potential NPE in Replicator (#15003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation ```java public CompletableFuture getFuture() { return null; } ``` The return value null may cause potential NPE in ` getFuture`. The method does similarly： CompletableFuture remove()` & . ### Modifications Use ` return CompletableFuture.completedFuture(null);` instead of `return null;` (cherry picked from commit b604b4c9fb9fdae832bfabc61a0527ee0aec566f) --- .../org/apache/pulsar/broker/service/AbstractReplicator.java | 2 +- .../broker/service/nonpersistent/NonPersistentReplicator.java | 2 +- .../pulsar/broker/service/persistent/PersistentReplicator.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 0af749c0b737f..e4b00cb992e77 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -209,7 +209,7 @@ public synchronized CompletableFuture disconnect(boolean failIfHasBacklog) public CompletableFuture remove() { // No-op - return null; + return CompletableFuture.completedFuture(null); } protected boolean isWritable() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index ce1fe3443f5f3..b863e9eb3c2cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -229,7 +229,7 @@ public MessageImpl getNextMessage() { @Override public CompletableFuture getFuture() { - return null; + return CompletableFuture.completedFuture(null); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 4d79c9a37cf72..1db685ccbec34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -503,7 +503,7 @@ public MessageImpl getNextMessage() { @Override public CompletableFuture getFuture() { - return null; + return CompletableFuture.completedFuture(null); } } From 59a1360c06f68390f5e65f6d0708a852e3523d23 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sun, 3 Apr 2022 03:27:20 +0800 Subject: [PATCH 467/823] [fix][broker] Fix getPendingAckInternalStats redirect issue. (#14876) (cherry picked from commit 417d1e50957bee1b0f88b8172d7831af496ed001) --- .../broker/admin/impl/TransactionsBase.java | 93 +++++++------------ .../pulsar/broker/admin/v3/Transactions.java | 33 ++++++- .../admin/v3/AdminApiTransactionTest.java | 22 +++++ 3 files changed, 82 insertions(+), 66 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index 504ce92de45c2..308f18e8bc31c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -38,9 +38,6 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; -import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; -import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.web.RestException; @@ -508,69 +505,41 @@ protected void internalGetCoordinatorInternalStats(AsyncResponse asyncResponse, } } - protected void internalGetPendingAckInternalStats(AsyncResponse asyncResponse, boolean authoritative, - TopicName topicName, String subName, boolean metadata) { - try { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - validateTopicOwnership(topicName, authoritative); - CompletableFuture> topicFuture = pulsar().getBrokerService() - .getTopics().get(topicName.toString()); - if (topicFuture != null) { - topicFuture.whenComplete((optionalTopic, e) -> { - - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } + protected CompletableFuture internalGetPendingAckInternalStats( + boolean authoritative, TopicName topicName, String subName, boolean metadata) { + if (!pulsar().getConfig().isTransactionCoordinatorEnabled()) { + return FutureUtil.failedFuture(new RestException(SERVICE_UNAVAILABLE, + "This Broker is not configured with transactionCoordinatorEnabled=true.")); + } + return validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> { + CompletableFuture> topicFuture = pulsar().getBrokerService() + .getTopics().get(topicName.toString()); + if (topicFuture == null) { + return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); + } + return topicFuture.thenCompose(optionalTopic -> { if (!optionalTopic.isPresent()) { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, - "Topic is not owned by this broker!")); - return; - } - Topic topicObject = optionalTopic.get(); - if (topicObject instanceof PersistentTopic) { - try { - ManagedLedger managedLedger = - ((PersistentTopic) topicObject).getPendingAckManagedLedger(subName).get(); - TransactionPendingAckInternalStats stats = - new TransactionPendingAckInternalStats(); - TransactionLogStats pendingAckLogStats = new TransactionLogStats(); - pendingAckLogStats.managedLedgerName = managedLedger.getName(); - pendingAckLogStats.managedLedgerInternalStats = - managedLedger.getManagedLedgerInternalStats(metadata).get(); - stats.pendingAckLogStats = pendingAckLogStats; - asyncResponse.resume(stats); - } catch (Exception exception) { - if (exception instanceof ExecutionException) { - if (exception.getCause() instanceof ServiceUnitNotReadyException) { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - exception.getCause())); - return; - } else if (exception.getCause() instanceof NotAllowedException) { - asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, - exception.getCause())); - return; - } else if (exception.getCause() instanceof SubscriptionNotFoundException) { - asyncResponse.resume(new RestException(NOT_FOUND, exception.getCause())); - return; - } - } - asyncResponse.resume(new RestException(exception)); - } + return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); } else { - asyncResponse.resume(new RestException(BAD_REQUEST, "Topic is not a persistent topic!")); + Topic topicObject = optionalTopic.get(); + return ((PersistentTopic) topicObject).getPendingAckManagedLedger(subName) + .thenCompose(managedLedger -> managedLedger.getManagedLedgerInternalStats(metadata) + .thenApply(internalStats -> { + TransactionLogStats pendingAckLogStats = new TransactionLogStats(); + pendingAckLogStats.managedLedgerName = managedLedger.getName(); + pendingAckLogStats.managedLedgerInternalStats = internalStats; + return pendingAckLogStats; + }) + .thenApply(pendingAckLogStats -> { + TransactionPendingAckInternalStats stats = + new TransactionPendingAckInternalStats(); + stats.pendingAckLogStats = pendingAckLogStats; + return stats; + })); } }); - } else { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, "Topic is not owned by this broker!")); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); - } - } catch (Exception e) { - asyncResponse.resume(new RestException(e.getCause())); - } + }); } protected void validateTopicName(String property, String namespace, String encodedTopic) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java index 94411f6d16df8..bbd79036ecf71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.broker.admin.v3; +import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; @@ -33,14 +36,17 @@ import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.container.Suspended; import javax.ws.rs.core.MediaType; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.admin.impl.TransactionsBase; -import org.apache.pulsar.common.naming.TopicDomain; -import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.common.util.FutureUtil; @Path("/transactions") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Api(value = "/transactions", description = "Transactions admin apis", tags = "transactions") +@Slf4j public class Transactions extends TransactionsBase { @GET @@ -222,7 +228,26 @@ public void getPendingAckInternalStats(@Suspended final AsyncResponse asyncRespo @PathParam("topic") @Encoded String encodedTopic, @PathParam("subName") String subName, @QueryParam("metadata") @DefaultValue("false") boolean metadata) { - internalGetPendingAckInternalStats(asyncResponse, authoritative, - TopicName.get(TopicDomain.persistent.value(), tenant, namespace, encodedTopic), subName, metadata); + try { + validateTopicName(tenant, namespace, encodedTopic); + internalGetPendingAckInternalStats(authoritative, topicName, subName, metadata) + .thenAccept(stats -> asyncResponse.resume(stats)) + .exceptionally(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.error("[{}] Failed to get pending ack internal stats {}", clientAppId(), topicName, cause); + if (cause instanceof BrokerServiceException.ServiceUnitNotReadyException) { + asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, cause)); + } else if (cause instanceof BrokerServiceException.NotAllowedException) { + asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, cause)); + } else if (cause instanceof BrokerServiceException.SubscriptionNotFoundException) { + asyncResponse.resume(new RestException(NOT_FOUND, cause)); + } else { + asyncResponse.resume(new RestException(cause)); + } + return null; + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 93a2d0e188e17..1f796553bdcad 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -22,6 +22,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -56,12 +57,14 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class AdminApiTransactionTest extends MockedPulsarServiceBaseTest { @@ -374,6 +377,25 @@ public void testGetPendingAckInternalStats() throws Exception { TransactionImpl transaction = (TransactionImpl) getTransaction(); final String topic = "persistent://public/default/testGetPendingAckInternalStats"; final String subName = "test"; + try { + admin.transactions() + .getPendingAckInternalStatsAsync(topic, subName, true).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } + try { + pulsar.getBrokerService().getTopic(topic, false); + admin.transactions() + .getPendingAckInternalStatsAsync(topic, subName, true).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic).create(); Consumer consumer = pulsarClient.newConsumer(Schema.BYTES).topic(topic) From fed8b8c324685d907e9867eac5a14ed8a5fefa78 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sun, 3 Apr 2022 21:50:45 +0800 Subject: [PATCH 468/823] [fix][broker] Fix creating system namespace topic failure. (#14949) (cherry picked from commit f3b87b65c6946eb197c1eece22cff8ff04e16fcb) --- .../pulsar/broker/service/BrokerService.java | 3 +- .../systopic/PartitionedSystemTopicTest.java | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 9f14da56653ce..261d006ebd09c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -24,7 +24,6 @@ import static org.apache.commons.collections.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.broker.PulsarService.isTransactionSystemTopic; -import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -2606,7 +2605,7 @@ public boolean isAllowAutoTopicCreation(final String topic) { public boolean isAllowAutoTopicCreation(final TopicName topicName) { //System topic can always be created automatically - if (pulsar.getConfiguration().isSystemTopicEnabled() && checkTopicIsEventsNames(topicName)) { + if (pulsar.getConfiguration().isSystemTopicEnabled() && isSystemTopic(topicName)) { return true; } AutoTopicCreationOverride autoTopicCreationOverride = getAutoTopicCreationOverride(topicName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index bbd3cae711704..ff45c140f56f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -19,21 +19,28 @@ package org.apache.pulsar.broker.systopic; import com.google.common.collect.Sets; +import lombok.Cleanup; import org.apache.commons.lang.RandomStringUtils; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; - +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; @Test(groups = "broker") public class PartitionedSystemTopicTest extends BrokerTestBase { @@ -104,4 +111,28 @@ public void testConsumerCreationWhenEnablingTopicPolicy() throws Exception { } } + @Test + public void testProduceAndConsumeUnderSystemNamespace() throws Exception { + TenantInfo tenantInfo = TenantInfo + .builder() + .adminRoles(Sets.newHashSet("admin")) + .allowedClusters(Sets.newHashSet("test")) + .build(); + admin.tenants().createTenant("pulsar", tenantInfo); + admin.namespaces().createNamespace("pulsar/system", 2); + @Cleanup + Producer producer = pulsarClient.newProducer().topic("pulsar/system/__topic-1").create(); + producer.send("test".getBytes(StandardCharsets.UTF_8)); + @Cleanup + Consumer consumer = pulsarClient + .newConsumer() + .topic("pulsar/system/__topic-1") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("sub1") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + Message receive = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertNotNull(receive); + } + } From ce56c0fb1c2a86f94c17c193cff1c4d9ba458108 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 5 Apr 2022 03:48:05 +0800 Subject: [PATCH 469/823] allow download package from package management service (#14814) (cherry picked from commit 7cc5cf1fadf8835fcc3166494f0e05632894da43) --- .../functions/worker/FunctionActioner.java | 22 +++++---- .../worker/FunctionActionerTest.java | 49 +++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 5fa554097257a..758157976f11f 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -18,6 +18,14 @@ */ package org.apache.pulsar.functions.worker; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.pulsar.common.functions.Utils.FILE; +import static org.apache.pulsar.common.functions.Utils.HTTP; +import static org.apache.pulsar.common.functions.Utils.hasPackageTypePrefix; +import static org.apache.pulsar.common.functions.Utils.isFunctionPackageUrlSupported; +import static org.apache.pulsar.functions.auth.FunctionAuthUtils.getFunctionAuthData; +import static org.apache.pulsar.functions.utils.FunctionCommon.getSinkType; +import static org.apache.pulsar.functions.utils.FunctionCommon.getSourceType; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.io.MoreFiles; import com.google.common.io.RecursiveDeleteOption; @@ -65,14 +73,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.pulsar.common.functions.Utils.FILE; -import static org.apache.pulsar.common.functions.Utils.HTTP; -import static org.apache.pulsar.common.functions.Utils.isFunctionPackageUrlSupported; -import static org.apache.pulsar.functions.auth.FunctionAuthUtils.getFunctionAuthData; -import static org.apache.pulsar.functions.utils.FunctionCommon.getSinkType; -import static org.apache.pulsar.functions.utils.FunctionCommon.getSourceType; - @Data @Slf4j public class FunctionActioner { @@ -192,7 +192,8 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu return instanceConfig; } - private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaData functionMetaData, int instanceId) throws FileNotFoundException, IOException { + private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaData functionMetaData, + int instanceId) throws FileNotFoundException, IOException, PulsarAdminException { FunctionDetails details = functionMetaData.getFunctionDetails(); File pkgDir = pkgFile.getParentFile(); @@ -211,12 +212,15 @@ private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaDa } while (tempPkgFile.exists() || !tempPkgFile.createNewFile()); String pkgLocationPath = functionMetaData.getPackageLocation().getPackagePath(); boolean downloadFromHttp = isPkgUrlProvided && pkgLocationPath.startsWith(HTTP); + boolean downloadFromPackageManagementService = isPkgUrlProvided && hasPackageTypePrefix(pkgLocationPath); log.info("{}/{}/{} Function package file {} will be downloaded from {}", tempPkgFile, details.getTenant(), details.getNamespace(), details.getName(), downloadFromHttp ? pkgLocationPath : functionMetaData.getPackageLocation()); if(downloadFromHttp) { FunctionCommon.downloadFromHttpUrl(pkgLocationPath, tempPkgFile); + } else if (downloadFromPackageManagementService) { + getPulsarAdmin().packages().download(pkgLocationPath, tempPkgFile.getPath()); } else { FileOutputStream tempPkgFos = new FileOutputStream(tempPkgFile); WorkerUtils.downloadFromBookkeeper( diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java index dc49036a5bebb..7502e247d9908 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.verify; import org.apache.distributedlog.api.namespace.Namespace; +import org.apache.pulsar.client.admin.Packages; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.auth.FunctionAuthProvider; @@ -220,4 +221,52 @@ public void testFunctionAuthDisabled() throws Exception { verify(functionAuthProvider.get(), times(0)).cleanUpAuthData(any(), any()); } + @Test + public void testStartFunctionWithPackageUrl() throws Exception { + + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setWorkerId("worker-1"); + workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName()); + workerConfig.setFunctionRuntimeFactoryConfigs( + ObjectMapperFactory.getThreadLocal().convertValue( + new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class)); + workerConfig.setPulsarServiceUrl("pulsar://localhost:6650"); + workerConfig.setStateStorageServiceUrl("foo"); + workerConfig.setFunctionAssignmentTopicName("assignments"); + String downloadDir = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath(); + workerConfig.setDownloadDirectory(downloadDir); + + RuntimeFactory factory = mock(RuntimeFactory.class); + Runtime runtime = mock(Runtime.class); + doReturn(runtime).when(factory).createContainer(any(), any(), any(), any()); + doNothing().when(runtime).start(); + Namespace dlogNamespace = mock(Namespace.class); + final String exceptionMsg = "dl namespace not-found"; + doThrow(new IllegalArgumentException(exceptionMsg)).when(dlogNamespace).openLog(any()); + PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class); + Packages packages = mock(Packages.class); + doReturn(packages).when(pulsarAdmin).packages(); + doNothing().when(packages).download(any(), any()); + + @SuppressWarnings("resource") + FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), pulsarAdmin); + + // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call + // RuntimeSpawner + String pkgPathLocation = "function://public/default/test-function@latest"; + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() + .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") + .setNamespace("test-namespace").setName("func-1")) + .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath(pkgPathLocation).build()) + .build(); + Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) + .build(); + FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class); + doReturn(instance).when(functionRuntimeInfo).getFunctionInstance(); + + actioner.startFunction(functionRuntimeInfo); + verify(runtime, times(1)).start(); + } + } From 03f49481e1c7d141d3c94309ffb5bf5f7761b547 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 6 Apr 2022 15:22:19 +0800 Subject: [PATCH 470/823] [fix][broker] Avoid heartbeat topic to offload. (#15008) (cherry picked from commit cdb67e436225fce52105986c8f3906ed331f6f4d) --- .../mledger/impl/ManagedLedgerImpl.java | 4 +++ .../mledger/impl/OffloadPrefixTest.java | 2 +- .../pulsar/broker/service/BrokerService.java | 28 +++++++++------ .../systopic/PartitionedSystemTopicTest.java | 35 +++++++++++++++++++ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 193ce42338527..8d5f0a6ed8c62 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2779,6 +2779,10 @@ public void offloadFailed(ManagedLedgerException e, Object ctx) { @Override public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ctx) { + if (config.getLedgerOffloader() != null && config.getLedgerOffloader() == NullLedgerOffloader.INSTANCE) { + callback.offloadFailed(new ManagedLedgerException("NullLedgerOffloader"), ctx); + return; + } PositionImpl requestOffloadTo = (PositionImpl) pos; if (!isValidPosition(requestOffloadTo) && // Also consider the case where the last ledger is currently diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index a7092e4ec46f6..a0930e4244215 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -85,7 +85,7 @@ public void testNullOffloader() throws Exception { ledger.offloadPrefix(p); fail("Should have thrown an exception"); } catch (ManagedLedgerException e) { - assertEquals(e.getCause().getClass(), CompletionException.class); + assertEquals(e.getMessage(), "NullLedgerOffloader"); } assertEquals(ledger.getLedgersInfoAsList().size(), 5); assertEquals(ledger.getLedgersInfoAsList().stream() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 261d006ebd09c..a4b745d7b6576 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -85,6 +85,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -101,6 +102,7 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl; import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources; @@ -1496,18 +1498,22 @@ public CompletableFuture getManagedLedgerConfig(TopicName t topicLevelOffloadPolicies, OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), getPulsar().getConfig().getProperties()); - if (topicLevelOffloadPolicies != null) { - try { - LedgerOffloader topicLevelLedgerOffLoader = - pulsar().createManagedLedgerOffloader(offloadPolicies); - managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); - } catch (PulsarServerException e) { - throw new RuntimeException(e); + if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { + managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); + } else { + if (topicLevelOffloadPolicies != null) { + try { + LedgerOffloader topicLevelLedgerOffLoader = + pulsar().createManagedLedgerOffloader(offloadPolicies); + managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); + } catch (PulsarServerException e) { + throw new RuntimeException(e); + } + } else { + //If the topic level policy is null, use the namespace level + managedLedgerConfig + .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); } - } else { - //If the topic level policy is null, use the namespace level - managedLedgerConfig - .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); } managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index ff45c140f56f5..d506f7127678f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -20,8 +20,14 @@ import com.google.common.collect.Sets; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.LedgerOffloader; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; import org.apache.commons.lang.RandomStringUtils; +import org.apache.pulsar.broker.admin.impl.BrokersBase; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -30,8 +36,13 @@ import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -135,4 +146,28 @@ public void testProduceAndConsumeUnderSystemNamespace() throws Exception { Assert.assertNotNull(receive); } + @Test + public void testHealthCheckTopicNotOffload() throws Exception { + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), + pulsar.getConfig()); + TopicName topicName = TopicName.get("persistent", namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopic(topicName.toString(), true).get().get(); + ManagedLedgerConfig config = persistentTopic.getManagedLedger().getConfig(); + config.setLedgerOffloader(NullLedgerOffloader.INSTANCE); + admin.brokers().healthcheck(TopicVersion.V2); + admin.topics().triggerOffload(topicName.toString(), MessageId.earliest); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(persistentTopic.getManagedLedger().getOffloadedSize(), 0); + }); + LedgerOffloader ledgerOffloader = Mockito.mock(LedgerOffloader.class); + config.setLedgerOffloader(ledgerOffloader); + Assert.assertEquals(config.getLedgerOffloader(), ledgerOffloader); + admin.topicPolicies().setMaxConsumers(topicName.toString(), 2); + Awaitility.await().pollDelay(5, TimeUnit.SECONDS).untilAsserted(() -> { + Assert.assertEquals(persistentTopic.getManagedLedger().getConfig().getLedgerOffloader(), + NullLedgerOffloader.INSTANCE); + }); + } + } From 2c765d0d76e181ff5ce470295f96a16cfb307ad9 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Wed, 6 Apr 2022 15:33:08 +0800 Subject: [PATCH 471/823] [improve][transaction] support configurable ``transactionBufferClientOperationTimeoutInMills`` (#15011) (cherry picked from commit 1c6ea125d629beb40bd5f381002f22619edf43df) --- .../java/org/apache/pulsar/broker/ServiceConfiguration.java | 6 ++++++ .../main/java/org/apache/pulsar/broker/PulsarService.java | 3 ++- .../buffer/impl/TransactionBufferClientImpl.java | 4 ++-- .../buffer/impl/TransactionBufferHandlerImpl.java | 6 +++--- .../transaction/buffer/TransactionBufferClientTest.java | 6 +++--- .../buffer/TransactionBufferHandlerImplTest.java | 6 ++++-- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 4b3b2e1783177..fa8464376fce6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2222,6 +2222,12 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private int transactionBufferClientMaxConcurrentRequests = 1000; + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "The transaction buffer client's operation timeout in milliseconds." + ) + private long transactionBufferClientOperationTimeoutInMills = 3000L; + /**** --- KeyStore TLS config variables --- ****/ @FieldContext( category = CATEGORY_KEYSTORE_TLS, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index ac4759d79484d..6451e518b883a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -733,7 +733,8 @@ config, localMetadataStore, getZkClient(), this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); transactionBufferClient = TransactionBufferClientImpl.create(getClient(), transactionTimer, - config.getTransactionBufferClientMaxConcurrentRequests()); + config.getTransactionBufferClientMaxConcurrentRequests(), + config.getTransactionBufferClientOperationTimeoutInMills()); transactionMetadataStoreService = new TransactionMetadataStoreService(TransactionMetadataStoreProvider .newProvider(config.getTransactionMetadataStoreProviderClassName()), this, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java index 454e9a6d53beb..060476e573c2a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java @@ -40,9 +40,9 @@ private TransactionBufferClientImpl(TransactionBufferHandler tbHandler) { } public static TransactionBufferClient create(PulsarClient pulsarClient, HashedWheelTimer timer, - int maxConcurrentRequests) { + int maxConcurrentRequests, long operationTimeoutInMills) { TransactionBufferHandler handler = new TransactionBufferHandlerImpl(pulsarClient, timer, - maxConcurrentRequests); + maxConcurrentRequests, operationTimeoutInMills); return new TransactionBufferClientImpl(handler); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 6ea53a3edd263..b80a273bc6fcd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -75,12 +75,12 @@ public CompletableFuture load(String topic) { } }); - public TransactionBufferHandlerImpl(PulsarClient pulsarClient, - HashedWheelTimer timer, int maxConcurrentRequests) { + public TransactionBufferHandlerImpl(PulsarClient pulsarClient, HashedWheelTimer timer, + int maxConcurrentRequests, long operationTimeoutInMills) { this.pulsarClient = pulsarClient; this.outstandingRequests = new ConcurrentSkipListMap<>(); this.pendingRequests = new GrowableArrayBlockingQueue<>(); - this.operationTimeoutInMills = 3000L; + this.operationTimeoutInMills = operationTimeoutInMills; this.timer = timer; this.requestCredits = Math.max(100, maxConcurrentRequests); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index fa1b9e7f287e6..d85f6b42f23f6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -81,7 +81,7 @@ protected void setup() throws Exception { admin.namespaces().createNamespace(namespace, 10); admin.topics().createPartitionedTopic(partitionedTopicName.getPartitionedTopicName(), partitions); tbClient = TransactionBufferClientImpl.create(pulsarClient, - new HashedWheelTimer(new DefaultThreadFactory("transaction-buffer")), 1000); + new HashedWheelTimer(new DefaultThreadFactory("transaction-buffer")), 1000, 3000); } @Override @@ -160,7 +160,7 @@ public void testTransactionBufferClientTimeout() throws Exception { @Cleanup("stop") HashedWheelTimer hashedWheelTimer = new HashedWheelTimer(); TransactionBufferHandlerImpl transactionBufferHandler = - new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000); + new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000, 3000); CompletableFuture endFuture = transactionBufferHandler.endTxnOnTopic("test", 1, 1, TxnAction.ABORT, 1); @@ -203,7 +203,7 @@ public void testTransactionBufferChannelUnActive() { @Cleanup("stop") HashedWheelTimer hashedWheelTimer = new HashedWheelTimer(); TransactionBufferHandlerImpl transactionBufferHandler = - new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000); + new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000, 3000); try { transactionBufferHandler.endTxnOnTopic("test", 1, 1, TxnAction.ABORT, 1).get(); fail(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java index ef0cf037772b6..5241342635b88 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java @@ -42,7 +42,8 @@ public class TransactionBufferHandlerImplTest { public void testRequestCredits() { PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); when(pulsarClient.getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); - TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarClient, null, 1000)); + TransactionBufferHandlerImpl handler = spy( + new TransactionBufferHandlerImpl(pulsarClient, null, 1000, 3000)); doNothing().when(handler).endTxn(any()); for (int i = 0; i < 500; i++) { handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); @@ -61,7 +62,8 @@ public void testRequestCredits() { @Test public void testMinRequestCredits() { - TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(null, null, 50)); + TransactionBufferHandlerImpl handler = spy( + new TransactionBufferHandlerImpl(null, null, 50, 3000)); assertEquals(handler.getAvailableRequestCredits(), 100); } } From 9d0740b47a16cf21d8c53f74198d364a4ce4b516 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Wed, 6 Apr 2022 15:41:17 +0800 Subject: [PATCH 472/823] [improve][broker] Avoid using blocking calls for the async method ``checkTopicOwnership`` (#15023) (cherry picked from commit c59402ef09c870469a4d5ff835fa6222518704b9) --- .../pulsar/broker/namespace/NamespaceService.java | 2 +- .../pulsar/broker/namespace/OwnershipCache.java | 13 +++++++++++-- .../pulsar/broker/namespace/OwnershipCacheTest.java | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 058354cd01ac3..e576e864467e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1031,7 +1031,7 @@ private boolean isTopicOwned(TopicName topicName) { public CompletableFuture checkTopicOwnership(TopicName topicName) { return getBundleAsync(topicName) - .thenApply(ownershipCache::checkOwnership); + .thenCompose(ownershipCache::checkOwnershipAsync); } public void removeOwnedServiceUnit(NamespaceBundle nsBundle) throws Exception { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index daedb712e299c..fc014414f5ea2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -148,8 +148,13 @@ public OwnershipCache(PulsarService pulsar, NamespaceBundleFactory bundleFactory * @param bundle namespace bundle * @return future that will complete with check result */ - public boolean checkOwnership(NamespaceBundle bundle) { - return getOwnedBundle(bundle) != null; + public CompletableFuture checkOwnershipAsync(NamespaceBundle bundle) { + Optional> ownedBundleFuture = getOwnedBundleAsync(bundle); + if (!ownedBundleFuture.isPresent()) { + return CompletableFuture.completedFuture(false); + } + return ownedBundleFuture.get() + .thenApply(bd -> bd != null && bd.isActive()); } /** @@ -277,6 +282,10 @@ public OwnedBundle getOwnedBundle(NamespaceBundle bundle) { } } + public Optional> getOwnedBundleAsync(NamespaceBundle bundle) { + return Optional.ofNullable(ownedBundlesCache.getIfPresent(bundle)); + } + /** * Disable bundle in local cache and on zk. * diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java index 143d5ef78f510..dde25fa2eed29 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java @@ -401,7 +401,7 @@ public void testReestablishOwnership() throws Exception { assertFalse(data3.isDisabled()); assertNotNull(cache.getOwnedBundle(testFullBundle)); - assertTrue(cache.checkOwnership(testFullBundle)); + assertTrue(cache.checkOwnershipAsync(testFullBundle).get()); assertEquals(data2.getNativeUrl(), selfBrokerUrl); assertFalse(data2.isDisabled()); assertNotNull(cache.getOwnedBundle(testFullBundle)); From 98db8707cb0c6f083fdeafeeb1ab5c0cdb0c25cf Mon Sep 17 00:00:00 2001 From: Xiaoyu Hou Date: Wed, 6 Apr 2022 20:49:01 +0800 Subject: [PATCH 473/823] [fix][broker] Return if reset in progress (#14978) ### Motivation - Fix bug. If cursor reset in progress, the callback method will call `resetFailed` but without return. This will cause the callback be invoked again ### Modifications - Just add `return` after callback invoke `resetFailed` If cursor reset in progress (cherry picked from commit 81da8d3cd199fd6c1e4510a1c1c2ac71418efd5e) --- .../org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index adb29bb37bb96..e2f271a3eed29 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1077,6 +1077,7 @@ protected void internalResetCursor(PositionImpl position, AsyncCallbacks.ResetCu resetCursorCallback.resetFailed( new ManagedLedgerException.ConcurrentFindCursorPositionException("reset already in progress"), position); + return; } } From 839b66385e638226ac75133b90c0f626cba42375 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 7 Apr 2022 10:20:28 +0800 Subject: [PATCH 474/823] [fix][transaction] Fix transaction REST API redirect issue. (#15017) (cherry picked from commit 8b8bf154aecf07c45fede2679445ef0563b49c5b) --- .../broker/admin/impl/TransactionsBase.java | 510 +++++++----------- .../pulsar/broker/admin/v3/Transactions.java | 67 ++- .../admin/v3/AdminApiTransactionTest.java | 92 +++- 3 files changed, 341 insertions(+), 328 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index 308f18e8bc31c..b225cd2e2661a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -18,11 +18,9 @@ */ package org.apache.pulsar.broker.admin.impl; -import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; -import static javax.ws.rs.core.Response.Status.TEMPORARY_REDIRECT; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.HashMap; @@ -46,6 +44,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorStats; import org.apache.pulsar.common.policies.data.TransactionInBufferStats; @@ -53,6 +52,7 @@ import org.apache.pulsar.common.policies.data.TransactionLogStats; import org.apache.pulsar.common.policies.data.TransactionMetadata; import org.apache.pulsar.common.policies.data.TransactionPendingAckInternalStats; +import org.apache.pulsar.common.policies.data.TransactionPendingAckStats; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; @@ -67,216 +67,97 @@ public abstract class TransactionsBase extends AdminResource { protected void internalGetCoordinatorStats(AsyncResponse asyncResponse, boolean authoritative, Integer coordinatorId) { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - if (coordinatorId != null) { - validateTopicOwnership(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(coordinatorId), - authoritative); - TransactionMetadataStore transactionMetadataStore = - pulsar().getTransactionMetadataStoreService().getStores() - .get(TransactionCoordinatorID.get(coordinatorId)); - if (transactionMetadataStore == null) { - asyncResponse.resume(new RestException(NOT_FOUND, - "Transaction coordinator not found! coordinator id : " + coordinatorId)); + if (coordinatorId != null) { + validateTopicOwnership(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(coordinatorId), + authoritative); + TransactionMetadataStore transactionMetadataStore = + pulsar().getTransactionMetadataStoreService().getStores() + .get(TransactionCoordinatorID.get(coordinatorId)); + if (transactionMetadataStore == null) { + asyncResponse.resume(new RestException(NOT_FOUND, + "Transaction coordinator not found! coordinator id : " + coordinatorId)); + return; + } + asyncResponse.resume(transactionMetadataStore.getCoordinatorStats()); + } else { + getPartitionedTopicMetadataAsync(TopicName.TRANSACTION_COORDINATOR_ASSIGN, + false, false).thenAccept(partitionMetadata -> { + if (partitionMetadata.partitions == 0) { + asyncResponse.resume(new RestException(Response.Status.NOT_FOUND, + "Transaction coordinator not found")); return; } - asyncResponse.resume(transactionMetadataStore.getCoordinatorStats()); - } else { - getPartitionedTopicMetadataAsync(TopicName.TRANSACTION_COORDINATOR_ASSIGN, - false, false).thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions == 0) { - asyncResponse.resume(new RestException(Response.Status.NOT_FOUND, - "Transaction coordinator not found")); + List> transactionMetadataStoreInfoFutures = + Lists.newArrayList(); + for (int i = 0; i < partitionMetadata.partitions; i++) { + try { + transactionMetadataStoreInfoFutures + .add(pulsar().getAdminClient().transactions().getCoordinatorStatsByIdAsync(i)); + } catch (PulsarServerException e) { + asyncResponse.resume(new RestException(e)); return; } - List> transactionMetadataStoreInfoFutures = - Lists.newArrayList(); - for (int i = 0; i < partitionMetadata.partitions; i++) { + } + Map stats = new HashMap<>(); + FutureUtil.waitForAll(transactionMetadataStoreInfoFutures).whenComplete((result, e) -> { + if (e != null) { + asyncResponse.resume(new RestException(e)); + return; + } + + for (int i = 0; i < transactionMetadataStoreInfoFutures.size(); i++) { try { - transactionMetadataStoreInfoFutures - .add(pulsar().getAdminClient().transactions().getCoordinatorStatsByIdAsync(i)); - } catch (PulsarServerException e) { - asyncResponse.resume(new RestException(e)); + stats.put(i, transactionMetadataStoreInfoFutures.get(i).get()); + } catch (Exception exception) { + asyncResponse.resume(new RestException(exception.getCause())); return; } } - Map stats = new HashMap<>(); - FutureUtil.waitForAll(transactionMetadataStoreInfoFutures).whenComplete((result, e) -> { - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } - - for (int i = 0; i < transactionMetadataStoreInfoFutures.size(); i++) { - try { - stats.put(i, transactionMetadataStoreInfoFutures.get(i).get()); - } catch (Exception exception) { - asyncResponse.resume(new RestException(exception.getCause())); - return; - } - } - asyncResponse.resume(stats); - }); - }).exceptionally(ex -> { - log.error("[{}] Failed to get transaction coordinator state.", clientAppId(), ex); - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; + asyncResponse.resume(stats); }); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); + }).exceptionally(ex -> { + log.error("[{}] Failed to get transaction coordinator state.", clientAppId(), ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } } - protected void internalGetTransactionInPendingAckStats(AsyncResponse asyncResponse, boolean authoritative, - long mostSigBits, long leastSigBits, String subName) { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - validateTopicOwnership(topicName, authoritative); - CompletableFuture> topicFuture = pulsar().getBrokerService() - .getTopics().get(topicName.toString()); - if (topicFuture != null) { - topicFuture.whenComplete((optionalTopic, e) -> { - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } - if (!optionalTopic.isPresent()) { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, - "Topic is not owned by this broker!")); - return; - } - Topic topicObject = optionalTopic.get(); - if (topicObject instanceof PersistentTopic) { - asyncResponse.resume(((PersistentTopic) topicObject) - .getTransactionInPendingAckStats(new TxnID(mostSigBits, leastSigBits), subName)); - } else { - asyncResponse.resume(new RestException(BAD_REQUEST, "Topic is not a persistent topic!")); - } - }); - } else { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, "Topic is not owned by this broker!")); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); - } + protected CompletableFuture internalGetTransactionInPendingAckStats( + boolean authoritative, long mostSigBits, long leastSigBits, String subName) { + return getExistingPersistentTopicAsync(authoritative) + .thenApply(topic -> topic.getTransactionInPendingAckStats(new TxnID(mostSigBits, leastSigBits), + subName)); } - protected void internalGetTransactionInBufferStats(AsyncResponse asyncResponse, boolean authoritative, - long mostSigBits, long leastSigBits) { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - validateTopicOwnership(topicName, authoritative); - CompletableFuture> topicFuture = pulsar().getBrokerService() - .getTopics().get(topicName.toString()); - if (topicFuture != null) { - topicFuture.whenComplete((optionalTopic, e) -> { - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } - if (!optionalTopic.isPresent()) { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, - "Topic is not owned by this broker!")); - return; - } - Topic topicObject = optionalTopic.get(); - if (topicObject instanceof PersistentTopic) { - TransactionInBufferStats transactionInBufferStats = ((PersistentTopic) topicObject) - .getTransactionInBufferStats(new TxnID(mostSigBits, leastSigBits)); - asyncResponse.resume(transactionInBufferStats); - } else { - asyncResponse.resume(new RestException(BAD_REQUEST, "Topic is not a persistent topic!")); - } - }); - } else { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, "Topic is not owned by this broker!")); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); - } + protected CompletableFuture internalGetTransactionInBufferStats( + boolean authoritative, long mostSigBits, long leastSigBits) { + return getExistingPersistentTopicAsync(authoritative) + .thenApply(topic -> topic.getTransactionInBufferStats(new TxnID(mostSigBits, leastSigBits))); } - protected void internalGetTransactionBufferStats(AsyncResponse asyncResponse, boolean authoritative) { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - validateTopicOwnership(topicName, authoritative); - CompletableFuture> topicFuture = pulsar().getBrokerService() - .getTopics().get(topicName.toString()); - if (topicFuture != null) { - topicFuture.whenComplete((optionalTopic, e) -> { - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } - - if (!optionalTopic.isPresent()) { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, - "Topic is not owned by this broker!")); - return; - } - Topic topicObject = optionalTopic.get(); - if (topicObject instanceof PersistentTopic) { - asyncResponse.resume(((PersistentTopic) topicObject).getTransactionBufferStats()); - } else { - asyncResponse.resume(new RestException(BAD_REQUEST, "Topic is not a persistent topic!")); - } - }); - } else { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, "Topic is not owned by this broker!")); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, "Broker don't support transaction!")); - } + protected CompletableFuture internalGetTransactionBufferStats(boolean authoritative) { + return getExistingPersistentTopicAsync(authoritative) + .thenApply(topic -> topic.getTransactionBufferStats()); } - protected void internalGetPendingAckStats(AsyncResponse asyncResponse, boolean authoritative, String subName) { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - validateTopicOwnership(topicName, authoritative); - CompletableFuture> topicFuture = pulsar().getBrokerService() - .getTopics().get(topicName.toString()); - if (topicFuture != null) { - topicFuture.whenComplete((optionalTopic, e) -> { - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } - - if (!optionalTopic.isPresent()) { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, - "Topic is not owned by this broker!")); - return; - } - Topic topicObject = optionalTopic.get(); - if (topicObject instanceof PersistentTopic) { - asyncResponse.resume(((PersistentTopic) topicObject).getTransactionPendingAckStats(subName)); - } else { - asyncResponse.resume(new RestException(BAD_REQUEST, "Topic is not a persistent topic!")); - } - }); - } else { - asyncResponse.resume(new RestException(TEMPORARY_REDIRECT, "Topic is not owned by this broker!")); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, "Broker don't support transaction!")); - } + protected CompletableFuture internalGetPendingAckStats( + boolean authoritative, String subName) { + return getExistingPersistentTopicAsync(authoritative) + .thenApply(topic -> topic.getTransactionPendingAckStats(subName)); } protected void internalGetTransactionMetadata(AsyncResponse asyncResponse, boolean authoritative, int mostSigBits, long leastSigBits) { try { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - validateTopicOwnership(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(mostSigBits), - authoritative); - CompletableFuture transactionMetadataFuture = new CompletableFuture<>(); - TxnMeta txnMeta = pulsar().getTransactionMetadataStoreService() - .getTxnMeta(new TxnID(mostSigBits, leastSigBits)).get(); - getTransactionMetadata(txnMeta, transactionMetadataFuture); - asyncResponse.resume(transactionMetadataFuture.get(10, TimeUnit.SECONDS)); - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); - } + validateTopicOwnership(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(mostSigBits), + authoritative); + CompletableFuture transactionMetadataFuture = new CompletableFuture<>(); + TxnMeta txnMeta = pulsar().getTransactionMetadataStoreService() + .getTxnMeta(new TxnID(mostSigBits, leastSigBits)).get(); + getTransactionMetadata(txnMeta, transactionMetadataFuture); + asyncResponse.resume(transactionMetadataFuture.get(10, TimeUnit.SECONDS)); } catch (Exception e) { if (e instanceof ExecutionException) { if (e.getCause() instanceof CoordinatorNotFoundException @@ -378,91 +259,87 @@ private void getTransactionMetadata(TxnMeta txnMeta, protected void internalGetSlowTransactions(AsyncResponse asyncResponse, boolean authoritative, long timeout, Integer coordinatorId) { try { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - if (coordinatorId != null) { - validateTopicOwnership(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(coordinatorId), - authoritative); - TransactionMetadataStore transactionMetadataStore = - pulsar().getTransactionMetadataStoreService().getStores() - .get(TransactionCoordinatorID.get(coordinatorId)); - if (transactionMetadataStore == null) { - asyncResponse.resume(new RestException(NOT_FOUND, - "Transaction coordinator not found! coordinator id : " + coordinatorId)); + if (coordinatorId != null) { + validateTopicOwnership(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(coordinatorId), + authoritative); + TransactionMetadataStore transactionMetadataStore = + pulsar().getTransactionMetadataStoreService().getStores() + .get(TransactionCoordinatorID.get(coordinatorId)); + if (transactionMetadataStore == null) { + asyncResponse.resume(new RestException(NOT_FOUND, + "Transaction coordinator not found! coordinator id : " + coordinatorId)); + return; + } + List transactions = transactionMetadataStore.getSlowTransactions(timeout); + List> completableFutures = new ArrayList<>(); + for (TxnMeta txnMeta : transactions) { + CompletableFuture completableFuture = new CompletableFuture<>(); + getTransactionMetadata(txnMeta, completableFuture); + completableFutures.add(completableFuture); + } + + FutureUtil.waitForAll(completableFutures).whenComplete((v, e) -> { + if (e != null) { + asyncResponse.resume(new RestException(e.getCause())); return; } - List transactions = transactionMetadataStore.getSlowTransactions(timeout); - List> completableFutures = new ArrayList<>(); - for (TxnMeta txnMeta : transactions) { - CompletableFuture completableFuture = new CompletableFuture<>(); - getTransactionMetadata(txnMeta, completableFuture); - completableFutures.add(completableFuture); - } - FutureUtil.waitForAll(completableFutures).whenComplete((v, e) -> { + Map transactionMetadata = new HashMap<>(); + for (CompletableFuture future : completableFutures) { + try { + transactionMetadata.put(future.get().txnId, future.get()); + } catch (Exception exception) { + asyncResponse.resume(new RestException(exception.getCause())); + return; + } + } + asyncResponse.resume(transactionMetadata); + }); + } else { + getPartitionedTopicMetadataAsync(TopicName.TRANSACTION_COORDINATOR_ASSIGN, + false, false).thenAccept(partitionMetadata -> { + if (partitionMetadata.partitions == 0) { + asyncResponse.resume(new RestException(Response.Status.NOT_FOUND, + "Transaction coordinator not found")); + return; + } + List>> completableFutures = + Lists.newArrayList(); + for (int i = 0; i < partitionMetadata.partitions; i++) { + try { + completableFutures + .add(pulsar().getAdminClient().transactions() + .getSlowTransactionsByCoordinatorIdAsync(i, timeout, + TimeUnit.MILLISECONDS)); + } catch (PulsarServerException e) { + asyncResponse.resume(new RestException(e)); + return; + } + } + Map transactionMetadataMaps = new HashMap<>(); + FutureUtil.waitForAll(completableFutures).whenComplete((result, e) -> { if (e != null) { - asyncResponse.resume(new RestException(e.getCause())); + asyncResponse.resume(new RestException(e)); return; } - Map transactionMetadata = new HashMap<>(); - for (CompletableFuture future : completableFutures) { + for (CompletableFuture> transactionMetadataMap + : completableFutures) { try { - transactionMetadata.put(future.get().txnId, future.get()); + transactionMetadataMaps.putAll(transactionMetadataMap.get()); } catch (Exception exception) { asyncResponse.resume(new RestException(exception.getCause())); return; } } - asyncResponse.resume(transactionMetadata); - }); - } else { - getPartitionedTopicMetadataAsync(TopicName.TRANSACTION_COORDINATOR_ASSIGN, - false, false).thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions == 0) { - asyncResponse.resume(new RestException(Response.Status.NOT_FOUND, - "Transaction coordinator not found")); - return; - } - List>> completableFutures = - Lists.newArrayList(); - for (int i = 0; i < partitionMetadata.partitions; i++) { - try { - completableFutures - .add(pulsar().getAdminClient().transactions() - .getSlowTransactionsByCoordinatorIdAsync(i, timeout, - TimeUnit.MILLISECONDS)); - } catch (PulsarServerException e) { - asyncResponse.resume(new RestException(e)); - return; - } - } - Map transactionMetadataMaps = new HashMap<>(); - FutureUtil.waitForAll(completableFutures).whenComplete((result, e) -> { - if (e != null) { - asyncResponse.resume(new RestException(e)); - return; - } - - for (CompletableFuture> transactionMetadataMap - : completableFutures) { - try { - transactionMetadataMaps.putAll(transactionMetadataMap.get()); - } catch (Exception exception) { - asyncResponse.resume(new RestException(exception.getCause())); - return; - } - } - asyncResponse.resume(transactionMetadataMaps); - }); - }).exceptionally(ex -> { - log.error("[{}] Failed to get transaction coordinator state.", clientAppId(), ex); - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; + asyncResponse.resume(transactionMetadataMaps); }); + }).exceptionally(ex -> { + log.error("[{}] Failed to get transaction coordinator state.", clientAppId(), ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); - } - } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, "Broker don't support transaction!")); } } catch (Exception e) { asyncResponse.resume(new RestException(e)); @@ -472,33 +349,28 @@ protected void internalGetSlowTransactions(AsyncResponse asyncResponse, protected void internalGetCoordinatorInternalStats(AsyncResponse asyncResponse, boolean authoritative, boolean metadata, int coordinatorId) { try { - if (pulsar().getConfig().isTransactionCoordinatorEnabled()) { - TopicName topicName = TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(coordinatorId); - validateTopicOwnership(topicName, authoritative); - TransactionMetadataStore metadataStore = pulsar().getTransactionMetadataStoreService() - .getStores().get(TransactionCoordinatorID.get(coordinatorId)); - if (metadataStore == null) { - asyncResponse.resume(new RestException(NOT_FOUND, - "Transaction coordinator not found! coordinator id : " + coordinatorId)); - return; - } - if (metadataStore instanceof MLTransactionMetadataStore) { - ManagedLedger managedLedger = ((MLTransactionMetadataStore) metadataStore).getManagedLedger(); - TransactionCoordinatorInternalStats transactionCoordinatorInternalStats = - new TransactionCoordinatorInternalStats(); - TransactionLogStats transactionLogStats = new TransactionLogStats(); - transactionLogStats.managedLedgerName = managedLedger.getName(); - transactionLogStats.managedLedgerInternalStats = - managedLedger.getManagedLedgerInternalStats(metadata).get(); - transactionCoordinatorInternalStats.transactionLogStats = transactionLogStats; - asyncResponse.resume(transactionCoordinatorInternalStats); - } else { - asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, - "Broker don't use MLTransactionMetadataStore!")); - } + TopicName topicName = TopicName.TRANSACTION_COORDINATOR_ASSIGN.getPartition(coordinatorId); + validateTopicOwnership(topicName, authoritative); + TransactionMetadataStore metadataStore = pulsar().getTransactionMetadataStoreService() + .getStores().get(TransactionCoordinatorID.get(coordinatorId)); + if (metadataStore == null) { + asyncResponse.resume(new RestException(NOT_FOUND, + "Transaction coordinator not found! coordinator id : " + coordinatorId)); + return; + } + if (metadataStore instanceof MLTransactionMetadataStore) { + ManagedLedger managedLedger = ((MLTransactionMetadataStore) metadataStore).getManagedLedger(); + TransactionCoordinatorInternalStats transactionCoordinatorInternalStats = + new TransactionCoordinatorInternalStats(); + TransactionLogStats transactionLogStats = new TransactionLogStats(); + transactionLogStats.managedLedgerName = managedLedger.getName(); + transactionLogStats.managedLedgerInternalStats = + managedLedger.getManagedLedgerInternalStats(metadata).get(); + transactionCoordinatorInternalStats.transactionLogStats = transactionLogStats; + asyncResponse.resume(transactionCoordinatorInternalStats); } else { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); + asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, + "Broker don't use MLTransactionMetadataStore!")); } } catch (Exception e) { asyncResponse.resume(new RestException(e.getCause())); @@ -506,40 +378,46 @@ protected void internalGetCoordinatorInternalStats(AsyncResponse asyncResponse, } protected CompletableFuture internalGetPendingAckInternalStats( - boolean authoritative, TopicName topicName, String subName, boolean metadata) { + boolean authoritative, String subName, boolean metadata) { + return getExistingPersistentTopicAsync(authoritative) + .thenCompose(topic -> topic.getPendingAckManagedLedger(subName)) + .thenCompose(managedLedger -> + managedLedger.getManagedLedgerInternalStats(metadata) + .thenApply(internalStats -> { + TransactionLogStats pendingAckLogStats = new TransactionLogStats(); + pendingAckLogStats.managedLedgerName = managedLedger.getName(); + pendingAckLogStats.managedLedgerInternalStats = internalStats; + return pendingAckLogStats; + }) + .thenApply(pendingAckLogStats -> { + TransactionPendingAckInternalStats stats = new TransactionPendingAckInternalStats(); + stats.pendingAckLogStats = pendingAckLogStats; + return stats; + }) + ); + } + + protected CompletableFuture getExistingPersistentTopicAsync(boolean authoritative) { + return validateTopicOwnershipAsync(topicName, authoritative).thenCompose(__ -> { + CompletableFuture> topicFuture = pulsar().getBrokerService() + .getTopics().get(topicName.toString()); + if (topicFuture == null) { + return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); + } + return topicFuture.thenCompose(optionalTopic -> { + if (!optionalTopic.isPresent()) { + return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); + } + return CompletableFuture.completedFuture((PersistentTopic) optionalTopic.get()); + }); + }); + } + + protected void checkTransactionCoordinatorEnabled() { if (!pulsar().getConfig().isTransactionCoordinatorEnabled()) { - return FutureUtil.failedFuture(new RestException(SERVICE_UNAVAILABLE, - "This Broker is not configured with transactionCoordinatorEnabled=true.")); + throw new RestException(SERVICE_UNAVAILABLE, + "This Broker is not configured with transactionCoordinatorEnabled=true."); } - return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> { - CompletableFuture> topicFuture = pulsar().getBrokerService() - .getTopics().get(topicName.toString()); - if (topicFuture == null) { - return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); - } - return topicFuture.thenCompose(optionalTopic -> { - if (!optionalTopic.isPresent()) { - return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); - } else { - Topic topicObject = optionalTopic.get(); - return ((PersistentTopic) topicObject).getPendingAckManagedLedger(subName) - .thenCompose(managedLedger -> managedLedger.getManagedLedgerInternalStats(metadata) - .thenApply(internalStats -> { - TransactionLogStats pendingAckLogStats = new TransactionLogStats(); - pendingAckLogStats.managedLedgerName = managedLedger.getName(); - pendingAckLogStats.managedLedgerInternalStats = internalStats; - return pendingAckLogStats; - }) - .thenApply(pendingAckLogStats -> { - TransactionPendingAckInternalStats stats = - new TransactionPendingAckInternalStats(); - stats.pendingAckLogStats = pendingAckLogStats; - return stats; - })); - } - }); - }); } protected void validateTopicName(String property, String namespace, String encodedTopic) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java index bbd79036ecf71..9cb825b9f8e1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java @@ -61,6 +61,7 @@ public void getCoordinatorStats(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("coordinatorId") Integer coordinatorId) { + checkTransactionCoordinatorEnabled(); internalGetCoordinatorStats(asyncResponse, authoritative, coordinatorId); } @@ -82,9 +83,19 @@ public void getTransactionInBufferStats(@Suspended final AsyncResponse asyncResp @PathParam("topic") @Encoded String encodedTopic, @PathParam("mostSigBits") String mostSigBits, @PathParam("leastSigBits") String leastSigBits) { - validateTopicName(tenant, namespace, encodedTopic); - internalGetTransactionInBufferStats(asyncResponse, authoritative, - Long.parseLong(mostSigBits), Long.parseLong(leastSigBits)); + try { + checkTransactionCoordinatorEnabled(); + validateTopicName(tenant, namespace, encodedTopic); + internalGetTransactionInBufferStats(authoritative, Long.parseLong(mostSigBits), + Long.parseLong(leastSigBits)) + .thenAccept(stat -> asyncResponse.resume(stat)) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } } @GET @@ -106,9 +117,19 @@ public void getTransactionInPendingAckStats(@Suspended final AsyncResponse async @PathParam("mostSigBits") String mostSigBits, @PathParam("leastSigBits") String leastSigBits, @PathParam("subName") String subName) { - validateTopicName(tenant, namespace, encodedTopic); - internalGetTransactionInPendingAckStats(asyncResponse, authoritative, Long.parseLong(mostSigBits), - Long.parseLong(leastSigBits), subName); + try { + checkTransactionCoordinatorEnabled(); + validateTopicName(tenant, namespace, encodedTopic); + internalGetTransactionInPendingAckStats(authoritative, Long.parseLong(mostSigBits), + Long.parseLong(leastSigBits), subName) + .thenAccept(stat -> asyncResponse.resume(stat)) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } } @GET @@ -127,8 +148,18 @@ public void getTransactionBufferStats(@Suspended final AsyncResponse asyncRespon @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic) { - validateTopicName(tenant, namespace, encodedTopic); - internalGetTransactionBufferStats(asyncResponse, authoritative); + try { + checkTransactionCoordinatorEnabled(); + validateTopicName(tenant, namespace, encodedTopic); + internalGetTransactionBufferStats(authoritative) + .thenAccept(stat -> asyncResponse.resume(stat)) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } } @GET @@ -148,8 +179,18 @@ public void getPendingAckStats(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @PathParam("subName") String subName) { - validateTopicName(tenant, namespace, encodedTopic); - internalGetPendingAckStats(asyncResponse, authoritative, subName); + try { + checkTransactionCoordinatorEnabled(); + validateTopicName(tenant, namespace, encodedTopic); + internalGetPendingAckStats(authoritative, subName) + .thenAccept(stats -> asyncResponse.resume(stats)) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } } @GET @@ -168,6 +209,7 @@ public void getTransactionMetadata(@Suspended final AsyncResponse asyncResponse, @DefaultValue("false") boolean authoritative, @PathParam("mostSigBits") String mostSigBits, @PathParam("leastSigBits") String leastSigBits) { + checkTransactionCoordinatorEnabled(); internalGetTransactionMetadata(asyncResponse, authoritative, Integer.parseInt(mostSigBits), Long.parseLong(leastSigBits)); } @@ -188,6 +230,7 @@ public void getSlowTransactions(@Suspended final AsyncResponse asyncResponse, @DefaultValue("false") boolean authoritative, @PathParam("timeout") String timeout, @QueryParam("coordinatorId") Integer coordinatorId) { + checkTransactionCoordinatorEnabled(); internalGetSlowTransactions(asyncResponse, authoritative, Long.parseLong(timeout), coordinatorId); } @@ -205,6 +248,7 @@ public void getCoordinatorInternalStats(@Suspended final AsyncResponse asyncResp @DefaultValue("false") boolean authoritative, @PathParam("coordinatorId") String coordinatorId, @QueryParam("metadata") @DefaultValue("false") boolean metadata) { + checkTransactionCoordinatorEnabled(); internalGetCoordinatorInternalStats(asyncResponse, authoritative, metadata, Integer.parseInt(coordinatorId)); } @@ -229,8 +273,9 @@ public void getPendingAckInternalStats(@Suspended final AsyncResponse asyncRespo @PathParam("subName") String subName, @QueryParam("metadata") @DefaultValue("false") boolean metadata) { try { + checkTransactionCoordinatorEnabled(); validateTopicName(tenant, namespace, encodedTopic); - internalGetPendingAckInternalStats(authoritative, topicName, subName, metadata) + internalGetPendingAckInternalStats(authoritative, subName, metadata) .thenAccept(stats -> asyncResponse.resume(stats)) .exceptionally(ex -> { Throwable cause = FutureUtil.unwrapCompletionException(ex); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 1f796553bdcad..6b72916f41e37 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -20,6 +20,7 @@ import com.google.common.collect.Sets; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.http.HttpStatus; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -122,6 +123,25 @@ public void testGetTransactionInBufferStats() throws Exception { initTransaction(2); TransactionImpl transaction = (TransactionImpl) getTransaction(); final String topic = "persistent://public/default/testGetTransactionInBufferStats"; + try { + admin.transactions() + .getTransactionInBufferStatsAsync(new TxnID(1, 1), topic).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } + try { + pulsar.getBrokerService().getTopic(topic, false); + admin.transactions() + .getTransactionInBufferStatsAsync(new TxnID(1, 1), topic).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic).sendTimeout(0, TimeUnit.SECONDS).create(); MessageId messageId = producer.newMessage(transaction).value("Hello pulsar!".getBytes()).send(); @@ -147,6 +167,27 @@ public void testGetTransactionPendingAckStats() throws Exception { initTransaction(2); final String topic = "persistent://public/default/testGetTransactionInBufferStats"; final String subName = "test"; + try { + admin.transactions() + .getTransactionInPendingAckStatsAsync(new TxnID(1, + 2), topic, subName).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } + try { + pulsar.getBrokerService().getTopic(topic, false); + admin.transactions() + .getTransactionInPendingAckStatsAsync(new TxnID(1, + 2), topic, subName).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic).create(); Consumer consumer = pulsarClient.newConsumer(Schema.BYTES).topic(topic) @@ -253,8 +294,26 @@ public void testGetTransactionBufferStats() throws Exception { final String topic = "persistent://public/default/testGetTransactionBufferStats"; final String subName1 = "test1"; final String subName2 = "test2"; + try { + admin.transactions() + .getTransactionBufferStatsAsync(topic).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } + try { + pulsar.getBrokerService().getTopic(topic, false); + admin.transactions() + .getTransactionBufferStatsAsync(topic).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } admin.topics().createNonPartitionedTopic(topic); - Producer producer = pulsarClient.newProducer(Schema.BYTES) .sendTimeout(0, TimeUnit.SECONDS).topic(topic).create(); Consumer consumer1 = pulsarClient.newConsumer(Schema.BYTES).topic(topic) @@ -290,6 +349,25 @@ public void testGetPendingAckStats(String ackType) throws Exception { initTransaction(2); final String topic = "persistent://public/default/testGetPendingAckStats"; final String subName = "test1"; + try { + admin.transactions() + .getPendingAckStatsAsync(topic, subName).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } + try { + pulsar.getBrokerService().getTopic(topic, false); + admin.transactions() + .getPendingAckStatsAsync(topic, subName).get(); + fail("Should failed here"); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); + PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); + assertEquals(cause.getMessage(), "Topic not found"); + } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES) @@ -430,6 +508,18 @@ public void testGetPendingAckInternalStats() throws Exception { assertNull(managedLedgerInternalStats.ledgers.get(0).metadata); } + @Test(timeOut = 20000) + public void testTransactionNotEnabled() throws Exception { + stopBroker(); + conf.setTransactionCoordinatorEnabled(false); + super.internalSetup(); + try { + admin.transactions().getCoordinatorInternalStats(1, false); + } catch (PulsarAdminException ex) { + assertEquals(ex.getStatusCode(), HttpStatus.SC_SERVICE_UNAVAILABLE); + } + } + private static void verifyCoordinatorStats(String state, long sequenceId, long lowWaterMark) { assertEquals(state, "Ready"); From e5546279ed07ebcf2650375f2c6c791ed439778a Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 8 Apr 2022 10:07:36 +0800 Subject: [PATCH 475/823] handle NPE when getLeader returns null (#15058) (cherry picked from commit 2e2cd4480d6ee7c6183826686a93a6e155ac3b7b) --- .../apache/pulsar/functions/worker/LeaderService.java | 3 ++- .../pulsar/functions/worker/PulsarWorkerService.java | 4 +++- .../apache/pulsar/functions/worker/WorkerUtils.java | 11 ++++++++++- .../functions/worker/rest/api/FunctionsImpl.java | 2 +- .../pulsar/functions/worker/rest/api/WorkerImpl.java | 9 ++++++++- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java index fb11fab785b8c..645cdcb2bae96 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java @@ -103,7 +103,8 @@ public void becameActive(Consumer consumer, int partitionId) { // attempt to acquire exclusive publishers to both the metadata topic and assignments topic // we should keep trying to acquire exclusive producers as long as we are still the leader - Supplier checkIsStillLeader = () -> membershipManager.getLeader().getWorkerId().equals(workerConfig.getWorkerId()); + Supplier checkIsStillLeader = WorkerUtils.getIsStillLeaderSupplier(membershipManager, + workerConfig.getWorkerId()); Producer scheduleManagerExclusiveProducer = null; Producer functionMetaDataManagerExclusiveProducer = null; try { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index 3056a0dc64baa..0f5c402fbceed 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -503,7 +503,9 @@ public void start(AuthenticationService authenticationService, log.info("/** Initializing Runtime Manager **/"); MessageId lastAssignmentMessageId = functionRuntimeManager.initialize(); - Supplier checkIsStillLeader = () -> membershipManager.getLeader().getWorkerId().equals(workerConfig.getWorkerId()); + + Supplier checkIsStillLeader = WorkerUtils.getIsStillLeaderSupplier(membershipManager, + workerConfig.getWorkerId()); // Setting references to managers in scheduler schedulerManager.setFunctionMetaDataManager(functionMetaDataManager); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java index 4f0287f19ffc7..4f3ee0428666c 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java @@ -41,8 +41,9 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.common.conf.InternalConfigurationData; -import org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl; +import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl; +import org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.runtime.Runtime; import org.apache.pulsar.functions.runtime.RuntimeSpawner; @@ -368,4 +369,12 @@ public static Producer createExclusiveProducerWithRetry(PulsarClient cli public static class NotLeaderAnymore extends Exception { } + + public static Supplier getIsStillLeaderSupplier(final MembershipManager membershipManager, + final String workerId) { + return () -> { + WorkerInfo workerInfo = membershipManager.getLeader(); + return workerInfo != null && workerInfo.getWorkerId().equals(workerId); + }; + } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 0e3a28a07576c..506ca452857a5 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -702,7 +702,7 @@ public void updateFunctionOnWorkerLeader(final String tenant, // Redirect if we are not the leader if (!worker().getLeaderService().isLeader()) { WorkerInfo workerInfo = worker().getMembershipManager().getLeader(); - if (workerInfo.getWorkerId().equals(worker().getWorkerConfig().getWorkerId())) { + if (workerInfo == null || workerInfo.getWorkerId().equals(worker().getWorkerConfig().getWorkerId())) { throw new RestException(Response.Status.SERVICE_UNAVAILABLE, "Leader not yet ready. Please retry again"); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java index 6c2180cee7076..8255061d2b420 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java @@ -230,7 +230,11 @@ public void rebalance(final URI uri, final String clientRole) { } } else { WorkerInfo workerInfo = worker().getMembershipManager().getLeader(); - URI redirect = UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).build(); + if (workerInfo == null) { + throw new RestException(Status.INTERNAL_SERVER_ERROR, "Leader cannot be determined"); + } + URI redirect = + UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).build(); throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); } } @@ -337,6 +341,9 @@ private URI buildRedirectUriForDrainRelatedOp(final URI uri, String workerId) { // Use the leader-URI path in both cases for the redirect to the leader. String leaderPath = "admin/v2/worker/leader/drain"; WorkerInfo workerInfo = worker().getMembershipManager().getLeader(); + if (workerInfo == null) { + throw new RestException(Status.INTERNAL_SERVER_ERROR, "Leader cannot be determined"); + } URI redirect = UriBuilder.fromUri(uri) .host(workerInfo.getWorkerHostname()) .port(workerInfo.getPort()) From cfe2d2c2be7e78b6b67cc4a4d198d96470c8d2fa Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 8 Apr 2022 11:40:25 +0800 Subject: [PATCH 476/823] Fix when nextValidLedger is null cause npe (#13975) (cherry picked from commit 5cf3fa0d8050b16305ee060820b884d69ec8a828) --- .../org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index e2f271a3eed29..ee9474a0019e9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1738,7 +1738,8 @@ public void asyncMarkDelete(final Position position, Map propertie try { long ledgerEntries = ledger.getLedgerInfo(markDeletePosition.getLedgerId()).get().getEntries(); Long nextValidLedger = ledger.getNextValidLedger(ledger.getLastConfirmedEntry().getLedgerId()); - shouldCursorMoveForward = (markDeletePosition.getEntryId() + 1 >= ledgerEntries) + shouldCursorMoveForward = nextValidLedger != null + && (markDeletePosition.getEntryId() + 1 >= ledgerEntries) && (newPosition.getLedgerId() == nextValidLedger); } catch (Exception e) { log.warn("Failed to get ledger entries while setting mark-delete-position", e); From d13baa7eb5859731d067dbeb24df2fb33dd8e3d0 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:59:50 +0800 Subject: [PATCH 477/823] [fix][Transaction] Fix transaction admin redirect get 500 due to getCause (#14965) ### Motivation Transaction admin `getCoordinatorInternalStats` add a getCause when try catch. Which make redirect fail. * Exception is thrown by `validateTopicOwnership`, sync() call get() in `sync(()-> validateTopicOwnershipAsync(topicName, authoritative));` * Exception already got cause by in `validateTopicOwnershipAsync` ### Modifications Delete get cause if not ExecutionException (cherry picked from commit 74df947894548ba6e610a71ec03a9d87bce59f35) --- .../broker/admin/impl/TransactionsBase.java | 2 +- .../AdminApiTransactionMultiBrokerTest.java | 67 +++++++++++++++++++ .../admin/v3/AdminApiTransactionTest.java | 1 + 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index b225cd2e2661a..8eff6815404cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -373,7 +373,7 @@ protected void internalGetCoordinatorInternalStats(AsyncResponse asyncResponse, "Broker don't use MLTransactionMetadataStore!")); } } catch (Exception e) { - asyncResponse.resume(new RestException(e.getCause())); + resumeAsyncResponseExceptionally(asyncResponse, e); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java new file mode 100644 index 0000000000000..add277fc524d7 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin.v3; + +import java.util.Map; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-admin") +public class AdminApiTransactionMultiBrokerTest extends TransactionTestBase { + + private static final int NUM_BROKERS = 16; + private static final int NUM_PARTITIONS = 3; + + @BeforeMethod + protected void setup() throws Exception { + setUpBase(NUM_BROKERS, NUM_PARTITIONS, NAMESPACE1 + "/test", 0); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testRedirectOfGetCoordinatorInternalStats() throws Exception { + Map map = admin.lookups() + .lookupPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()); + while (map.containsValue(getPulsarServiceList().get(0).getBrokerServiceUrl())) { + admin.topics().deletePartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), NUM_PARTITIONS); + map = admin.lookups().lookupPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString()); + } + //init tc stores + pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(true) + .build(); + for (int i = 0; i < NUM_PARTITIONS; i++) { + admin.transactions().getCoordinatorInternalStats(i, false); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 6b72916f41e37..ede74be63d868 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -67,6 +67,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +@Test(groups = "broker-admin") public class AdminApiTransactionTest extends MockedPulsarServiceBaseTest { @BeforeMethod From 944d9b8a45b47385e130aa768d0eb5eaa19a5841 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Fri, 8 Apr 2022 16:01:11 +0800 Subject: [PATCH 478/823] [fix][broker] Fix rewind failed when ``redeliverUnacknowledgedMessages`` (#15046) According to many PRs flaky-test ``SimpleProducerConsumerTest#testRedeliveryFailOverConsumer``, the broker logs as below. ``` 2022-04-07T11:19:16,561+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/0 epoch 0 2022-04-07T11:19:16,562+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:0:-1 epoch 0 2022-04-07T11:19:16,562+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/1 epoch 0 2022-04-07T11:19:16,562+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:1:-1 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/2 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/3 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/4 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/5 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/6 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/7 epoch 0 2022-04-07T11:19:16,576+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/8 epoch 0 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:2:-1 epoch 0 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:3:-1 epoch 0 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:4:-1 epoch 0 2022-04-07T11:19:16,577+0800 [main] INFO org.apache.pulsar.client.api.SimpleProducerConsumerTest - ----Consumer receive and ack message 3/0 epoch 0 2022-04-07T11:19:16,577+0800 [main] INFO org.apache.pulsar.client.api.SimpleProducerConsumerTest - ----Consumer receive and ack message 3/1 epoch 0 2022-04-07T11:19:16,577+0800 [main] INFO org.apache.pulsar.client.api.SimpleProducerConsumerTest - ----Consumer receive and ack message 3/2 epoch 0 2022-04-07T11:19:16,577+0800 [main] INFO org.apache.pulsar.client.api.SimpleProducerConsumerTest - ----Consumer receive and ack message 3/3 epoch 0 2022-04-07T11:19:16,577+0800 [main] INFO org.apache.pulsar.client.api.SimpleProducerConsumerTest - ----Trigger unack messages redeliver and clear receive queue---- 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:5:-1 epoch 0 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:6:-1 epoch 0 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:7:-1 epoch 0 2022-04-07T11:19:16,577+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:8:-1 epoch 0 2022-04-07T11:19:16,578+0800 [broker-topic-workers-OrderedScheduler-8-0] INFO org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer - [persistent://my-property/my-ns/unacked-topic / subscriber-1-Consumer{subscription=PersistentSubscription{topic=persistent://my-property/my-ns/unacked-topic, name=subscriber-1}, consumerId=0, consumerName=dde4a, address=/127.0.0.1:49153}] Ignoring reDeliverUnAcknowledgedMessages: cancelPendingRequest on cursor failed 2022-04-07T11:19:16,578+0800 [pulsar-io-308-1] INFO org.apache.pulsar.broker.service.PulsarCommandSenderImpl - Send message to consumer, message id 3/9 epoch 0 2022-04-07T11:19:16,578+0800 [pulsar-client-io-336-1] INFO org.apache.pulsar.client.impl.ConsumerImpl - receiver queue received message, message id 3:9:-1 epoch 0 2022-04-07T11:19:17,580+0800 [main] WARN org.apache.pulsar.client.impl.ConsumerBase - Consumer filter old epoch message, topic : [persistent://my-property/my-ns/unacked-topic], messageId : [3:5:-1:0], messageConsumerEpoch : [0], consumerEpoch : [1] 2022-04-07T11:19:17,580+0800 [main] WARN org.apache.pulsar.client.impl.ConsumerBase - Consumer filter old epoch message, topic : [persistent://my-property/my-ns/unacked-topic], messageId : [3:6:-1:0], messageConsumerEpoch : [0], consumerEpoch : [1] 2022-04-07T11:19:17,580+0800 [main] WARN org.apache.pulsar.client.impl.ConsumerBase - Consumer filter old epoch message, topic : [persistent://my-property/my-ns/unacked-topic], messageId : [3:7:-1:0], messageConsumerEpoch : [0], consumerEpoch : [1] 2022-04-07T11:19:17,580+0800 [main] WARN org.apache.pulsar.client.impl.ConsumerBase - Consumer filter old epoch message, topic : [persistent://my-property/my-ns/unacked-topic], messageId : [3:8:-1:0], messageConsumerEpoch : [0], consumerEpoch : [1] 2022-04-07T11:19:17,580+0800 [main] WARN org.apache.pulsar.client.impl.ConsumerBase - Consumer filter old epoch message, topic : [persistent://my-property/my-ns/unacked-topic], messageId : [3:9:-1:0], messageConsumerEpoch : [0], consumerEpoch : [1] ``` In normal logic, the consumer will continue to receive messages from the broker. When the consumer triggers `redeliverUnacknowledgedMessages`, it will immediately increase the consumer epoch, and the request will rewind the broker cursor to trigger the broker to redeliver these messages with the new epoch. And then, the consumer will filter old messages by epoch. But in this case, we will find the following abnormal log, which means that the cursor pending read has been cancelled(cursor.cancelPendingReadRequest() return false), and then havePendingRead is not set to false. causing no cursor rewind at the end and didn't trigger the Dispatcher to re-push the messages. > Ignoring reDeliverUnAcknowledgedMessages: cancelPendingRequest on cursor failed Due to this abnormal behaviour causing the consumer will never receive redelivered messages with epoch 1 as below. ``` message id 3:4:-1 epoch 1 message id 3:5:-1 epoch 1 message id 3:6:-1 epoch 1 message id 3:7:-1 epoch 1 message id 3:8:-1 epoch 1 message id 3:9:-1 epoch 1 ``` Relative code: https://github.com/apache/pulsar/blob/81da8d3cd199fd6c1e4510a1c1c2ac71418efd5e/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java#L317-L328 https://github.com/apache/pulsar/blob/81da8d3cd199fd6c1e4510a1c1c2ac71418efd5e/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java#L136-L141 https://github.com/apache/pulsar/blob/81da8d3cd199fd6c1e4510a1c1c2ac71418efd5e/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java#L847-L851 - Force to rewind cursor when `redeliverUnacknowledgedMessages` (cherry picked from commit 22d6beb15b9d44e630e46931f5905d19f33fe724) --- ...sistentDispatcherSingleActiveConsumer.java | 19 ++++++------------- .../api/SimpleProducerConsumerTest.java | 6 +++--- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 6900a54e35c2c..8d0adcf0ddb41 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -295,20 +295,13 @@ private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consu name, consumer); return; } - - cancelPendingRead(); - - if (!havePendingRead) { - cursor.rewind(); - if (log.isDebugEnabled()) { - log.debug("[{}-{}] Cursor rewinded, redelivering unacknowledged messages. ", name, consumer); - } - readMoreEntries(consumer); - } else { - log.info("[{}-{}] Ignoring reDeliverUnAcknowledgedMessages: cancelPendingRequest on cursor failed", name, - consumer); + cursor.cancelPendingReadRequest(); + havePendingRead = false; + cursor.rewind(); + if (log.isDebugEnabled()) { + log.debug("[{}-{}] Cursor rewinded, redelivering unacknowledged messages. ", name, consumer); } - + readMoreEntries(consumer); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index f5677d362737c..8a47c6fbb62c7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -2441,7 +2441,7 @@ public void testRedeliveryFailOverConsumer(boolean ackReceiptEnabled) throws Exc Message msg; List> messages1 = Lists.newArrayList(); for (int i = 0; i < consumeMsgInParts; i++) { - msg = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + msg = consumer.receive(); if (msg != null) { messages1.add(msg); consumer.acknowledge(msg); @@ -2457,7 +2457,7 @@ public void testRedeliveryFailOverConsumer(boolean ackReceiptEnabled) throws Exc // (1.b) consume second consumeMsgInParts msgs and trigger redeliver messages1.clear(); for (int i = 0; i < consumeMsgInParts; i++) { - msg = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + msg = consumer.receive(); if (msg != null) { messages1.add(msg); consumer.acknowledge(msg); @@ -2480,7 +2480,7 @@ public void testRedeliveryFailOverConsumer(boolean ackReceiptEnabled) throws Exc int remainingMsgs = (2 * receiverQueueSize) - (2 * consumeMsgInParts); messages1.clear(); for (int i = 0; i < remainingMsgs; i++) { - msg = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + msg = consumer.receive(); if (msg != null) { messages1.add(msg); consumer.acknowledge(msg); From 1df54b93be662228dc45f0180a938b1f22476e7a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 Apr 2022 18:53:03 +0800 Subject: [PATCH 479/823] [improve][transaction] Optimize topic lookup when TC end tx. (#14991) When TC ends tx, it has to look up the topic. The original way uses pulsar-client to do this. It will look up every topic. If using bundle cache to find the topic owner broker, it can avoid lookup every topic and then decrease the lookup time. - Using bundle cache to find the topic owner broker. If occurs an error, fall back to look up the topic. - Remove the topic cache, because using bundle cache, the original cache is useless. (cherry picked from commit 46baae6a5e89a5feed171d231b59042a2ce9e1f8) --- .../apache/pulsar/broker/PulsarService.java | 2 +- .../impl/TransactionBufferClientImpl.java | 9 +- .../impl/TransactionBufferHandlerImpl.java | 155 +++++++++--------- .../transaction/TransactionProduceTest.java | 29 ++++ .../transaction/TransactionTestBase.java | 2 +- .../buffer/TransactionBufferClientTest.java | 37 ++++- .../TransactionBufferHandlerImplTest.java | 41 +++-- .../pulsar/client/impl/PulsarClientImpl.java | 10 +- 8 files changed, 175 insertions(+), 110 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 6451e518b883a..60cd75501d0dc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -732,7 +732,7 @@ config, localMetadataStore, getZkClient(), this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient()); this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); - transactionBufferClient = TransactionBufferClientImpl.create(getClient(), transactionTimer, + transactionBufferClient = TransactionBufferClientImpl.create(this, transactionTimer, config.getTransactionBufferClientMaxConcurrentRequests(), config.getTransactionBufferClientOperationTimeoutInMills()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java index 060476e573c2a..c531f9f1871d6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java @@ -21,7 +21,8 @@ import io.netty.util.HashedWheelTimer; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.client.api.transaction.TransactionBufferClient; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.transaction.TransactionBufferHandler; @@ -39,9 +40,9 @@ private TransactionBufferClientImpl(TransactionBufferHandler tbHandler) { this.tbHandler = tbHandler; } - public static TransactionBufferClient create(PulsarClient pulsarClient, HashedWheelTimer timer, - int maxConcurrentRequests, long operationTimeoutInMills) { - TransactionBufferHandler handler = new TransactionBufferHandlerImpl(pulsarClient, timer, + public static TransactionBufferClient create(PulsarService pulsarService, HashedWheelTimer timer, + int maxConcurrentRequests, long operationTimeoutInMills) throws PulsarServerException { + TransactionBufferHandler handler = new TransactionBufferHandlerImpl(pulsarService, timer, maxConcurrentRequests, operationTimeoutInMills); return new TransactionBufferClientImpl(handler); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index b80a273bc6fcd..3f9a083787bb5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -18,21 +18,23 @@ */ package org.apache.pulsar.broker.transaction.buffer.impl; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import io.netty.buffer.ByteBuf; import io.netty.util.HashedWheelTimer; import io.netty.util.Recycler; import io.netty.util.ReferenceCountUtil; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionBufferClientException; import org.apache.pulsar.client.api.transaction.TxnID; @@ -42,7 +44,10 @@ import org.apache.pulsar.common.api.proto.CommandEndTxnOnPartitionResponse; import org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscriptionResponse; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; @Slf4j @@ -53,31 +58,17 @@ public class TransactionBufferHandlerImpl implements TransactionBufferHandler { private final AtomicLong requestIdGenerator = new AtomicLong(); private final long operationTimeoutInMills; private final HashedWheelTimer timer; - private final PulsarClient pulsarClient; + private final PulsarService pulsarService; + private final PulsarClientImpl pulsarClient; private static final AtomicIntegerFieldUpdater REQUEST_CREDITS_UPDATER = AtomicIntegerFieldUpdater.newUpdater(TransactionBufferHandlerImpl.class, "requestCredits"); private volatile int requestCredits; - private final LoadingCache> lookupCache = CacheBuilder.newBuilder() - .maximumSize(100000) - .expireAfterAccess(30, TimeUnit.MINUTES) - .build(new CacheLoader>() { - @Override - public CompletableFuture load(String topic) { - CompletableFuture siFuture = getClientCnx(topic); - siFuture.whenComplete((si, cause) -> { - if (null != cause) { - lookupCache.invalidate(topic); - } - }); - return siFuture; - } - }); - - public TransactionBufferHandlerImpl(PulsarClient pulsarClient, HashedWheelTimer timer, - int maxConcurrentRequests, long operationTimeoutInMills) { - this.pulsarClient = pulsarClient; + public TransactionBufferHandlerImpl(PulsarService pulsarService, HashedWheelTimer timer, + int maxConcurrentRequests, long operationTimeoutInMills) throws PulsarServerException { + this.pulsarService = pulsarService; + this.pulsarClient = (PulsarClientImpl) pulsarService.getClient(); this.outstandingRequests = new ConcurrentSkipListMap<>(); this.pendingRequests = new GrowableArrayBlockingQueue<>(); this.operationTimeoutInMills = operationTimeoutInMills; @@ -97,15 +88,9 @@ public CompletableFuture endTxnOnTopic(String topic, long txnIdMostBits, ByteBuf cmd = Commands.newEndTxnOnPartition(requestId, txnIdLeastBits, txnIdMostBits, topic, action, lowWaterMark); - try { - OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb, lookupCache.get(topic)); - if (checkRequestCredits(op)) { - endTxn(op); - } - } catch (ExecutionException e) { - log.error("[{}] failed to get client cnx from lookup cache", topic, e); - lookupCache.invalidate(topic); - cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage())); + OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb, getClientCnx(topic)); + if (checkRequestCredits(op)) { + endTxn(op); } return cb; } @@ -122,15 +107,9 @@ public CompletableFuture endTxnOnSubscription(String topic, String subscr long requestId = requestIdGenerator.getAndIncrement(); ByteBuf cmd = Commands.newEndTxnOnSubscription(requestId, txnIdLeastBits, txnIdMostBits, topic, subscription, action, lowWaterMark); - try { - OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb, lookupCache.get(topic)); - if (checkRequestCredits(op)) { - endTxn(op); - } - } catch (ExecutionException e) { - log.error("[{}] failed to get client cnx from lookup cache", topic, e); - lookupCache.invalidate(topic); - cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage())); + OpRequestSend op = OpRequestSend.create(requestId, topic, cmd, cb, getClientCnx(topic)); + if (checkRequestCredits(op)) { + endTxn(op); } return cb; } @@ -150,8 +129,8 @@ private boolean checkRequestCredits(OpRequestSend op) { } public void endTxn(OpRequestSend op) { - op.cnx.whenComplete((clientCnx, throwable) -> { - if (throwable == null) { + op.cnx.whenComplete((clientCnx, ex) -> { + if (ex == null) { if (clientCnx.ctx().channel().isActive()) { clientCnx.registerTransactionBufferHandler(TransactionBufferHandlerImpl.this); outstandingRequests.put(op.requestId, op); @@ -166,16 +145,19 @@ public void endTxn(OpRequestSend op) { op.cmd.retain(); clientCnx.ctx().writeAndFlush(op.cmd, clientCnx.ctx().voidPromise()); } else { - invalidateLookupCache(op); op.cb.completeExceptionally( new PulsarClientException.LookupException(op.topic + " endTxn channel is not active")); onResponse(op); } } else { - log.error("endTxn error topic: [{}]", op.topic, throwable); - invalidateLookupCache(op); - op.cb.completeExceptionally( - new PulsarClientException.LookupException(throwable.getMessage())); + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.error("endTxn error topic: [{}]", op.topic, cause); + if (cause instanceof PulsarClientException.BrokerMetadataException) { + op.cb.complete(null); + } else { + op.cb.completeExceptionally( + new PulsarClientException.LookupException(cause.getMessage())); + } onResponse(op); } }); @@ -202,12 +184,9 @@ public void handleEndTxnOnTopicResponse(long requestId, CommandEndTxnOnPartition log.error("[{}] Got end txn on topic response for request {} error {}", op.topic, response.getRequestId(), response.getError()); - invalidateLookupCache(op); op.cb.completeExceptionally(ClientCnx.getPulsarClientException(response.getError(), response.getMessage())); } - } catch (Exception e) { - log.error("[{}] Got exception when complete EndTxnOnTopic op for request {}", op.topic, e); } finally { onResponse(op); } @@ -235,12 +214,9 @@ public void handleEndTxnOnSubscriptionResponse(long requestId, } else { log.error("[{}] Got end txn on subscription response for request {} error {}", op.topic, response.getRequestId(), response.getError()); - invalidateLookupCache(op); op.cb.completeExceptionally(ClientCnx.getPulsarClientException(response.getError(), response.getMessage())); } - } catch (Exception e) { - log.error("[{}] Got exception when complete EndTxnOnSub op for request {}", op.topic, e); } finally { onResponse(op); } @@ -262,21 +238,14 @@ private void checkPendingRequests() { if (REQUEST_CREDITS_UPDATER.compareAndSet(this, permits, permits - 1)) { OpRequestSend polled = pendingRequests.poll(); if (polled != null) { - try { - if (polled.cnx != lookupCache.get(polled.topic)) { - OpRequestSend invalid = polled; - polled = OpRequestSend.create(invalid.requestId, invalid.topic, invalid.cmd, invalid.cb, - lookupCache.get(invalid.topic)); - invalid.recycle(); - } - endTxn(polled); - } catch (ExecutionException e) { - log.error("[{}] failed to get client cnx from lookup cache", polled.topic, e); - lookupCache.invalidate(polled.topic); - polled.cb.completeExceptionally(new PulsarClientException.LookupException( - e.getCause().getMessage())); - REQUEST_CREDITS_UPDATER.incrementAndGet(this); + CompletableFuture clientCnx = getClientCnx(polled.topic); + if (polled.cnx != clientCnx) { + OpRequestSend invalid = polled; + polled = OpRequestSend.create(invalid.requestId, invalid.topic, invalid.cmd, invalid.cb, + clientCnx); + invalid.recycle(); } + endTxn(polled); } else { REQUEST_CREDITS_UPDATER.incrementAndGet(this); } @@ -287,16 +256,6 @@ private void checkPendingRequests() { } } - private void invalidateLookupCache(OpRequestSend op) { - try { - if (lookupCache.get(op.topic) == op.cnx) { - lookupCache.invalidate(op.topic); - } - } catch (ExecutionException e) { - lookupCache.invalidate(op.topic); - } - } - public static final class OpRequestSend { long requestId; @@ -336,8 +295,42 @@ protected OpRequestSend newObject(Handle handle) { }; } + public CompletableFuture getClientCnxWithLookup(String topic) { + return pulsarClient.getConnection(topic); + } + public CompletableFuture getClientCnx(String topic) { - return ((PulsarClientImpl) pulsarClient).getConnection(topic); + NamespaceService namespaceService = pulsarService.getNamespaceService(); + CompletableFuture nsBundle = namespaceService.getBundleAsync(TopicName.get(topic)); + return nsBundle + .thenCompose(bundle -> namespaceService.getOwnerAsync(bundle)) + .thenCompose(data -> { + if (data.isPresent()) { + NamespaceEphemeralData ephemeralData = data.get(); + try { + if (!ephemeralData.isDisabled()) { + URI uri; + if (pulsarClient.getConfiguration().isUseTls()) { + uri = new URI(ephemeralData.getNativeUrlTls()); + } else { + uri = new URI(ephemeralData.getNativeUrl()); + } + InetSocketAddress brokerAddress = + InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()); + return pulsarClient.getConnection(brokerAddress, brokerAddress); + } else { + // Bundle is unloading, lookup topic + return getClientCnxWithLookup(topic); + } + } catch (URISyntaxException e) { + // Should never go here + return getClientCnxWithLookup(topic); + } + } else { + // Bundle is not loaded yet, lookup topic + return getClientCnxWithLookup(topic); + } + }); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index cbae03b1a8b94..350dfa8c3d061 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -89,6 +89,35 @@ public void produceAndCommitTest() throws Exception { produceTest(true); } + @Test + public void testDeleteNamespaceBeforeCommit() throws Exception { + final String topic = NAMESPACE1 + "/testDeleteTopicBeforeCommit"; + PulsarClient pulsarClient = this.pulsarClient; + Transaction tnx = pulsarClient.newTransaction() + .withTransactionTimeout(60, TimeUnit.SECONDS) + .build().get(); + long txnIdMostBits = ((TransactionImpl) tnx).getTxnIdMostBits(); + long txnIdLeastBits = ((TransactionImpl) tnx).getTxnIdLeastBits(); + Assert.assertTrue(txnIdMostBits > -1); + Assert.assertTrue(txnIdLeastBits > -1); + + @Cleanup + Producer outProducer = pulsarClient + .newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .enableBatching(false) + .create(); + + String content = "Hello Txn"; + outProducer.newMessage(tnx).value(content.getBytes(UTF_8)).send(); + + try { + admin.namespaces().deleteNamespace(NAMESPACE1, true); + } catch (Exception ignore) {} + tnx.commit().get(); + } + @Test public void produceAndAbortTest() throws Exception { produceTest(false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index d7a0d3dd7ad2a..7cae6ca3ec30b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -156,7 +156,7 @@ protected void startBroker() throws Exception { conf.setConfigurationStoreServers("localhost:3181"); conf.setAllowAutoTopicCreationType("non-partitioned"); conf.setBookkeeperClientExposeStatsToPrometheus(true); - + conf.setForceDeleteNamespaceAllowed(true); conf.setBrokerShutdownTimeoutMs(0L); conf.setBrokerServicePort(Optional.of(0)); conf.setBrokerServicePortTls(Optional.of(0)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index d85f6b42f23f6..66460778dc22d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -30,10 +30,13 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionBufferClient; import org.apache.pulsar.client.api.transaction.TransactionBufferClientException; @@ -45,6 +48,8 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; @@ -80,7 +85,7 @@ protected void setup() throws Exception { new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); admin.namespaces().createNamespace(namespace, 10); admin.topics().createPartitionedTopic(partitionedTopicName.getPartitionedTopicName(), partitions); - tbClient = TransactionBufferClientImpl.create(pulsarClient, + tbClient = TransactionBufferClientImpl.create(pulsarServiceList.get(0), new HashedWheelTimer(new DefaultThreadFactory("transaction-buffer")), 1000, 3000); } @@ -145,22 +150,30 @@ public void testAbortOnSubscription() throws ExecutionException, InterruptedExce @Test public void testTransactionBufferClientTimeout() throws Exception { - PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + PulsarService pulsarService = pulsarServiceList.get(0); + PulsarClient mockClient = mock(PulsarClientImpl.class); CompletableFuture completableFuture = new CompletableFuture<>(); ClientCnx clientCnx = mock(ClientCnx.class); completableFuture.complete(clientCnx); - when(mockClient.getConnection(anyString())).thenReturn(completableFuture); + when(((PulsarClientImpl)mockClient).getConnection(anyString())).thenReturn(completableFuture); ChannelHandlerContext cnx = mock(ChannelHandlerContext.class); when(clientCnx.ctx()).thenReturn(cnx); Channel channel = mock(Channel.class); when(cnx.channel()).thenReturn(channel); + when(pulsarService.getClient()).thenAnswer(new Answer(){ + + @Override + public PulsarClient answer(InvocationOnMock invocation) throws Throwable { + return mockClient; + } + }); when(channel.isActive()).thenReturn(true); @Cleanup("stop") HashedWheelTimer hashedWheelTimer = new HashedWheelTimer(); TransactionBufferHandlerImpl transactionBufferHandler = - new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000, 3000); + new TransactionBufferHandlerImpl(pulsarService, hashedWheelTimer, 1000, 3000); CompletableFuture endFuture = transactionBufferHandler.endTxnOnTopic("test", 1, 1, TxnAction.ABORT, 1); @@ -187,23 +200,31 @@ public void testTransactionBufferClientTimeout() throws Exception { } @Test - public void testTransactionBufferChannelUnActive() { - PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + public void testTransactionBufferChannelUnActive() throws PulsarServerException { + PulsarService pulsarService = pulsarServiceList.get(0); + PulsarClient mockClient = mock(PulsarClientImpl.class); CompletableFuture completableFuture = new CompletableFuture<>(); ClientCnx clientCnx = mock(ClientCnx.class); completableFuture.complete(clientCnx); - when(mockClient.getConnection(anyString())).thenReturn(completableFuture); + when(((PulsarClientImpl)mockClient).getConnection(anyString())).thenReturn(completableFuture); ChannelHandlerContext cnx = mock(ChannelHandlerContext.class); when(clientCnx.ctx()).thenReturn(cnx); Channel channel = mock(Channel.class); when(cnx.channel()).thenReturn(channel); when(channel.isActive()).thenReturn(false); + when(pulsarService.getClient()).thenAnswer(new Answer(){ + + @Override + public PulsarClient answer(InvocationOnMock invocation) throws Throwable { + return mockClient; + } + }); @Cleanup("stop") HashedWheelTimer hashedWheelTimer = new HashedWheelTimer(); TransactionBufferHandlerImpl transactionBufferHandler = - new TransactionBufferHandlerImpl(mockClient, hashedWheelTimer, 1000, 3000); + new TransactionBufferHandlerImpl(pulsarServiceList.get(0), hashedWheelTimer, 1000, 3000); try { transactionBufferHandler.endTxnOnTopic("test", 1, 1, TxnAction.ABORT, 1).get(); fail(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java index 5241342635b88..af4e442f617a3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java @@ -18,42 +18,55 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; - +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.testng.annotations.Test; +import java.util.Optional; import java.util.concurrent.CompletableFuture; @Test(groups = "broker") public class TransactionBufferHandlerImplTest { @Test - public void testRequestCredits() { - PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); - when(pulsarClient.getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); - TransactionBufferHandlerImpl handler = spy( - new TransactionBufferHandlerImpl(pulsarClient, null, 1000, 3000)); + public void testRequestCredits() throws PulsarServerException { + PulsarClient pulsarClient = mock(PulsarClientImpl.class); + PulsarService pulsarService = mock(PulsarService.class); + NamespaceService namespaceService = mock(NamespaceService.class); + when(pulsarService.getNamespaceService()).thenReturn(namespaceService); + when(pulsarService.getClient()).thenReturn(pulsarClient); + when(namespaceService.getBundleAsync(any())).thenReturn(CompletableFuture.completedFuture(mock(NamespaceBundle.class))); + Optional opData = Optional.empty(); + when(namespaceService.getOwnerAsync(any())).thenReturn(CompletableFuture.completedFuture(opData)); + when(((PulsarClientImpl)pulsarClient).getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); + TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarService, null, 1000, 3000)); doNothing().when(handler).endTxn(any()); + doReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))).when(handler).getClientCnx(anyString()); for (int i = 0; i < 500; i++) { - handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); + handler.endTxnOnTopic("public/default/t", 1L, 1L, TxnAction.COMMIT, 1L); } assertEquals(handler.getAvailableRequestCredits(), 500); for (int i = 0; i < 500; i++) { - handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); + handler.endTxnOnTopic("public/default/t", 1L, 1L, TxnAction.COMMIT, 1L); } assertEquals(handler.getAvailableRequestCredits(), 0); - handler.endTxnOnTopic("t", 1L, 1L, TxnAction.COMMIT, 1L); + handler.endTxnOnTopic("public/default/t", 1L, 1L, TxnAction.COMMIT, 1L); assertEquals(handler.getPendingRequestsCount(), 1); handler.onResponse(null); assertEquals(handler.getAvailableRequestCredits(), 0); @@ -61,9 +74,11 @@ public void testRequestCredits() { } @Test - public void testMinRequestCredits() { - TransactionBufferHandlerImpl handler = spy( - new TransactionBufferHandlerImpl(null, null, 50, 3000)); + public void testMinRequestCredits() throws PulsarServerException { + PulsarClient pulsarClient = mock(PulsarClientImpl.class); + PulsarService pulsarService = mock(PulsarService.class); + when(pulsarService.getClient()).thenReturn(pulsarClient); + TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarService, null, 50, 3000)); assertEquals(handler.getAvailableRequestCredits(), 100); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 0ab7a1376081f..7c318f25a8ea5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -30,7 +30,8 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; - +import java.io.IOException; +import java.net.InetSocketAddress; import java.time.Clock; import java.util.ArrayList; import java.util.Collections; @@ -828,7 +829,12 @@ public synchronized void updateServiceUrl(String serviceUrl) throws PulsarClient public CompletableFuture getConnection(final String topic) { TopicName topicName = TopicName.get(topic); return lookup.getBroker(topicName) - .thenCompose(pair -> cnxPool.getConnection(pair.getLeft(), pair.getRight())); + .thenCompose(pair -> getConnection(pair.getLeft(), pair.getRight())); + } + + public CompletableFuture getConnection(final InetSocketAddress logicalAddress, + final InetSocketAddress physicalAddress) { + return cnxPool.getConnection(logicalAddress, physicalAddress); } /** visible for pulsar-functions **/ From a6537f44fb7bba8efa578ee59ddd4594d13cf37a Mon Sep 17 00:00:00 2001 From: Dezhi LIiu <33149602+liudezhi2098@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:57:56 +0800 Subject: [PATCH 480/823] [broker] fix parameter saslJaasBrokerSectionName in broker.conf (#15110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation when the parameter saslJaasBrokerSectionName is set, it does not take effect ， becase `ServiceConfiguration` is saslJaasServerSectionName not match. ``` @FieldContext( category = CATEGORY_SASL_AUTH, doc = "Service Principal, for login context name. Default value is \"PulsarBroker\"." ) private String saslJaasServerSectionName = SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME; ``` ### Modifications alert conf `# Service Principal, for login context name.` `# Default value `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker".` `saslJaasServerSectionName=` (cherry picked from commit 8ff87ac9523ac74f8a009cd89f3ace69e412eef1) --- conf/broker.conf | 2 +- deployment/terraform-ansible/templates/broker.conf | 2 +- site2/docs/functions-worker.md | 4 ++-- site2/docs/reference-configuration.md | 2 +- site2/docs/security-kerberos.md | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 4fd2834e84590..e36892e358427 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -724,7 +724,7 @@ saslJaasClientAllowedIds= # Service Principal, for login context name. # Default value `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker". -saslJaasBrokerSectionName= +saslJaasServerSectionName= ### --- HTTP Server config --- ### diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index 9936105c21095..39adc6b0a2076 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -613,7 +613,7 @@ saslJaasClientAllowedIds= # Service Principal, for login context name. # Default value `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker". -saslJaasBrokerSectionName= +saslJaasServerSectionName= ### --- HTTP Server config --- ### diff --git a/site2/docs/functions-worker.md b/site2/docs/functions-worker.md index 05944165c4374..0a1c985a76b05 100644 --- a/site2/docs/functions-worker.md +++ b/site2/docs/functions-worker.md @@ -178,13 +178,13 @@ authenticationEnabled: true authenticationProviders: ['org.apache.pulsar.broker.authentication.AuthenticationProviderTls'] ``` -For *SASL Authentication* provider, add `saslJaasClientAllowedIds` and `saslJaasBrokerSectionName` +For *SASL Authentication* provider, add `saslJaasClientAllowedIds` and `saslJaasServerSectionName` under `properties` if needed. ``` properties: saslJaasClientAllowedIds: .*pulsar.* - saslJaasBrokerSectionName: Broker + saslJaasServerSectionName: Broker ``` For *Token Authentication* provider, add necessary settings for `properties` if needed. diff --git a/site2/docs/reference-configuration.md b/site2/docs/reference-configuration.md index 72f134ae7286f..6fe8565f9e9c6 100644 --- a/site2/docs/reference-configuration.md +++ b/site2/docs/reference-configuration.md @@ -545,7 +545,7 @@ You can set the log level and configuration in the [log4j2.yaml](https://github |tokenAudienceClaim| The token audience "claim" name, e.g. "aud". It is used to get the audience from token. If it is not set, the audience is not verified. || | tokenAudience | The token audience stands for this broker. The field `tokenAudienceClaim` of a valid token need contains this parameter.| | |saslJaasClientAllowedIds|This is a regexp, which limits the range of possible ids which can connect to the Broker using SASL. By default, it is set to `SaslConstants.JAAS_CLIENT_ALLOWED_IDS_DEFAULT`, which is ".*pulsar.*", so only clients whose id contains 'pulsar' are allowed to connect.|N/A| -|saslJaasBrokerSectionName|Service Principal, for login context name. By default, it is set to `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker".|N/A| +|saslJaasServerSectionName|Service Principal, for login context name. By default, it is set to `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker".|N/A| |httpMaxRequestSize|If the value is larger than 0, it rejects all HTTP requests with bodies larged than the configured limit.|-1| |exposePreciseBacklogInPrometheus| Enable expose the precise backlog stats, set false to use published counter and consumed counter to calculate, this would be more efficient but may be inaccurate. |false| |bookkeeperMetadataServiceUri|Metadata service uri is what BookKeeper used for loading corresponding metadata driver and resolving its metadata service location. This value can be fetched using `bookkeeper shell whatisinstanceid` command in BookKeeper cluster. For example: `zk+hierarchical://localhost:2181/ledgers`. The metadata service uri list can also be semicolon separated values like: `zk+hierarchical://zk1:2181;zk2:2181;zk3:2181/ledgers`.|N/A| diff --git a/site2/docs/security-kerberos.md b/site2/docs/security-kerberos.md index f7fe4c5f650e5..897bf3bb0dd9f 100644 --- a/site2/docs/security-kerberos.md +++ b/site2/docs/security-kerberos.md @@ -113,7 +113,7 @@ You can have 2 separate JAAS configuration files: - Set `authenticationEnabled` to `true`; - Set `authenticationProviders` to choose `AuthenticationProviderSasl`; - Set `saslJaasClientAllowedIds` regex for principal that is allowed to connect to broker; - - Set `saslJaasBrokerSectionName` that corresponds to the section in JAAS configuration file for broker; + - Set `saslJaasServerSectionName` that corresponds to the section in JAAS configuration file for broker; To make Pulsar internal admin client work properly, you need to set the configuration in the `broker.conf` file as below: - Set `brokerClientAuthenticationPlugin` to client plugin `AuthenticationSasl`; @@ -125,7 +125,7 @@ You can have 2 separate JAAS configuration files: authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl saslJaasClientAllowedIds=.*client.* -saslJaasBrokerSectionName=PulsarBroker +saslJaasServerSectionName=PulsarBroker ## Authentication settings of the broker itself. Used when the broker connects to other brokers brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl @@ -306,7 +306,7 @@ In the `proxy.conf` file, set Kerberos related configuration. Here is an example authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl saslJaasClientAllowedIds=.*client.* -saslJaasBrokerSectionName=PulsarProxy +saslJaasServerSectionName=PulsarProxy ## related to be authenticated by broker brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl @@ -326,7 +326,7 @@ The broker side configuration file is the same with the above `broker.conf`, you authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl saslJaasClientAllowedIds=.*client.* -saslJaasBrokerSectionName=PulsarBroker +saslJaasServerSectionName=PulsarBroker ``` ## Regarding authorization and role token From ac0a242a0e93fc8166f162fffcd74f6964e5b166 Mon Sep 17 00:00:00 2001 From: Dezhi LIiu <33149602+liudezhi2098@users.noreply.github.com> Date: Wed, 13 Apr 2022 08:58:14 +0800 Subject: [PATCH 481/823] fix debug log authenticate role error (#14784) (cherry picked from commit 97b7e77a9075159dd58a279d64f834e85472f437) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 87eea84af8c59..d8255f6015daf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -815,8 +815,13 @@ protected void handleConnect(CommandConnect connect) { authenticationData = authState.getAuthDataSource(); if (log.isDebugEnabled()) { - log.debug("[{}] Authenticate role : {}", remoteAddress, - authState != null ? authState.getAuthRole() : null); + String role = ""; + if (authState != null && authState.isComplete()) { + role = authState.getAuthRole(); + } else { + role = "authentication incomplete or null"; + } + log.debug("[{}] Authenticate role : {}", remoteAddress, role); } state = doAuthentication(clientData, clientProtocolVersion, clientVersion); From 549692f8e4f7ceac114b504e7fc354faf1e97a06 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Wed, 13 Apr 2022 10:34:15 +0800 Subject: [PATCH 482/823] [fix][security] Remove log4j for CVE-2022-23307 (#15109) (cherry picked from commit a4c4aea993aabab5231d4136b7eba366bee9e778) --- pom.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pom.xml b/pom.xml index 8b7eb2bf343ce..9f3bfcd45333b 100644 --- a/pom.xml +++ b/pom.xml @@ -176,7 +176,6 @@ flexible messaging model and an intuitive client API. 2.8.0 1.15 2.1 - 1.2.17 2.1.9 3.1.0 2.9.1 @@ -785,18 +784,6 @@ flexible messaging model and an intuitive client API. ${jackson.databind.version} - - log4j - log4j - ${log4j.version} - - - com.sun.jmx - jmxri - - - - org.hdrhistogram HdrHistogram From 3adb09978e4b20db1e0175348c19a86a04a26c77 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 13 Apr 2022 18:00:03 +0800 Subject: [PATCH 483/823] [fix][txn] TransactionMetadataService don't connect again if store exist (#15114) ### Motivation && Modification We don`t need to openTransactionMetadataStore if store exist. (cherry picked from commit ccbcb86e3037bb39cf204d3c430f0978330e50b8) --- .../apache/pulsar/broker/TransactionMetadataStoreService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 97fae3be9f669..679bcf99368e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -179,6 +179,8 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc // so we should jude the store exist again. if (stores.get(tcId) != null) { completableFuture.complete(null); + tcLoadSemaphore.release(); + return; } openTransactionMetadataStore(tcId).thenAccept((store) -> internalPinnedExecutor.execute(() -> { From 2dab3bae59a6a9298cd6d9cc520faf8a6e513ba1 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Wed, 13 Apr 2022 21:43:47 +0800 Subject: [PATCH 484/823] [improve][txn] Avoid create multiple future and exception handler. (#15089) (cherry picked from commit 4aeeed5dab9dfe9493526f36d539b3ef29cf6fe5) --- .../TransactionMetadataStoreService.java | 228 +++++++----------- 1 file changed, 92 insertions(+), 136 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 679bcf99368e1..7297c334c4cab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -25,12 +25,12 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; -import java.util.ArrayList; import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutorService; @@ -38,6 +38,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; @@ -338,12 +340,13 @@ public CompletableFuture updateTxnStatus(TxnID txnId, TxnStatus newStatus, } public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolean isTimeout) { - CompletableFuture completableFuture = new CompletableFuture<>(); - return endTransaction(txnID, txnAction, isTimeout, completableFuture); + CompletableFuture future = new CompletableFuture<>(); + endTransaction(txnID, txnAction, isTimeout, future); + return future; } - public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolean isTimeout, - CompletableFuture completableFuture) { + public void endTransaction(TxnID txnID, int txnAction, boolean isTimeout, + CompletableFuture future) { TxnStatus newStatus; switch (txnAction) { case TxnAction.COMMIT_VALUE: @@ -356,90 +359,60 @@ public CompletableFuture endTransaction(TxnID txnID, int txnAction, boolea TransactionCoordinatorException.UnsupportedTxnActionException exception = new TransactionCoordinatorException.UnsupportedTxnActionException(txnID, txnAction); LOG.error(exception.getMessage()); - completableFuture.completeExceptionally(exception); - return completableFuture; + future.completeExceptionally(exception); + return; } - - getTxnMeta(txnID).thenAccept(txnMeta -> { - TxnStatus txnStatus = txnMeta.status(); - if (txnStatus == TxnStatus.OPEN) { - updateTxnStatus(txnID, newStatus, TxnStatus.OPEN, isTimeout).thenAccept(v -> - endTxnInTransactionBuffer(txnID, txnAction).thenAccept(a -> - completableFuture.complete(null)).exceptionally(e -> { - if (!isRetryableException(e.getCause())) { - LOG.error("EndTxnInTransactionBuffer fail! TxnId : {}, " - + "TxnAction : {}", txnID, txnAction, e); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("EndTxnInTransactionBuffer retry! TxnId : {}, " - + "TxnAction : {}", txnID, txnAction, e); - } - transactionOpRetryTimer.newTimeout(timeout -> - endTransaction(txnID, txnAction, isTimeout, completableFuture), - endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); - return null; - - } - completableFuture.completeExceptionally(e.getCause()); - return null; - })).exceptionally(e -> { - if (!isRetryableException(e.getCause())) { - LOG.error("EndTransaction UpdateTxnStatus fail! TxnId : {}, " - + "TxnAction : {}", txnID, txnAction, e); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("EndTransaction UpdateTxnStatus op retry! TxnId : {}, " - + "TxnAction : {}", txnID, txnAction, e); - } - transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, - isTimeout, completableFuture), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); - return null; - + getTxnMeta(txnID) + .thenCompose(txnMeta -> { + if (txnMeta.status() == TxnStatus.OPEN) { + return updateTxnStatus(txnID, newStatus, TxnStatus.OPEN, isTimeout) + .thenCompose(__ -> endTxnInTransactionBuffer(txnID, txnAction)); + } + return fakeAsyncCheckTxnStatus(txnMeta.status(), txnAction, txnID, newStatus) + .thenCompose(__ -> endTxnInTransactionBuffer(txnID, txnAction)); + }).whenComplete((__, ex)-> { + if (ex == null) { + future.complete(null); + return; + } + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (!isRetryableException(realCause)) { + LOG.error("End transaction fail! TxnId : {}, " + + "TxnAction : {}", txnID, txnAction, realCause); + future.completeExceptionally(ex); + return; } - completableFuture.completeExceptionally(e.getCause()); - return null; - }); - } else { - if ((txnStatus == COMMITTING && txnAction == TxnAction.COMMIT.getValue()) - || (txnStatus == ABORTING && txnAction == TxnAction.ABORT.getValue())) { - endTxnInTransactionBuffer(txnID, txnAction).thenAccept(k -> - completableFuture.complete(null)).exceptionally(e -> { - if (isRetryableException(e.getCause())) { - if (LOG.isDebugEnabled()) { - LOG.debug("EndTxnInTransactionBuffer retry! TxnId : {}, " - + "TxnAction : {}", txnID, txnAction, e); - } - transactionOpRetryTimer.newTimeout(timeout -> - endTransaction(txnID, txnAction, isTimeout, completableFuture), - endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); - return null; - } else { - LOG.error("EndTxnInTransactionBuffer fail! TxnId : {}, " - + "TxnAction : {}", txnID, txnAction, e); - } - completableFuture.completeExceptionally(e.getCause()); - return null; - }); - } else { if (LOG.isDebugEnabled()) { - LOG.debug("EndTxnInTransactionBuffer op retry! TxnId : {}, TxnAction : {}", txnID, txnAction); + LOG.debug("EndTxnInTransactionBuffer retry! TxnId : {}, " + + "TxnAction : {}", txnID, txnAction, realCause); } - completableFuture.completeExceptionally(new InvalidTxnStatusException(txnID, newStatus, txnStatus)); - } - } - }).exceptionally(e -> { - if (isRetryableException(e.getCause())) { - if (LOG.isDebugEnabled()) { - LOG.debug("End transaction op retry! TxnId : {}, TxnAction : {}", txnID, txnAction, e); - } - transactionOpRetryTimer.newTimeout(timeout -> endTransaction(txnID, txnAction, isTimeout, - completableFuture), endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); - return null; + transactionOpRetryTimer.newTimeout(timeout -> + endTransaction(txnID, txnAction, isTimeout, future), + endTransactionRetryIntervalTime, TimeUnit.MILLISECONDS); + }); + } + + private CompletionStage fakeAsyncCheckTxnStatus(TxnStatus txnStatus, int txnAction, + TxnID txnID, TxnStatus expectStatus) { + boolean isLegal; + switch (txnStatus) { + case COMMITTING: + isLegal = (txnAction == TxnAction.COMMIT.getValue()); + break; + case ABORTING: + isLegal = (txnAction == TxnAction.ABORT.getValue()); + break; + default: + isLegal = false; + } + if (!isLegal) { + if (LOG.isDebugEnabled()) { + LOG.debug("EndTxnInTransactionBuffer op retry! TxnId : {}, TxnAction : {}", txnID, txnAction); } - completableFuture.completeExceptionally(e.getCause()); - return null; - }); - return completableFuture; + return FutureUtil.failedFuture( + new InvalidTxnStatusException(txnID, expectStatus, txnStatus)); + } + return CompletableFuture.completedFuture(null); } // when managedLedger fence will remove this tc and reload @@ -470,59 +443,42 @@ public void endTransactionForTimeout(TxnID txnID) { } private CompletableFuture endTxnInTransactionBuffer(TxnID txnID, int txnAction) { - CompletableFuture resultFuture = new CompletableFuture<>(); - List> completableFutureList = new ArrayList<>(); - this.getTxnMeta(txnID).whenComplete((txnMeta, throwable) -> { - if (throwable != null) { - resultFuture.completeExceptionally(throwable); - return; - } - long lowWaterMark = getLowWaterMark(txnID); - - txnMeta.ackedPartitions().forEach(tbSub -> { - CompletableFuture actionFuture = new CompletableFuture<>(); - if (TxnAction.COMMIT_VALUE == txnAction) { - actionFuture = tbClient.commitTxnOnSubscription( - tbSub.getTopic(), tbSub.getSubscription(), txnID.getMostSigBits(), - txnID.getLeastSigBits(), lowWaterMark); - } else if (TxnAction.ABORT_VALUE == txnAction) { - actionFuture = tbClient.abortTxnOnSubscription( - tbSub.getTopic(), tbSub.getSubscription(), txnID.getMostSigBits(), - txnID.getLeastSigBits(), lowWaterMark); - } else { - actionFuture.completeExceptionally(new Throwable("Unsupported txnAction " + txnAction)); - } - completableFutureList.add(actionFuture); - }); - - txnMeta.producedPartitions().forEach(partition -> { - CompletableFuture actionFuture = new CompletableFuture<>(); - if (TxnAction.COMMIT_VALUE == txnAction) { - actionFuture = tbClient.commitTxnOnTopic(partition, txnID.getMostSigBits(), - txnID.getLeastSigBits(), lowWaterMark); - } else if (TxnAction.ABORT_VALUE == txnAction) { - actionFuture = tbClient.abortTxnOnTopic(partition, txnID.getMostSigBits(), - txnID.getLeastSigBits(), lowWaterMark); - } else { - actionFuture.completeExceptionally(new Throwable("Unsupported txnAction " + txnAction)); - } - completableFutureList.add(actionFuture); - }); - - try { - FutureUtil.waitForAll(completableFutureList).whenComplete((ignored, waitThrowable) -> { - if (waitThrowable != null) { - resultFuture.completeExceptionally(waitThrowable); - return; - } - resultFuture.complete(null); + return getTxnMeta(txnID) + .thenCompose(txnMeta -> { + long lowWaterMark = getLowWaterMark(txnID); + Stream> onSubFutureStream = txnMeta.ackedPartitions().stream().map(tbSub -> { + switch (txnAction) { + case TxnAction.COMMIT_VALUE: + return tbClient.commitTxnOnSubscription( + tbSub.getTopic(), tbSub.getSubscription(), txnID.getMostSigBits(), + txnID.getLeastSigBits(), lowWaterMark); + case TxnAction.ABORT_VALUE: + return tbClient.abortTxnOnSubscription( + tbSub.getTopic(), tbSub.getSubscription(), txnID.getMostSigBits(), + txnID.getLeastSigBits(), lowWaterMark); + default: + return FutureUtil.failedFuture( + new IllegalStateException("Unsupported txnAction " + txnAction)); + } + }); + Stream> onTopicFutureStream = + txnMeta.producedPartitions().stream().map(partition -> { + switch (txnAction) { + case TxnAction.COMMIT_VALUE: + return tbClient.commitTxnOnTopic(partition, txnID.getMostSigBits(), + txnID.getLeastSigBits(), lowWaterMark); + case TxnAction.ABORT_VALUE: + return tbClient.abortTxnOnTopic(partition, txnID.getMostSigBits(), + txnID.getLeastSigBits(), lowWaterMark); + default: + return FutureUtil.failedFuture( + new IllegalStateException("Unsupported txnAction " + txnAction)); + } + }); + return FutureUtil.waitForAll(Stream.concat(onSubFutureStream, onTopicFutureStream) + .collect(Collectors.toList())) + .thenCompose(__ -> endTxnInTransactionMetadataStore(txnID, txnAction)); }); - } catch (Exception e) { - resultFuture.completeExceptionally(e); - } - }); - - return resultFuture.thenCompose((future) -> endTxnInTransactionMetadataStore(txnID, txnAction)); } private static boolean isRetryableException(Throwable e) { From 70c4d0d97616ef15c189802c5a93e43029dbddce Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 14 Apr 2022 14:59:46 +0800 Subject: [PATCH 485/823] [C++] Fix UnknownError might be returned for a partitioned producer (#15161) Fixes #15078 When a partitioned producer is created and some of the partitioned failed to create, `closeAsync` will be called immediately, even if other partitions were still in progress of creating the associated single producers. Since `closeAsync` is called before calling `setFailed` on the `partitionedProducerCreatedPromise_` field, there is a race condition that all single producers are closed before the promise is set. Then the promise will be set with `ResultUnknownError`, see https://github.com/apache/pulsar/blob/4aeeed5dab9dfe9493526f36d539b3ef29cf6fe5/pulsar-client-cpp/lib/PartitionedProducerImpl.cc#L317. Only after all single producers failed or succeeded then call `closeAsync` if one of them failed. And ensure `partitionedProducerCreatedPromise_` was completed before calling `closeAsync`. This PR also makes the state of a partitioned producer atomic because using a mutex to protect it makes code hard to write. Create a separate namespace `public/test-backlog-quotas` to test the case when the backlog quota exceeds. Then add `testBacklogQuotasExceeded` test to make some backlog via creating a consumer and sending some messages to a partition of the topic. In this test, only 1 partition has backlog and will fail with the related error. So the test verifies that `createProducer` could return a correct error instead of `ResultUnknownError`. (cherry picked from commit 0f8559611914a17efe222bf6ed1dd621d5a9cf45) --- .../lib/PartitionedProducerImpl.cc | 55 +++++++------------ .../lib/PartitionedProducerImpl.h | 12 +--- .../pulsar-test-service-start.sh | 4 ++ pulsar-client-cpp/tests/ProducerTest.cc | 43 +++++++++++++++ 4 files changed, 71 insertions(+), 43 deletions(-) diff --git a/pulsar-client-cpp/lib/PartitionedProducerImpl.cc b/pulsar-client-cpp/lib/PartitionedProducerImpl.cc index bdd23ed6c91c2..9f197f0d45fd4 100644 --- a/pulsar-client-cpp/lib/PartitionedProducerImpl.cc +++ b/pulsar-client-cpp/lib/PartitionedProducerImpl.cc @@ -136,28 +136,29 @@ void PartitionedProducerImpl::handleSinglePartitionProducerCreated(Result result unsigned int partitionIndex) { // to indicate, we are doing cleanup using closeAsync after producer create // has failed and the invocation of closeAsync is not from client - CloseCallback closeCallback = NULL; - Lock lock(mutex_); + const auto numPartitions = getNumPartitionsWithLock(); + assert(numProducersCreated_ <= numPartitions && partitionIndex <= numPartitions); + if (state_ == Failed) { - // Ignore, we have already informed client that producer creation failed + // We have already informed client that producer creation failed + if (++numProducersCreated_ == numPartitions) { + closeAsync(nullptr); + } return; } - const auto numPartitions = getNumPartitionsWithLock(); - assert(numProducersCreated_ <= numPartitions); + if (result != ResultOk) { - state_ = Failed; - lock.unlock(); - closeAsync(closeCallback); - partitionedProducerCreatedPromise_.setFailed(result); LOG_ERROR("Unable to create Producer for partition - " << partitionIndex << " Error - " << result); + partitionedProducerCreatedPromise_.setFailed(result); + state_ = Failed; + if (++numProducersCreated_ == numPartitions) { + closeAsync(nullptr); + } return; } - assert(partitionIndex <= numPartitions); - numProducersCreated_++; - if (numProducersCreated_ == numPartitions) { + if (++numProducersCreated_ == numPartitions) { state_ = Ready; - lock.unlock(); if (partitionsUpdateTimer_) { runPartitionUpdateTask(); } @@ -181,7 +182,7 @@ void PartitionedProducerImpl::createLazyPartitionProducer(unsigned int partition // override void PartitionedProducerImpl::sendAsync(const Message& msg, SendCallback callback) { - if (!assertState(Ready)) { + if (state_ != Ready) { callback(ResultAlreadyClosed, msg.getMessageId()); return; } @@ -211,18 +212,7 @@ void PartitionedProducerImpl::sendAsync(const Message& msg, SendCallback callbac } // override -void PartitionedProducerImpl::shutdown() { setState(Closed); } - -void PartitionedProducerImpl::setState(const PartitionedProducerState state) { - Lock lock(mutex_); - state_ = state; - lock.unlock(); -} - -bool PartitionedProducerImpl::assertState(const PartitionedProducerState state) { - Lock lock(mutex_); - return state_ == state; -} +void PartitionedProducerImpl::shutdown() { state_ = Closed; } const std::string& PartitionedProducerImpl::getProducerName() const { Lock producersLock(producersMutex_); @@ -251,7 +241,10 @@ int64_t PartitionedProducerImpl::getLastSequenceId() const { * create one or many producers for partitions. So, we have to notify with ERROR on createProducerFailure */ void PartitionedProducerImpl::closeAsync(CloseCallback closeCallback) { - setState(Closing); + if (state_ == Closing || state_ == Closed) { + return; + } + state_ = Closing; unsigned int producerAlreadyClosed = 0; @@ -280,7 +273,7 @@ void PartitionedProducerImpl::closeAsync(CloseCallback closeCallback) { * handleSinglePartitionProducerCreated */ if (producerAlreadyClosed == numProducers && closeCallback) { - setState(Closed); + state_ = Closed; closeCallback(ResultOk); } } @@ -288,14 +281,12 @@ void PartitionedProducerImpl::closeAsync(CloseCallback closeCallback) { void PartitionedProducerImpl::handleSinglePartitionProducerClose(Result result, const unsigned int partitionIndex, CloseCallback callback) { - Lock lock(mutex_); if (state_ == Failed) { // we should have already notified the client by callback return; } if (result != ResultOk) { state_ = Failed; - lock.unlock(); LOG_ERROR("Closing the producer failed for partition - " << partitionIndex); if (callback) { callback(result); @@ -309,7 +300,6 @@ void PartitionedProducerImpl::handleSinglePartitionProducerClose(Result result, // closed all successfully if (!numProducersCreated_) { state_ = Closed; - lock.unlock(); // set the producerCreatedPromise to failure, if client called // closeAsync and it's not failure to create producer, the promise // is set second time here, first time it was successful. So check @@ -395,7 +385,6 @@ void PartitionedProducerImpl::getPartitionMetadata() { void PartitionedProducerImpl::handleGetPartitions(Result result, const LookupDataResultPtr& lookupDataResult) { - Lock stateLock(mutex_); if (state_ != Ready) { return; } @@ -428,11 +417,9 @@ void PartitionedProducerImpl::handleGetPartitions(Result result, } bool PartitionedProducerImpl::isConnected() const { - Lock stateLock(mutex_); if (state_ != Ready) { return false; } - stateLock.unlock(); Lock producersLock(producersMutex_); const auto producers = producers_; diff --git a/pulsar-client-cpp/lib/PartitionedProducerImpl.h b/pulsar-client-cpp/lib/PartitionedProducerImpl.h index 874d6cda5265e..0a8c10e221303 100644 --- a/pulsar-client-cpp/lib/PartitionedProducerImpl.h +++ b/pulsar-client-cpp/lib/PartitionedProducerImpl.h @@ -30,7 +30,7 @@ namespace pulsar { class PartitionedProducerImpl : public ProducerImplBase, public std::enable_shared_from_this { public: - enum PartitionedProducerState + enum State { Pending, Ready, @@ -73,8 +73,6 @@ class PartitionedProducerImpl : public ProducerImplBase, void notifyResult(CloseCallback closeCallback); - void setState(PartitionedProducerState state); - friend class PulsarFriend; private: @@ -83,7 +81,7 @@ class PartitionedProducerImpl : public ProducerImplBase, const TopicNamePtr topicName_; const std::string topic_; - unsigned int numProducersCreated_ = 0; + std::atomic_uint numProducersCreated_{0}; /* * set when one or more Single Partition Creation fails, close will cleanup and fail the create callbackxo @@ -99,10 +97,7 @@ class PartitionedProducerImpl : public ProducerImplBase, mutable std::mutex producersMutex_; MessageRoutingPolicyPtr routerPolicy_; - // mutex_ is used to share state_, and numProducersCreated_ - mutable std::mutex mutex_; - - PartitionedProducerState state_ = Pending; + std::atomic state_{Pending}; // only set this promise to value, when producers on all partitions are created. Promise partitionedProducerCreatedPromise_; @@ -124,7 +119,6 @@ class PartitionedProducerImpl : public ProducerImplBase, void runPartitionUpdateTask(); void getPartitionMetadata(); void handleGetPartitions(const Result result, const LookupDataResultPtr& partitionMetadata); - bool assertState(const PartitionedProducerState state); }; } // namespace pulsar diff --git a/pulsar-client-cpp/pulsar-test-service-start.sh b/pulsar-client-cpp/pulsar-test-service-start.sh index 248d628b9c2a9..2bee18e64b9c5 100755 --- a/pulsar-client-cpp/pulsar-test-service-start.sh +++ b/pulsar-client-cpp/pulsar-test-service-start.sh @@ -106,6 +106,10 @@ $PULSAR_DIR/bin/pulsar-admin namespaces grant-permission public/default-4 \ --role "anonymous" $PULSAR_DIR/bin/pulsar-admin namespaces set-encryption-required public/default-4 -e +# Create "public/test-backlog-quotas" to test backlog quotas policy +$PULSAR_DIR/bin/pulsar-admin namespaces create public/test-backlog-quotas \ + --clusters standalone + # Create "private" tenant $PULSAR_DIR/bin/pulsar-admin tenants create private -r "" -c "standalone" diff --git a/pulsar-client-cpp/tests/ProducerTest.cc b/pulsar-client-cpp/tests/ProducerTest.cc index 210f01345d4af..14461429da311 100644 --- a/pulsar-client-cpp/tests/ProducerTest.cc +++ b/pulsar-client-cpp/tests/ProducerTest.cc @@ -240,4 +240,47 @@ TEST(ProducerTest, testSendAsyncCloseAsyncConcurrentlyWithLazyProducers) { client.close(); LOG_INFO("End of run " << run); } +} + +TEST(ProducerTest, testBacklogQuotasExceeded) { + std::string ns = "public/test-backlog-quotas"; + std::string topic = ns + "/testBacklogQuotasExceeded" + std::to_string(time(nullptr)); + + int res = makePutRequest(adminUrl + "admin/v2/persistent/" + topic + "/partitions", "5"); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + LOG_INFO("Created topic " << topic << " with 5 partitions"); + + auto setBacklogPolicy = [&ns](const std::string& policy, int limitSize) { + const auto body = + R"({"policy":")" + policy + R"(","limitSize":)" + std::to_string(limitSize) + "}"; + int res = makePostRequest(adminUrl + "admin/v2/namespaces/" + ns + "/backlogQuota", body); + LOG_INFO(res << " | Change the backlog policy to: " << body); + ASSERT_TRUE(res == 204 || res == 409); + }; + + Client client(serviceUrl); + + // Create a topic with backlog size that is greater than 1024 + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe(topic, "sub", consumer)); // create a cursor + Producer producer; + + const auto partition = topic + "-partition-0"; + ASSERT_EQ(ResultOk, client.createProducer(partition, producer)); + ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent(std::string(1024L, 'a')).build())); + ASSERT_EQ(ResultOk, producer.close()); + + setBacklogPolicy("producer_request_hold", 1024); + ASSERT_EQ(ResultProducerBlockedQuotaExceededError, client.createProducer(topic, producer)); + ASSERT_EQ(ResultProducerBlockedQuotaExceededError, client.createProducer(partition, producer)); + + setBacklogPolicy("producer_exception", 1024); + ASSERT_EQ(ResultProducerBlockedQuotaExceededException, client.createProducer(topic, producer)); + ASSERT_EQ(ResultProducerBlockedQuotaExceededException, client.createProducer(partition, producer)); + + setBacklogPolicy("consumer_backlog_eviction", 1024); + ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); + ASSERT_EQ(ResultOk, client.createProducer(partition, producer)); + + client.close(); } \ No newline at end of file From 06a0a7520bfe2c55c8c322774c36fe9e87cc068e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 20 Apr 2022 12:17:02 +0800 Subject: [PATCH 486/823] [branch-2.9][C++] Fix broken testBacklogQuotasExceeded The C++ test is broken, see https://github.com/apache/pulsar/pull/15220#issuecomment-1103441676 for details. It might be caused by a deadlock that was fixed by https://github.com/apache/pulsar/pull/13885. Since that PR has many dependencies, it cannot be cherry-picked into branch-2.9 currently. Therefore, apply some changes to the broken test as a workaround --- pulsar-client-cpp/tests/ProducerTest.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client-cpp/tests/ProducerTest.cc b/pulsar-client-cpp/tests/ProducerTest.cc index 14461429da311..258811fcdaf33 100644 --- a/pulsar-client-cpp/tests/ProducerTest.cc +++ b/pulsar-client-cpp/tests/ProducerTest.cc @@ -269,6 +269,7 @@ TEST(ProducerTest, testBacklogQuotasExceeded) { ASSERT_EQ(ResultOk, client.createProducer(partition, producer)); ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent(std::string(1024L, 'a')).build())); ASSERT_EQ(ResultOk, producer.close()); + ASSERT_EQ(ResultOk, consumer.close()); setBacklogPolicy("producer_request_hold", 1024); ASSERT_EQ(ResultProducerBlockedQuotaExceededError, client.createProducer(topic, producer)); @@ -283,4 +284,4 @@ TEST(ProducerTest, testBacklogQuotasExceeded) { ASSERT_EQ(ResultOk, client.createProducer(partition, producer)); client.close(); -} \ No newline at end of file +} From 9ca52dddba364603676e28e31a43ef237a1cbb69 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 20 Apr 2022 12:19:37 +0800 Subject: [PATCH 487/823] [Branch-2.9][test] Fix AdminTest#test500Error. (#15220) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- .../java/org/apache/pulsar/broker/service/AbstractTopic.java | 2 +- .../test/java/org/apache/pulsar/broker/admin/AdminTest.java | 5 +---- pulsar-sql/presto-distribution/LICENSE | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index deefa4eab91ad..974aeb47d78c9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -445,7 +445,7 @@ The Apache Software License, Version 2.0 - org.eclipse.jetty.websocket-websocket-servlet-9.4.43.v20210629.jar - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.43.v20210629.jar - org.eclipse.jetty-jetty-alpn-server-9.4.43.v20210629.jar - * SnakeYaml -- org.yaml-snakeyaml-1.27.jar + * SnakeYaml -- org.yaml-snakeyaml-1.30.jar * RocksDB - org.rocksdb-rocksdbjni-6.10.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index caa0ea29cd3c7..dedca9cced414 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -789,7 +789,7 @@ public PublishRateLimiter getBrokerPublishRateLimiter() { } public void updateMaxPublishRate(Policies policies) { - updatePublishDispatcher(Optional.of(policies)); + updatePublishDispatcher(Optional.ofNullable(policies)); } private void updatePublishDispatcher(Optional optPolicies) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index f9300a7fcac72..82e5b90b54f14 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -844,10 +844,7 @@ public void test500Error() throws Exception { AsyncResponse response1 = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(RestException.class); NamespaceName namespaceName = NamespaceName.get(property, cluster, namespace); - NamespaceService ns = spy(pulsar.getNamespaceService()); - Field namespaceField = pulsar.getClass().getDeclaredField("nsService"); - namespaceField.setAccessible(true); - namespaceField.set(pulsar, ns); + NamespaceService ns = pulsar.getNamespaceService(); CompletableFuture> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("500 error contains error message")); doReturn(future).when(ns).getListOfTopics(namespaceName, CommandGetTopicsOfNamespace.Mode.ALL); diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index b6f2eefe59521..c31f1442ae9b9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -394,7 +394,7 @@ The Apache Software License, Version 2.0 * RocksDB JNI - rocksdbjni-6.10.2.jar * SnakeYAML - - snakeyaml-1.27.jar + - snakeyaml-1.30.jar * Bean Validation API - validation-api-2.0.1.Final.jar * Objectsize From 0aad9a12339de434ce149cd57d524a3b45b1acc8 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 19 Apr 2022 22:18:52 +0800 Subject: [PATCH 488/823] [fix][java-client] Fix performance regression with message listener (#15162) https://github.com/apache/pulsar/pull/13023 has introduced a performance regression. For each message, we are switching from external thread pool -> internal thread poll -> external thread pool. Previously we want to control the outstanding messages of a consumer using a listener, so after #11455, the message will not move from the receiver queue to the external executor. And #13023 changed the listener trigger in the internal thread pool to fix the ordering issue, so this is the root cause of the performance regression. Here is the frame graph to show the thread frame of the internal thread and external thread. [framegraph.html.txt](https://github.com/apache/pulsar/files/8483765/framegraph.html.txt) And also fix the performance issue for multiple topic consumers and key-shared subscriptions which enabled message listeners. Before this change, the messages are processed serially. After this change, We can improve parallelism on the premise of ensuring order. - Remove the isListenerHandlingMessage control - Move the messages from the receiver queue to the queue of external executor but not increase permits - Increase permits before call message listener (cherry picked from commit 83cd7911ec43085be223b18e9d3d58a4638fd36f) --- .../pulsar/client/impl/RawReaderImpl.java | 1 + .../pulsar/client/impl/ConsumerBase.java | 37 ++++++++++--------- .../pulsar/client/impl/ConsumerImpl.java | 25 ++++++++++--- .../client/impl/MultiTopicsConsumerImpl.java | 6 +-- .../apache/pulsar/client/impl/ReaderImpl.java | 2 +- .../client/impl/ZeroQueueConsumerImpl.java | 2 +- 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 6b032370898e9..7c75f09977376 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -116,6 +116,7 @@ static class RawConsumerImpl extends ConsumerImpl { client.externalExecutorProvider(), TopicName.getPartitionIndex(conf.getSingleTopic()), false, + false, consumerFuture, MessageId.earliest, 0 /* startMessageRollbackDurationInSec */, diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index b50a570009398..062b277165bb6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -84,7 +84,6 @@ public abstract class ConsumerBase extends HandlerState implements Consumer conf, int receiverQueueSize, ExecutorProvider executorProvider, @@ -920,33 +919,34 @@ protected void tryTriggerListener() { } private void triggerListener() { + // The messages are added into the receiver queue by the internal pinned executor, + // so need to use internal pinned executor to avoid race condition which message + // might be added into the receiver queue but not able to read here. internalPinnedExecutor.execute(() -> { try { - // Listener should only have one pending/running executable to process a message - // See https://github.com/apache/pulsar/issues/11008 for context. - if (!isListenerHandlingMessage) { - final Message msg = internalReceive(0, TimeUnit.MILLISECONDS); + Message msg; + do { + msg = internalReceive(0, TimeUnit.MILLISECONDS); if (msg != null) { - isListenerHandlingMessage = true; // Trigger the notification on the message listener in a separate thread to avoid blocking the // internal pinned executor thread while the message processing happens + final Message finalMsg = msg; if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) { executorProvider.getExecutor(peekMessageKey(msg)).execute(() -> - callMessageListener(msg)); + callMessageListener(finalMsg)); } else { getExternalExecutor(msg).execute(() -> { - callMessageListener(msg); + callMessageListener(finalMsg); }); } + } else { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription); + } } - } + } while (msg != null); } catch (PulsarClientException e) { log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e); - return; - } - - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription); } }); } @@ -957,13 +957,16 @@ protected void callMessageListener(Message msg) { log.debug("[{}][{}] Calling message listener for message {}", topic, subscription, msg.getMessageId()); } + ConsumerImpl receivedConsumer = (msg instanceof TopicMessageImpl) + ? ((TopicMessageImpl) msg).receivedByconsumer : (ConsumerImpl) this; + // Increase the permits here since we will not increase permits while receive messages from consumer + // after enabled message listener. + receivedConsumer.increaseAvailablePermits((MessageImpl) (msg instanceof TopicMessageImpl + ? ((TopicMessageImpl) msg).getMessage() : msg)); listener.received(ConsumerBase.this, msg); } catch (Throwable t) { log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, msg.getMessageId(), t); - } finally { - isListenerHandlingMessage = false; - triggerListener(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 21a3151d04aff..85f7e05c4605d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -137,6 +137,7 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final int partitionIndex; private final boolean hasParentConsumer; + private final boolean parentConsumerHasListener; private final int receiverQueueRefillThreshold; @@ -210,8 +211,8 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, Schema schema, ConsumerInterceptors interceptors, boolean createTopicIfDoesNotExist) { - return newConsumerImpl(client, topic, conf, executorProvider, partitionIndex, hasParentConsumer, subscribeFuture, - startMessageId, schema, interceptors, createTopicIfDoesNotExist, 0); + return newConsumerImpl(client, topic, conf, executorProvider, partitionIndex, hasParentConsumer, false, + subscribeFuture, startMessageId, schema, interceptors, createTopicIfDoesNotExist, 0); } static ConsumerImpl newConsumerImpl(PulsarClientImpl client, @@ -220,6 +221,7 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, + boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId, Schema schema, @@ -233,14 +235,16 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, createTopicIfDoesNotExist); } else { return new ConsumerImpl<>(client, topic, conf, executorProvider, partitionIndex, hasParentConsumer, - subscribeFuture, startMessageId, startMessageRollbackDurationInSec /* rollback time in sec to start msgId */, + parentConsumerHasListener, + subscribeFuture, startMessageId, + startMessageRollbackDurationInSec /* rollback time in sec to start msgId */, schema, interceptors, createTopicIfDoesNotExist); } } protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, - CompletableFuture> subscribeFuture, MessageId startMessageId, + boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId, long startMessageRollbackDurationInSec, Schema schema, ConsumerInterceptors interceptors, boolean createTopicIfDoesNotExist) { super(client, topic, conf, conf.getReceiverQueueSize(), executorProvider, subscribeFuture, schema, interceptors); @@ -254,6 +258,7 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat this.partitionIndex = partitionIndex; this.hasParentConsumer = hasParentConsumer; this.receiverQueueRefillThreshold = conf.getReceiverQueueSize() / 2; + this.parentConsumerHasListener = parentConsumerHasListener; this.priorityLevel = conf.getPriorityLevel(); this.readCompacted = conf.isReadCompacted(); this.subscriptionInitialPosition = conf.getSubscriptionInitialPosition(); @@ -1450,7 +1455,9 @@ protected synchronized void messageProcessed(Message msg) { if (msgCnx != currentCnx) { // The processed message did belong to the old queue that was cleared after reconnection. } else { - increaseAvailablePermits(currentCnx); + if (listener == null && !parentConsumerHasListener) { + increaseAvailablePermits(currentCnx); + } stats.updateNumMsgsReceived(msg); trackMessage(msg); @@ -1481,6 +1488,14 @@ protected void trackMessage(MessageId messageId) { } } + void increaseAvailablePermits(MessageImpl msg) { + ClientCnx currentCnx = cnx(); + ClientCnx msgCnx = msg.getCnx(); + if (msgCnx == currentCnx) { + increaseAvailablePermits(currentCnx); + } + } + void increaseAvailablePermits(ClientCnx currentCnx) { increaseAvailablePermits(currentCnx, 1); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 1d479976950ce..6bfbb71c72c00 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -975,7 +975,7 @@ private void doSubscribeTopicPartitions(Schema schema, CompletableFuture> subFuture = new CompletableFuture<>(); ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, partitionName, configurationData, client.externalExecutorProvider(), - partitionIndex, true, subFuture, + partitionIndex, true, listener != null, subFuture, startMessageId, schema, interceptors, createIfDoesNotExist, startMessageRollbackDurationInSec); synchronized (pauseMutex) { @@ -1002,7 +1002,7 @@ private void doSubscribeTopicPartitions(Schema schema, } else { ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, topicName, internalConfig, client.externalExecutorProvider(), -1, - true, subFuture, startMessageId, schema, interceptors, + true, listener != null, subFuture, startMessageId, schema, interceptors, createIfDoesNotExist, startMessageRollbackDurationInSec); synchronized (pauseMutex) { @@ -1298,7 +1298,7 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl( client, partitionName, configurationData, client.externalExecutorProvider(), - partitionIndex, true, subFuture, startMessageId, schema, interceptors, + partitionIndex, true, listener != null, subFuture, startMessageId, schema, interceptors, true /* createTopicIfDoesNotExist */, startMessageRollbackDurationInSec); synchronized (pauseMutex) { if (paused) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java index 37d346ef93209..72c74e8875dde 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java @@ -116,7 +116,7 @@ public void reachedEndOfTopic(Consumer consumer) { final int partitionIdx = TopicName.getPartitionIndex(readerConfiguration.getTopicName()); consumer = new ConsumerImpl<>(client, readerConfiguration.getTopicName(), consumerConfiguration, - executorProvider, partitionIdx, false, consumerFuture, + executorProvider, partitionIdx, false, false, consumerFuture, readerConfiguration.getStartMessageId(), readerConfiguration.getStartMessageFromRollbackDurationInSec(), schema, null, true /* createTopicIfDoesNotExist */); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java index 5324c33d01127..6375e37a89cc9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java @@ -54,7 +54,7 @@ public ZeroQueueConsumerImpl(PulsarClientImpl client, String topic, ConsumerConf CompletableFuture> subscribeFuture, MessageId startMessageId, Schema schema, ConsumerInterceptors interceptors, boolean createTopicIfDoesNotExist) { - super(client, topic, conf, executorProvider, partitionIndex, hasParentConsumer, subscribeFuture, + super(client, topic, conf, executorProvider, partitionIndex, hasParentConsumer, false, subscribeFuture, startMessageId, 0 /* startMessageRollbackDurationInSec */, schema, interceptors, createTopicIfDoesNotExist); } From 6b6a32d6cb1f6f11d5c7e82cffd30a64ee11492e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 21 Apr 2022 11:09:05 +0800 Subject: [PATCH 489/823] [branch-2.9][broker] Full-support set ssl provider, ciphers and protocols (#15226) --- conf/broker.conf | 8 +- conf/standalone.conf | 8 +- .../pulsar/broker/ServiceConfiguration.java | 4 +- .../service/PulsarChannelInitializer.java | 15 +- .../internal/http/AsyncHttpConnector.java | 26 +++- .../apache/pulsar/client/impl/HttpClient.java | 38 +++-- .../client/impl/PulsarChannelInitializer.java | 47 +++++-- .../util/NettyClientSslContextRefresher.java | 43 ++++-- .../util/NettyServerSslContextBuilder.java | 16 ++- .../pulsar/common/util/SecurityUtility.java | 71 ++++++---- .../common/util/netty/SslContextTest.java | 132 ++++++++++++++++++ .../test/resources/ssl/jetty_client_key.jks | Bin 0 -> 2679 bytes .../test/resources/ssl/jetty_client_trust.jks | Bin 0 -> 1207 bytes .../test/resources/ssl/jetty_server_key.jks | Bin 0 -> 2679 bytes .../test/resources/ssl/jetty_server_trust.jks | Bin 0 -> 1207 bytes .../src/test/resources/ssl/my-ca/ca.pem | 18 +++ .../test/resources/ssl/my-ca/client-ca.pem | 19 +++ .../test/resources/ssl/my-ca/client-key.pem | 28 ++++ .../test/resources/ssl/my-ca/server-ca.pem | 19 +++ .../test/resources/ssl/my-ca/server-key.pem | 28 ++++ .../proxy/server/ProxyConfiguration.java | 4 +- .../server/ServiceChannelInitializer.java | 19 ++- 22 files changed, 452 insertions(+), 91 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java create mode 100644 pulsar-common/src/test/resources/ssl/jetty_client_key.jks create mode 100644 pulsar-common/src/test/resources/ssl/jetty_client_trust.jks create mode 100644 pulsar-common/src/test/resources/ssl/jetty_server_key.jks create mode 100644 pulsar-common/src/test/resources/ssl/jetty_server_trust.jks create mode 100644 pulsar-common/src/test/resources/ssl/my-ca/ca.pem create mode 100644 pulsar-common/src/test/resources/ssl/my-ca/client-ca.pem create mode 100644 pulsar-common/src/test/resources/ssl/my-ca/client-key.pem create mode 100644 pulsar-common/src/test/resources/ssl/my-ca/server-ca.pem create mode 100644 pulsar-common/src/test/resources/ssl/my-ca/server-key.pem diff --git a/conf/broker.conf b/conf/broker.conf index e36892e358427..849121f9fe8c9 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -594,13 +594,15 @@ tlsCiphers= # authentication. tlsRequireTrustedClientCertOnConnect=false +# Specify the TLS provider for the broker service: +# When using TLS authentication with CACert, the valid value is either OPENSSL or JDK. +# When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc. +tlsProvider= + ### --- KeyStore TLS config variables --- ### # Enable TLS with KeyStore type configuration in broker. tlsEnabledWithKeyStore=false -# TLS Provider for KeyStore type -tlsProvider= - # TLS KeyStore type configuration in broker: JKS, PKCS12 tlsKeyStoreType=JKS diff --git a/conf/standalone.conf b/conf/standalone.conf index 577a6ffad42cd..16312125a0254 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -357,13 +357,15 @@ tlsCiphers= # authentication. tlsRequireTrustedClientCertOnConnect=false +# Specify the TLS provider for the broker service: +# When using TLS authentication with CACert, the valid value is either OPENSSL or JDK. +# When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc. +tlsProvider= + ### --- KeyStore TLS config variables --- ### # Enable TLS with KeyStore type configuration in broker. tlsEnabledWithKeyStore=false -# TLS Provider for KeyStore type -tlsProvider= - # TLS KeyStore type configuration in broker: JKS, PKCS12 tlsKeyStoreType=JKS diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index fa8464376fce6..1e7eac5ef9a1a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2237,7 +2237,9 @@ public class ServiceConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_KEYSTORE_TLS, - doc = "TLS Provider for KeyStore type" + doc = "TLS Provider for Specify the SSL provider for the broker service: \n" + + "When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.\n" + + "When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc." ) private String tlsProvider = null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java index 831e56f4f5d53..e75c518a50f02 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java @@ -28,6 +28,7 @@ import io.netty.handler.flow.FlowControlHandler; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslProvider; import java.net.SocketAddress; import java.util.concurrent.TimeUnit; import lombok.Builder; @@ -92,10 +93,18 @@ public PulsarChannelInitializer(PulsarService pulsar, PulsarChannelOptions opts) serviceConfig.getTlsProtocols(), serviceConfig.getTlsCertRefreshCheckDurationSec()); } else { - sslCtxRefresher = new NettyServerSslContextBuilder(serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustCertsFilePath(), serviceConfig.getTlsCertificateFilePath(), + SslProvider sslProvider = null; + if (serviceConfig.getTlsProvider() != null) { + sslProvider = SslProvider.valueOf(serviceConfig.getTlsProvider()); + } + sslCtxRefresher = new NettyServerSslContextBuilder( + sslProvider, + serviceConfig.isTlsAllowInsecureConnection(), + serviceConfig.getTlsTrustCertsFilePath(), + serviceConfig.getTlsCertificateFilePath(), serviceConfig.getTlsKeyFilePath(), - serviceConfig.getTlsCiphers(), serviceConfig.getTlsProtocols(), + serviceConfig.getTlsCiphers(), + serviceConfig.getTlsProtocols(), serviceConfig.isTlsRequireTrustedClientCertOnConnect(), serviceConfig.getTlsCertRefreshCheckDurationSec()); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java index 3e17a38da2b34..1f302f6586cd5 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java @@ -21,6 +21,7 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslProvider; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -137,21 +138,32 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, JsseSslEngineFactory sslEngineFactory = new JsseSslEngineFactory(sslCtx); confBuilder.setSslEngineFactory(sslEngineFactory); } else { + SslProvider sslProvider = null; + if (conf.getSslProvider() != null) { + sslProvider = SslProvider.valueOf(conf.getSslProvider()); + } SslContext sslCtx = null; if (authData.hasDataForTls()) { sslCtx = authData.getTlsTrustStoreStream() == null ? SecurityUtility.createAutoRefreshSslContextForClient( - conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), - conf.getTlsTrustCertsFilePath(), authData.getTlsCerificateFilePath(), - authData.getTlsPrivateKeyFilePath(), null, autoCertRefreshTimeSeconds, delayer) + sslProvider, + conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), + conf.getTlsTrustCertsFilePath(), authData.getTlsCerificateFilePath(), + authData.getTlsPrivateKeyFilePath(), null, autoCertRefreshTimeSeconds, delayer) : SecurityUtility.createNettySslContextForClient( - conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), - authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), - authData.getTlsPrivateKey()); + sslProvider, + conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), + authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), + authData.getTlsPrivateKey(), + conf.getTlsCiphers(), + conf.getTlsProtocols()); } else { sslCtx = SecurityUtility.createNettySslContextForClient( + sslProvider, conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), - conf.getTlsTrustCertsFilePath()); + conf.getTlsTrustCertsFilePath(), + conf.getTlsCiphers(), + conf.getTlsProtocols()); } confBuilder.setSslContext(sslCtx); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index c295975ecd36a..2a7e434cd3885 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -18,24 +18,24 @@ */ package org.apache.pulsar.client.impl; +import io.netty.channel.EventLoopGroup; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslProvider; import java.io.Closeable; import java.io.IOException; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; +import java.security.GeneralSecurityException; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.CompletableFuture; - -import io.netty.channel.EventLoopGroup; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.ssl.SslContext; import javax.net.ssl.SSLContext; import lombok.extern.slf4j.Slf4j; - import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; @@ -111,25 +111,33 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, JsseSslEngineFactory sslEngineFactory = new JsseSslEngineFactory(sslCtx); confBuilder.setSslEngineFactory(sslEngineFactory); } else { + SslProvider sslProvider = null; + if (conf.getSslProvider() != null) { + sslProvider = SslProvider.valueOf(conf.getSslProvider()); + } SslContext sslCtx = null; if (authData.hasDataForTls()) { sslCtx = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), authData.getTlsCertificates(), - authData.getTlsPrivateKey()) - : SecurityUtility.createNettySslContextForClient(conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), - authData.getTlsPrivateKey()); - } - else { + ? SecurityUtility.createNettySslContextForClient(sslProvider, + conf.isTlsAllowInsecureConnection(), + conf.getTlsTrustCertsFilePath(), authData.getTlsCertificates(), + authData.getTlsPrivateKey(), conf.getTlsCiphers(), conf.getTlsProtocols()) + : SecurityUtility.createNettySslContextForClient(sslProvider, + conf.isTlsAllowInsecureConnection(), + authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), + authData.getTlsPrivateKey(), conf.getTlsCiphers(), conf.getTlsProtocols()); + } else { sslCtx = SecurityUtility.createNettySslContextForClient( + sslProvider, conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath()); + conf.getTlsTrustCertsFilePath(), conf.getTlsCiphers(), conf.getTlsProtocols()); } confBuilder.setSslContext(sslCtx); } confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); + } catch (GeneralSecurityException e) { + throw new PulsarClientException.InvalidConfigurationException(e); } catch (Exception e) { throw new PulsarClientException.InvalidConfigurationException(e); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index b7a5fbadb4237..497793d792d8e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -18,6 +18,13 @@ */ package org.apache.pulsar.client.impl; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslProvider; import java.net.InetSocketAddress; import java.util.Objects; import java.util.concurrent.CompletableFuture; @@ -36,13 +43,6 @@ import org.apache.pulsar.common.util.SecurityUtility; import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.LengthFieldBasedFrameDecoder; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslHandler; - @Slf4j public class PulsarChannelInitializer extends ChannelInitializer { @@ -93,19 +93,36 @@ public PulsarChannelInitializer(ClientConfigurationData conf, Supplier(() -> { try { + SslProvider sslProvider = null; + if (conf.getSslProvider() != null) { + sslProvider = SslProvider.valueOf(conf.getSslProvider()); + } + // Set client certificate if available AuthenticationDataProvider authData = conf.getAuthentication().getAuthData(); if (authData.hasDataForTls()) { return authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - authData.getTlsCertificates(), authData.getTlsPrivateKey()) - : SecurityUtility.createNettySslContextForClient(conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), - authData.getTlsCertificates(), authData.getTlsPrivateKey()); + ? SecurityUtility.createNettySslContextForClient( + sslProvider, + conf.isTlsAllowInsecureConnection(), + conf.getTlsTrustCertsFilePath(), + authData.getTlsCertificates(), + authData.getTlsPrivateKey(), + conf.getTlsCiphers(), + conf.getTlsProtocols()) + : SecurityUtility.createNettySslContextForClient(sslProvider, + conf.isTlsAllowInsecureConnection(), + authData.getTlsTrustStoreStream(), + authData.getTlsCertificates(), authData.getTlsPrivateKey(), + conf.getTlsCiphers(), + conf.getTlsProtocols()); } else { - return SecurityUtility.createNettySslContextForClient(conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath()); + return SecurityUtility.createNettySslContextForClient( + sslProvider, + conf.isTlsAllowInsecureConnection(), + conf.getTlsTrustCertsFilePath(), + conf.getTlsCiphers(), + conf.getTlsProtocols()); } } catch (Exception e) { throw new RuntimeException("Failed to create TLS context", e); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java index 560746df7f608..e1fef9aaa9b10 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java @@ -19,10 +19,12 @@ package org.apache.pulsar.common.util; import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslProvider; import java.io.FileNotFoundException; import java.io.IOException; import java.security.GeneralSecurityException; import java.security.cert.X509Certificate; +import java.util.Set; import javax.net.ssl.SSLException; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.AuthenticationDataProvider; @@ -33,19 +35,33 @@ @Slf4j public class NettyClientSslContextRefresher extends SslContextAutoRefreshBuilder { private volatile SslContext sslNettyContext; - private boolean tlsAllowInsecureConnection; + private final boolean tlsAllowInsecureConnection; protected final FileModifiedTimeUpdater tlsTrustCertsFilePath; - private AuthenticationDataProvider authData; + protected final FileModifiedTimeUpdater tlsCertsFilePath; + protected final FileModifiedTimeUpdater tlsPrivateKeyFilePath; + private final AuthenticationDataProvider authData; + private final SslProvider sslProvider; + private final Set ciphers; + private final Set protocols; - public NettyClientSslContextRefresher(boolean allowInsecure, + public NettyClientSslContextRefresher(SslProvider sslProvider, boolean allowInsecure, String trustCertsFilePath, AuthenticationDataProvider authData, + Set ciphers, + Set protocols, long delayInSeconds) throws IOException, GeneralSecurityException { super(delayInSeconds); this.tlsAllowInsecureConnection = allowInsecure; this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); this.authData = authData; + this.tlsCertsFilePath = new FileModifiedTimeUpdater( + authData != null ? authData.getTlsCerificateFilePath() : null); + this.tlsPrivateKeyFilePath = new FileModifiedTimeUpdater( + authData != null ? authData.getTlsPrivateKeyFilePath() : null); + this.sslProvider = sslProvider; + this.ciphers = ciphers; + this.protocols = protocols; } @Override @@ -53,15 +69,16 @@ public synchronized SslContext update() throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { if (authData != null && authData.hasDataForTls()) { this.sslNettyContext = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(this.tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), (X509Certificate[]) authData.getTlsCertificates(), - authData.getTlsPrivateKey()) - : SecurityUtility.createNettySslContextForClient(this.tlsAllowInsecureConnection, - authData.getTlsTrustStoreStream(), (X509Certificate[]) authData.getTlsCertificates(), - authData.getTlsPrivateKey()); + ? SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, + tlsTrustCertsFilePath.getFileName(), (X509Certificate[]) authData.getTlsCertificates(), + authData.getTlsPrivateKey(), this.ciphers, this.protocols) + : SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, + authData.getTlsTrustStoreStream(), (X509Certificate[]) authData.getTlsCertificates(), + authData.getTlsPrivateKey(), this.ciphers, this.protocols); } else { - this.sslNettyContext = SecurityUtility.createNettySslContextForClient(this.tlsAllowInsecureConnection, - this.tlsTrustCertsFilePath.getFileName()); + this.sslNettyContext = + SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, + this.tlsTrustCertsFilePath.getFileName(), this.ciphers, this.protocols); } return this.sslNettyContext; } @@ -73,6 +90,8 @@ public SslContext getSslContext() { @Override public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh(); + return tlsTrustCertsFilePath.checkAndRefresh() || tlsCertsFilePath.checkAndRefresh() + || tlsPrivateKeyFilePath.checkAndRefresh(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java index 250e628f0def7..e9fbb1f5e3ecf 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java @@ -19,6 +19,7 @@ package org.apache.pulsar.common.util; import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslProvider; import java.io.FileNotFoundException; import java.io.IOException; import java.security.GeneralSecurityException; @@ -36,8 +37,10 @@ public class NettyServerSslContextBuilder extends SslContextAutoRefreshBuilder tlsCiphers; protected final Set tlsProtocols; protected final boolean tlsRequireTrustedClientCertOnConnect; + protected final SslProvider sslProvider; - public NettyServerSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath, + public NettyServerSslContextBuilder(SslProvider sslProvider, boolean allowInsecure, String trustCertsFilePath, + String certificateFilePath, String keyFilePath, Set ciphers, Set protocols, boolean requireTrustedClientCertOnConnect, long delayInSeconds) { @@ -49,14 +52,17 @@ public NettyServerSslContextBuilder(boolean allowInsecure, String trustCertsFile this.tlsCiphers = ciphers; this.tlsProtocols = protocols; this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; + this.sslProvider = sslProvider; } @Override public synchronized SslContext update() - throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { - this.sslNettyContext = SecurityUtility.createNettySslContextForServer(tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), tlsKeyFilePath.getFileName(), - tlsCiphers, tlsProtocols, tlsRequireTrustedClientCertOnConnect); + throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { + this.sslNettyContext = + SecurityUtility.createNettySslContextForServer(this.sslProvider, tlsAllowInsecureConnection, + tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), + tlsKeyFilePath.getFileName(), + tlsCiphers, tlsProtocols, tlsRequireTrustedClientCertOnConnect); return this.sslNettyContext; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index db8f861c789a3..d5a0c5a576722 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -21,6 +21,7 @@ import io.netty.handler.ssl.ClientAuth; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslProvider; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import java.io.BufferedReader; import java.io.File; @@ -111,7 +112,7 @@ public static Provider getProvider() { return getBCProviderFromClassPath(); } catch (Exception e) { log.warn("Not able to get Bouncy Castle provider for both FIPS and Non-FIPS from class path:", e); - throw new RuntimeException(e); + return null; } } @@ -201,10 +202,13 @@ public static SSLContext createSslContext(boolean allowInsecureConnection, Certi return createSslContext(allowInsecureConnection, trustCertificates, (Certificate[]) null, (PrivateKey) null); } - public static SslContext createNettySslContextForClient(boolean allowInsecureConnection, String trustCertsFilePath) + public static SslContext createNettySslContextForClient(SslProvider sslProvider, boolean allowInsecureConnection, + String trustCertsFilePath, Set ciphers, + Set protocols) throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { - return createNettySslContextForClient(allowInsecureConnection, trustCertsFilePath, (Certificate[]) null, - (PrivateKey) null); + return createNettySslContextForClient(sslProvider, allowInsecureConnection, trustCertsFilePath, + (Certificate[]) null, + (PrivateKey) null, ciphers, protocols); } public static SSLContext createSslContext(boolean allowInsecureConnection, String trustCertsFilePath, @@ -230,12 +234,15 @@ public static SSLContext createSslContext(boolean allowInsecureConnection, Strin * @throws FileNotFoundException * @throws IOException */ - public static SslContext createAutoRefreshSslContextForClient(boolean allowInsecureConnection, - String trustCertsFilePath, String certFilePath, String keyFilePath, String sslContextAlgorithm, - int refreshDurationSec, ScheduledExecutorService executor) + public static SslContext createAutoRefreshSslContextForClient(SslProvider sslProvider, + boolean allowInsecureConnection, + String trustCertsFilePath, String certFilePath, + String keyFilePath, String sslContextAlgorithm, + int refreshDurationSec, + ScheduledExecutorService executor) throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { KeyManagerProxy keyManager = new KeyManagerProxy(certFilePath, keyFilePath, refreshDurationSec, executor); - SslContextBuilder sslContexBuilder = SslContextBuilder.forClient(); + SslContextBuilder sslContexBuilder = SslContextBuilder.forClient().sslProvider(sslProvider); sslContexBuilder.keyManager(keyManager); if (allowInsecureConnection) { sslContexBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); @@ -246,46 +253,62 @@ public static SslContext createAutoRefreshSslContextForClient(boolean allowInsec return sslContexBuilder.build(); } - public static SslContext createNettySslContextForClient(boolean allowInsecureConnection, String trustCertsFilePath, - String certFilePath, String keyFilePath) + public static SslContext createNettySslContextForClient(SslProvider sslProvider, boolean allowInsecureConnection, + String trustCertsFilePath, + String certFilePath, String keyFilePath, + Set ciphers, + Set protocols) throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { X509Certificate[] certificates = loadCertificatesFromPemFile(certFilePath); PrivateKey privateKey = loadPrivateKeyFromPemFile(keyFilePath); - return createNettySslContextForClient(allowInsecureConnection, trustCertsFilePath, certificates, privateKey); + return createNettySslContextForClient(sslProvider, allowInsecureConnection, trustCertsFilePath, certificates, + privateKey, ciphers, protocols); } - public static SslContext createNettySslContextForClient(boolean allowInsecureConnection, String trustCertsFilePath, - Certificate[] certificates, PrivateKey privateKey) + public static SslContext createNettySslContextForClient(SslProvider sslProvider, boolean allowInsecureConnection, + String trustCertsFilePath, + Certificate[] certificates, PrivateKey privateKey, + Set ciphers, + Set protocols) throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { if (StringUtils.isNotBlank(trustCertsFilePath)) { try (FileInputStream trustCertsStream = new FileInputStream(trustCertsFilePath)) { - return createNettySslContextForClient(allowInsecureConnection, trustCertsStream, certificates, - privateKey); + return createNettySslContextForClient(sslProvider, allowInsecureConnection, trustCertsStream, + certificates, + privateKey, ciphers, protocols); } } else { - return createNettySslContextForClient(allowInsecureConnection, (InputStream) null, certificates, - privateKey); + return createNettySslContextForClient(sslProvider, allowInsecureConnection, (InputStream) null, + certificates, + privateKey, ciphers, protocols); } } - public static SslContext createNettySslContextForClient(boolean allowInsecureConnection, - InputStream trustCertsStream, Certificate[] certificates, PrivateKey privateKey) + public static SslContext createNettySslContextForClient(SslProvider sslProvider, boolean allowInsecureConnection, + InputStream trustCertsStream, Certificate[] certificates, + PrivateKey privateKey, Set ciphers, + Set protocols) throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { - SslContextBuilder builder = SslContextBuilder.forClient(); + SslContextBuilder builder = SslContextBuilder.forClient().sslProvider(sslProvider); setupTrustCerts(builder, allowInsecureConnection, trustCertsStream); setupKeyManager(builder, privateKey, (X509Certificate[]) certificates); + setupCiphers(builder, ciphers); + setupProtocols(builder, protocols); return builder.build(); } - public static SslContext createNettySslContextForServer(boolean allowInsecureConnection, String trustCertsFilePath, - String certFilePath, String keyFilePath, Set ciphers, Set protocols, - boolean requireTrustedClientCertOnConnect) + public static SslContext createNettySslContextForServer(SslProvider sslProvider, boolean allowInsecureConnection, + String trustCertsFilePath, + String certFilePath, String keyFilePath, + Set ciphers, Set protocols, + boolean requireTrustedClientCertOnConnect) throws GeneralSecurityException, SSLException, FileNotFoundException, IOException { X509Certificate[] certificates = loadCertificatesFromPemFile(certFilePath); PrivateKey privateKey = loadPrivateKeyFromPemFile(keyFilePath); - SslContextBuilder builder = SslContextBuilder.forServer(privateKey, (X509Certificate[]) certificates); + SslContextBuilder builder = + SslContextBuilder.forServer(privateKey, (X509Certificate[]) certificates).sslProvider(sslProvider); setupCiphers(builder, ciphers); setupProtocols(builder, protocols); if (StringUtils.isNotBlank(trustCertsFilePath)) { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java new file mode 100644 index 0000000000000..0fbd2521ae08d --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.common.util.netty; + +import static org.testng.Assert.assertThrows; +import com.google.common.io.Resources; +import io.netty.handler.ssl.SslProvider; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.HashSet; +import java.util.Set; +import javax.net.ssl.SSLException; +import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.KeyStoreParams; +import org.apache.pulsar.common.util.NettyClientSslContextRefresher; +import org.apache.pulsar.common.util.NettyServerSslContextBuilder; +import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class SslContextTest { + @DataProvider(name = "caCertSslContextDataProvider") + public static Object[][] getSslContextDataProvider() { + Set ciphers = new HashSet<>(); + ciphers.add("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384"); + ciphers.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + ciphers.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"); + ciphers.add("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"); + ciphers.add("TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"); + + // Note: OPENSSL doesn't support these ciphers. + return new Object[][]{ + new Object[]{SslProvider.JDK, ciphers}, + new Object[]{SslProvider.JDK, null}, + + new Object[]{SslProvider.OPENSSL, ciphers}, + new Object[]{SslProvider.OPENSSL, null}, + + new Object[]{null, ciphers}, + new Object[]{null, null}, + }; + } + + @DataProvider(name = "cipherDataProvider") + public static Object[] getCipher() { + Set cipher = new HashSet<>(); + cipher.add("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384"); + cipher.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + cipher.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"); + cipher.add("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"); + cipher.add("TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"); + + return new Object[]{null, cipher}; + } + + @Test(dataProvider = "cipherDataProvider") + public void testServerKeyStoreSSLContext(Set cipher) throws Exception { + NettySSLContextAutoRefreshBuilder contextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder(null, + "JKS", Resources.getResource("ssl/jetty_server_key.jks").getPath(), + "jetty_server_pwd", false, "JKS", + Resources.getResource("ssl/jetty_server_trust.jks").getPath(), + "jetty_server_pwd", true, cipher, + null, 600); + contextAutoRefreshBuilder.update(); + } + + private static class ClientAuthenticationData implements AuthenticationDataProvider { + @Override + public KeyStoreParams getTlsKeyStoreParams() { + return null; + } + } + + @Test(dataProvider = "cipherDataProvider") + public void testClientKeyStoreSSLContext(Set cipher) throws Exception { + NettySSLContextAutoRefreshBuilder contextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder(null, + false, "JKS", Resources.getResource("ssl/jetty_server_trust.jks").getPath(), + "jetty_server_pwd", cipher, null, 0, new ClientAuthenticationData()); + contextAutoRefreshBuilder.update(); + } + + @Test(dataProvider = "caCertSslContextDataProvider") + public void testServerCaCertSslContextWithSslProvider(SslProvider sslProvider, Set ciphers) + throws GeneralSecurityException, IOException { + NettyServerSslContextBuilder sslContext = new NettyServerSslContextBuilder(sslProvider, + true, Resources.getResource("ssl/my-ca/ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-key.pem").getPath(), + ciphers, + null, + true, 60); + if (ciphers != null) { + if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { + assertThrows(SSLException.class, sslContext::update); + return; + } + } + sslContext.update(); + } + + @Test(dataProvider = "caCertSslContextDataProvider") + public void testClientCaCertSslContextWithSslProvider(SslProvider sslProvider, Set ciphers) + throws GeneralSecurityException, IOException { + NettyClientSslContextRefresher sslContext = new NettyClientSslContextRefresher(sslProvider, + true, Resources.getResource("ssl/my-ca/ca.pem").getPath(), + null, ciphers, null, 0); + if (ciphers != null) { + if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { + assertThrows(SSLException.class, sslContext::update); + return; + } + } + sslContext.update(); + } +} diff --git a/pulsar-common/src/test/resources/ssl/jetty_client_key.jks b/pulsar-common/src/test/resources/ssl/jetty_client_key.jks new file mode 100644 index 0000000000000000000000000000000000000000..2b8ea64347ddc536209770871a3629a62e806b04 GIT binary patch literal 2679 zcma)+c{J1u8^>qH%$Q+FmaJK_T+7gpGDC*!%h)1Z*+RA{WQ)i;cTAa5A>lS-Np=(2 z)r2TS!ia1oyM`#tgmk@~^S~?o&*%GmzkhxKB2Od)%mom6&~T(G*^>O7 z56lfNB=Q`E5_!abVG)1`tNt$uMuQSzdB1S>ugQS({>O#l0YeIjFvDLE2L$}>fbjw6 z0G)qM6rctb?(Jm`w&pdWD`__usFO&%hhSMa1oE2z1S|`n;D~=GLXa>Ja1@U8CR>92 zxu9TGsDQ;l0;cFo?FM$EYqFP0BdOQ zOW2Hc$=;m$0C^bRSZl3fEO;l&2s5JHyEm2{!fO*0THA6&$|mJUHHYCPsG4s=^7)gK zoZ{8nam4&_g$GSLyevUJRY`AuovLjT-!LlGJ9OH*Q*vE&eWi_1BXry|Sz7F;a&0yA zQ(s4V(7f$I!CPzI8|jv2%W3y*B(QdGBF;%zQ}k>$8kM)zc3=}UXe;ogMXKv>q~(^a>#WN9j$ zG-M6PdmVnfS1SLFTJ(@^#o9f~WQUsmmgFJI?cbJ{;vy}>H%#ajU4P`!!+pmUbU8`e zhc2lW)bGcKOT2haTIPTOzhxV)+_cC>1Vo`9nAcsc65^0QDzQ|@euELH}pPzW6?DY_xNg`rMG>Pe*&e| zbsLj!VYL|VvOa1$xkA&hz{qAXQpXf$xVtOhp~;#``vI&F;~_Mn-SEZU+asl|p27S9 z8%VTwoB9@Sq`#On|93t!$qDqJ5@9tPDVo+D`PTE`a73gm(M7<0=ldMHSR$|y>r66g zB1|WBo%yMqQB4p^hv-LHUCshs@I)*gh*!U082}z4VBouwYw$ zR)kFQau3;4i#0zw*q#XpYajO}!NROW~6{VO4=o-!qZcmN&9_8n`8K=+f)IT6ijJJ}<;8?@-* z`f(s;zEDsu$ZfT9Z7U&}`99UKAk1N^>f&NYE}sPuOkF8Cv_+COa^kpinjaSHS4nDA zwxVY!Uhqt&-|LSj$BoLo(K964putKVh)TB;7?uyDOI!<{t@y${qHTp2u;L3ff8au~ z+xHlwtuII%xRhoWof=;EWU=PuxUF-OcQ(a;q`BhM6pb1Vi~~=A%pLD1`e8E=z4LahXJTapqc1TI{oLc(_$?W89?rL-9dJ@$x<(TCZTnw&#zVOvehNeC6 z^|DBg5MBRL@gpgx)yR2}OJ=dP+x9WsOk$Hkfbff(WF8GlF8<}F{i)mi`b$if^cxC& zL(>2d|GO@c!kQ>)9#9A<7IX>Z4~hlk{*gp9g`t8@gb;sOP0UFhO&ttYO9u-801@i* z_Y*g~kO+1Bh3p_;(643jj{^Q*C9U>nO9xX*|H7^Kpr`s@4N+EoKhpl6k~Z$cCx?C< zX~HV8M=umjI3^RhhN`6)+|T^nf#Z5}rZ*PbjWevmFrx}XmI@E5kJFk4refCI)79;W z`~wW%A_jJO3fij$k0?G6o78JDq$R0+9GB#IZ?zFB{;l+d!sc=C{Lp)wb=mTV2(~lf zd%5tyEB*mLF+>PnaTiKT)vU{Yy~d9S-c0z$#W{bf{NlE!G~Yxz#Po~y7Ydh8HO=Cs zufvxB<}FIfx>K}{-LdpGt@3p-lt>h+{Q+dCgud|ekicK(GU3HafzshO{U#PZ4Cfks zzNQ=z2JS2n2-Zs><-|?In^Xl~|9zS_W!AWNw}4-@>XJH%3xnsZ1oqfob&HCdS8mWa z{OX;PeBIg&zolM#V@!Hd8!Th4rCMFyq|y5NSF#KxMfot#y=*v^@QKhJuAe1_;h>eF zA9Hl{lkb7dt0Q+t6%rKLQAalVn$(`f`80@=;*@<$%Chx1;*2SW^my&Dfwasg*rr@w zW*=7_%jXmQbL+6nSUJ8~MRhLsGGUcMp2|xWO_#|g)ok?~&y9Oqww&71d}R@P?u5^q zFeLcI{rguAsbbnr&;M*0e%LR^9OxJhpnf`Os_C~?bj5i`d+kZz-P@wJ#E~G2TKTo| zn$Ee!){KtNG-OI^AdD`p8R-wPkAJ(;YjS_>(au8MwMP(e+@1!QENz zQT=$w6$jf1IhA$xtHuz7B4)2-RHZchN99rVxNY3IH1vfRPd$|0KW}ml@+&$Y+J~z- zA!O-Ta1H6IG({E1g}ETNP4ydS_u(M=TRd>zZ=mz7~; zzDlv@U%3_XP+8*XW-_uQp`I8SklWk)juwrxp9LV%@xfPpwRXeq>6Tzw-H&f7!lQad ztnI_YP52g?HtI{=-f7gC`a5ux6`?k3CU}-&)X6M9_MvwLgnBlCH~jI1-slxO^0b3c z9vyN=`IckGzR4LYR07qZtzYNFhv47Z;1(f_juXWTPH#MxLh;ipSB1%;XnnPO$&o$& z=HEs3jK6DlkSY!KYO!r0W}RAfDsaby`;60|+l&3X#4fY9SZaN|edJ%^8g_4*yKo9hB0UwLkk>X_cg1#}x>sw+RSQap z@)k;sbCS=yZzmQGYv}Haeb~B1Uye!{%9>}Fs2)=hYH|0SbQ5!#_PVIIUTDhk44UrG zo_+BJFaYELBpjy3&jpcyfuO>OwKJQG@jt literal 0 HcmV?d00001 diff --git a/pulsar-common/src/test/resources/ssl/jetty_client_trust.jks b/pulsar-common/src/test/resources/ssl/jetty_client_trust.jks new file mode 100644 index 0000000000000000000000000000000000000000..166a2e00fb371dd160b46c58c7fdda651b80476d GIT binary patch literal 1207 zcmV;o1W5ZZf&{Yy0Ru3C1Z)NgDuzgg_YDCD0ic2eT?B#zSulbGRWO1CQ3eSrhDe6@ z4FLxRpn?QKFoFa=0s#Opf&@1P2`Yw2hW8Bt2LUi<1_>&LNQUx*>HEztr3ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q5^M!Qal>5Fu9lG2=y3IQM{p1Hj5fkrUO$PsVgh_z>aVy6Owl z_B9+#ZC|>KfcUL|7_CY&aO4xIf|96gWjHehp;Y4!>sS@?EWjQ=d+XsZg^7dp^TeY- zzmjwZ9@gMR;!r&giVFFc?)qDf>!lsdCxaBa+rIK=D%ZxSF@q;=}sH zU`O19^Av45I@o%cH&)LRjP0uu45d#_5!XkhqG@7f7r*OlNP|c%>f_<5_}+};6V)Ye zo<&eUjqm%MmS6YE*Eo86+8~=D9_0!c34C{~kf;Ik0n2H34li!D_#G%HS5EXBPa4g1 zMZbWlwCjx`h<`-x=4tGu75Mxk3(#Z84s zDUF@87lhI{F91%P>^+1ypM3H?3ir^XTWH#SfMogz_CvvQTOondxHfV=Mc!=BF@qE1 z+wFg<&CWA^go4=Eq76dLZn3Y69?Q3+Us-dr-jn^MvD> zth`d6)$l^5<1*&malqV@mqlV3Aw%p<8c2hSEPo3oLUzYf*nhxH6!dw2KB-v(ZJF~j zgI~)+$p&SJ%m!Z`1=CSOu?$E(F|qkajvaE1toh*9V(7iR+j^&|mzeX{a)Ed7?+r?Q z>IpT!G`B9+XGP=uj+ZiF8k+HF2^-*-Ph^^l5{+)6_0oLnGnN!U*enujJW)y>3wQz2 z;f5ykk#&1Ld=Ooq6@lzy^K}rOV#U|&>&8T{xFg~Wj|Vr@Eo$U&!2IKra#PjkEK$)S_Fi%(FtbThGYxZF5Z`UpSYg?e2E)u!vTys ze;3jUqo#IR=b`1Pf;YU|nS!fyv?ss>JBZ&EA&9{AB-j1(#1-J8QE&}mmGv0I*j9+L ze@zWFisk;5)z#5mRS*q6Dv4u?poAuood3M$$9g4CmOT$O&@YwLl;|2%=|;o8+emn9ebOhnC?~#}$I^h@!r!!V#h5^OS=4h@@+Rx&ql41i_-s*j zNR#b7c`+Vl@obamJi@l89PjWCP^gsBpoZ_fZe5gs{MoW~Qe*y3wWxU=YUB_kDt!Tg zFkP!&rzfLU$z!l8PPH3s6n)}QGnb}TW4pQ4tBiBYKnSwgXOZ+Fv!V858yjfYFG4qK zc=86Fmswe&6(HIi;SGFvXHjAYapq$y7DoYcNW3>e4KzOF50RjU7hM@CGG!p;- literal 0 HcmV?d00001 diff --git a/pulsar-common/src/test/resources/ssl/jetty_server_key.jks b/pulsar-common/src/test/resources/ssl/jetty_server_key.jks new file mode 100644 index 0000000000000000000000000000000000000000..b6189b75c8ad01656e2d5e9102de21d82183b61f GIT binary patch literal 2679 zcma)+X*d*&7RP6pF$~#Fq~@K3!Jx5Z8)e_JXJ;5?DO<>vEsQZkVXWCT$@1EgY?Uly zA4{^NM7CmNDe4v3=6arcpZ9(4r+Yt~=RD{4KmYULeE6gB&>$utGa3(-W?_>e8WML; z0UU9%^P@acFxtCpe;S$+O7Xt$>GBF{!nScm12Mg=JFEX)#0qAoqY_3E@ zpa(MuC%W^VX`cow3oFs zo><#J99M7b-8cJe736RVi8p6?H;e0!c2%V{3MKW9k@>K{>ooT(Mq>S=E(e)=(vGU~ z7O1DD%$vW;Bv~wMwQXoGULh955gQEPQ6MZMs=qH@6* z5uS$`^Iw2p$PR7A#4+$f?gyZA@{10{Tw$IoCuQRkN_|w?L^TnkXPiHB>PiPU#@~5Y z!9mF1LG@fNpYL=2QUdU?`?19gmN(@u`&sKl1Knp_h3*+SJKr@yAU*o~5!%{BgT*sYXC1;_qRcBeNkSefY;KcqnX_c5C<#P9as7(m8& zr6jy+>7rW8w3jxFeSsYtc9KT#2lwF~QVzDb?MM=Y_4@9VG7b#L9%xeMN9a#Ih9(Nl zuaqPVyLu)|ioncWPmAZv@eFd4Y6Im?(xT{(XNB|5NH4qo>@^X~B33{6j+ zj3o!x8}PX~jjA(o`4!)!9j5CwOS#AAyjA|SH~xn$vu^d4(kw%gQ1^W1o*~;FOR|`? zP~|H7wlYqCQ8lBP;|8x4#(w6_=irHkEA_9pNo9p&DFVuZ+0psqzoX&R%!%#6eeAj= zP~4)>1XvVo$P0^&pbw?Oja-m=*j-|KAZ$KI=mEm77`QEJqL{raHVK;TQcHVE?MB*U`f}%s>8wj9#+AFw|u+6 zI#Z>9su`^hZOZN^zkcey#8J7%TXPCg{lEiZ_T6TbIyVI@IDc>YMcA`!R6~^!&%!VH zio01fD{z?hTCFLJlh1PRrOLs|#a+r%vb^bJhbUV4K+=iujYH&B7=hlrraC)j5_?Gd zXzgqnx)ky(pptTx3p!r^mtkITP$|AaKs zb0_Pdzc&>U7_obEXsIt2&-_-qA200c`=i=p@SQ8t;*`ZIIh>LIX#c?$yNnxGx-@CG9V!L7FJ4KSuGqX&B+6|BaW5Xv0*68j*#pS{@pz^mx^gY>19M-NN5 zrdu}_)JjON(q`$K^JzTBGCk{OU{XGDln_a=ZOJVYd-bwwdgM6`Xzk@vK0_bGO}5&f z-SY^}80&9+E6cvf4C|N6SgZZ)jFgHB`URl|)_g2t(QFvM4E%wr2k@tr=I)Yv3MDkj zO723hARR^mkM0}vU$|-?shl}5+{NKKz5Oir;wo-DkB>RB79(V7*$qdk9Xq)lpD||0 zX5wlNM^%&J$5&;GhCxo#+$ymdAF4nHrK$p*-|1&GQq1?KpO^nwJ+~?`cS(8qDOz59 zYQ1ob{l2g!Fr5J-8DB2^^Okl{;#SFWh`+sNXVajcdQ9dfY8KvR;nsnw$JJ@Xs8(&g zKiV3RqlNdu3&TGj1Sug??R$EGAzGOng-@KoJbojDry?5aN>~q;7s)v6(}4 zd75i6-ME&)pkQs#EsOfpHa9`ZY>YXM;OHU6<b~0Kfjx$$TqxRP9Q?laVx({SEw|p zsq>Yr49M9W-J9Z$2Ftt)w{0&O{Vod*@~smt_}FRP`sJ(ILS&W8ed8r*tMoXBF?ODy z?E+S1Yj(=r+0^BD!^bawE!w`X;m+t+-3Y+2P7IL1*FrdD9J`b59n+Qzg~>4fytnR7 zd_FG%;)6q&#__}pRBW_!=uGWOwKEb49x(9eJ z()s{(+pgj?)%NF+lJ3#n!k8a>kGF<>*e&j%B|2rN9Nu9^s?~(t<36b*!Z0=F$ZD{M zc;dmmOJ&^l9+c01mQrS6;DJsJ1)qpA_p4#>A+8$R>BAU)S~C}p=2qb)=dwVE3PgCy zWT$&(%A&8J&!gE`z$i{;rZZpwh`%^7JT?I}quq=3p8No(`+aSzJp}R3&tU&3V+y$Q VG@ehyINfadmm$iEi5W&LNQU1{2WbhzDFg0%tyClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q6%HmdJvi=!x)|)(Hp2!0dp61HeN(zLVt~!UK$+y>FT6hquWc zr{0$V-CjGyITpE#ztPRgd(`p)YFNC@C_yu^0N01Xe*q8KQ;UmCk%nnl+PypV^2qf+ zR5yLpp)DMAmu}*~$K;662hCCZ_{m5P2>T^RxB|+ZzD-qP4)!Vf&Fe=T+xF@JBdWoF z{S1eg(YE3~A(q!fiujgl`J!f@`vN?5e2(Wnr{@>fso0GN%?f~^he4CIysbR?m{@;T z)lx{Nv%{~>sw_e}Vu z;OD|xTR{ud@3X3lmr#6Ro};!0(F_X4b$)6B2CCVUvrbIJI%)!`k3jnD-^f9Hn&cc$Ef*nIU{&(ehk z4R22VpYJy+S=az>ufLVyHVq|?fZZWknJoD@*scGvYVB122|{OxC-%g;Spmd?ygmT& z1UuMmhg!~l@A_`k$I(#?v*m2MZNoK?deqnSWE-bKmMedv9-u*ob~U|IwIh`Ueg_MH zeXQjav8gD=;3htb*_c^RFSCP}^|s%9NRC#c)!eor%_kzk)yd}&&o)?oP{(!!!N~pu3E;* zY2Ww4*mK2JS_z11Dc^y@vC#}G7yiV?JYXeTBoq<&&RHuhLchZfDRcTptBPE03vJ(o zg<1-94BM}s)}9%Pf<9eXy6+H`36S!$HUV+2q-G3Ll#XAY>s2f$UhNZ&>62r_1S!<| z=y+ro10#ie1Fkq;i~FdCDE%T{rOjo-J9%3z?ah?$iQi?twj8Vg1Qlj?lMK93ubTbKujdc^9ACs zc%-%U0zjYXYlz>Txj2xl%&TWzVevX=IVS)B literal 0 HcmV?d00001 diff --git a/pulsar-common/src/test/resources/ssl/my-ca/ca.pem b/pulsar-common/src/test/resources/ssl/my-ca/ca.pem new file mode 100644 index 0000000000000..3d5a80e234784 --- /dev/null +++ b/pulsar-common/src/test/resources/ssl/my-ca/ca.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC9DCCAdygAwIBAgIUNbNkV2+K2Hf4Q1V5gdAENZQiLokwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAxMGUHVsc2FyMCAXDTIyMDExNDA0MjgwMFoYDzIxMjIwMTE2 +MDQyODAwWjARMQ8wDQYDVQQDEwZQdWxzYXIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDBR2K5EKVziLqdsz78efEW4lOwKiJ32e97uxn1Z6oKgkgImpVP +Z9aoJB4EwSnDg+6FV2YULdWPm7C6W33tDmWRaU/Hlo/cOejnK8UmiMu/EyDpE2Wj +n0RimGmwOkBi2IWIcIzWMmPDZ9kZc65OUeEmwZedKRy62PQyfCeNU4OOHQn3PXjI +NbXJZD5TvBmn4SJn2RP9EgmIPaBAh/Mng045ZeHHLhwMKC8EOyHc2aB7AL6brymR +xzsiYWdcJn4mqqMvT82mVvhkgAMOcR4CXYF8eYnsG6ZbDHb13CawcvLVREJZk7AB +XZi9Rd5xczxHILM8rdkIZfunaG1X5hbih5wJAgMBAAGjQjBAMA4GA1UdDwEB/wQE +AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTCC1lYG+62cUPjNk9q4jCm +Ps65njANBgkqhkiG9w0BAQsFAAOCAQEAKV2Lpu5cH5EsG53EWsYxEKvuQZ0LTxCE +wCDf/NxJaQbzfv0tsbZatMge0vcZ/5r8tZZoOC+pGTwk6MaRbEFH8PmvlH1LIQvu +Y34/YQZOy8wBTWwaIfFMnYWc0iAFoFt2Lzuq+GOI+svTFp729Ae8r7UxY/f9Lioc +ttdGr7vA6PpcIMoEIPjVp+m41uL9IDfX8eOxg4gVlwtqpbHdTzMrOz0YY+3qH/WK +6Qffw4pwitzAEj2zCn2lvGC5cbpd13SAaqtB3xL/Aet0SS2r3g9qDo1RruQhXUng +06U/Hqtn5K1fNQv3pivi3Jg5z1DfJWHkH37luAoIlOZHRmPK6rhp/g== +-----END CERTIFICATE----- diff --git a/pulsar-common/src/test/resources/ssl/my-ca/client-ca.pem b/pulsar-common/src/test/resources/ssl/my-ca/client-ca.pem new file mode 100644 index 0000000000000..adcae3393ade1 --- /dev/null +++ b/pulsar-common/src/test/resources/ssl/my-ca/client-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHDCCAgSgAwIBAgIUJJpmKX3DnbUwJ7tUhCt8MTiwz0owDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAxMGUHVsc2FyMCAXDTIyMDExNDA0MjgwMFoYDzIxMjExMjIx +MDQyODAwWjARMQ8wDQYDVQQDEwZQdWxzYXIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDZN+CNZ1i1WaXulbwSASOfXErWXhGV9DHqavPp3DohgQdundfS +648T/X80uWQlyxu4L4j0oc97jtzc1AyZFXj5nocVsveEO9aDjnYCc5NdBNJLQHgl +IO59fEpTd55NO24g9a8/sxgn0ADCenMlngk1Ou+2QJBONw7W12/WUSUg6ICe+b+x +qPzgApue16oGw9HxhPwa3oEvVZrEnFIWLjsSWtezhgFHMCH9/ngk0KlRyes/EZCz +ZgkO5mgii2fmNDg+yuWUfw7Q0x6BJskGIrxisJiJBRR1+DIvJqgqxJsNmeeEQrZK +YHBukj5RWDFOpOHgqFbPsv45sVKoLrGFrMnNAgMBAAGjajBoMA4GA1UdDwEB/wQE +AwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQW +BBSwkx93xjYP4I+dcFF3xS9NLesmFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJ +KoZIhvcNAQELBQADggEBAAK3ZF63w46pT76QIOeSM3ocUm6izvW/IrxLUESfgRC4 +gg0/5VfPiHHUe6orn15KuPXHe7xCUFqc2oFn5aIU1B/6iOPeNItvMJidU0a3UAiw +hFK9MSFgESNBiEnu1dE5tPcIIxTyCFQ/8loeY3dsdcNVoguH/2J9v/XcMMga46A1 +wudaaa1nb+ZYnXkRuyObKVJQN7EqC+4edinMOTPBbF9wtRMAMBRHXXENXb9zFthi +Dbdn4YvadYsNHxh5ar+hQn/HSPMuCUPY/uUqxtBagb6aS0YnSoUscSLs1Jizg5NX +d+QV8X/5E6W4xWnptUZwVxOemkdnr6A8MH1eQKKFZTM= +-----END CERTIFICATE----- diff --git a/pulsar-common/src/test/resources/ssl/my-ca/client-key.pem b/pulsar-common/src/test/resources/ssl/my-ca/client-key.pem new file mode 100644 index 0000000000000..5b08b151c8094 --- /dev/null +++ b/pulsar-common/src/test/resources/ssl/my-ca/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDZN+CNZ1i1WaXu +lbwSASOfXErWXhGV9DHqavPp3DohgQdundfS648T/X80uWQlyxu4L4j0oc97jtzc +1AyZFXj5nocVsveEO9aDjnYCc5NdBNJLQHglIO59fEpTd55NO24g9a8/sxgn0ADC +enMlngk1Ou+2QJBONw7W12/WUSUg6ICe+b+xqPzgApue16oGw9HxhPwa3oEvVZrE +nFIWLjsSWtezhgFHMCH9/ngk0KlRyes/EZCzZgkO5mgii2fmNDg+yuWUfw7Q0x6B +JskGIrxisJiJBRR1+DIvJqgqxJsNmeeEQrZKYHBukj5RWDFOpOHgqFbPsv45sVKo +LrGFrMnNAgMBAAECggEATeVZ45uiFja16J9NuG8sJSPluoY1bD8L/3KnUcAmIImy +7powIXVT8+k+StwI6/ywThbN2FyGmVqcHZz1f5hRr8KH0uJBHOyQetEFxM9Jk1v9 +Rfsymq36mImP5erJnAyp66vvUrqY+P4Ap71duam4x5wBBqyUk1fvPGA5vPOQiwHs +TN9JHizGobY25fpigWKIMamyE7HWXEUzVdOo83ZiNx53ths+WcF/kqto2v5LtyfJ +HgoPocfZI8tRz9tfgc8zOkvyjsvgdd6rLhd0r2oExnyQBJdktGFpQZMGambU328u +NqcdJscjP/HWAHRzuSdOvCMOEn8E5GIjcWEnQqOmSQKBgQDcpb655/UdcVxrv2Ou +8juucDJMpf6i/UcmlXVXx+3zGSuQZcCC2fupe3JcxPdK7bo65YlC3OoRihggh2sS +cnFMNHMfyoE3G/doXIr3QyL9UAQt4yb+7Nz7jRXYcg4Ytv+FVS6BSzIDEK17v+es +GuWDM3JwtigtzYS4tRh7lgmuBwKBgQD8BXp7yIyVv657B8OJJSoeGataziFPhZux +WKoS3gq24169ZWXwLc+nwrdgvBNrRaHuX+cYh93RF9+2WZrRcRL41XqN938adasY +zPsfOJa9IOgUzQtGUMSe1/WqvHfcvqZCqYq4u/LSdf+I67woP4tCqqn4E928aIZb +6PjLH+dUiwKBgH1ntn7y1t1lEKIspPtJsaHzIqNttMvuKAJF7+t0Nkl0hM4NBt1Y +BzDMeLNBP0vW0YGn89uMs3xEgHH8hV52rO4i4UuwTMCFpJgsAM+H2NsgHz/1WrSI +6xANn9zk9h4V5CRjxYq2sjYLxI4RBBtNLiTjmKd24F8n78cLJl8XZ2kBAoGAGoHF +ATH1v2ZaxqvpYApdpK7UfAeEL2YBGyUVNkjOXbAKbec1Uo6u8ZkkSnNdo4G+Z2EE +4Gqh5PUa3YYNJ4w6D5v8eOQYJUNNDJ26p+z+xcOpRU7PqcSi+YYDW8LY5InU2NwW +MBnsj0BD8TXCI4WTcx6aI/KK9t8TiqU1Tb/8R8MCgYANVinOLz2enB+Qzu4o88W/ +witKHI3D9+z/uWjp0Q4rwmr3OL4FD9vZWvL4qwbDgpfLirJ4e3UVfN1/FoytAKlk +Kykf8oDWciCIdxStt/yUpgQv78IL3vM5d9B8Qb7KCRtJ0BIXGJ7Gle3xJeuduZLe ++F+hwI3Dpv5HPqa9o6ttJw== +-----END PRIVATE KEY----- diff --git a/pulsar-common/src/test/resources/ssl/my-ca/server-ca.pem b/pulsar-common/src/test/resources/ssl/my-ca/server-ca.pem new file mode 100644 index 0000000000000..df5f69298e258 --- /dev/null +++ b/pulsar-common/src/test/resources/ssl/my-ca/server-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHDCCAgSgAwIBAgIUVQHD0/oi9Ca50HA7DFLYOO2wEzYwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAxMGUHVsc2FyMCAXDTIyMDExNDA0MjgwMFoYDzIxMjExMjIx +MDQyODAwWjARMQ8wDQYDVQQDEwZQdWxzYXIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDBcqDkMhjLd9ik//UQijqbajQP5t6dvVZNn9gODQrS9oB/URur +NzCcPWYPJZfEJlTkV8mlmgq4dBjwghpy5ALOGiERk55JPIN4cy01hQ6j7YSPFvMv +BjqZvm5dpGDNTr7GY7THegMM1wpk9EaUOm7tBOHtf6ZnANjSMcQM74RCSBt0Koqw +06CKVDCbgJ5NNE1LgwYeVQAwtQAhY8rqqQKJvCorFbq7OiisFBnz5pRBT6N4kMo1 +9LZo3Oe2F2w9eH9vacQ0NjSOCNXqal9Xl/Pwy9JgKKppwZ/3nCgRc+yfjrnkRz0f +b+llb2NpR5Ge+tNMakqelE8bDSw/5BPjRPftAgMBAAGjajBoMA4GA1UdDwEB/wQE +AwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMB0GA1UdDgQW +BBRXws5mmLbW+xOLflUyUZ0I0uN96zAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJ +KoZIhvcNAQELBQADggEBAKMklpYJIkp4icz9Ea5wWQiRXWb94lGdyCA833VHeGB2 +fKvNXj1d6lEiy26pOjhDmycroKelj70WqOsqVgi4xh4Y9sj6pwb8Q423Tu3qNO1k +qaScTar2DANSigNzqlSbLshPWQ2ZyDwkvZPuqPgHzOXekzbUGwxgCiySaQkl2mCS +mBaG3XnESwiMIKkLphEv0MAvTVaImbSRWYEQ4OECwcHXxx+14wK8NLcdDIHcSzki +8Eq24CxDOeL5QxciGMi5tylsdCpT+D/BXTKiu46yoRjXUsTLYL53yUZZIqQ3A4CV +enZ/vHhP0Ev9RcRigFTqrBm7EC3b2AUpvqgRMnPwQZo= +-----END CERTIFICATE----- diff --git a/pulsar-common/src/test/resources/ssl/my-ca/server-key.pem b/pulsar-common/src/test/resources/ssl/my-ca/server-key.pem new file mode 100644 index 0000000000000..a3f3a36b73c37 --- /dev/null +++ b/pulsar-common/src/test/resources/ssl/my-ca/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBcqDkMhjLd9ik +//UQijqbajQP5t6dvVZNn9gODQrS9oB/URurNzCcPWYPJZfEJlTkV8mlmgq4dBjw +ghpy5ALOGiERk55JPIN4cy01hQ6j7YSPFvMvBjqZvm5dpGDNTr7GY7THegMM1wpk +9EaUOm7tBOHtf6ZnANjSMcQM74RCSBt0Koqw06CKVDCbgJ5NNE1LgwYeVQAwtQAh +Y8rqqQKJvCorFbq7OiisFBnz5pRBT6N4kMo19LZo3Oe2F2w9eH9vacQ0NjSOCNXq +al9Xl/Pwy9JgKKppwZ/3nCgRc+yfjrnkRz0fb+llb2NpR5Ge+tNMakqelE8bDSw/ +5BPjRPftAgMBAAECggEBAJm2JsgMUo1ihn/dbnIdFCKoCgRUs7FtYCVADOJlVKN7 +AXGpFi4/JV4Qn4cLnQNcXfovE2iF9VzJy4NYLgH60YvJUVtxC8Yv0lukUVkEiDST +p9A3MTa9YVUG7xVzZwPcPVTQpzYV6lSKjpTXUTm5EKk/RvJ7itKv5plmt9x7eYFb +/JwqXo1Z6C4gfIFR85LWmrCsNUK5T9oooLz88D6+ZH3+fWlr75RDff2kqdLshMTs +N0Ov7NXcRFeruFs/IPrgTxjBMeNa2LFdYVPeeQ41L4uOI49uVBAmSn1be+THvDoj +Do+6wTEF/h6/VLoOaIFZZdHlqd4is+xcEg8gwVkCn2ECgYEAxqVvGKc9qaqEVwBx +U5Ru9OFx0NqEBvkYZRbCg1REcMFd3lqFTHvHiF3pmCp0XgLJKYuy42618IJXhj6D +Y15/p9jX0025MpnH/AdwpO6x5pv6gb/JOMnHOnq8sI3R+V6TVsv1WZj0sOj94mF0 ++Od++bQkUnSlfE4X7v+cJfo/Q8UCgYEA+Uz1yOyI9Dv1dEdBMdBA8MTriYU0uJCV +dVKzL/uC9XyguVBWu1HX0MvEKyjPRycvLB7TuQqAFLgCtC8EEuPGBpWtyXOm9Jxw +ToCfUZFuBQeMuf4vZcFgJjiEKTdKBxrvjkhyIhPR6JAy0WUr8Ry+ZtqvmG5NOEz5 +ptm1tznYngkCgYEAlckeyV8p/uqF2biKu3QcamgoU0zB6yQfAfK0fySmasNTzZtC +EhbvsOLnhgbVMiI1ny8ol5fedtlBuAchOWeDKIQ40as0r3QHuQG/LY6S9Im+zeFY +kIqNwInWB+cYYkmvHe6zNXlBYLh+4BmOgzTDqPPtw4MTWXTlVSDGlFhrJeUCgYBX +7rlS4Xt9ChkNpoRsWZROWGbr3rw1zWmqND1X01Lh28+lDZ1J/RguYXET+BUEd+G/ +oi/zuKxsomrxuxOoxgZ3FBx0TgK5jORgDCYl0zIHPB57DBkTvx123cBf+Ux3LR0K +BqubMXp8mUATc6gIJ6dRCBmfnmhGT4BPRcM+mXy6YQKBgGEGH37VABus+Oi3g1bk +qEAaUI1asRLJIfbY2ImxEroLIQAbTFuIQUsZTKpT7jJZubjYvy1Fev0LU/n7Kv2w +7ym41z70ro5uxwUBfJjnF3RtgncNcftn4b3siNzvBfKEBuhegMeS5YAbBIwABUpR +4mVpm9BLOiX4yENIT6JdUQFc +-----END PRIVATE KEY----- diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 0231251e3f79c..a29158f39026c 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -410,7 +410,9 @@ public class ProxyConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_KEYSTORE_TLS, - doc = "TLS Provider" + doc = "Specify the TLS provider for the broker service: \n" + + "When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.\n" + + "When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc." ) private String tlsProvider = null; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index a033a87912d87..12abf871b5078 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -21,6 +21,7 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -79,7 +80,13 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s serviceConfig.getTlsProtocols(), serviceConfig.getTlsCertRefreshCheckDurationSec()); } else { - serverSslCtxRefresher = new NettyServerSslContextBuilder(serviceConfig.isTlsAllowInsecureConnection(), + SslProvider sslProvider = null; + if (serviceConfig.getTlsProvider() != null) { + sslProvider = SslProvider.valueOf(serviceConfig.getTlsProvider()); + } + serverSslCtxRefresher = new NettyServerSslContextBuilder( + sslProvider, + serviceConfig.isTlsAllowInsecureConnection(), serviceConfig.getTlsTrustCertsFilePath(), serviceConfig.getTlsCertificateFilePath(), serviceConfig.getTlsKeyFilePath(), serviceConfig.getTlsCiphers(), serviceConfig.getTlsProtocols(), serviceConfig.isTlsRequireTrustedClientCertOnConnect(), @@ -109,11 +116,19 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s serviceConfig.getTlsCertRefreshCheckDurationSec(), authData); } else { + SslProvider sslProvider = null; + if (serviceConfig.getBrokerClientSslProvider() != null) { + sslProvider = SslProvider.valueOf(serviceConfig.getBrokerClientSslProvider()); + } clientSslCtxRefresher = new NettyClientSslContextRefresher( + sslProvider, serviceConfig.isTlsAllowInsecureConnection(), serviceConfig.getBrokerClientTrustCertsFilePath(), authData, - serviceConfig.getTlsCertRefreshCheckDurationSec()); + serviceConfig.getBrokerClientTlsCiphers(), + serviceConfig.getBrokerClientTlsProtocols(), + serviceConfig.getTlsCertRefreshCheckDurationSec() + ); } } else { this.clientSslCtxRefresher = null; From 4fb5180f010cc2dab0698977c51755988e7d6a02 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 19 Apr 2022 19:14:44 +0300 Subject: [PATCH 490/823] Upgrade Netty to 4.1.76.Final, Netty Tcnative, grpc and protobuf (#15212) * Upgrade Netty to 4.1.76.Final and Netty Tcnative to 2.0.51.Final Fixes #14015 - release notes https://netty.io/news/2022/04/12/4-1-76-Final.html - contains fix for https://github.com/netty/netty/issues/11695 * Upgrade grpc to 1.45.1 and protobuf to 3.19.2 - grpc < 1.45.1 is not compatible with Netty > 4.1.74.Final - https://github.com/grpc/grpc-java/pull/9004 (cherry picked from commit 332a3c74c03184bb8d1298450ff21e5dc93be375) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 94 ++++++++++--------- pom.xml | 35 ++++++- pulsar-sql/presto-distribution/LICENSE | 54 +++++++---- 4 files changed, 116 insertions(+), 69 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index eca2dadddf384..aef37e4055867 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -105,7 +105,7 @@ io.netty netty-common - 4.1.74.Final + 4.1.76.Final test diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 974aeb47d78c9..c45a4e74d8278 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -352,26 +352,31 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.11.jar * Netty - - io.netty-netty-buffer-4.1.74.Final.jar - - io.netty-netty-codec-4.1.74.Final.jar - - io.netty-netty-codec-dns-4.1.74.Final.jar - - io.netty-netty-codec-http-4.1.74.Final.jar - - io.netty-netty-codec-http2-4.1.74.Final.jar - - io.netty-netty-codec-socks-4.1.74.Final.jar - - io.netty-netty-codec-haproxy-4.1.74.Final.jar - - io.netty-netty-common-4.1.74.Final.jar - - io.netty-netty-handler-4.1.74.Final.jar - - io.netty-netty-handler-proxy-4.1.74.Final.jar - - io.netty-netty-resolver-4.1.74.Final.jar - - io.netty-netty-resolver-dns-4.1.74.Final.jar - - io.netty-netty-transport-4.1.74.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.74.Final.jar - - io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.74.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.74.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.74.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar - - io.netty-netty-tcnative-classes-2.0.48.Final.jar + - io.netty-netty-buffer-4.1.76.Final.jar + - io.netty-netty-codec-4.1.76.Final.jar + - io.netty-netty-codec-dns-4.1.76.Final.jar + - io.netty-netty-codec-http-4.1.76.Final.jar + - io.netty-netty-codec-http2-4.1.76.Final.jar + - io.netty-netty-codec-socks-4.1.76.Final.jar + - io.netty-netty-codec-haproxy-4.1.76.Final.jar + - io.netty-netty-common-4.1.76.Final.jar + - io.netty-netty-handler-4.1.76.Final.jar + - io.netty-netty-handler-proxy-4.1.76.Final.jar + - io.netty-netty-resolver-4.1.76.Final.jar + - io.netty-netty-resolver-dns-4.1.76.Final.jar + - io.netty-netty-transport-4.1.76.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.76.Final.jar + - io.netty-netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.76.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.76.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.51.Final.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar @@ -461,26 +466,26 @@ The Apache Software License, Version 2.0 - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.4.32.jar - org.jetbrains-annotations-13.0.jar * gRPC - - io.grpc-grpc-all-1.42.1.jar - - io.grpc-grpc-auth-1.42.1.jar - - io.grpc-grpc-context-1.42.1.jar - - io.grpc-grpc-core-1.42.1.jar - - io.grpc-grpc-netty-1.42.1.jar - - io.grpc-grpc-protobuf-1.42.1.jar - - io.grpc-grpc-protobuf-lite-1.42.1.jar - - io.grpc-grpc-stub-1.42.1.jar - - io.grpc-grpc-alts-1.42.1.jar - - io.grpc-grpc-api-1.42.1.jar - - io.grpc-grpc-grpclb-1.42.1.jar - - io.grpc-grpc-netty-shaded-1.42.1.jar - - io.grpc-grpc-services-1.42.1.jar - - io.grpc-grpc-xds-1.42.1.jar - - io.grpc-grpc-rls-1.42.1.jar + - io.grpc-grpc-all-1.45.1.jar + - io.grpc-grpc-auth-1.45.1.jar + - io.grpc-grpc-context-1.45.1.jar + - io.grpc-grpc-core-1.45.1.jar + - io.grpc-grpc-netty-1.45.1.jar + - io.grpc-grpc-protobuf-1.45.1.jar + - io.grpc-grpc-protobuf-lite-1.45.1.jar + - io.grpc-grpc-stub-1.45.1.jar + - io.grpc-grpc-alts-1.45.1.jar + - io.grpc-grpc-api-1.45.1.jar + - io.grpc-grpc-grpclb-1.45.1.jar + - io.grpc-grpc-netty-shaded-1.45.1.jar + - io.grpc-grpc-services-1.45.1.jar + - io.grpc-grpc-xds-1.45.1.jar + - io.grpc-grpc-rls-1.45.1.jar * Perfmark - io.perfmark-perfmark-api-0.19.0.jar * OpenCensus - - io.opencensus-opencensus-api-0.18.0.jar - - io.opencensus-opencensus-contrib-http-util-0.24.0.jar + - io.opencensus-opencensus-api-0.28.0.jar + - io.opencensus-opencensus-contrib-http-util-0.28.0.jar - io.opencensus-opencensus-proto-0.2.0.jar * Jodah - net.jodah-typetools-0.5.0.jar @@ -522,17 +527,18 @@ The Apache Software License, Version 2.0 * Snappy Java - org.xerial.snappy-snappy-java-1.1.7.jar * Google HTTP Client - - com.google.http-client-google-http-client-jackson2-1.38.0.jar - - com.google.http-client-google-http-client-1.38.0.jar - - com.google.auto.value-auto-value-annotations-1.7.4.jar + - com.google.http-client-google-http-client-jackson2-1.41.0.jar + - com.google.http-client-google-http-client-gson-1.41.0.jar + - com.google.http-client-google-http-client-1.41.0.jar + - com.google.auto.value-auto-value-annotations-1.9.jar - com.google.re2j-re2j-1.5.jar * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar BSD 3-clause "New" or "Revised" License * Google auth library - - com.google.auth-google-auth-library-credentials-0.22.2.jar -- licenses/LICENSE-google-auth-library.txt - - com.google.auth-google-auth-library-oauth2-http-0.22.2.jar -- licenses/LICENSE-google-auth-library.txt + - com.google.auth-google-auth-library-credentials-1.4.0.jar -- licenses/LICENSE-google-auth-library.txt + - com.google.auth-google-auth-library-oauth2-http-1.4.0.jar -- licenses/LICENSE-google-auth-library.txt * LevelDB -- (included in org.rocksdb.*.jar) -- licenses/LICENSE-LevelDB.txt * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- licenses/LICENSE-JSR305.txt * JLine -- jline-jline-2.14.6.jar -- licenses/LICENSE-JLine.txt @@ -551,8 +557,8 @@ MIT License Protocol Buffers License * Protocol Buffers - - com.google.protobuf-protobuf-java-3.16.1.jar -- licenses/LICENSE-protobuf.txt - - com.google.protobuf-protobuf-java-util-3.16.1.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-3.19.2.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-util-3.19.2.jar -- licenses/LICENSE-protobuf.txt CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt * Java Annotations API diff --git a/pom.xml b/pom.xml index 9f3bfcd45333b..f8722466cd98d 100644 --- a/pom.xml +++ b/pom.xml @@ -109,8 +109,8 @@ flexible messaging model and an intuitive client API. 1.1.7 3.2.5 5.1.0 - 4.1.74.Final - 2.0.48.Final + 4.1.76.Final + 2.0.51.Final 9.4.43.v20210629 2.5.2 2.34 @@ -130,9 +130,10 @@ flexible messaging model and an intuitive client API. 8.37 1.4.13 0.5.0 - 3.16.1 + 3.19.2 ${protobuf3.version} - 1.42.1 + 1.45.1 + 1.41.0 0.19.0 ${grpc.version} 2.8.9 @@ -159,7 +160,7 @@ flexible messaging model and an intuitive client API. 1.7.1.Final 42.2.25 0.11.1 - 0.18.0 + 0.28.0 2.3.0 30.1-jre 1.0 @@ -954,6 +955,24 @@ flexible messaging model and an intuitive client API. ${grpc.version} + + com.google.http-client + google-http-client + ${google-http-client.version} + + + + com.google.http-client + google-http-client-jackson2 + ${google-http-client.version} + + + + com.google.http-client + google-http-client-gson + ${google-http-client.version} + + io.perfmark perfmark-api @@ -1104,6 +1123,12 @@ flexible messaging model and an intuitive client API. ${opencensus.version} + + io.opencensus + opencensus-contrib-http-util + ${opencensus.version} + + io.opencensus opencensus-contrib-grpc-metrics diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index c31f1442ae9b9..687feeb3918a9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -233,25 +233,40 @@ The Apache Software License, Version 2.0 - commons-lang3-3.11.jar * Netty - netty-3.10.6.Final.jar - - netty-buffer-4.1.74.Final.jar - - netty-codec-4.1.74.Final.jar - - netty-codec-dns-4.1.74.Final.jar - - netty-codec-http-4.1.74.Final.jar - - netty-codec-haproxy-4.1.74.Final.jar - - netty-codec-socks-4.1.74.Final.jar - - netty-handler-proxy-4.1.74.Final.jar - - netty-common-4.1.74.Final.jar - - netty-handler-4.1.74.Final.jar + - netty-buffer-4.1.76.Final.jar + - netty-codec-4.1.76.Final.jar + - netty-codec-dns-4.1.76.Final.jar + - netty-codec-http-4.1.76.Final.jar + - netty-codec-haproxy-4.1.76.Final.jar + - netty-codec-socks-4.1.76.Final.jar + - netty-handler-proxy-4.1.76.Final.jar + - netty-common-4.1.76.Final.jar + - netty-handler-4.1.76.Final.jar - netty-reactive-streams-2.0.4.jar - - netty-resolver-4.1.74.Final.jar - - netty-resolver-dns-4.1.74.Final.jar - - netty-tcnative-boringssl-static-2.0.48.Final.jar - - netty-tcnative-classes-2.0.48.Final.jar - - netty-transport-4.1.74.Final.jar - - netty-transport-classes-epoll-4.1.74.Final.jar - - netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.74.Final.jar - - netty-transport-native-unix-common-4.1.74.Final-linux-x86_64.jar + - netty-resolver-4.1.76.Final.jar + - netty-resolver-dns-4.1.76.Final.jar + - netty-tcnative-boringssl-static-2.0.51.Final.jar + - netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.51.Final.jar + - netty-transport-4.1.76.Final.jar + - netty-transport-classes-epoll-4.1.76.Final.jar + - netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.76.Final.jar + - netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar + - netty-codec-http2-4.1.76.Final.jar + * GRPC + - grpc-api-1.45.1.jar + - grpc-context-1.45.1.jar + - grpc-core-1.45.1.jar + - grpc-grpclb-1.45.1.jar + - grpc-netty-1.45.1.jar + - grpc-protobuf-1.45.1.jar + - grpc-protobuf-lite-1.45.1.jar + - grpc-stub-1.45.1.jar * Joda Time - joda-time-2.10.5.jar * Jetty @@ -460,7 +475,8 @@ The Apache Software License, Version 2.0 Protocol Buffers License * Protocol Buffers - - protobuf-java-3.16.1.jar + - protobuf-java-3.19.2.jar + - protobuf-java-util-3.19.2.jar BSD 3-clause "New" or "Revised" License * RE2J TD -- re2j-td-1.4.jar From c6dd300dc53645158d17812de0c64f6d010f093d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 20 Apr 2022 14:55:09 +0300 Subject: [PATCH 491/823] [Build] Use grpc-bom to align grpc library versions (#15234) (cherry picked from commit 081f0d1d20165007de1ed8b0ebd130605e094910) --- pom.xml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index f8722466cd98d..a6cc5f8f8ea15 100644 --- a/pom.xml +++ b/pom.xml @@ -925,6 +925,14 @@ flexible messaging model and an intuitive client API. ${typetools.version} + + io.grpc + grpc-bom + ${grpc.version} + pom + import + + io.grpc grpc-all @@ -949,12 +957,6 @@ flexible messaging model and an intuitive client API. - - io.grpc - grpc-core - ${grpc.version} - - com.google.http-client google-http-client @@ -986,18 +988,6 @@ flexible messaging model and an intuitive client API. - - io.grpc - grpc-stub - ${grpc.version} - - - - io.grpc - grpc-protobuf-lite - ${grpc.version} - - com.google.protobuf protobuf-bom @@ -1241,7 +1231,6 @@ flexible messaging model and an intuitive client API. ${kotlin-stdlib.version} - From e99ed2d5152a71ea0f5c5ac5f67c20a224c3203d Mon Sep 17 00:00:00 2001 From: ran Date: Sun, 24 Apr 2022 09:44:02 +0800 Subject: [PATCH 492/823] fix LICENSE for branch-2.9 (#15262) --- pulsar-sql/presto-distribution/LICENSE | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 687feeb3918a9..9fb5b74e2f771 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -257,16 +257,6 @@ The Apache Software License, Version 2.0 - netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar - netty-transport-native-unix-common-4.1.76.Final.jar - netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar - - netty-codec-http2-4.1.76.Final.jar - * GRPC - - grpc-api-1.45.1.jar - - grpc-context-1.45.1.jar - - grpc-core-1.45.1.jar - - grpc-grpclb-1.45.1.jar - - grpc-netty-1.45.1.jar - - grpc-protobuf-1.45.1.jar - - grpc-protobuf-lite-1.45.1.jar - - grpc-stub-1.45.1.jar * Joda Time - joda-time-2.10.5.jar * Jetty @@ -476,7 +466,6 @@ The Apache Software License, Version 2.0 Protocol Buffers License * Protocol Buffers - protobuf-java-3.19.2.jar - - protobuf-java-util-3.19.2.jar BSD 3-clause "New" or "Revised" License * RE2J TD -- re2j-td-1.4.jar From dc14b8bec49eea90860d6b2f0bba4d22f245c68e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Sun, 24 Apr 2022 11:34:50 +0800 Subject: [PATCH 493/823] [Broker] Fix call sync method in onPoliciesUpdate method (#15227) --- .../pulsar/broker/service/Consumer.java | 29 ++-- .../pulsar/broker/service/Producer.java | 31 ++-- .../nonpersistent/NonPersistentTopic.java | 37 +++-- .../service/persistent/PersistentTopic.java | 154 ++++++++++-------- 4 files changed, 144 insertions(+), 107 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 591b71c71f6b2..b8358d06f3a70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -765,21 +765,26 @@ public String toString() { .add("consumerName", consumerName).add("address", this.cnx.clientAddress()).toString(); } - public void checkPermissions() { + public CompletableFuture checkPermissionsAsync() { TopicName topicName = TopicName.get(subscription.getTopicName()); if (cnx.getBrokerService().getAuthorizationService() != null) { - try { - if (cnx.getBrokerService().getAuthorizationService().canConsume(topicName, appId, - cnx.getAuthenticationData(), subscription.getName())) { - return; - } - } catch (Exception e) { - log.warn("[{}] Get unexpected error while autorizing [{}] {}", appId, subscription.getTopicName(), - e.getMessage(), e); - } - log.info("[{}] is not allowed to consume from topic [{}] anymore", appId, subscription.getTopicName()); - disconnect(); + return cnx.getBrokerService().getAuthorizationService().canConsumeAsync(topicName, appId, + cnx.getAuthenticationData(), subscription.getName()) + .handle((ok, e) -> { + if (e != null) { + log.warn("[{}] Get unexpected error while autorizing [{}] {}", appId, + subscription.getTopicName(), e.getMessage(), e); + } + + if (ok == null || !ok) { + log.info("[{}] is not allowed to consume from topic [{}] anymore", appId, + subscription.getTopicName()); + disconnect(); + } + return null; + }); } + return CompletableFuture.completedFuture(null); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index d72f904019a4e..8043695bb9911 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -647,21 +647,26 @@ long getPendingPublishAcks() { return pendingPublishAcks; } - public void checkPermissions() { + public CompletableFuture checkPermissionsAsync() { TopicName topicName = TopicName.get(topic.getName()); if (cnx.getBrokerService().getAuthorizationService() != null) { - try { - if (cnx.getBrokerService().getAuthorizationService().canProduce(topicName, appId, - cnx.getAuthenticationData())) { - return; - } - } catch (Exception e) { - log.warn("[{}] Get unexpected error while autorizing [{}] {}", appId, topic.getName(), e.getMessage(), - e); - } - log.info("[{}] is not allowed to produce on topic [{}] anymore", appId, topic.getName()); - disconnect(); - } + return cnx.getBrokerService().getAuthorizationService() + .canProduceAsync(topicName, appId, cnx.getAuthenticationData()) + .handle((ok, ex) -> { + if (ex != null) { + log.warn("[{}] Get unexpected error while autorizing [{}] {}", appId, topic.getName(), + ex.getMessage(), ex); + } + + if (ok == null || !ok) { + log.info("[{}] is not allowed to produce on topic [{}] anymore", appId, topic.getName()); + disconnect(); + } + + return null; + }); + } + return CompletableFuture.completedFuture(null); } public void checkEncryption() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 886140eb25b78..472cf1cc200b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -25,6 +25,7 @@ import com.google.common.collect.Maps; import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -978,21 +979,29 @@ public CompletableFuture onPoliciesUpdate(Policies data) { isAllowAutoUpdateSchema = data.is_allow_auto_update_schema; schemaValidationEnforced = data.schema_validation_enforced; - producers.values().forEach(producer -> { - producer.checkPermissions(); - producer.checkEncryption(); - }); - subscriptions.forEach((subName, sub) -> sub.getConsumers().forEach(Consumer::checkPermissions)); + List> producerCheckFutures = new ArrayList<>(producers.size()); + producers.values().forEach(producer -> producerCheckFutures.add( + producer.checkPermissionsAsync().thenRun(producer::checkEncryption))); - if (data.inactive_topic_policies != null) { - this.inactiveTopicPolicies = data.inactive_topic_policies; - } else { - ServiceConfiguration cfg = brokerService.getPulsar().getConfiguration(); - resetInactiveTopicPolicies(cfg.getBrokerDeleteInactiveTopicsMode() - , cfg.getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(), - cfg.isBrokerDeleteInactiveTopicsEnabled()); - } - return checkReplicationAndRetryOnFailure(); + return FutureUtil.waitForAll(producerCheckFutures).thenCompose((__) -> { + List> consumerCheckFutures = new ArrayList<>(); + subscriptions.forEach((subName, sub) -> sub.getConsumers().forEach(consumer -> { + consumerCheckFutures.add(consumer.checkPermissionsAsync()); + })); + + return FutureUtil.waitForAll(consumerCheckFutures) + .thenCompose((___) -> { + if (data.inactive_topic_policies != null) { + this.inactiveTopicPolicies = data.inactive_topic_policies; + } else { + ServiceConfiguration cfg = brokerService.getPulsar().getConfiguration(); + resetInactiveTopicPolicies(cfg.getBrokerDeleteInactiveTopicsMode() + , cfg.getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(), + cfg.isBrokerDeleteInactiveTopicsEnabled()); + } + return checkReplicationAndRetryOnFailure(); + }); + }); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4f1b8cc31d880..bdfb2b2e2b2f0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2456,40 +2456,49 @@ public CompletableFuture onPoliciesUpdate(Policies data) { this.updateMaxPublishRate(data); - producers.values().forEach(producer -> { - producer.checkPermissions(); - producer.checkEncryption(); - }); - subscriptions.forEach((subName, sub) -> { - sub.getConsumers().forEach(Consumer::checkPermissions); - Dispatcher dispatcher = sub.getDispatcher(); - // If the topic-level policy already exists, the namespace-level policy cannot override - // the topic-level policy. - if (dispatcher != null - && (!topicPolicies.isPresent() || !topicPolicies.get().isSubscriptionDispatchRateSet())) { - dispatcher.getRateLimiter().ifPresent(rateLimiter -> rateLimiter.onPoliciesUpdate(data)); - } - }); - replicators.forEach((name, replicator) -> - replicator.getRateLimiter().ifPresent(DispatchRateLimiter::updateDispatchRate) - ); - checkMessageExpiry(); - CompletableFuture replicationFuture = checkReplicationAndRetryOnFailure(); - CompletableFuture dedupFuture = checkDeduplicationStatus(); - CompletableFuture persistentPoliciesFuture = checkPersistencePolicies(); - // update rate-limiter if policies updated - if (this.dispatchRateLimiter.isPresent()) { - if (!topicPolicies.isPresent() || !topicPolicies.get().isDispatchRateSet()) { - dispatchRateLimiter.get().onPoliciesUpdate(data); - } - } - if (this.subscribeRateLimiter.isPresent()) { - subscribeRateLimiter.get().onPoliciesUpdate(data); - } + List> producerCheckFutures = new ArrayList<>(producers.size()); + producers.values().forEach(producer -> producerCheckFutures.add( + producer.checkPermissionsAsync().thenRun(producer::checkEncryption))); + + return FutureUtil.waitForAll(producerCheckFutures).thenCompose((__) -> { + List> subscriptionCheckFutures = new ArrayList<>((int) subscriptions.size()); + subscriptions.forEach((subName, sub) -> { + List> consumerCheckFutures = new ArrayList<>(sub.getConsumers().size()); + sub.getConsumers().forEach(consumer -> consumerCheckFutures.add(consumer.checkPermissionsAsync())); + subscriptionCheckFutures.add(FutureUtil.waitForAll(consumerCheckFutures).thenRun(() -> { + Dispatcher dispatcher = sub.getDispatcher(); + // If the topic-level policy already exists, the namespace-level policy cannot override + // the topic-level policy. + if (dispatcher != null && (!topicPolicies.isPresent() || !topicPolicies.get() + .isSubscriptionDispatchRateSet())) { + dispatcher.getRateLimiter() + .ifPresent(rateLimiter -> rateLimiter.onPoliciesUpdate(data)); + } + })); + }); + return FutureUtil.waitForAll(subscriptionCheckFutures).thenCompose((___) -> { + replicators.forEach((name, replicator) -> + replicator.getRateLimiter().ifPresent(DispatchRateLimiter::updateDispatchRate) + ); + checkMessageExpiry(); + CompletableFuture replicationFuture = checkReplicationAndRetryOnFailure(); + CompletableFuture dedupFuture = checkDeduplicationStatus(); + CompletableFuture persistentPoliciesFuture = checkPersistencePolicies(); + // update rate-limiter if policies updated + if (this.dispatchRateLimiter.isPresent()) { + if (!topicPolicies.isPresent() || !topicPolicies.get().isDispatchRateSet()) { + dispatchRateLimiter.get().onPoliciesUpdate(data); + } + } + if (this.subscribeRateLimiter.isPresent()) { + subscribeRateLimiter.get().onPoliciesUpdate(data); + } - return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture, - preCreateSubscriptionForCompactionIfNeeded()); + return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture, + preCreateSubscriptionForCompactionIfNeeded()); + }); + }); } /** @@ -3114,48 +3123,57 @@ public void onUpdate(TopicPolicies policies) { } }); - subscriptions.forEach((subName, sub) -> { - sub.getConsumers().forEach(Consumer::checkPermissions); - Dispatcher dispatcher = sub.getDispatcher(); - if (dispatcher != null) { - dispatcher.updateRateLimiter(policies.getSubscriptionDispatchRate()); + List> consumerCheckFutures = new ArrayList<>(); + subscriptions.forEach((subName, sub) -> sub.getConsumers().forEach(consumer -> { + consumerCheckFutures.add(consumer.checkPermissionsAsync().thenRun(() -> { + Dispatcher dispatcher = sub.getDispatcher(); + if (dispatcher != null) { + dispatcher.updateRateLimiter(policies.getSubscriptionDispatchRate()); + } + })); + })); + + FutureUtil.waitForAll(consumerCheckFutures).thenRun(() -> { + if (policies.getPublishRate() != null) { + updatePublishDispatcher(policies.getPublishRate()); + } else { + updateMaxPublishRate(namespacePolicies.orElse(null)); } - }); - if (policies.getPublishRate() != null) { - updatePublishDispatcher(policies.getPublishRate()); - } else { - updateMaxPublishRate(namespacePolicies.orElse(null)); - } + if (policies.isInactiveTopicPoliciesSet()) { + inactiveTopicPolicies = policies.getInactiveTopicPolicies(); + } else if (namespacePolicies.isPresent() && namespacePolicies.get().inactive_topic_policies != null) { + //topic-level policies is null , so use namespace-level + inactiveTopicPolicies = namespacePolicies.get().inactive_topic_policies; + } else { + //namespace-level policies is null , so use broker level + ServiceConfiguration cfg = brokerService.getPulsar().getConfiguration(); + resetInactiveTopicPolicies(cfg.getBrokerDeleteInactiveTopicsMode() + , cfg.getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(), + cfg.isBrokerDeleteInactiveTopicsEnabled()); + } - if (policies.isInactiveTopicPoliciesSet()) { - inactiveTopicPolicies = policies.getInactiveTopicPolicies(); - } else if (namespacePolicies.isPresent() && namespacePolicies.get().inactive_topic_policies != null) { - //topic-level policies is null , so use namespace-level - inactiveTopicPolicies = namespacePolicies.get().inactive_topic_policies; - } else { - //namespace-level policies is null , so use broker level - ServiceConfiguration cfg = brokerService.getPulsar().getConfiguration(); - resetInactiveTopicPolicies(cfg.getBrokerDeleteInactiveTopicsMode() - , cfg.getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(), - cfg.isBrokerDeleteInactiveTopicsEnabled()); - } - updateUnackedMessagesAppliedOnSubscription(namespacePolicies.orElse(null)); - initializeTopicSubscribeRateLimiterIfNeeded(Optional.ofNullable(policies)); - if (this.subscribeRateLimiter.isPresent()) { - subscribeRateLimiter.ifPresent(subscribeRateLimiter -> - subscribeRateLimiter.onSubscribeRateUpdate(policies.getSubscribeRate())); - } - replicators.forEach((name, replicator) -> replicator.getRateLimiter() - .ifPresent(DispatchRateLimiter::updateDispatchRate)); - updateUnackedMessagesExceededOnConsumer(namespacePolicies.orElse(null)); + updateUnackedMessagesAppliedOnSubscription(namespacePolicies.orElse(null)); + initializeTopicSubscribeRateLimiterIfNeeded(Optional.ofNullable(policies)); + if (this.subscribeRateLimiter.isPresent()) { + subscribeRateLimiter.ifPresent(subscribeRateLimiter -> + subscribeRateLimiter.onSubscribeRateUpdate(policies.getSubscribeRate())); + } + replicators.forEach((name, replicator) -> replicator.getRateLimiter() + .ifPresent(DispatchRateLimiter::updateDispatchRate)); + updateUnackedMessagesExceededOnConsumer(namespacePolicies.orElse(null)); - checkDeduplicationStatus(); + checkDeduplicationStatus(); - preCreateSubscriptionForCompactionIfNeeded(); + preCreateSubscriptionForCompactionIfNeeded(); - // update managed ledger config - checkPersistencePolicies(); + // update managed ledger config + checkPersistencePolicies(); + }).exceptionally(e -> { + Throwable t = e instanceof CompletionException ? e.getCause() : e; + log.error("[{}] update topic policy error: {}", topic, t.getMessage(), t); + return null; + }); } private Optional getNamespacePolicies() { From 421e7c7ff36137eb9c7960ed56b83e1a7521a7a1 Mon Sep 17 00:00:00 2001 From: ran Date: Wed, 27 Apr 2022 07:50:28 +0800 Subject: [PATCH 494/823] [cherry-pick][pulsar-sql] Java version trim agent presto332 branch2.9 (#15326) Fix #14951 The presto 332 couldn't parse Java version like this `11.0.14.1`, so add a Java version trim agent to walk around the problem. This is a temporary patch, after the presto upgrade to 332+, we could remove this. Add a Java version trim agent to format the system property `java.version` for the Pulsar SQL plugin. (cherry picked from commit a1aa18f1a856d074bb23d68f5c330d3558bdf6cb) --- conf/presto/jvm.config | 1 + pulsar-sql/java-version-trim-agent/pom.xml | 58 +++++++++++++++++++ .../sql/agent/TrimJavaVersionAgent.java | 51 ++++++++++++++++ .../apache/pulsar/sql/agent/package-info.java | 22 +++++++ pulsar-sql/pom.xml | 1 + pulsar-sql/presto-distribution/pom.xml | 7 +++ .../src/assembly/assembly.xml | 5 ++ pulsar-sql/presto-pulsar/pom.xml | 5 ++ .../conf/presto/jvm.config | 1 + .../presto/TestPrestoQueryTieredStorage.java | 1 - .../suites/PulsarSQLTestSuite.java | 1 + 11 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 pulsar-sql/java-version-trim-agent/pom.xml create mode 100644 pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/TrimJavaVersionAgent.java create mode 100644 pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/package-info.java diff --git a/conf/presto/jvm.config b/conf/presto/jvm.config index 4a431b63367e3..3e1cdd6a24bc7 100644 --- a/conf/presto/jvm.config +++ b/conf/presto/jvm.config @@ -27,3 +27,4 @@ -XX:+ExitOnOutOfMemoryError -Dpresto-temporarily-allow-java8=true -Djdk.attach.allowAttachSelf=true +-javaagent:java-version-trim-agent.jar \ No newline at end of file diff --git a/pulsar-sql/java-version-trim-agent/pom.xml b/pulsar-sql/java-version-trim-agent/pom.xml new file mode 100644 index 0000000000000..2f1171408b601 --- /dev/null +++ b/pulsar-sql/java-version-trim-agent/pom.xml @@ -0,0 +1,58 @@ + + + + pulsar-sql + org.apache.pulsar + 2.9.2 + + 4.0.0 + + java-version-trim-agent + Pulsar SQL :: Java Version Trim Agent + + + java-version-trim-agent + + + org.apache.maven.plugins + maven-jar-plugin + 3.1.0 + + + + true + + + org.apache.pulsar.sql.agent.TrimJavaVersionAgent + org.apache.pulsar.sql.agent.TrimJavaVersionAgent + true + true + + + + + + + + \ No newline at end of file diff --git a/pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/TrimJavaVersionAgent.java b/pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/TrimJavaVersionAgent.java new file mode 100644 index 0000000000000..7cb422bd27a13 --- /dev/null +++ b/pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/TrimJavaVersionAgent.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.agent; + +import java.lang.instrument.Instrumentation; +import java.util.logging.Logger; + +/** + * The presto 332 couldn't parse Java version like this `11.0.14.1`, + * so add java version trim agent to walk around the problem. + * + * After the presto upgrade to 332+, we could remove this. + */ +public class TrimJavaVersionAgent { + + private static final Logger logger = Logger.getLogger(TrimJavaVersionAgent.class.getName()); + + private static final String JAVA_VERSION = "java.version"; + + public static String trimJavaVersion(String javaVersion) { + String[] arr = javaVersion.split("\\."); + if (arr.length <= 3) { + return javaVersion; + } + return arr[0] + "." + arr[1] + "." + arr[2]; + } + + public static void premain(String agentArgs, Instrumentation inst) { + String javaVersion = System.getProperty(JAVA_VERSION); + String trimVersion = trimJavaVersion(javaVersion); + logger.info("original java version " + javaVersion + " => trim java version " + trimVersion); + System.setProperty(JAVA_VERSION, trimVersion); + } + +} diff --git a/pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/package-info.java b/pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/package-info.java new file mode 100644 index 0000000000000..7d60a6fc6fbb2 --- /dev/null +++ b/pulsar-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Implementation of the connector to the Presto engine. + */ +package org.apache.pulsar.sql.agent; diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 31ec8436754ff..5aac5ffa4564a 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -34,6 +34,7 @@ presto-pulsar presto-pulsar-plugin + java-version-trim-agent presto-distribution diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 0d9c145dff5bb..b074172568578 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -200,6 +200,13 @@ ${jackson.version} + + ${project.groupId} + java-version-trim-agent + ${project.version} + provided + + diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml index 4f1bac20dcee4..bc1fe5ed46037 100644 --- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml +++ b/pulsar-sql/presto-distribution/src/assembly/assembly.xml @@ -40,6 +40,11 @@ bin/ 644 + + ${basedir}/../java-version-trim-agent/target/java-version-trim-agent.jar + java-version-trim-agent.jar + / + diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 48376baa59270..129f0854bc81f 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -107,6 +107,11 @@ ${presto.version} + + javax.annotation + javax.annotation-api + + io.prestosql presto-main diff --git a/tests/docker-images/latest-version-image/conf/presto/jvm.config b/tests/docker-images/latest-version-image/conf/presto/jvm.config index 28db36a3937da..e8a0cec43ba0e 100644 --- a/tests/docker-images/latest-version-image/conf/presto/jvm.config +++ b/tests/docker-images/latest-version-image/conf/presto/jvm.config @@ -28,3 +28,4 @@ -XX:+ExitOnOutOfMemoryError -Dpresto-temporarily-allow-java8=true -Djdk.attach.allowAttachSelf=true +-javaagent:java-version-trim-agent.jar \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java index 4c129fadce002..7e4aae47bf0d5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java @@ -41,7 +41,6 @@ import org.testng.Assert; import org.testng.annotations.Test; - /** * Test presto query from tiered storage, the Pulsar SQL is cluster mode. */ diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java index 762fff751b069..902de4dd4ad8c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; + import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; From e04c19eec38978c58ff38cbcdcb056f94cbe9bb7 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 29 Apr 2022 10:20:31 +0800 Subject: [PATCH 495/823] Add log when update namespace policies with error. (#15056) --- .../broker/service/nonpersistent/NonPersistentTopic.java | 3 +++ .../pulsar/broker/service/persistent/PersistentTopic.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 472cf1cc200b1..b9a85ab97c520 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -1001,6 +1001,9 @@ public CompletableFuture onPoliciesUpdate(Policies data) { } return checkReplicationAndRetryOnFailure(); }); + }).exceptionally(ex -> { + log.error("[{}] update namespace polices : {} error", this.getName(), data, ex); + throw FutureUtil.wrapToCompletionException(ex); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index bdfb2b2e2b2f0..333438849e6ad 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2496,8 +2496,11 @@ public CompletableFuture onPoliciesUpdate(Policies data) { } return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture, - preCreateSubscriptionForCompactionIfNeeded()); + preCreateSubscriptionForCompactionIfNeeded()); }); + }).exceptionally(ex -> { + log.error("[{}] update namespace polices : {} error", this.getName(), data, ex); + throw FutureUtil.wrapToCompletionException(ex); }); } From 025500ae8cc93665b88e3b3bee8020b1dcf80711 Mon Sep 17 00:00:00 2001 From: lin chen <1572139390@qq.com> Date: Tue, 1 Mar 2022 21:16:52 +0800 Subject: [PATCH 496/823] Support shrink in ConcurrentLongHashMap (#14497) (cherry picked from commit 297941964ed739e35ca68aa46d74410cf112b7bc) --- .../mledger/impl/ManagedLedgerImpl.java | 7 +- .../TransactionMetadataStoreService.java | 5 +- .../pulsar/broker/service/ServerCnx.java | 10 +- .../apache/pulsar/client/impl/ClientCnx.java | 23 ++- .../impl/TransactionMetaStoreHandler.java | 5 +- .../TransactionCoordinatorClientImpl.java | 6 +- .../collections/ConcurrentLongHashMap.java | 139 ++++++++++++++++-- .../ConcurrentLongHashMapTest.java | 122 +++++++++++++-- 8 files changed, 275 insertions(+), 42 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 8d5f0a6ed8c62..62edcd5ca9de7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -152,8 +152,11 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { protected Map propertiesMap; protected final MetaStore store; - final ConcurrentLongHashMap> ledgerCache = new ConcurrentLongHashMap<>( - 16 /* initial capacity */, 1 /* number of sections */); + final ConcurrentLongHashMap> ledgerCache = + ConcurrentLongHashMap.>newBuilder() + .expectedItems(16) // initial capacity + .concurrencyLevel(1) // number of sections + .build(); protected final NavigableMap ledgers = new ConcurrentSkipListMap<>(); private volatile Stat ledgersStat; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 7297c334c4cab..cd18839798935 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -107,8 +107,9 @@ public TransactionMetadataStoreService(TransactionMetadataStoreProvider transact this.tbClient = tbClient; this.timeoutTrackerFactory = new TransactionTimeoutTrackerFactoryImpl(this, timer); this.transactionOpRetryTimer = timer; - this.tcLoadSemaphores = new ConcurrentLongHashMap<>(); - this.pendingConnectRequests = new ConcurrentLongHashMap<>(); + this.tcLoadSemaphores = ConcurrentLongHashMap.newBuilder().build(); + this.pendingConnectRequests = + ConcurrentLongHashMap.>>newBuilder().build(); this.internalPinnedExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d8255f6015daf..d8a51f60d65b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -241,8 +241,14 @@ public ServerCnx(PulsarService pulsar, String listenerName) { ServiceConfiguration conf = pulsar.getConfiguration(); // This maps are not heavily contended since most accesses are within the cnx thread - this.producers = new ConcurrentLongHashMap<>(8, 1); - this.consumers = new ConcurrentLongHashMap<>(8, 1); + this.producers = ConcurrentLongHashMap.>newBuilder() + .expectedItems(8) + .concurrencyLevel(1) + .build(); + this.consumers = ConcurrentLongHashMap.>newBuilder() + .expectedItems(8) + .concurrencyLevel(1) + .build(); this.replicatorPrefix = conf.getReplicatorPrefix(); this.maxNonPersistentPendingMessages = conf.getMaxConcurrentNonPersistentMessagePerConnection(); this.proxyRoles = conf.getProxyRoles(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index cae1594219eed..00b8e4f7561dd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -110,13 +110,28 @@ public class ClientCnx extends PulsarHandler { private State state; private final ConcurrentLongHashMap> pendingRequests = - new ConcurrentLongHashMap<>(16, 1); + ConcurrentLongHashMap.>newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); // LookupRequests that waiting in client side. private final Queue>>> waitingLookupRequests; - private final ConcurrentLongHashMap> producers = new ConcurrentLongHashMap<>(16, 1); - private final ConcurrentLongHashMap> consumers = new ConcurrentLongHashMap<>(16, 1); - private final ConcurrentLongHashMap transactionMetaStoreHandlers = new ConcurrentLongHashMap<>(16, 1); + private final ConcurrentLongHashMap> producers = + ConcurrentLongHashMap.>newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + private final ConcurrentLongHashMap> consumers = + ConcurrentLongHashMap.>newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + private final ConcurrentLongHashMap transactionMetaStoreHandlers = + ConcurrentLongHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); private final CompletableFuture connectionFuture = new CompletableFuture(); private final ConcurrentLinkedQueue requestTimeoutQueue = new ConcurrentLinkedQueue<>(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index a30c2350908c0..5b91a1cd84b75 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -60,7 +60,10 @@ public class TransactionMetaStoreHandler extends HandlerState implements Connect private final long transactionCoordinatorId; private final ConnectionHandler connectionHandler; private final ConcurrentLongHashMap> pendingRequests = - new ConcurrentLongHashMap<>(16, 1); + ConcurrentLongHashMap.>newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); private final ConcurrentLinkedQueue timeoutQueue; protected final Timer timer; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java index 8db80545ad257..e8baec784a7b1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java @@ -52,7 +52,11 @@ public class TransactionCoordinatorClientImpl implements TransactionCoordinatorC private final PulsarClientImpl pulsarClient; private TransactionMetaStoreHandler[] handlers; - private ConcurrentLongHashMap handlerMap = new ConcurrentLongHashMap<>(16, 1); + private ConcurrentLongHashMap handlerMap = + ConcurrentLongHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); private final AtomicLong epoch = new AtomicLong(0); private static final AtomicReferenceFieldUpdater STATE_UPDATER = diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java index cd285221bc862..a4779357a440b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java @@ -44,33 +44,112 @@ public class ConcurrentLongHashMap { private static final Object EmptyValue = null; private static final Object DeletedValue = new Object(); - private static final float MapFillFactor = 0.66f; - private static final int DefaultExpectedItems = 256; private static final int DefaultConcurrencyLevel = 16; + private static final float DefaultMapFillFactor = 0.66f; + private static final float DefaultMapIdleFactor = 0.15f; + + private static final float DefaultExpandFactor = 2; + private static final float DefaultShrinkFactor = 2; + + private static final boolean DefaultAutoShrink = false; + + public static Builder newBuilder() { + return new Builder<>(); + } + + /** + * Builder of ConcurrentLongHashMap. + */ + public static class Builder { + int expectedItems = DefaultExpectedItems; + int concurrencyLevel = DefaultConcurrencyLevel; + float mapFillFactor = DefaultMapFillFactor; + float mapIdleFactor = DefaultMapIdleFactor; + float expandFactor = DefaultExpandFactor; + float shrinkFactor = DefaultShrinkFactor; + boolean autoShrink = DefaultAutoShrink; + + public Builder expectedItems(int expectedItems) { + this.expectedItems = expectedItems; + return this; + } + + public Builder concurrencyLevel(int concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + return this; + } + + public Builder mapFillFactor(float mapFillFactor) { + this.mapFillFactor = mapFillFactor; + return this; + } + + public Builder mapIdleFactor(float mapIdleFactor) { + this.mapIdleFactor = mapIdleFactor; + return this; + } + + public Builder expandFactor(float expandFactor) { + this.expandFactor = expandFactor; + return this; + } + + public Builder shrinkFactor(float shrinkFactor) { + this.shrinkFactor = shrinkFactor; + return this; + } + + public Builder autoShrink(boolean autoShrink) { + this.autoShrink = autoShrink; + return this; + } + + public ConcurrentLongHashMap build() { + return new ConcurrentLongHashMap<>(expectedItems, concurrencyLevel, + mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); + } + } + private final Section[] sections; + @Deprecated public ConcurrentLongHashMap() { this(DefaultExpectedItems); } + @Deprecated public ConcurrentLongHashMap(int expectedItems) { this(expectedItems, DefaultConcurrencyLevel); } + @Deprecated public ConcurrentLongHashMap(int expectedItems, int concurrencyLevel) { + this(expectedItems, concurrencyLevel, DefaultMapFillFactor, DefaultMapIdleFactor, + DefaultAutoShrink, DefaultExpandFactor, DefaultShrinkFactor); + } + + public ConcurrentLongHashMap(int expectedItems, int concurrencyLevel, + float mapFillFactor, float mapIdleFactor, + boolean autoShrink, float expandFactor, float shrinkFactor) { checkArgument(expectedItems > 0); checkArgument(concurrencyLevel > 0); checkArgument(expectedItems >= concurrencyLevel); + checkArgument(mapFillFactor > 0 && mapFillFactor < 1); + checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); + checkArgument(mapFillFactor > mapIdleFactor); + checkArgument(expandFactor > 1); + checkArgument(shrinkFactor > 1); int numSections = concurrencyLevel; int perSectionExpectedItems = expectedItems / numSections; - int perSectionCapacity = (int) (perSectionExpectedItems / MapFillFactor); + int perSectionCapacity = (int) (perSectionExpectedItems / mapFillFactor); this.sections = (Section[]) new Section[numSections]; for (int i = 0; i < numSections; i++) { - sections[i] = new Section<>(perSectionCapacity); + sections[i] = new Section<>(perSectionCapacity, mapFillFactor, mapIdleFactor, + autoShrink, expandFactor, shrinkFactor); } } @@ -195,20 +274,35 @@ private static final class Section extends StampedLock { private volatile V[] values; private volatile int capacity; + private final int initCapacity; private static final AtomicIntegerFieldUpdater
    SIZE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(Section.class, "size"); private volatile int size; private int usedBuckets; - private int resizeThreshold; - - Section(int capacity) { + private int resizeThresholdUp; + private int resizeThresholdBelow; + private final float mapFillFactor; + private final float mapIdleFactor; + private final float expandFactor; + private final float shrinkFactor; + private final boolean autoShrink; + + Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, + float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); + this.initCapacity = this.capacity; this.keys = new long[this.capacity]; this.values = (V[]) new Object[this.capacity]; this.size = 0; this.usedBuckets = 0; - this.resizeThreshold = (int) (this.capacity * MapFillFactor); + this.autoShrink = autoShrink; + this.mapFillFactor = mapFillFactor; + this.mapIdleFactor = mapIdleFactor; + this.expandFactor = expandFactor; + this.shrinkFactor = shrinkFactor; + this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); + this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); } V get(long key, int keyHash) { @@ -322,9 +416,10 @@ V put(long key, V value, int keyHash, boolean onlyIfAbsent, LongFunction valu ++bucket; } } finally { - if (usedBuckets >= resizeThreshold) { + if (usedBuckets > resizeThresholdUp) { try { - rehash(); + int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); + rehash(newCapacity); } finally { unlockWrite(stamp); } @@ -373,7 +468,20 @@ private V remove(long key, Object value, int keyHash) { } } finally { - unlockWrite(stamp); + if (autoShrink && size < resizeThresholdBelow) { + try { + int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); + int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); + if (newCapacity < capacity && newResizeThresholdUp > size) { + // shrink the hashmap + rehash(newCapacity); + } + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } } } @@ -385,6 +493,9 @@ void clear() { Arrays.fill(values, EmptyValue); this.size = 0; this.usedBuckets = 0; + if (autoShrink) { + rehash(initCapacity); + } } finally { unlockWrite(stamp); } @@ -439,9 +550,8 @@ public void forEach(EntryProcessor processor) { } } - private void rehash() { + private void rehash(int newCapacity) { // Expand the hashmap - int newCapacity = capacity * 2; long[] newKeys = new long[newCapacity]; V[] newValues = (V[]) new Object[newCapacity]; @@ -458,7 +568,8 @@ private void rehash() { values = newValues; capacity = newCapacity; usedBuckets = size; - resizeThreshold = (int) (capacity * MapFillFactor); + resizeThresholdUp = (int) (capacity * mapFillFactor); + resizeThresholdBelow = (int) (capacity * mapIdleFactor); } private static void insertKeyValueNoLock(long[] keys, V[] values, long key, V value) { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java index 14d8395ae8c8a..6cf126cf2ff15 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java @@ -48,21 +48,29 @@ public class ConcurrentLongHashMapTest { @Test public void testConstructor() { try { - new ConcurrentLongHashMap(0); + ConcurrentLongHashMap.newBuilder() + .expectedItems(0) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok } try { - new ConcurrentLongHashMap(16, 0); + ConcurrentLongHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(0) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok } try { - new ConcurrentLongHashMap(4, 8); + ConcurrentLongHashMap.newBuilder() + .expectedItems(4) + .concurrencyLevel(8) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok @@ -71,7 +79,9 @@ public void testConstructor() { @Test public void simpleInsertions() { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(16); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(16) + .build(); assertTrue(map.isEmpty()); assertNull(map.put(1, "one")); @@ -97,9 +107,64 @@ public void simpleInsertions() { assertEquals(map.size(), 3); } + @Test + public void testClear() { + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertNull(map.put(1, "v1")); + assertNull(map.put(2, "v2")); + assertNull(map.put(3, "v3")); + + assertTrue(map.capacity() == 8); + map.clear(); + assertTrue(map.capacity() == 4); + } + + @Test + public void testExpandAndShrink() { + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertNull(map.put(1, "v1")); + assertNull(map.put(2, "v2")); + assertNull(map.put(3, "v3")); + + // expand hashmap + assertTrue(map.capacity() == 8); + + assertTrue(map.remove(1, "v1")); + // not shrink + assertTrue(map.capacity() == 8); + assertTrue(map.remove(2, "v2")); + // shrink hashmap + assertTrue(map.capacity() == 4); + + // expand hashmap + assertNull(map.put(4, "v4")); + assertNull(map.put(5, "v5")); + assertTrue(map.capacity() == 8); + + //verify that the map does not keep shrinking at every remove() operation + assertNull(map.put(6, "v6")); + assertTrue(map.remove(6, "v6")); + assertTrue(map.capacity() == 8); + } + @Test public void testRemove() { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .build(); assertTrue(map.isEmpty()); assertNull(map.put(1, "one")); @@ -115,7 +180,10 @@ public void testRemove() { @Test public void testNegativeUsedBucketCount() { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(16, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); map.put(0, "zero"); assertEquals(1, map.getUsedBucketCount()); @@ -130,7 +198,10 @@ public void testNegativeUsedBucketCount() { @Test public void testRehashing() { int n = 16; - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(n / 2, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(map.capacity(), n); assertEquals(map.size(), 0); @@ -145,7 +216,10 @@ public void testRehashing() { @Test public void testRehashingWithDeletes() { int n = 16; - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(n / 2, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(map.capacity(), n); assertEquals(map.size(), 0); @@ -167,7 +241,8 @@ public void testRehashingWithDeletes() { @Test public void concurrentInsertions() throws Throwable { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -201,7 +276,8 @@ public void concurrentInsertions() throws Throwable { @Test public void concurrentInsertionsAndReads() throws Throwable { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -235,7 +311,10 @@ public void concurrentInsertionsAndReads() throws Throwable { @Test public void stressConcurrentInsertionsAndReads() throws Throwable { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(4, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(4) + .concurrencyLevel(1) + .build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); final int writeThreads = 16; @@ -286,7 +365,8 @@ public void stressConcurrentInsertionsAndReads() throws Throwable { @Test public void testIteration() { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .build(); assertEquals(map.keys(), Collections.emptyList()); assertEquals(map.values(), Collections.emptyList()); @@ -330,7 +410,10 @@ public void testIteration() { @Test public void testHashConflictWithDeletion() { final int Buckets = 16; - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(Buckets, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(Buckets) + .concurrencyLevel(1) + .build(); // Pick 2 keys that fall into the same bucket long key1 = 1; @@ -363,7 +446,8 @@ public void testHashConflictWithDeletion() { @Test public void testPutIfAbsent() { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .build(); assertNull(map.putIfAbsent(1, "one")); assertEquals(map.get(1), "one"); @@ -373,7 +457,10 @@ public void testPutIfAbsent() { @Test public void testComputeIfAbsent() { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(16, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); AtomicInteger counter = new AtomicInteger(); LongFunction provider = key -> counter.getAndIncrement(); @@ -395,7 +482,10 @@ public void testComputeIfAbsent() { static final int N = 100_000; public void benchConcurrentLongHashMap() throws Exception { - ConcurrentLongHashMap map = new ConcurrentLongHashMap<>(N, 1); + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(N) + .concurrencyLevel(1) + .build(); for (long i = 0; i < Iterations; i++) { for (int j = 0; j < N; j++) { From a052f938130bb3fa228887fca2a2b505c4266457 Mon Sep 17 00:00:00 2001 From: lin chen <1572139390@qq.com> Date: Sun, 6 Mar 2022 04:58:14 +0800 Subject: [PATCH 497/823] Optimize memory usage: support to shrink for pendingAcks map (#14515) (cherry picked from commit e747b8f16b0b660231ff27a8c2100d67ad7c79a6) --- .../pulsar/broker/ServiceConfiguration.java | 8 + .../pulsar/broker/service/Consumer.java | 11 +- .../ConcurrentLongLongPairHashMap.java | 673 ++++++++++++++++++ .../ConcurrentLongLongPairHashMapTest.java | 427 +++++++++++ 4 files changed, 1116 insertions(+), 3 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 1e7eac5ef9a1a..dc310d5056fcc 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -580,6 +580,14 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private boolean isAllowAutoUpdateSchemaEnabled = true; + @FieldContext( + category = CATEGORY_SERVER, + doc = "Whether to enable the automatic shrink of pendingAcks map, " + + "the default is false, which means it is not enabled. " + + "When there are a large number of share or key share consumers in the cluster, " + + "it can be enabled to reduce the memory consumption caused by pendingAcks.") + private boolean autoShrinkForConsumerPendingAcksMap = false; + @FieldContext( category = CATEGORY_SERVER, dynamic = true, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index b8358d06f3a70..f7ed211fb2c7a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -37,8 +37,6 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap.LongPair; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -59,6 +57,8 @@ import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,7 +177,12 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo stats.metadata = this.metadata; if (Subscription.isIndividualAckMode(subType)) { - this.pendingAcks = new ConcurrentLongLongPairHashMap(256, 1); + this.pendingAcks = ConcurrentLongLongPairHashMap.newBuilder() + .autoShrink(subscription.getTopic().getBrokerService() + .getPulsar().getConfiguration().isAutoShrinkForConsumerPendingAcksMap()) + .expectedItems(256) + .concurrencyLevel(1) + .build(); } else { // We don't need to keep track of pending acks if the subscription is not shared this.pendingAcks = null; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java new file mode 100644 index 0000000000000..eac7268ba672d --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java @@ -0,0 +1,673 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.locks.StampedLock; + +/** + * Concurrent hash map where both keys and values are composed of pairs of longs. + * + *

    (long,long) --> (long,long) + * + *

    Provides similar methods as a {@code ConcurrentMap} but since it's an open hash map with linear probing, + * no node allocations are required to store the keys and values, and no boxing is required. + * + *

    Keys MUST be >= 0. + */ +public class ConcurrentLongLongPairHashMap { + + private static final long EmptyKey = -1L; + private static final long DeletedKey = -2L; + + private static final long ValueNotFound = -1L; + + + private static final int DefaultExpectedItems = 256; + private static final int DefaultConcurrencyLevel = 16; + + private static final float DefaultMapFillFactor = 0.66f; + private static final float DefaultMapIdleFactor = 0.15f; + + private static final float DefaultExpandFactor = 2; + private static final float DefaultShrinkFactor = 2; + + private static final boolean DefaultAutoShrink = false; + + private final Section[] sections; + + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder of ConcurrentLongLongPairHashMap. + */ + public static class Builder { + int expectedItems = DefaultExpectedItems; + int concurrencyLevel = DefaultConcurrencyLevel; + float mapFillFactor = DefaultMapFillFactor; + float mapIdleFactor = DefaultMapIdleFactor; + float expandFactor = DefaultExpandFactor; + float shrinkFactor = DefaultShrinkFactor; + boolean autoShrink = DefaultAutoShrink; + + public Builder expectedItems(int expectedItems) { + this.expectedItems = expectedItems; + return this; + } + + public Builder concurrencyLevel(int concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + return this; + } + + public Builder mapFillFactor(float mapFillFactor) { + this.mapFillFactor = mapFillFactor; + return this; + } + + public Builder mapIdleFactor(float mapIdleFactor) { + this.mapIdleFactor = mapIdleFactor; + return this; + } + + public Builder expandFactor(float expandFactor) { + this.expandFactor = expandFactor; + return this; + } + + public Builder shrinkFactor(float shrinkFactor) { + this.shrinkFactor = shrinkFactor; + return this; + } + + public Builder autoShrink(boolean autoShrink) { + this.autoShrink = autoShrink; + return this; + } + + public ConcurrentLongLongPairHashMap build() { + return new ConcurrentLongLongPairHashMap(expectedItems, concurrencyLevel, + mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); + } + } + + /** + * A BiConsumer Long pair. + */ + public interface BiConsumerLongPair { + void accept(long key1, long key2, long value1, long value2); + } + + /** + * A Long pair function. + */ + public interface LongLongPairFunction { + long apply(long key1, long key2); + } + + /** + * A Long pair predicate. + */ + public interface LongLongPairPredicate { + boolean test(long key1, long key2, long value1, long value2); + } + + private ConcurrentLongLongPairHashMap(int expectedItems, int concurrencyLevel, + float mapFillFactor, float mapIdleFactor, + boolean autoShrink, float expandFactor, float shrinkFactor) { + checkArgument(expectedItems > 0); + checkArgument(concurrencyLevel > 0); + checkArgument(expectedItems >= concurrencyLevel); + checkArgument(mapFillFactor > 0 && mapFillFactor < 1); + checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); + checkArgument(mapFillFactor > mapIdleFactor); + checkArgument(expandFactor > 1); + checkArgument(shrinkFactor > 1); + + int numSections = concurrencyLevel; + int perSectionExpectedItems = expectedItems / numSections; + int perSectionCapacity = (int) (perSectionExpectedItems / mapFillFactor); + this.sections = new Section[numSections]; + + for (int i = 0; i < numSections; i++) { + sections[i] = new Section(perSectionCapacity, mapFillFactor, mapIdleFactor, + autoShrink, expandFactor, shrinkFactor); + } + } + + public long size() { + long size = 0; + for (Section s : sections) { + size += s.size; + } + return size; + } + + public long capacity() { + long capacity = 0; + for (Section s : sections) { + capacity += s.capacity; + } + return capacity; + } + + public boolean isEmpty() { + for (Section s : sections) { + if (s.size != 0) { + return false; + } + } + + return true; + } + + long getUsedBucketCount() { + long usedBucketCount = 0; + for (Section s : sections) { + usedBucketCount += s.usedBuckets; + } + return usedBucketCount; + } + + /** + * @param key1 + * @param key2 + * @return the value or -1 if the key was not present. + */ + public LongPair get(long key1, long key2) { + checkBiggerEqualZero(key1); + long h = hash(key1, key2); + return getSection(h).get(key1, key2, (int) h); + } + + public boolean containsKey(long key1, long key2) { + return get(key1, key2) != null; + } + + public boolean put(long key1, long key2, long value1, long value2) { + checkBiggerEqualZero(key1); + checkBiggerEqualZero(value1); + long h = hash(key1, key2); + return getSection(h).put(key1, key2, value1, value2, (int) h, false); + } + + public boolean putIfAbsent(long key1, long key2, long value1, long value2) { + checkBiggerEqualZero(key1); + checkBiggerEqualZero(value1); + long h = hash(key1, key2); + return getSection(h).put(key1, key2, value1, value2, (int) h, true); + } + + /** + * Remove an existing entry if found. + * + * @param key1 + * @param key2 + * @return the value associated with the key or -1 if key was not present. + */ + public boolean remove(long key1, long key2) { + checkBiggerEqualZero(key1); + long h = hash(key1, key2); + return getSection(h).remove(key1, key2, ValueNotFound, ValueNotFound, (int) h); + } + + public boolean remove(long key1, long key2, long value1, long value2) { + checkBiggerEqualZero(key1); + checkBiggerEqualZero(value1); + long h = hash(key1, key2); + return getSection(h).remove(key1, key2, value1, value2, (int) h); + } + + private Section getSection(long hash) { + // Use 32 msb out of long to get the section + final int sectionIdx = (int) (hash >>> 32) & (sections.length - 1); + return sections[sectionIdx]; + } + + public void clear() { + for (Section s : sections) { + s.clear(); + } + } + + public void forEach(BiConsumerLongPair processor) { + for (Section s : sections) { + s.forEach(processor); + } + } + + /** + * @return a new list of all keys (makes a copy). + */ + public List keys() { + List keys = Lists.newArrayListWithExpectedSize((int) size()); + forEach((key1, key2, value1, value2) -> keys.add(new LongPair(key1, key2))); + return keys; + } + + public List values() { + List values = Lists.newArrayListWithExpectedSize((int) size()); + forEach((key1, key2, value1, value2) -> values.add(new LongPair(value1, value2))); + return values; + } + + public Map asMap() { + Map map = Maps.newHashMapWithExpectedSize((int) size()); + forEach((key1, key2, value1, value2) -> map.put(new LongPair(key1, key2), new LongPair(value1, value2))); + return map; + } + + // A section is a portion of the hash map that is covered by a single + @SuppressWarnings("serial") + private static final class Section extends StampedLock { + // Keys and values are stored interleaved in the table array + private volatile long[] table; + + private volatile int capacity; + private final int initCapacity; + private static final AtomicIntegerFieldUpdater

    SIZE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(Section.class, "size"); + + private volatile int size; + private int usedBuckets; + private int resizeThresholdUp; + private int resizeThresholdBelow; + private final float mapFillFactor; + private final float mapIdleFactor; + private final float expandFactor; + private final float shrinkFactor; + private final boolean autoShrink; + + Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, + float expandFactor, float shrinkFactor) { + this.capacity = alignToPowerOfTwo(capacity); + this.initCapacity = this.capacity; + this.table = new long[4 * this.capacity]; + this.size = 0; + this.usedBuckets = 0; + this.autoShrink = autoShrink; + this.mapFillFactor = mapFillFactor; + this.mapIdleFactor = mapIdleFactor; + this.expandFactor = expandFactor; + this.shrinkFactor = shrinkFactor; + this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); + this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); + Arrays.fill(table, EmptyKey); + } + + LongPair get(long key1, long key2, int keyHash) { + long stamp = tryOptimisticRead(); + boolean acquiredLock = false; + int bucket = signSafeMod(keyHash, capacity); + + try { + while (true) { + // First try optimistic locking + long storedKey1 = table[bucket]; + long storedKey2 = table[bucket + 1]; + long storedValue1 = table[bucket + 2]; + long storedValue2 = table[bucket + 3]; + + if (!acquiredLock && validate(stamp)) { + // The values we have read are consistent + if (key1 == storedKey1 && key2 == storedKey2) { + return new LongPair(storedValue1, storedValue2); + } else if (storedKey1 == EmptyKey) { + // Not found + return null; + } + } else { + // Fallback to acquiring read lock + if (!acquiredLock) { + stamp = readLock(); + acquiredLock = true; + + bucket = signSafeMod(keyHash, capacity); + storedKey1 = table[bucket]; + storedKey2 = table[bucket + 1]; + storedValue1 = table[bucket + 2]; + storedValue2 = table[bucket + 3]; + } + + if (key1 == storedKey1 && key2 == storedKey2) { + return new LongPair(storedValue1, storedValue2); + } else if (storedKey1 == EmptyKey) { + // Not found + return null; + } + } + + bucket = (bucket + 4) & (table.length - 1); + } + } finally { + if (acquiredLock) { + unlockRead(stamp); + } + } + } + + boolean put(long key1, long key2, long value1, long value2, int keyHash, boolean onlyIfAbsent) { + long stamp = writeLock(); + int bucket = signSafeMod(keyHash, capacity); + + // Remember where we find the first available spot + int firstDeletedKey = -1; + + try { + while (true) { + long storedKey1 = table[bucket]; + long storedKey2 = table[bucket + 1]; + + if (key1 == storedKey1 && key2 == storedKey2) { + if (!onlyIfAbsent) { + // Over written an old value for same key + table[bucket + 2] = value1; + table[bucket + 3] = value2; + return true; + } else { + return false; + } + } else if (storedKey1 == EmptyKey) { + // Found an empty bucket. This means the key is not in the map. If we've already seen a deleted + // key, we should write at that position + if (firstDeletedKey != -1) { + bucket = firstDeletedKey; + } else { + ++usedBuckets; + } + + table[bucket] = key1; + table[bucket + 1] = key2; + table[bucket + 2] = value1; + table[bucket + 3] = value2; + SIZE_UPDATER.incrementAndGet(this); + return true; + } else if (storedKey1 == DeletedKey) { + // The bucket contained a different deleted key + if (firstDeletedKey == -1) { + firstDeletedKey = bucket; + } + } + + bucket = (bucket + 4) & (table.length - 1); + } + } finally { + if (usedBuckets > resizeThresholdUp) { + try { + // Expand the hashmap + int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); + rehash(newCapacity); + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } + } + } + + private boolean remove(long key1, long key2, long value1, long value2, int keyHash) { + long stamp = writeLock(); + int bucket = signSafeMod(keyHash, capacity); + + try { + while (true) { + long storedKey1 = table[bucket]; + long storedKey2 = table[bucket + 1]; + long storedValue1 = table[bucket + 2]; + long storedValue2 = table[bucket + 3]; + if (key1 == storedKey1 && key2 == storedKey2) { + if (value1 == ValueNotFound || (value1 == storedValue1 && value2 == storedValue2)) { + SIZE_UPDATER.decrementAndGet(this); + + cleanBucket(bucket); + return true; + } else { + return false; + } + } else if (storedKey1 == EmptyKey) { + // Key wasn't found + return false; + } + + bucket = (bucket + 4) & (table.length - 1); + } + + } finally { + if (autoShrink && size < resizeThresholdBelow) { + try { + int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); + int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); + if (newCapacity < capacity && newResizeThresholdUp > size) { + // shrink the hashmap + rehash(newCapacity); + } + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } + } + } + + private void cleanBucket(int bucket) { + int nextInArray = (bucket + 4) & (table.length - 1); + if (table[nextInArray] == EmptyKey) { + table[bucket] = EmptyKey; + table[bucket + 1] = EmptyKey; + table[bucket + 2] = ValueNotFound; + table[bucket + 3] = ValueNotFound; + --usedBuckets; + + // Cleanup all the buckets that were in `DeletedKey` state, so that we can reduce unnecessary expansions + bucket = (bucket - 4) & (table.length - 1); + while (table[bucket] == DeletedKey) { + table[bucket] = EmptyKey; + table[bucket + 1] = EmptyKey; + table[bucket + 2] = ValueNotFound; + table[bucket + 3] = ValueNotFound; + --usedBuckets; + + bucket = (bucket - 4) & (table.length - 1); + } + } else { + table[bucket] = DeletedKey; + table[bucket + 1] = DeletedKey; + table[bucket + 2] = ValueNotFound; + table[bucket + 3] = ValueNotFound; + } + } + + void clear() { + long stamp = writeLock(); + + try { + Arrays.fill(table, EmptyKey); + this.size = 0; + this.usedBuckets = 0; + if (autoShrink) { + rehash(initCapacity); + } + } finally { + unlockWrite(stamp); + } + } + + public void forEach(BiConsumerLongPair processor) { + long stamp = tryOptimisticRead(); + + long[] table = this.table; + boolean acquiredReadLock = false; + + try { + + // Validate no rehashing + if (!validate(stamp)) { + // Fallback to read lock + stamp = readLock(); + acquiredReadLock = true; + table = this.table; + } + + // Go through all the buckets for this section + for (int bucket = 0; bucket < table.length; bucket += 4) { + long storedKey1 = table[bucket]; + long storedKey2 = table[bucket + 1]; + long storedValue1 = table[bucket + 2]; + long storedValue2 = table[bucket + 3]; + + if (!acquiredReadLock && !validate(stamp)) { + // Fallback to acquiring read lock + stamp = readLock(); + acquiredReadLock = true; + + storedKey1 = table[bucket]; + storedKey2 = table[bucket + 1]; + storedValue1 = table[bucket + 2]; + storedValue2 = table[bucket + 3]; + } + + if (storedKey1 != DeletedKey && storedKey1 != EmptyKey) { + processor.accept(storedKey1, storedKey2, storedValue1, storedValue2); + } + } + } finally { + if (acquiredReadLock) { + unlockRead(stamp); + } + } + } + + private void rehash(int newCapacity) { + long[] newTable = new long[4 * newCapacity]; + Arrays.fill(newTable, EmptyKey); + + // Re-hash table + for (int i = 0; i < table.length; i += 4) { + long storedKey1 = table[i]; + long storedKey2 = table[i + 1]; + long storedValue1 = table[i + 2]; + long storedValue2 = table[i + 3]; + if (storedKey1 != EmptyKey && storedKey1 != DeletedKey) { + insertKeyValueNoLock(newTable, newCapacity, storedKey1, storedKey2, storedValue1, storedValue2); + } + } + + table = newTable; + usedBuckets = size; + // Capacity needs to be updated after the values, so that we won't see + // a capacity value bigger than the actual array size + capacity = newCapacity; + resizeThresholdUp = (int) (capacity * mapFillFactor); + resizeThresholdBelow = (int) (capacity * mapIdleFactor); + } + + private static void insertKeyValueNoLock(long[] table, int capacity, long key1, long key2, long value1, + long value2) { + int bucket = signSafeMod(hash(key1, key2), capacity); + + while (true) { + long storedKey1 = table[bucket]; + + if (storedKey1 == EmptyKey) { + // The bucket is empty, so we can use it + table[bucket] = key1; + table[bucket + 1] = key2; + table[bucket + 2] = value1; + table[bucket + 3] = value2; + return; + } + + bucket = (bucket + 4) & (table.length - 1); + } + } + } + + private static final long HashMixer = 0xc6a4a7935bd1e995L; + private static final int R = 47; + + static final long hash(long key1, long key2) { + long hash = key1 * HashMixer; + hash ^= hash >>> R; + hash *= HashMixer; + hash += 31 + (key2 * HashMixer); + hash ^= hash >>> R; + hash *= HashMixer; + return hash; + } + + static final int signSafeMod(long n, int max) { + return (int) (n & (max - 1)) << 2; + } + + private static int alignToPowerOfTwo(int n) { + return (int) Math.pow(2, 32 - Integer.numberOfLeadingZeros(n - 1)); + } + + private static void checkBiggerEqualZero(long n) { + if (n < 0L) { + throw new IllegalArgumentException("Keys and values must be >= 0"); + } + } + + /** + * A pair of long values. + */ + public static class LongPair implements Comparable { + public final long first; + public final long second; + + public LongPair(long first, long second) { + this.first = first; + this.second = second; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof LongPair) { + LongPair other = (LongPair) obj; + return first == other.first && second == other.second; + } + return false; + } + + @Override + public int hashCode() { + return (int) hash(first, second); + } + + @Override + public int compareTo(LongPair o) { + if (first != o.first) { + return Long.compare(first, o.first); + } else { + return Long.compare(second, o.second); + } + } + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java new file mode 100644 index 0000000000000..98a96804d25e0 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java @@ -0,0 +1,427 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; +import org.junit.Test; + +/** + * Test the concurrent long-long pair hashmap class. + */ +public class ConcurrentLongLongPairHashMapTest { + + @Test + public void testConstructor() { + try { + ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(0) + .build(); + fail("should have thrown exception"); + } catch (IllegalArgumentException e) { + // ok + } + + try { + ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(0) + .build(); + fail("should have thrown exception"); + } catch (IllegalArgumentException e) { + // ok + } + + try { + ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(4) + .concurrencyLevel(8) + .build(); + fail("should have thrown exception"); + } catch (IllegalArgumentException e) { + // ok + } + } + + @Test + public void simpleInsertions() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(16) + .build(); + assertTrue(map.isEmpty()); + assertTrue(map.put(1, 1, 11, 11)); + assertFalse(map.isEmpty()); + + assertTrue(map.put(2, 2, 22, 22)); + assertTrue(map.put(3, 3, 33, 33)); + + assertEquals(map.size(), 3); + + assertEquals(map.get(1, 1), new LongPair(11, 11)); + assertEquals(map.size(), 3); + + assertTrue(map.remove(1, 1)); + assertEquals(map.size(), 2); + assertEquals(map.get(1, 1), null); + assertEquals(map.get(5, 5), null); + assertEquals(map.size(), 2); + + assertTrue(map.put(1, 1, 11, 11)); + assertEquals(map.size(), 3); + assertTrue(map.put(1, 1, 111, 111)); + assertEquals(map.size(), 3); + } + + @Test + public void testRemove() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap + .newBuilder() + .build(); + + assertTrue(map.isEmpty()); + assertTrue(map.put(1, 1, 11, 11)); + assertFalse(map.isEmpty()); + + assertFalse(map.remove(0, 0)); + assertFalse(map.remove(1, 1, 111, 111)); + + assertFalse(map.isEmpty()); + assertTrue(map.remove(1, 1, 11, 11)); + assertTrue(map.isEmpty()); + } + + @Test + public void testClear() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertTrue(map.put(1, 1, 11, 11)); + assertTrue(map.put(2, 2, 22, 22)); + assertTrue(map.put(3, 3, 33, 33)); + + assertTrue(map.capacity() == 8); + map.clear(); + assertTrue(map.capacity() == 4); + } + + @Test + public void testExpandAndShrink() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.put(1, 1, 11, 11)); + assertTrue(map.put(2, 2, 22, 22)); + assertTrue(map.put(3, 3, 33, 33)); + + // expand hashmap + assertTrue(map.capacity() == 8); + + assertTrue(map.remove(1, 1, 11, 11)); + // not shrink + assertTrue(map.capacity() == 8); + assertTrue(map.remove(2, 2, 22, 22)); + // shrink hashmap + assertTrue(map.capacity() == 4); + + // expand hashmap + assertTrue(map.put(4, 4, 44, 44)); + assertTrue(map.put(5, 5, 55, 55)); + assertTrue(map.capacity() == 8); + + //verify that the map does not keep shrinking at every remove() operation + assertTrue(map.put(6, 6, 66, 66)); + assertTrue(map.remove(6, 6, 66, 66)); + assertTrue(map.capacity() == 8); + } + + @Test + public void testNegativeUsedBucketCount() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + + map.put(0, 0, 0, 0); + assertEquals(1, map.getUsedBucketCount()); + map.put(0, 0, 1, 1); + assertEquals(1, map.getUsedBucketCount()); + map.remove(0, 0); + assertEquals(0, map.getUsedBucketCount()); + map.remove(0, 0); + assertEquals(0, map.getUsedBucketCount()); + } + + @Test + public void testRehashing() { + int n = 16; + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); + assertEquals(map.capacity(), n); + assertEquals(map.size(), 0); + + for (int i = 0; i < n; i++) { + map.put(i, i, i, i); + } + + assertEquals(map.capacity(), 2 * n); + assertEquals(map.size(), n); + } + + @Test + public void testRehashingWithDeletes() { + int n = 16; + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); + assertEquals(map.capacity(), n); + assertEquals(map.size(), 0); + + for (int i = 0; i < n / 2; i++) { + map.put(i, i, i, i); + } + + for (int i = 0; i < n / 2; i++) { + map.remove(i, i); + } + + for (int i = n; i < (2 * n); i++) { + map.put(i, i, i, i); + } + + assertEquals(map.capacity(), 2 * n); + assertEquals(map.size(), n); + } + + @Test + public void concurrentInsertions() throws Throwable { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .build(); + ExecutorService executor = Executors.newCachedThreadPool(); + + final int nThreads = 16; + final int n = 100_000; + long value = 55; + + List> futures = new ArrayList<>(); + for (int i = 0; i < nThreads; i++) { + final int threadIdx = i; + + futures.add(executor.submit(() -> { + Random random = new Random(); + + for (int j = 0; j < n; j++) { + long key1 = Math.abs(random.nextLong()); + // Ensure keys are uniques + key1 -= key1 % (threadIdx + 1); + + long key2 = Math.abs(random.nextLong()); + // Ensure keys are uniques + key2 -= key2 % (threadIdx + 1); + + map.put(key1, key2, value, value); + } + })); + } + + for (Future future : futures) { + future.get(); + } + + assertEquals(map.size(), n * nThreads); + + executor.shutdown(); + } + + @Test + public void concurrentInsertionsAndReads() throws Throwable { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .build(); + ExecutorService executor = Executors.newCachedThreadPool(); + + final int nThreads = 16; + final int n = 100_000; + final long value = 55; + + List> futures = new ArrayList<>(); + for (int i = 0; i < nThreads; i++) { + final int threadIdx = i; + + futures.add(executor.submit(() -> { + Random random = new Random(); + + for (int j = 0; j < n; j++) { + long key1 = Math.abs(random.nextLong()); + // Ensure keys are uniques + key1 -= key1 % (threadIdx + 1); + + long key2 = Math.abs(random.nextLong()); + // Ensure keys are uniques + key2 -= key2 % (threadIdx + 1); + + map.put(key1, key2, value, value); + } + })); + } + + for (Future future : futures) { + future.get(); + } + + assertEquals(map.size(), n * nThreads); + + executor.shutdown(); + } + + @Test + public void testIteration() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .build(); + + assertEquals(map.keys(), Collections.emptyList()); + assertEquals(map.values(), Collections.emptyList()); + + map.put(0, 0, 0, 0); + + assertEquals(map.keys(), Lists.newArrayList(new LongPair(0, 0))); + assertEquals(map.values(), Lists.newArrayList(new LongPair(0, 0))); + + map.remove(0, 0); + + assertEquals(map.keys(), Collections.emptyList()); + assertEquals(map.values(), Collections.emptyList()); + + map.put(0, 0, 0, 0); + map.put(1, 1, 11, 11); + map.put(2, 2, 22, 22); + + List keys = map.keys(); + Collections.sort(keys); + assertEquals(keys, Lists.newArrayList(new LongPair(0, 0), new LongPair(1, 1), new LongPair(2, 2))); + + List values = map.values(); + Collections.sort(values); + assertEquals(values, Lists.newArrayList(new LongPair(0, 0), new LongPair(11, 11), new LongPair(22, 22))); + + map.put(1, 1, 111, 111); + + keys = map.keys(); + Collections.sort(keys); + assertEquals(keys, Lists.newArrayList(new LongPair(0, 0), new LongPair(1, 1), new LongPair(2, 2))); + + values = map.values(); + Collections.sort(values); + assertEquals(values, Lists.newArrayList(new LongPair(0, 0), new LongPair(22, 22), new LongPair(111, 111))); + + map.clear(); + assertTrue(map.isEmpty()); + } + + @Test + public void testPutIfAbsent() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .build(); + + assertTrue(map.putIfAbsent(1, 1, 11, 11)); + assertEquals(map.get(1, 1), new LongPair(11, 11)); + + assertFalse(map.putIfAbsent(1, 1, 111, 111)); + assertEquals(map.get(1, 1), new LongPair(11, 11)); + } + + @Test + public void testIvalidKeys() { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + + + try { + map.put(-5, 3, 4, 4); + fail("should have failed"); + } catch (IllegalArgumentException e) { + // ok + } + + try { + map.get(-1, 0); + fail("should have failed"); + } catch (IllegalArgumentException e) { + // ok + } + + try { + map.containsKey(-1, 0); + fail("should have failed"); + } catch (IllegalArgumentException e) { + // ok + } + + try { + map.putIfAbsent(-1, 1, 1, 1); + fail("should have failed"); + } catch (IllegalArgumentException e) { + // ok + } + } + + @Test + public void testAsMap() { + ConcurrentLongLongPairHashMap lmap = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + lmap.put(1, 1, 11, 11); + lmap.put(2, 2, 22, 22); + lmap.put(3, 3, 33, 33); + + Map map = new HashMap<>(); + map.put(new LongPair(1, 1), new LongPair(11, 11)); + map.put(new LongPair(2, 2), new LongPair(22, 22)); + map.put(new LongPair(3, 3), new LongPair(33, 33)); + + assertEquals(map, lmap.asMap()); + } +} From f256491296907b575c62137327c10a1b49ccb3a7 Mon Sep 17 00:00:00 2001 From: LinChen <1572139390@qq.com> Date: Mon, 14 Mar 2022 23:23:47 +0800 Subject: [PATCH 498/823] support shrink for map or set (#14663) * support shrink for map or set * check style * check style (cherry picked from commit 1d10dff757ac7b9a203c14d2085a480495fb141b) --- .../impl/ManagedLedgerOfflineBacklog.java | 3 +- .../loadbalance/impl/LoadManagerShared.java | 20 ++- .../impl/ModularLoadManagerImpl.java | 18 ++- .../impl/SimpleLoadManagerImpl.java | 18 ++- .../broker/namespace/NamespaceService.java | 13 +- .../apache/pulsar/broker/rest/TopicsBase.java | 3 +- .../pulsar/broker/service/BrokerService.java | 42 +++-- .../nonpersistent/NonPersistentTopic.java | 12 +- .../persistent/MessageDeduplication.java | 12 +- .../service/persistent/PersistentTopic.java | 20 ++- .../stats/ClusterReplicationMetrics.java | 3 +- .../AntiAffinityNamespaceGroupTest.java | 15 +- .../impl/LoadManagerSharedTest.java | 13 +- .../broker/service/PersistentTopicTest.java | 24 ++- .../pulsar/client/impl/ConsumerBase.java | 3 +- .../pulsar/client/impl/ConsumerImpl.java | 3 +- .../client/impl/PartitionedProducerImpl.java | 3 +- .../pulsar/client/impl/ProducerBase.java | 3 +- .../AcknowledgementsGroupingTrackerTest.java | 3 +- .../collections/ConcurrentLongPairSet.java | 148 ++++++++++++++++-- .../collections/ConcurrentOpenHashMap.java | 140 +++++++++++++++-- .../collections/ConcurrentOpenHashSet.java | 140 +++++++++++++++-- .../ConcurrentSortedLongPairSet.java | 5 +- .../ConcurrentLongPairSetTest.java | 111 +++++++++++-- .../ConcurrentOpenHashMapTest.java | 125 +++++++++++++-- .../ConcurrentOpenHashSetTest.java | 73 ++++++++- .../pulsar/sql/presto/PulsarRecordCursor.java | 3 +- .../pulsar/websocket/WebSocketService.java | 23 ++- .../pulsar/websocket/stats/ProxyStats.java | 4 +- 29 files changed, 860 insertions(+), 143 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java index e00dd47a73974..d258c1ca339ca 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java @@ -220,7 +220,8 @@ private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, fin BookKeeper bk = factory.getBookKeeper(); final CountDownLatch allCursorsCounter = new CountDownLatch(1); final long errorInReadingCursor = -1; - ConcurrentOpenHashMap ledgerRetryMap = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap ledgerRetryMap = + ConcurrentOpenHashMap.newBuilder().build(); final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue(); final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 74165c78dfa6e..20b14a9d2202c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -190,7 +190,9 @@ public static void fillNamespaceToBundlesMap(final Set bundles, bundles.forEach(bundleName -> { final String namespaceName = getNamespaceNameFromBundleName(bundleName); final String bundleRange = getBundleRangeFromBundleName(bundleName); - target.computeIfAbsent(namespaceName, k -> new ConcurrentOpenHashSet<>()).add(bundleRange); + target.computeIfAbsent(namespaceName, + k -> ConcurrentOpenHashSet.newBuilder().build()) + .add(bundleRange); }); } @@ -263,8 +265,12 @@ public static void removeMostServicingBrokersForNamespace( for (final String broker : candidates) { int bundles = (int) brokerToNamespaceToBundleRange - .computeIfAbsent(broker, k -> new ConcurrentOpenHashMap<>()) - .computeIfAbsent(namespaceName, k -> new ConcurrentOpenHashSet<>()).size(); + .computeIfAbsent(broker, + k -> ConcurrentOpenHashMap.>newBuilder().build()) + .computeIfAbsent(namespaceName, + k -> ConcurrentOpenHashSet.newBuilder().build()) + .size(); leastBundles = Math.min(leastBundles, bundles); if (leastBundles == 0) { break; @@ -276,8 +282,12 @@ public static void removeMostServicingBrokersForNamespace( final int finalLeastBundles = leastBundles; candidates.removeIf( - broker -> brokerToNamespaceToBundleRange.computeIfAbsent(broker, k -> new ConcurrentOpenHashMap<>()) - .computeIfAbsent(namespaceName, k -> new ConcurrentOpenHashSet<>()).size() > finalLeastBundles); + broker -> brokerToNamespaceToBundleRange.computeIfAbsent(broker, + k -> ConcurrentOpenHashMap.>newBuilder().build()) + .computeIfAbsent(namespaceName, + k -> ConcurrentOpenHashSet.newBuilder().build()) + .size() > finalLeastBundles); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 7bcfcab0e49c7..68833199bf47c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -201,7 +201,10 @@ public class ModularLoadManagerImpl implements ModularLoadManager { */ public ModularLoadManagerImpl() { brokerCandidateCache = new HashSet<>(); - brokerToNamespaceToBundleRange = new ConcurrentOpenHashMap<>(); + brokerToNamespaceToBundleRange = + ConcurrentOpenHashMap.>>newBuilder() + .build(); defaultStats = new NamespaceBundleStats(); filterPipeline = new ArrayList<>(); loadData = new LoadData(); @@ -547,7 +550,10 @@ private void updateBundleData() { brokerData.getTimeAverageData().reset(statsMap.keySet(), bundleData, defaultStats); final ConcurrentOpenHashMap> namespaceToBundleRange = brokerToNamespaceToBundleRange - .computeIfAbsent(broker, k -> new ConcurrentOpenHashMap<>()); + .computeIfAbsent(broker, k -> + ConcurrentOpenHashMap.>newBuilder() + .build()); synchronized (namespaceToBundleRange) { namespaceToBundleRange.clear(); LoadManagerShared.fillNamespaceToBundlesMap(statsMap.keySet(), namespaceToBundleRange); @@ -830,9 +836,13 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); final ConcurrentOpenHashMap> namespaceToBundleRange = brokerToNamespaceToBundleRange - .computeIfAbsent(broker.get(), k -> new ConcurrentOpenHashMap<>()); + .computeIfAbsent(broker.get(), + k -> ConcurrentOpenHashMap.>newBuilder() + .build()); synchronized (namespaceToBundleRange) { - namespaceToBundleRange.computeIfAbsent(namespaceName, k -> new ConcurrentOpenHashSet<>()) + namespaceToBundleRange.computeIfAbsent(namespaceName, + k -> ConcurrentOpenHashSet.newBuilder().build()) .add(bundleRange); } return broker; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index e1829e68aed0e..212d299953e40 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -200,7 +200,10 @@ public SimpleLoadManagerImpl() { bundleLossesCache = new HashSet<>(); brokerCandidateCache = new HashSet<>(); availableBrokersCache = new HashSet<>(); - brokerToNamespaceToBundleRange = new ConcurrentOpenHashMap<>(); + brokerToNamespaceToBundleRange = + ConcurrentOpenHashMap.>>newBuilder() + .build(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override public boolean isEnablePersistentTopics(String brokerUrl) { @@ -833,8 +836,12 @@ private synchronized ResourceUnit findBrokerForPlacement(Multimap new ConcurrentOpenHashMap<>()) - .computeIfAbsent(namespaceName, k -> new ConcurrentOpenHashSet<>()).add(bundleRange); + k -> ConcurrentOpenHashMap.>newBuilder() + .build()) + .computeIfAbsent(namespaceName, k -> + ConcurrentOpenHashSet.newBuilder().build()) + .add(bundleRange); ranking.addPreAllocatedServiceUnit(serviceUnitId, quota); resourceUnitRankings.put(selectedRU, ranking); } @@ -1252,7 +1259,10 @@ private synchronized void updateBrokerToNamespaceToBundle() { final Set preallocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles(); final ConcurrentOpenHashMap> namespaceToBundleRange = brokerToNamespaceToBundleRange - .computeIfAbsent(broker.replace("http://", ""), k -> new ConcurrentOpenHashMap<>()); + .computeIfAbsent(broker.replace("http://", ""), + k -> ConcurrentOpenHashMap.>newBuilder() + .build()); namespaceToBundleRange.clear(); LoadManagerShared.fillNamespaceToBundlesMap(loadedBundles, namespaceToBundleRange); LoadManagerShared.fillNamespaceToBundlesMap(preallocatedBundles, namespaceToBundleRange); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index e576e864467e6..25d935d835505 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -166,7 +166,8 @@ public NamespaceService(PulsarService pulsar) { this.loadManager = pulsar.getLoadManager(); this.bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32()); this.ownershipCache = new OwnershipCache(pulsar, bundleFactory, this); - this.namespaceClients = new ConcurrentOpenHashMap<>(); + this.namespaceClients = + ConcurrentOpenHashMap.newBuilder().build(); this.bundleOwnershipListeners = new CopyOnWriteArrayList<>(); this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class); this.localPoliciesCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalPolicies.class); @@ -356,9 +357,15 @@ public boolean registerNamespace(NamespaceName nsname, boolean ensureOwned) thro } private final ConcurrentOpenHashMap>> - findingBundlesAuthoritative = new ConcurrentOpenHashMap<>(); + findingBundlesAuthoritative = + ConcurrentOpenHashMap.>>newBuilder() + .build(); private final ConcurrentOpenHashMap>> - findingBundlesNotAuthoritative = new ConcurrentOpenHashMap<>(); + findingBundlesNotAuthoritative = + ConcurrentOpenHashMap.>>newBuilder() + .build(); /** * Main internal method to lookup and setup ownership of service unit to a broker. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index f89abf9bea30d..770d77794d5df 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -431,7 +431,8 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN partitionedTopicName, result.getLookupData()); } pulsar().getBrokerService().getOwningTopics().computeIfAbsent(partitionedTopicName - .getPartitionedTopicName(), (key) -> new ConcurrentOpenHashSet()) + .getPartitionedTopicName(), + (key) -> ConcurrentOpenHashSet.newBuilder().build()) .add(partitionedTopicName.getPartitionIndex()); completeLookup(Pair.of(Collections.emptyList(), false), redirectAddresses, future); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index a4b745d7b6576..a9f9c7fe090c1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -277,17 +277,28 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.preciseTopicPublishRateLimitingEnable = pulsar.getConfiguration().isPreciseTopicPublishRateLimiterEnable(); this.managedLedgerFactory = pulsar.getManagedLedgerFactory(); - this.topics = new ConcurrentOpenHashMap<>(); - this.replicationClients = new ConcurrentOpenHashMap<>(); - this.clusterAdmins = new ConcurrentOpenHashMap<>(); + this.topics = + ConcurrentOpenHashMap.>>newBuilder() + .build(); + this.replicationClients = + ConcurrentOpenHashMap.newBuilder().build(); + this.clusterAdmins = + ConcurrentOpenHashMap.newBuilder().build(); this.keepAliveIntervalSeconds = pulsar.getConfiguration().getKeepAliveIntervalSeconds(); - this.configRegisteredListeners = new ConcurrentOpenHashMap<>(); + this.configRegisteredListeners = + ConcurrentOpenHashMap.>newBuilder().build(); this.pendingTopicLoadingQueue = Queues.newConcurrentLinkedQueue(); - this.multiLayerTopicsMap = new ConcurrentOpenHashMap<>(); - this.owningTopics = new ConcurrentOpenHashMap<>(); + this.multiLayerTopicsMap = ConcurrentOpenHashMap.>>newBuilder() + .build(); + this.owningTopics = ConcurrentOpenHashMap.>newBuilder() + .build(); this.pulsarStats = new PulsarStats(pulsar); - this.offlineTopicStatCache = new ConcurrentOpenHashMap<>(); + this.offlineTopicStatCache = + ConcurrentOpenHashMap.newBuilder().build(); this.topicOrderedExecutor = OrderedScheduler.newSchedulerBuilder() .numThreads(pulsar.getConfiguration().getNumWorkerThreadsForNonPersistentTopic()) @@ -319,7 +330,8 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.backlogQuotaChecker = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-backlog-quota-checker")); this.authenticationService = new AuthenticationService(pulsar.getConfiguration()); - this.blockedDispatchers = new ConcurrentOpenHashSet<>(); + this.blockedDispatchers = + ConcurrentOpenHashSet.newBuilder().build(); // update dynamic configuration and register-listener updateConfigurationAndRegisterListeners(); this.lookupRequestSemaphore = new AtomicReference( @@ -1532,8 +1544,12 @@ private void addTopicToStatsMaps(TopicName topicName, Topic topic) { synchronized (multiLayerTopicsMap) { String serviceUnit = namespaceBundle.toString(); multiLayerTopicsMap // - .computeIfAbsent(topicName.getNamespace(), k -> new ConcurrentOpenHashMap<>()) // - .computeIfAbsent(serviceUnit, k -> new ConcurrentOpenHashMap<>()) // + .computeIfAbsent(topicName.getNamespace(), + k -> ConcurrentOpenHashMap.>newBuilder() + .build()) // + .computeIfAbsent(serviceUnit, + k -> ConcurrentOpenHashMap.newBuilder().build()) // .put(topicName.toString(), topic); } } @@ -2311,7 +2327,8 @@ public static boolean validateDynamicConfiguration(String key, String value) { } private static ConcurrentOpenHashMap prepareDynamicConfigurationMap() { - ConcurrentOpenHashMap dynamicConfigurationMap = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap dynamicConfigurationMap = + ConcurrentOpenHashMap.newBuilder().build(); for (Field field : ServiceConfiguration.class.getDeclaredFields()) { if (field != null && field.isAnnotationPresent(FieldContext.class)) { field.setAccessible(true); @@ -2324,7 +2341,8 @@ private static ConcurrentOpenHashMap prepareDynamicConfigur } private ConcurrentOpenHashMap getRuntimeConfigurationMap() { - ConcurrentOpenHashMap runtimeConfigurationMap = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap runtimeConfigurationMap = + ConcurrentOpenHashMap.newBuilder().build(); for (Field field : ServiceConfiguration.class.getDeclaredFields()) { if (field != null && field.isAnnotationPresent(FieldContext.class)) { field.setAccessible(true); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index b9a85ab97c520..c6ea96179d6f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -138,8 +138,16 @@ public void reset() { public NonPersistentTopic(String topic, BrokerService brokerService) { super(topic, brokerService); - this.subscriptions = new ConcurrentOpenHashMap<>(16, 1); - this.replicators = new ConcurrentOpenHashMap<>(16, 1); + this.subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + this.replicators = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); this.isFenced = false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 9913ddf91cf8f..bafe93a5ffe0b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -101,12 +101,20 @@ public MessageDupUnknownException() { // Map that contains the highest sequenceId that have been sent by each producers. The map will be updated before // the messages are persisted @VisibleForTesting - final ConcurrentOpenHashMap highestSequencedPushed = new ConcurrentOpenHashMap<>(16, 1); + final ConcurrentOpenHashMap highestSequencedPushed = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); // Map that contains the highest sequenceId that have been persistent by each producers. The map will be updated // after the messages are persisted @VisibleForTesting - final ConcurrentOpenHashMap highestSequencedPersisted = new ConcurrentOpenHashMap<>(16, 1); + final ConcurrentOpenHashMap highestSequencedPersisted = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); // Number of persisted entries after which to store a snapshot of the sequence ids map private final int snapshotInterval; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 333438849e6ad..756622c78dbdf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -255,8 +255,14 @@ public void reset() { public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) { super(topic, brokerService); this.ledger = ledger; - this.subscriptions = new ConcurrentOpenHashMap<>(16, 1); - this.replicators = new ConcurrentOpenHashMap<>(16, 1); + this.subscriptions = ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + this.replicators = ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); this.delayedDeliveryEnabled = brokerService.pulsar().getConfiguration().isDelayedDeliveryEnabled(); this.delayedDeliveryTickTimeMillis = brokerService.pulsar().getConfiguration().getDelayedDeliveryTickTimeMillis(); @@ -349,8 +355,14 @@ public CompletableFuture initialize() { super(topic, brokerService); this.ledger = ledger; this.messageDeduplication = messageDeduplication; - this.subscriptions = new ConcurrentOpenHashMap<>(16, 1); - this.replicators = new ConcurrentOpenHashMap<>(16, 1); + this.subscriptions = ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + this.replicators = ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); this.compactedTopic = new CompactedTopicImpl(brokerService.pulsar().getBookKeeperClient()); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java index 1086563085b7f..6718f074c67b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java @@ -35,7 +35,8 @@ public class ClusterReplicationMetrics { public ClusterReplicationMetrics(String localCluster, boolean metricsEnabled) { metricsList = new ArrayList<>(); this.localCluster = localCluster; - metricsMap = new ConcurrentOpenHashMap<>(); + metricsMap = ConcurrentOpenHashMap.newBuilder() + .build(); this.metricsEnabled = metricsEnabled; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java index 1429c7376f4ae..9e81a3e1db968 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java @@ -234,7 +234,8 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception { brokerToDomainMap.put("brokerName-3", "domain-1"); Set candidate = Sets.newHashSet(); - ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange = + ConcurrentOpenHashMap.>>newBuilder().build(); assertEquals(brokers.size(), totalBrokers); @@ -320,7 +321,8 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception { Set brokers = Sets.newHashSet(); Set candidate = Sets.newHashSet(); - ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange = + ConcurrentOpenHashMap.>>newBuilder().build(); brokers.add("broker-0"); brokers.add("broker-1"); brokers.add("broker-2"); @@ -367,9 +369,11 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception { private void selectBrokerForNamespace( ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange, String broker, String namespace, String assignedBundleName) { - ConcurrentOpenHashSet bundleSet = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet bundleSet = + ConcurrentOpenHashSet.newBuilder().build(); bundleSet.add(assignedBundleName); - ConcurrentOpenHashMap> nsToBundleMap = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap> nsToBundleMap = + ConcurrentOpenHashMap.>newBuilder().build(); nsToBundleMap.put(namespace, bundleSet); brokerToNamespaceToBundleRange.put(broker, nsToBundleMap); } @@ -469,7 +473,8 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception { Set brokers = Sets.newHashSet(); Set candidate = Sets.newHashSet(); - ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange = + ConcurrentOpenHashMap.>>newBuilder().build(); brokers.add("broker-0"); brokers.add("broker-1"); brokers.add("broker-2"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java index 716b9716425aa..d23772185f1e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java @@ -36,7 +36,10 @@ public void testRemoveMostServicingBrokersForNamespace() { String assignedBundle = namespace + "/0x00000000_0x40000000"; Set candidates = Sets.newHashSet(); - ConcurrentOpenHashMap>> map = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap>> map = + ConcurrentOpenHashMap.>>newBuilder() + .build(); LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); Assert.assertEquals(candidates.size(), 0); @@ -80,8 +83,12 @@ public void testRemoveMostServicingBrokersForNamespace() { private static void fillBrokerToNamespaceToBundleMap( ConcurrentOpenHashMap>> map, String broker, String namespace, String bundle) { - map.computeIfAbsent(broker, k -> new ConcurrentOpenHashMap<>()) - .computeIfAbsent(namespace, k -> new ConcurrentOpenHashSet<>()).add(bundle); + map.computeIfAbsent(broker, + k -> ConcurrentOpenHashMap.>newBuilder().build()) + .computeIfAbsent(namespace, + k -> ConcurrentOpenHashSet.newBuilder().build()) + .add(bundle); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 2314acdc3f51f..067cbccf273b0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -838,7 +838,11 @@ private void testMaxConsumersShared() throws Exception { addConsumerToSubscription.setAccessible(true); // for count consumers on topic - ConcurrentOpenHashMap subscriptions = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); subscriptions.put("sub-1", sub); subscriptions.put("sub-2", sub2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -937,7 +941,11 @@ private void testMaxConsumersFailover() throws Exception { addConsumerToSubscription.setAccessible(true); // for count consumers on topic - ConcurrentOpenHashMap subscriptions = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); subscriptions.put("sub-1", sub); subscriptions.put("sub-2", sub2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -1063,7 +1071,11 @@ public void testMaxSameAddressConsumers() throws Exception { addConsumerToSubscription.setAccessible(true); // for count consumers on topic - ConcurrentOpenHashMap subscriptions = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); subscriptions.put("sub1", sub1); subscriptions.put("sub2", sub2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -2088,7 +2100,11 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { public void testCheckInactiveSubscriptions() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); - ConcurrentOpenHashMap subscriptions = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); // This subscription is connected by consumer. PersistentSubscription nonDeletableSubscription1 = spy(new PersistentSubscription(topic, "nonDeletableSubscription1", cursorMock, false)); subscriptions.put(nonDeletableSubscription1.getName(), nonDeletableSubscription1); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 062b277165bb6..71fb2d6275610 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -99,7 +99,8 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat this.consumerEventListener = conf.getConsumerEventListener(); // Always use growable queue since items can exceed the advertised size this.incomingMessages = new GrowableArrayBlockingQueue<>(); - this.unAckedChunkedMessageIdSequenceMap = new ConcurrentOpenHashMap<>(); + this.unAckedChunkedMessageIdSequenceMap = + ConcurrentOpenHashMap.newBuilder().build(); this.executorProvider = executorProvider; this.externalPinnedExecutor = (ScheduledExecutorService) executorProvider.getExecutor(); this.internalPinnedExecutor = (ScheduledExecutorService) client.getInternalExecutorService(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 85f7e05c4605d..603dae6ecbe58 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -183,7 +183,8 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle protected volatile boolean paused; - protected ConcurrentOpenHashMap chunkedMessagesMap = new ConcurrentOpenHashMap<>(); + protected ConcurrentOpenHashMap chunkedMessagesMap = + ConcurrentOpenHashMap.newBuilder().build(); private int pendingChunkedMessageCount = 0; protected long expireTimeOfIncompleteChunkedMessageMillis = 0; private boolean expireChunkMessageTaskScheduled = false; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index 037f401634034..f8d3e8855047d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -73,7 +73,8 @@ public class PartitionedProducerImpl extends ProducerBase { public PartitionedProducerImpl(PulsarClientImpl client, String topic, ProducerConfigurationData conf, int numPartitions, CompletableFuture> producerCreatedFuture, Schema schema, ProducerInterceptors interceptors) { super(client, topic, conf, producerCreatedFuture, schema, interceptors); - this.producers = new ConcurrentOpenHashMap<>(); + this.producers = + ConcurrentOpenHashMap.>newBuilder().build(); this.topicMetadata = new TopicMetadataImpl(numPartitions); this.routerPolicy = getMessageRouter(); stats = client.getConfiguration().getStatsIntervalSeconds() > 0 ? new ProducerStatsRecorderImpl() : null; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java index d164d09c4c4b4..180319034daea 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java @@ -53,7 +53,8 @@ protected ProducerBase(PulsarClientImpl client, String topic, ProducerConfigurat this.conf = conf; this.schema = schema; this.interceptors = interceptors; - this.schemaCache = new ConcurrentOpenHashMap<>(); + this.schemaCache = + ConcurrentOpenHashMap.newBuilder().build(); if (!conf.isMultiSchema()) { multiSchemaMode = MultiSchemaMode.Disabled; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index c0b952a281a8e..d577f48357c89 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -60,7 +60,8 @@ public class AcknowledgementsGroupingTrackerTest { public void setup() throws NoSuchFieldException, IllegalAccessException { eventLoopGroup = new NioEventLoopGroup(1); consumer = mock(ConsumerImpl.class); - consumer.unAckedChunkedMessageIdSequenceMap = new ConcurrentOpenHashMap<>(); + consumer.unAckedChunkedMessageIdSequenceMap = + ConcurrentOpenHashMap.newBuilder().build(); cnx = spy(new ClientCnxTest(new ClientConfigurationData(), new NioEventLoopGroup())); PulsarClientImpl client = mock(PulsarClientImpl.class); doReturn(client).when(consumer).getClient(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java index f1806c511e2ee..abbe11576a9bf 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java @@ -45,8 +45,74 @@ public class ConcurrentLongPairSet implements LongPairSet { private static final int DefaultExpectedItems = 256; private static final int DefaultConcurrencyLevel = 16; + private static final float DefaultMapFillFactor = 0.66f; + private static final float DefaultMapIdleFactor = 0.15f; + + private static final float DefaultExpandFactor = 2; + private static final float DefaultShrinkFactor = 2; + + private static final boolean DefaultAutoShrink = false; + private final Section[] sections; + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder of ConcurrentLongPairSet. + */ + public static class Builder { + int expectedItems = DefaultExpectedItems; + int concurrencyLevel = DefaultConcurrencyLevel; + float mapFillFactor = DefaultMapFillFactor; + float mapIdleFactor = DefaultMapIdleFactor; + float expandFactor = DefaultExpandFactor; + float shrinkFactor = DefaultShrinkFactor; + boolean autoShrink = DefaultAutoShrink; + + public Builder expectedItems(int expectedItems) { + this.expectedItems = expectedItems; + return this; + } + + public Builder concurrencyLevel(int concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + return this; + } + + public Builder mapFillFactor(float mapFillFactor) { + this.mapFillFactor = mapFillFactor; + return this; + } + + public Builder mapIdleFactor(float mapIdleFactor) { + this.mapIdleFactor = mapIdleFactor; + return this; + } + + public Builder expandFactor(float expandFactor) { + this.expandFactor = expandFactor; + return this; + } + + public Builder shrinkFactor(float shrinkFactor) { + this.shrinkFactor = shrinkFactor; + return this; + } + + public Builder autoShrink(boolean autoShrink) { + this.autoShrink = autoShrink; + return this; + } + + public ConcurrentLongPairSet build() { + return new ConcurrentLongPairSet(expectedItems, concurrencyLevel, + mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); + } + } + + /** * Represents a function that accepts an object of the {@code LongPair} type. */ @@ -61,18 +127,33 @@ public interface LongPairConsumer { void accept(long v1, long v2); } + @Deprecated public ConcurrentLongPairSet() { this(DefaultExpectedItems); } + @Deprecated public ConcurrentLongPairSet(int expectedItems) { this(expectedItems, DefaultConcurrencyLevel); } + @Deprecated public ConcurrentLongPairSet(int expectedItems, int concurrencyLevel) { + this(expectedItems, concurrencyLevel, DefaultMapFillFactor, DefaultMapIdleFactor, + DefaultAutoShrink, DefaultExpandFactor, DefaultShrinkFactor); + } + + public ConcurrentLongPairSet(int expectedItems, int concurrencyLevel, + float mapFillFactor, float mapIdleFactor, + boolean autoShrink, float expandFactor, float shrinkFactor) { checkArgument(expectedItems > 0); checkArgument(concurrencyLevel > 0); checkArgument(expectedItems >= concurrencyLevel); + checkArgument(mapFillFactor > 0 && mapFillFactor < 1); + checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); + checkArgument(mapFillFactor > mapIdleFactor); + checkArgument(expandFactor > 1); + checkArgument(shrinkFactor > 1); int numSections = concurrencyLevel; int perSectionExpectedItems = expectedItems / numSections; @@ -80,10 +161,12 @@ public ConcurrentLongPairSet(int expectedItems, int concurrencyLevel) { this.sections = new Section[numSections]; for (int i = 0; i < numSections; i++) { - sections[i] = new Section(perSectionCapacity); + sections[i] = new Section(perSectionCapacity, mapFillFactor, mapIdleFactor, + autoShrink, expandFactor, shrinkFactor); } } + @Override public long size() { long size = 0; for (int i = 0; i < sections.length; i++) { @@ -214,18 +297,33 @@ private static final class Section extends StampedLock { private volatile long[] table; private volatile int capacity; + private final int initCapacity; private static final AtomicIntegerFieldUpdater
    SIZE_UPDATER = AtomicIntegerFieldUpdater .newUpdater(Section.class, "size"); private volatile int size; private int usedBuckets; - private int resizeThreshold; - - Section(int capacity) { + private int resizeThresholdUp; + private int resizeThresholdBelow; + private final float mapFillFactor; + private final float mapIdleFactor; + private final float expandFactor; + private final float shrinkFactor; + private final boolean autoShrink; + + Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, + float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); + this.initCapacity = this.capacity; this.table = new long[2 * this.capacity]; this.size = 0; this.usedBuckets = 0; - this.resizeThreshold = (int) (this.capacity * SetFillFactor); + this.autoShrink = autoShrink; + this.mapFillFactor = mapFillFactor; + this.mapIdleFactor = mapIdleFactor; + this.expandFactor = expandFactor; + this.shrinkFactor = shrinkFactor; + this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); + this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); Arrays.fill(table, EmptyItem); } @@ -314,9 +412,11 @@ boolean add(long item1, long item2, long hash) { bucket = (bucket + 2) & (table.length - 1); } } finally { - if (usedBuckets > resizeThreshold) { + if (usedBuckets > resizeThresholdUp) { try { - rehash(); + // Expand the hashmap + int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); + rehash(newCapacity); } finally { unlockWrite(stamp); } @@ -347,7 +447,20 @@ private boolean remove(long item1, long item2, int hash) { bucket = (bucket + 2) & (table.length - 1); } } finally { - unlockWrite(stamp); + if (autoShrink && size < resizeThresholdBelow) { + try { + int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); + int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); + if (newCapacity < capacity && newResizeThresholdUp > size) { + // shrink the hashmap + rehash(newCapacity); + } + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } } } @@ -379,6 +492,16 @@ private void cleanBucket(int bucket) { table[bucket] = EmptyItem; table[bucket + 1] = EmptyItem; --usedBuckets; + + // Cleanup all the buckets that were in `DeletedKey` state, + // so that we can reduce unnecessary expansions + bucket = (bucket - 1) & (table.length - 1); + while (table[bucket] == DeletedItem) { + table[bucket] = EmptyItem; + --usedBuckets; + + bucket = (bucket - 1) & (table.length - 1); + } } else { table[bucket] = DeletedItem; table[bucket + 1] = DeletedItem; @@ -392,6 +515,9 @@ void clear() { Arrays.fill(table, EmptyItem); this.size = 0; this.usedBuckets = 0; + if (autoShrink) { + rehash(initCapacity); + } } finally { unlockWrite(stamp); } @@ -431,9 +557,8 @@ public void forEach(LongPairConsumer processor) { } } - private void rehash() { + private void rehash(int newCapacity) { // Expand the hashmap - int newCapacity = capacity * 2; long[] newTable = new long[2 * newCapacity]; Arrays.fill(newTable, EmptyItem); @@ -451,7 +576,8 @@ private void rehash() { // Capacity needs to be updated after the values, so that we won't see // a capacity value bigger than the actual array size capacity = newCapacity; - resizeThreshold = (int) (capacity * SetFillFactor); + resizeThresholdUp = (int) (capacity * mapFillFactor); + resizeThresholdBelow = (int) (capacity * mapIdleFactor); } private static void insertKeyValueNoLock(long[] table, int capacity, long item1, long item2) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java index 2c7eed1b58e03..255844cf4badd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java @@ -64,33 +64,112 @@ public int hashCode() { } }; - private static final float MapFillFactor = 0.66f; - private static final int DefaultExpectedItems = 256; private static final int DefaultConcurrencyLevel = 16; + private static final float DefaultMapFillFactor = 0.66f; + private static final float DefaultMapIdleFactor = 0.15f; + + private static final float DefaultExpandFactor = 2; + private static final float DefaultShrinkFactor = 2; + + private static final boolean DefaultAutoShrink = false; + private final Section[] sections; + public static Builder newBuilder() { + return new Builder<>(); + } + + /** + * Builder of ConcurrentOpenHashMap. + */ + public static class Builder { + int expectedItems = DefaultExpectedItems; + int concurrencyLevel = DefaultConcurrencyLevel; + float mapFillFactor = DefaultMapFillFactor; + float mapIdleFactor = DefaultMapIdleFactor; + float expandFactor = DefaultExpandFactor; + float shrinkFactor = DefaultShrinkFactor; + boolean autoShrink = DefaultAutoShrink; + + public Builder expectedItems(int expectedItems) { + this.expectedItems = expectedItems; + return this; + } + + public Builder concurrencyLevel(int concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + return this; + } + + public Builder mapFillFactor(float mapFillFactor) { + this.mapFillFactor = mapFillFactor; + return this; + } + + public Builder mapIdleFactor(float mapIdleFactor) { + this.mapIdleFactor = mapIdleFactor; + return this; + } + + public Builder expandFactor(float expandFactor) { + this.expandFactor = expandFactor; + return this; + } + + public Builder shrinkFactor(float shrinkFactor) { + this.shrinkFactor = shrinkFactor; + return this; + } + + public Builder autoShrink(boolean autoShrink) { + this.autoShrink = autoShrink; + return this; + } + + public ConcurrentOpenHashMap build() { + return new ConcurrentOpenHashMap<>(expectedItems, concurrencyLevel, + mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); + } + } + + @Deprecated public ConcurrentOpenHashMap() { this(DefaultExpectedItems); } + @Deprecated public ConcurrentOpenHashMap(int expectedItems) { this(expectedItems, DefaultConcurrencyLevel); } + @Deprecated public ConcurrentOpenHashMap(int expectedItems, int concurrencyLevel) { + this(expectedItems, concurrencyLevel, DefaultMapFillFactor, DefaultMapIdleFactor, + DefaultAutoShrink, DefaultExpandFactor, DefaultShrinkFactor); + } + + public ConcurrentOpenHashMap(int expectedItems, int concurrencyLevel, + float mapFillFactor, float mapIdleFactor, + boolean autoShrink, float expandFactor, float shrinkFactor) { checkArgument(expectedItems > 0); checkArgument(concurrencyLevel > 0); checkArgument(expectedItems >= concurrencyLevel); + checkArgument(mapFillFactor > 0 && mapFillFactor < 1); + checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); + checkArgument(mapFillFactor > mapIdleFactor); + checkArgument(expandFactor > 1); + checkArgument(shrinkFactor > 1); int numSections = concurrencyLevel; int perSectionExpectedItems = expectedItems / numSections; - int perSectionCapacity = (int) (perSectionExpectedItems / MapFillFactor); + int perSectionCapacity = (int) (perSectionExpectedItems / mapFillFactor); this.sections = (Section[]) new Section[numSections]; for (int i = 0; i < numSections; i++) { - sections[i] = new Section<>(perSectionCapacity); + sections[i] = new Section<>(perSectionCapacity, mapFillFactor, mapIdleFactor, + autoShrink, expandFactor, shrinkFactor); } } @@ -208,18 +287,33 @@ private static final class Section extends StampedLock { private volatile Object[] table; private volatile int capacity; + private final int initCapacity; private static final AtomicIntegerFieldUpdater
    SIZE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(Section.class, "size"); private volatile int size; private int usedBuckets; - private int resizeThreshold; - - Section(int capacity) { + private int resizeThresholdUp; + private int resizeThresholdBelow; + private final float mapFillFactor; + private final float mapIdleFactor; + private final float expandFactor; + private final float shrinkFactor; + private final boolean autoShrink; + + Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, + float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); + this.initCapacity = this.capacity; this.table = new Object[2 * this.capacity]; this.size = 0; this.usedBuckets = 0; - this.resizeThreshold = (int) (this.capacity * MapFillFactor); + this.autoShrink = autoShrink; + this.mapFillFactor = mapFillFactor; + this.mapIdleFactor = mapIdleFactor; + this.expandFactor = expandFactor; + this.shrinkFactor = shrinkFactor; + this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); + this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); } V get(K key, int keyHash) { @@ -316,9 +410,11 @@ V put(K key, V value, int keyHash, boolean onlyIfAbsent, Function valuePro bucket = (bucket + 2) & (table.length - 1); } } finally { - if (usedBuckets > resizeThreshold) { + if (usedBuckets > resizeThresholdUp) { try { - rehash(); + // Expand the hashmap + int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); + rehash(newCapacity); } finally { unlockWrite(stamp); } @@ -363,7 +459,20 @@ private V remove(K key, Object value, int keyHash) { } } finally { - unlockWrite(stamp); + if (autoShrink && size < resizeThresholdBelow) { + try { + int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); + int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); + if (newCapacity < capacity && newResizeThresholdUp > size) { + // shrink the hashmap + rehash(newCapacity); + } + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } } } @@ -374,6 +483,9 @@ void clear() { Arrays.fill(table, EmptyKey); this.size = 0; this.usedBuckets = 0; + if (autoShrink) { + rehash(initCapacity); + } } finally { unlockWrite(stamp); } @@ -415,9 +527,8 @@ public void forEach(BiConsumer processor) { } } - private void rehash() { + private void rehash(int newCapacity) { // Expand the hashmap - int newCapacity = capacity * 2; Object[] newTable = new Object[2 * newCapacity]; // Re-hash table @@ -432,7 +543,8 @@ private void rehash() { table = newTable; capacity = newCapacity; usedBuckets = size; - resizeThreshold = (int) (capacity * MapFillFactor); + resizeThresholdUp = (int) (capacity * mapFillFactor); + resizeThresholdBelow = (int) (capacity * mapIdleFactor); } private static void insertKeyValueNoLock(Object[] table, int capacity, K key, V value) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java index 8b77d9052b3bb..6dc8552174e32 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java @@ -43,33 +43,112 @@ public class ConcurrentOpenHashSet { private static final Object EmptyValue = null; private static final Object DeletedValue = new Object(); - private static final float MapFillFactor = 0.66f; - private static final int DefaultExpectedItems = 256; private static final int DefaultConcurrencyLevel = 16; + private static final float DefaultMapFillFactor = 0.66f; + private static final float DefaultMapIdleFactor = 0.15f; + + private static final float DefaultExpandFactor = 2; + private static final float DefaultShrinkFactor = 2; + + private static final boolean DefaultAutoShrink = false; + private final Section[] sections; + public static Builder newBuilder() { + return new Builder<>(); + } + + /** + * Builder of ConcurrentOpenHashSet. + */ + public static class Builder { + int expectedItems = DefaultExpectedItems; + int concurrencyLevel = DefaultConcurrencyLevel; + float mapFillFactor = DefaultMapFillFactor; + float mapIdleFactor = DefaultMapIdleFactor; + float expandFactor = DefaultExpandFactor; + float shrinkFactor = DefaultShrinkFactor; + boolean autoShrink = DefaultAutoShrink; + + public Builder expectedItems(int expectedItems) { + this.expectedItems = expectedItems; + return this; + } + + public Builder concurrencyLevel(int concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + return this; + } + + public Builder mapFillFactor(float mapFillFactor) { + this.mapFillFactor = mapFillFactor; + return this; + } + + public Builder mapIdleFactor(float mapIdleFactor) { + this.mapIdleFactor = mapIdleFactor; + return this; + } + + public Builder expandFactor(float expandFactor) { + this.expandFactor = expandFactor; + return this; + } + + public Builder shrinkFactor(float shrinkFactor) { + this.shrinkFactor = shrinkFactor; + return this; + } + + public Builder autoShrink(boolean autoShrink) { + this.autoShrink = autoShrink; + return this; + } + + public ConcurrentOpenHashSet build() { + return new ConcurrentOpenHashSet<>(expectedItems, concurrencyLevel, + mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); + } + } + + @Deprecated public ConcurrentOpenHashSet() { this(DefaultExpectedItems); } + @Deprecated public ConcurrentOpenHashSet(int expectedItems) { this(expectedItems, DefaultConcurrencyLevel); } + @Deprecated public ConcurrentOpenHashSet(int expectedItems, int concurrencyLevel) { + this(expectedItems, concurrencyLevel, DefaultMapFillFactor, DefaultMapIdleFactor, + DefaultAutoShrink, DefaultExpandFactor, DefaultShrinkFactor); + } + + public ConcurrentOpenHashSet(int expectedItems, int concurrencyLevel, + float mapFillFactor, float mapIdleFactor, + boolean autoShrink, float expandFactor, float shrinkFactor) { checkArgument(expectedItems > 0); checkArgument(concurrencyLevel > 0); checkArgument(expectedItems >= concurrencyLevel); + checkArgument(mapFillFactor > 0 && mapFillFactor < 1); + checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); + checkArgument(mapFillFactor > mapIdleFactor); + checkArgument(expandFactor > 1); + checkArgument(shrinkFactor > 1); int numSections = concurrencyLevel; int perSectionExpectedItems = expectedItems / numSections; - int perSectionCapacity = (int) (perSectionExpectedItems / MapFillFactor); + int perSectionCapacity = (int) (perSectionExpectedItems / mapFillFactor); this.sections = (Section[]) new Section[numSections]; for (int i = 0; i < numSections; i++) { - sections[i] = new Section<>(perSectionCapacity); + sections[i] = new Section<>(perSectionCapacity, mapFillFactor, mapIdleFactor, + autoShrink, expandFactor, shrinkFactor); } } @@ -177,18 +256,33 @@ private static final class Section extends StampedLock { private volatile V[] values; private volatile int capacity; + private final int initCapacity; private static final AtomicIntegerFieldUpdater
    SIZE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(Section.class, "size"); private volatile int size; private int usedBuckets; - private int resizeThreshold; - - Section(int capacity) { + private int resizeThresholdUp; + private int resizeThresholdBelow; + private final float mapFillFactor; + private final float mapIdleFactor; + private final float expandFactor; + private final float shrinkFactor; + private final boolean autoShrink; + + Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, + float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); + this.initCapacity = this.capacity; this.values = (V[]) new Object[this.capacity]; this.size = 0; this.usedBuckets = 0; - this.resizeThreshold = (int) (this.capacity * MapFillFactor); + this.autoShrink = autoShrink; + this.mapFillFactor = mapFillFactor; + this.mapIdleFactor = mapIdleFactor; + this.expandFactor = expandFactor; + this.shrinkFactor = shrinkFactor; + this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); + this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); } boolean contains(V value, int keyHash) { @@ -284,9 +378,11 @@ boolean add(V value, int keyHash) { ++bucket; } } finally { - if (usedBuckets > resizeThreshold) { + if (usedBuckets > resizeThresholdUp) { try { - rehash(); + // Expand the hashmap + int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); + rehash(newCapacity); } finally { unlockWrite(stamp); } @@ -319,7 +415,20 @@ private boolean remove(V value, int keyHash) { } } finally { - unlockWrite(stamp); + if (autoShrink && size < resizeThresholdBelow) { + try { + int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); + int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); + if (newCapacity < capacity && newResizeThresholdUp > size) { + // shrink the hashmap + rehash(newCapacity); + } + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } } } @@ -330,6 +439,9 @@ void clear() { Arrays.fill(values, EmptyValue); this.size = 0; this.usedBuckets = 0; + if (autoShrink) { + rehash(initCapacity); + } } finally { unlockWrite(stamp); } @@ -402,9 +514,8 @@ public void forEach(Consumer processor) { } } - private void rehash() { + private void rehash(int newCapacity) { // Expand the hashmap - int newCapacity = capacity * 2; V[] newValues = (V[]) new Object[newCapacity]; // Re-hash table @@ -418,7 +529,8 @@ private void rehash() { values = newValues; capacity = newCapacity; usedBuckets = size; - resizeThreshold = (int) (capacity * MapFillFactor); + resizeThresholdUp = (int) (capacity * mapFillFactor); + resizeThresholdBelow = (int) (capacity * mapIdleFactor); } private static void insertValueNoLock(V[] values, V value) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java index 95e2302dcb7b9..e4cb668fc92d3 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java @@ -79,7 +79,10 @@ public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, int @Override public boolean add(long item1, long item2) { ConcurrentLongPairSet messagesToReplay = longPairSets.computeIfAbsent(item1, - (key) -> new ConcurrentLongPairSet(expectedItems, concurrencyLevel)); + (key) -> ConcurrentLongPairSet.newBuilder() + .expectedItems(expectedItems) + .concurrencyLevel(concurrencyLevel) + .build()); return messagesToReplay.add(item1, item2); } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java index 82cac712975ed..a8d3e1d06039a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java @@ -45,21 +45,29 @@ public class ConcurrentLongPairSetTest { @Test public void testConstructor() { try { - new ConcurrentLongPairSet(0); + ConcurrentLongPairSet.newBuilder() + .expectedItems(0) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok } try { - new ConcurrentLongPairSet(16, 0); + ConcurrentLongPairSet.newBuilder() + .expectedItems(16) + .concurrencyLevel(0) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok } try { - new ConcurrentLongPairSet(4, 8); + ConcurrentLongPairSet.newBuilder() + .expectedItems(4) + .concurrencyLevel(8) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok @@ -68,7 +76,9 @@ public void testConstructor() { @Test public void simpleInsertions() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(16); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(16) + .build(); assertTrue(set.isEmpty()); assertTrue(set.add(1, 1)); @@ -94,9 +104,64 @@ public void simpleInsertions() { assertEquals(set.size(), 3); } + @Test + public void testClear() { + ConcurrentLongPairSet map = ConcurrentLongPairSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertTrue(map.add(1, 1)); + assertTrue(map.add(2, 2)); + assertTrue(map.add(3, 3)); + + assertTrue(map.capacity() == 8); + map.clear(); + assertTrue(map.capacity() == 4); + } + + @Test + public void testExpandAndShrink() { + ConcurrentLongPairSet map = ConcurrentLongPairSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertTrue(map.add(1, 1)); + assertTrue(map.add(2, 2)); + assertTrue(map.add(3, 3)); + + // expand hashmap + assertTrue(map.capacity() == 8); + + assertTrue(map.remove(1, 1)); + // not shrink + assertTrue(map.capacity() == 8); + assertTrue(map.remove(2, 2)); + // shrink hashmap + assertTrue(map.capacity() == 4); + + // expand hashmap + assertTrue(map.add(4, 4)); + assertTrue(map.add(5, 5)); + assertTrue(map.capacity() == 8); + + //verify that the map does not keep shrinking at every remove() operation + assertTrue(map.add(6, 6)); + assertTrue(map.remove(6, 6)); + assertTrue(map.capacity() == 8); + } + + @Test public void testRemove() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); assertTrue(set.isEmpty()); assertTrue(set.add(1, 1)); @@ -111,7 +176,10 @@ public void testRemove() { @Test public void testRehashing() { int n = 16; - ConcurrentLongPairSet set = new ConcurrentLongPairSet(n / 2, 1); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(set.capacity(), n); assertEquals(set.size(), 0); @@ -126,7 +194,10 @@ public void testRehashing() { @Test public void testRehashingRemoval() { int n = 16; - ConcurrentLongPairSet set = new ConcurrentLongPairSet(n / 2, 1); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(set.capacity(), n); assertEquals(set.size(), 0); @@ -152,7 +223,10 @@ public void testRehashingRemoval() { @Test public void testRehashingWithDeletes() { int n = 16; - ConcurrentLongPairSet set = new ConcurrentLongPairSet(n / 2, 1); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(set.capacity(), n); assertEquals(set.size(), 0); @@ -177,7 +251,7 @@ public void testRehashingWithDeletes() { @Test public void concurrentInsertions() throws Throwable { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -210,7 +284,7 @@ public void concurrentInsertions() throws Throwable { @Test public void concurrentInsertionsAndReads() throws Throwable { - ConcurrentLongPairSet map = new ConcurrentLongPairSet(); + ConcurrentLongPairSet map = ConcurrentLongPairSet.newBuilder().build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -243,7 +317,7 @@ public void concurrentInsertionsAndReads() throws Throwable { @Test public void testIteration() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); assertEquals(set.items(), Collections.emptyList()); @@ -269,7 +343,7 @@ public void testIteration() { @Test public void testRemoval() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); set.add(0, 0); set.add(1, 1); @@ -295,7 +369,7 @@ public void testRemoval() { @Test public void testIfRemoval() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); set.add(0, 0); set.add(1, 1); @@ -319,7 +393,7 @@ public void testIfRemoval() { @Test public void testItems() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); int n = 100; int limit = 10; @@ -340,7 +414,10 @@ public void testItems() { @Test public void testHashConflictWithDeletion() { final int Buckets = 16; - ConcurrentLongPairSet set = new ConcurrentLongPairSet(Buckets, 1); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(Buckets) + .concurrencyLevel(1) + .build(); // Pick 2 keys that fall into the same bucket long key1 = 1; @@ -375,7 +452,7 @@ public void testHashConflictWithDeletion() { @Test public void testEqualsObjects() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); long t1 = 1; long t2 = 2; @@ -397,7 +474,7 @@ public void testEqualsObjects() { @Test public void testToString() { - ConcurrentLongPairSet set = new ConcurrentLongPairSet(); + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder().build(); set.add(0, 0); set.add(1, 1); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java index 254be51f29239..7919485d9b67d 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java @@ -49,21 +49,29 @@ public class ConcurrentOpenHashMapTest { @Test public void testConstructor() { try { - new ConcurrentOpenHashMap(0); + ConcurrentOpenHashMap.newBuilder() + .expectedItems(0) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok } try { - new ConcurrentOpenHashMap(16, 0); + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(0) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok } try { - new ConcurrentOpenHashMap(4, 8); + ConcurrentOpenHashMap.newBuilder() + .expectedItems(4) + .concurrencyLevel(8) + .build(); fail("should have thrown exception"); } catch (IllegalArgumentException e) { // ok @@ -72,7 +80,10 @@ public void testConstructor() { @Test public void simpleInsertions() { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(16); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .build(); assertTrue(map.isEmpty()); assertNull(map.put("1", "one")); @@ -98,9 +109,64 @@ public void simpleInsertions() { assertEquals(map.size(), 3); } + @Test + public void testClear() { + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertNull(map.put("k1", "v1")); + assertNull(map.put("k2", "v2")); + assertNull(map.put("k3", "v3")); + + assertTrue(map.capacity() == 8); + map.clear(); + assertTrue(map.capacity() == 4); + } + + @Test + public void testExpandAndShrink() { + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertNull(map.put("k1", "v1")); + assertNull(map.put("k2", "v2")); + assertNull(map.put("k3", "v3")); + + // expand hashmap + assertTrue(map.capacity() == 8); + + assertTrue(map.remove("k1", "v1")); + // not shrink + assertTrue(map.capacity() == 8); + assertTrue(map.remove("k2", "v2")); + // shrink hashmap + assertTrue(map.capacity() == 4); + + // expand hashmap + assertNull(map.put("k4", "v4")); + assertNull(map.put("k5", "v5")); + assertTrue(map.capacity() == 8); + + //verify that the map does not keep shrinking at every remove() operation + assertNull(map.put("k6", "v6")); + assertTrue(map.remove("k6", "v6")); + assertTrue(map.capacity() == 8); + } + @Test public void testRemove() { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder().build(); assertTrue(map.isEmpty()); assertNull(map.put("1", "one")); @@ -117,7 +183,10 @@ public void testRemove() { @Test public void testRehashing() { int n = 16; - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(n / 2, 1); + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(map.capacity(), n); assertEquals(map.size(), 0); @@ -132,7 +201,11 @@ public void testRehashing() { @Test public void testRehashingWithDeletes() { int n = 16; - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(n / 2, 1); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(n / 2) + .concurrencyLevel(1) + .build(); assertEquals(map.capacity(), n); assertEquals(map.size(), 0); @@ -154,7 +227,10 @@ public void testRehashingWithDeletes() { @Test public void concurrentInsertions() throws Throwable { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -188,7 +264,8 @@ public void concurrentInsertions() throws Throwable { @Test public void concurrentInsertionsAndReads() throws Throwable { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder().build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -222,7 +299,8 @@ public void concurrentInsertionsAndReads() throws Throwable { @Test public void testIteration() { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder().build(); assertEquals(map.keys(), Collections.emptyList()); assertEquals(map.values(), Collections.emptyList()); @@ -266,7 +344,10 @@ public void testIteration() { @Test public void testHashConflictWithDeletion() { final int Buckets = 16; - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(Buckets, 1); + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(Buckets) + .concurrencyLevel(1) + .build(); // Pick 2 keys that fall into the same bucket long key1 = 1; @@ -299,7 +380,8 @@ public void testHashConflictWithDeletion() { @Test public void testPutIfAbsent() { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder().build(); assertNull(map.putIfAbsent(1l, "one")); assertEquals(map.get(1l), "one"); @@ -309,7 +391,10 @@ public void testPutIfAbsent() { @Test public void testComputeIfAbsent() { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); AtomicInteger counter = new AtomicInteger(); Function provider = key -> counter.getAndIncrement(); @@ -350,7 +435,8 @@ public boolean equals(Object obj) { } } - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder().build(); T t1 = new T(1); T t1_b = new T(1); @@ -372,7 +458,11 @@ public boolean equals(Object obj) { @Test public void testNullValue() { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(16, 1); + ConcurrentOpenHashMap map = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); String key = "a"; assertThrows(NullPointerException.class, () -> map.put(key, null)); @@ -406,7 +496,10 @@ public void testNullValue() { static final int N = 1_000_000; public void benchConcurrentOpenHashMap() throws Exception { - ConcurrentOpenHashMap map = new ConcurrentOpenHashMap<>(N, 1); + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(N) + .concurrencyLevel(1) + .build(); for (long i = 0; i < Iterations; i++) { for (int j = 0; j < N; j++) { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java index 3c1d99668d733..af62948b64ad3 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java @@ -91,9 +91,66 @@ public void simpleInsertions() { assertEquals(set.size(), 3); } + @Test + public void testClear() { + ConcurrentOpenHashSet map = + ConcurrentOpenHashSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertTrue(map.add("k1")); + assertTrue(map.add("k2")); + assertTrue(map.add("k3")); + + assertTrue(map.capacity() == 8); + map.clear(); + assertTrue(map.capacity() == 4); + } + + @Test + public void testExpandAndShrink() { + ConcurrentOpenHashSet map = + ConcurrentOpenHashSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertTrue(map.capacity() == 4); + + assertTrue(map.add("k1")); + assertTrue(map.add("k2")); + assertTrue(map.add("k3")); + + // expand hashmap + assertTrue(map.capacity() == 8); + + assertTrue(map.remove("k1")); + // not shrink + assertTrue(map.capacity() == 8); + assertTrue(map.remove("k2")); + // shrink hashmap + assertTrue(map.capacity() == 4); + + // expand hashmap + assertTrue(map.add("k4")); + assertTrue(map.add("k5")); + assertTrue(map.capacity() == 8); + + //verify that the map does not keep shrinking at every remove() operation + assertTrue(map.add("k6")); + assertTrue(map.remove("k6")); + assertTrue(map.capacity() == 8); + } + @Test public void testRemove() { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet set = + ConcurrentOpenHashSet.newBuilder().build(); assertTrue(set.isEmpty()); assertTrue(set.add("1")); @@ -145,7 +202,8 @@ public void testRehashingWithDeletes() { @Test public void concurrentInsertions() throws Throwable { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet set = + ConcurrentOpenHashSet.newBuilder().build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -178,7 +236,8 @@ public void concurrentInsertions() throws Throwable { @Test public void concurrentInsertionsAndReads() throws Throwable { - ConcurrentOpenHashSet map = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet map = + ConcurrentOpenHashSet.newBuilder().build(); @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); @@ -211,7 +270,7 @@ public void concurrentInsertionsAndReads() throws Throwable { @Test public void testIteration() { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet set = ConcurrentOpenHashSet.newBuilder().build(); assertEquals(set.values(), Collections.emptyList()); @@ -237,7 +296,8 @@ public void testIteration() { @Test public void testRemoval() { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet set = + ConcurrentOpenHashSet.newBuilder().build(); set.add(0); set.add(1); @@ -315,7 +375,8 @@ public boolean equals(Object obj) { } } - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet set = + ConcurrentOpenHashSet.newBuilder().build(); T t1 = new T(1); T t1_b = new T(1); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 56ae17be97a7a..195f40fd326e6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -121,7 +121,8 @@ public class PulsarRecordCursor implements RecordCursor { PulsarDispatchingRowDecoderFactory decoderFactory; - protected ConcurrentOpenHashMap chunkedMessagesMap = new ConcurrentOpenHashMap<>(); + protected ConcurrentOpenHashMap chunkedMessagesMap = + ConcurrentOpenHashMap.newBuilder().build(); private static final Logger log = Logger.get(PulsarRecordCursor.class); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java index 0753dd282f6da..b2873d778ab2b 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java @@ -81,9 +81,17 @@ public WebSocketService(WebSocketProxyConfiguration config) { public WebSocketService(ClusterData localCluster, ServiceConfiguration config) { this.config = config; this.localCluster = localCluster; - this.topicProducerMap = new ConcurrentOpenHashMap<>(); - this.topicConsumerMap = new ConcurrentOpenHashMap<>(); - this.topicReaderMap = new ConcurrentOpenHashMap<>(); + this.topicProducerMap = + ConcurrentOpenHashMap.>newBuilder() + .build(); + this.topicConsumerMap = + ConcurrentOpenHashMap.>newBuilder() + .build(); + this.topicReaderMap = + ConcurrentOpenHashMap.>newBuilder() + .build(); this.proxyStats = new ProxyStats(this); } @@ -247,7 +255,8 @@ public boolean isAuthorizationEnabled() { public boolean addProducer(ProducerHandler producer) { return topicProducerMap - .computeIfAbsent(producer.getProducer().getTopic(), topic -> new ConcurrentOpenHashSet<>()) + .computeIfAbsent(producer.getProducer().getTopic(), + topic -> ConcurrentOpenHashSet.newBuilder().build()) .add(producer); } @@ -265,7 +274,8 @@ public boolean removeProducer(ProducerHandler producer) { public boolean addConsumer(ConsumerHandler consumer) { return topicConsumerMap - .computeIfAbsent(consumer.getConsumer().getTopic(), topic -> new ConcurrentOpenHashSet<>()) + .computeIfAbsent(consumer.getConsumer().getTopic(), topic -> + ConcurrentOpenHashSet.newBuilder().build()) .add(consumer); } @@ -282,7 +292,8 @@ public boolean removeConsumer(ConsumerHandler consumer) { } public boolean addReader(ReaderHandler reader) { - return topicReaderMap.computeIfAbsent(reader.getConsumer().getTopic(), topic -> new ConcurrentOpenHashSet<>()) + return topicReaderMap.computeIfAbsent(reader.getConsumer().getTopic(), topic -> + ConcurrentOpenHashSet.newBuilder().build()) .add(reader); } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java index 9bf5f4a68f6e5..8fa91130ae440 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java @@ -52,7 +52,9 @@ public ProxyStats(WebSocketService service) { super(); this.service = service; this.jvmMetrics = new JvmMetrics(service); - this.topicStats = new ConcurrentOpenHashMap<>(); + this.topicStats = + ConcurrentOpenHashMap.newBuilder() + .build(); this.metricsCollection = Lists.newArrayList(); this.tempMetricsCollection = Lists.newArrayList(); // schedule stat generation task every 1 minute From e38d75a0950de0dd5ab5dcc620a20cb332ee5fb8 Mon Sep 17 00:00:00 2001 From: LinChen <1572139390@qq.com> Date: Tue, 15 Mar 2022 11:41:08 +0800 Subject: [PATCH 499/823] Reduce unnecessary expansions for ConcurrentLong map and set (#14562) (cherry picked from commit 8e7006f899bd2b9ed9482ab2ce1ee35233957d03) --- .../collections/ConcurrentLongHashMap.java | 10 ++++++ .../collections/ConcurrentLongPairSet.java | 11 +++--- .../collections/ConcurrentOpenHashMap.java | 19 ++++++++++ .../collections/ConcurrentOpenHashSet.java | 18 ++++++++++ .../ConcurrentLongHashMapTest.java | 19 ++++++++++ .../ConcurrentLongPairSetTest.java | 19 ++++++++++ .../ConcurrentOpenHashMapTest.java | 19 ++++++++++ .../ConcurrentOpenHashSetTest.java | 36 ++++++++++++++----- 8 files changed, 138 insertions(+), 13 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java index a4779357a440b..6f2794468c464 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java @@ -451,6 +451,16 @@ private V remove(long key, Object value, int keyHash) { if (nextValueInArray == EmptyValue) { values[bucket] = (V) EmptyValue; --usedBuckets; + + // Cleanup all the buckets that were in `DeletedValue` state, + // so that we can reduce unnecessary expansions + int lastBucket = signSafeMod(bucket - 1, capacity); + while (values[lastBucket] == DeletedValue) { + values[lastBucket] = (V) EmptyValue; + --usedBuckets; + + lastBucket = signSafeMod(lastBucket - 1, capacity); + } } else { values[bucket] = (V) DeletedValue; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java index abbe11576a9bf..66ecaee4bfa5b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java @@ -493,14 +493,15 @@ private void cleanBucket(int bucket) { table[bucket + 1] = EmptyItem; --usedBuckets; - // Cleanup all the buckets that were in `DeletedKey` state, + // Cleanup all the buckets that were in `DeletedItem` state, // so that we can reduce unnecessary expansions - bucket = (bucket - 1) & (table.length - 1); - while (table[bucket] == DeletedItem) { - table[bucket] = EmptyItem; + int lastBucket = (bucket - 2) & (table.length - 1); + while (table[lastBucket] == DeletedItem) { + table[lastBucket] = EmptyItem; + table[lastBucket + 1] = EmptyItem; --usedBuckets; - bucket = (bucket - 1) & (table.length - 1); + lastBucket = (lastBucket - 2) & (table.length - 1); } } else { table[bucket] = DeletedItem; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java index 255844cf4badd..f82bf11a90e13 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java @@ -173,6 +173,14 @@ public ConcurrentOpenHashMap(int expectedItems, int concurrencyLevel, } } + long getUsedBucketCount() { + long usedBucketCount = 0; + for (Section s : sections) { + usedBucketCount += s.usedBuckets; + } + return usedBucketCount; + } + public long size() { long size = 0; for (Section s : sections) { @@ -441,6 +449,17 @@ private V remove(K key, Object value, int keyHash) { table[bucket] = EmptyKey; table[bucket + 1] = null; --usedBuckets; + + // Cleanup all the buckets that were in `DeletedKey` state, + // so that we can reduce unnecessary expansions + int lastBucket = (bucket - 2) & (table.length - 1); + while (table[lastBucket] == DeletedKey) { + table[lastBucket] = EmptyKey; + table[lastBucket + 1] = null; + --usedBuckets; + + lastBucket = (lastBucket - 2) & (table.length - 1); + } } else { table[bucket] = DeletedKey; table[bucket + 1] = null; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java index 6dc8552174e32..cf5ed7ccdc8d9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java @@ -152,6 +152,14 @@ public ConcurrentOpenHashSet(int expectedItems, int concurrencyLevel, } } + long getUsedBucketCount() { + long usedBucketCount = 0; + for (Section s : sections) { + usedBucketCount += s.usedBuckets; + } + return usedBucketCount; + } + public long size() { long size = 0; for (int i = 0; i < sections.length; i++) { @@ -477,6 +485,16 @@ private void cleanBucket(int bucket) { if (values[nextInArray] == EmptyValue) { values[bucket] = (V) EmptyValue; --usedBuckets; + + // Cleanup all the buckets that were in `DeletedValue` state, + // so that we can reduce unnecessary expansions + int lastBucket = signSafeMod(bucket - 1, capacity); + while (values[lastBucket] == DeletedValue) { + values[lastBucket] = (V) EmptyValue; + --usedBuckets; + + lastBucket = signSafeMod(lastBucket - 1, capacity); + } } else { values[bucket] = (V) DeletedValue; } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java index 6cf126cf2ff15..205cf91b47d12 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java @@ -107,6 +107,25 @@ public void simpleInsertions() { assertEquals(map.size(), 3); } + @Test + public void testReduceUnnecessaryExpansions() { + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .build(); + assertNull(map.put(1, "v1")); + assertNull(map.put(2, "v2")); + assertNull(map.put(3, "v3")); + assertNull(map.put(4, "v4")); + + assertTrue(map.remove(1, "v1")); + assertTrue(map.remove(2, "v2")); + assertTrue(map.remove(3, "v3")); + assertTrue(map.remove(4, "v4")); + + assertEquals(0, map.getUsedBucketCount()); + } + @Test public void testClear() { ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java index a8d3e1d06039a..86030f2161985 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java @@ -74,6 +74,25 @@ public void testConstructor() { } } + @Test + public void testReduceUnnecessaryExpansions() { + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .build(); + assertTrue(set.add(1, 1)); + assertTrue(set.add(2, 2)); + assertTrue(set.add(3, 3)); + assertTrue(set.add(4, 4)); + + assertTrue(set.remove(1, 1)); + assertTrue(set.remove(2, 2)); + assertTrue(set.remove(3, 3)); + assertTrue(set.remove(4, 4)); + + assertEquals(0, set.getUsedBucketCount()); + } + @Test public void simpleInsertions() { ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java index 7919485d9b67d..cec52ea3ded64 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java @@ -109,6 +109,25 @@ public void simpleInsertions() { assertEquals(map.size(), 3); } + @Test + public void testReduceUnnecessaryExpansions() { + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .build(); + assertNull(map.put("1", "1")); + assertNull(map.put("2", "2")); + assertNull(map.put("3", "3")); + assertNull(map.put("4", "4")); + + assertEquals(map.remove("1"), "1"); + assertEquals(map.remove("2"), "2"); + assertEquals(map.remove("3"), "3"); + assertEquals(map.remove("4"), "4"); + + assertEquals(0, map.getUsedBucketCount()); + } + @Test public void testClear() { ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java index af62948b64ad3..6c82293bec29a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java @@ -91,24 +91,44 @@ public void simpleInsertions() { assertEquals(set.size(), 3); } + @Test + public void testReduceUnnecessaryExpansions() { + ConcurrentOpenHashSet set = + ConcurrentOpenHashSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .build(); + + assertTrue(set.add("1")); + assertTrue(set.add("2")); + assertTrue(set.add("3")); + assertTrue(set.add("4")); + + assertTrue(set.remove("1")); + assertTrue(set.remove("2")); + assertTrue(set.remove("3")); + assertTrue(set.remove("4")); + assertEquals(0, set.getUsedBucketCount()); + } + @Test public void testClear() { - ConcurrentOpenHashSet map = + ConcurrentOpenHashSet set = ConcurrentOpenHashSet.newBuilder() .expectedItems(2) .concurrencyLevel(1) .autoShrink(true) .mapIdleFactor(0.25f) .build(); - assertTrue(map.capacity() == 4); + assertTrue(set.capacity() == 4); - assertTrue(map.add("k1")); - assertTrue(map.add("k2")); - assertTrue(map.add("k3")); + assertTrue(set.add("k1")); + assertTrue(set.add("k2")); + assertTrue(set.add("k3")); - assertTrue(map.capacity() == 8); - map.clear(); - assertTrue(map.capacity() == 4); + assertTrue(set.capacity() == 8); + set.clear(); + assertTrue(set.capacity() == 4); } @Test From cc17736dff5770bed1e9224343920ba2173d11c3 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 19 Apr 2022 14:31:20 +0800 Subject: [PATCH 500/823] [fix][txn] Fix potentially unfinishable future. (#15208) (cherry picked from commit 6aaabdb8acfc9ecf07b1f2799b9d8e2a980343a5) --- .../pulsar/broker/TransactionMetadataStoreService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index cd18839798935..902546958c54e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -243,7 +243,11 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc LOG.debug("Handle tc client connect added into pending queue! tcId : {}", tcId.toString()); } } - })); + })).exceptionally(ex -> { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + completableFuture.completeExceptionally(realCause); + return null; + }); } }); return completableFuture; From fab8c7c6d95220daa4da0e7a45b52a2c98949d83 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 20 Apr 2022 06:08:36 +0300 Subject: [PATCH 501/823] Skip unnecessary DNS resolution when creating AuthenticationDataHttp instance (#15221) (cherry picked from commit 14991c93533927c35dd3cba74fe52ba3d57f244b) --- .../pulsar/broker/authentication/AuthenticationDataHttp.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java index 958e5eab9c462..9ffb29c0376a0 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.authentication; +import io.netty.util.NetUtil; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -35,7 +36,9 @@ public AuthenticationDataHttp(HttpServletRequest request) { throw new IllegalArgumentException(); } this.request = request; - this.remoteAddress = new InetSocketAddress(request.getRemoteAddr(), request.getRemotePort()); + this.remoteAddress = + new InetSocketAddress(NetUtil.createInetAddressFromIpAddressString(request.getRemoteAddr()), + request.getRemotePort()); } /* From f91a920bad0d6ab6d335061e4d5e44efed61d552 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 20 Apr 2022 11:54:51 +0300 Subject: [PATCH 502/823] Improve skipping of DNS resolution when creating AuthenticationDataHttp instance (#15228) - improves solution added in #15221 - It's better to use the JDK provided InetSocketAddress.createUnresolved method to prevent unnecessary DNS resolution (cherry picked from commit e71b98ae157c4c108802661eaa72913e9c9e0bef) --- .../pulsar/broker/authentication/AuthenticationDataHttp.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java index 9ffb29c0376a0..8a8dda2f17715 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttp.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.authentication; -import io.netty.util.NetUtil; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -36,9 +35,7 @@ public AuthenticationDataHttp(HttpServletRequest request) { throw new IllegalArgumentException(); } this.request = request; - this.remoteAddress = - new InetSocketAddress(NetUtil.createInetAddressFromIpAddressString(request.getRemoteAddr()), - request.getRemotePort()); + this.remoteAddress = InetSocketAddress.createUnresolved(request.getRemoteAddr(), request.getRemotePort()); } /* From 433a48a22a744030bd50386284b27565d3e8abb8 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Thu, 21 Apr 2022 11:54:34 +0800 Subject: [PATCH 503/823] Fix topic closed normally but still call `closeFencedTopicForcefully`. (#15196) (#15202) Co-authored-by: druidliu Fixes #15196. ### Motivation If broker having conf `topicFencingTimeoutSeconds`>0, a topic is trigged closed and closed normally, `closeFencedTopicForcefully` should not be called. ### Modifications Cancel fenced topic monitoring task if topic close normally, which cancel running `closeFencedTopicForcefully`. ### Verifying this change - [ ] Make sure that the change passes the CI checks. This change added tests and can be verified as follows: - Add `org.apache.pulsar.broker.service.PersistentTopicTest#testTopicCloseFencingTimeout` ### Does this pull request potentially affect one of the following parts: *If `yes` was chosen, please highlight the changes* - Dependencies (does it add or upgrade a dependency): (yes / no) - The public API: (yes / no) - The schema: (yes / no / don't know) - The default values of configurations: (yes / no) - The wire protocol: (yes / no) - The rest endpoints: (yes / no) - The admin cli options: (yes / no) - Anything that affects deployment: (yes / no / don't know) ### Documentation Check the box below or label this PR directly. Need to update docs? - [ ] `doc-required` - [x] `no-need-doc` - [ ] `doc` - [ ] `doc-added` (cherry picked from commit e4a8de1eb9605e36060f0740a0097203a21e34ef) --- .../service/persistent/PersistentTopic.java | 13 +++++++---- .../broker/service/PersistentTopicTest.java | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 756622c78dbdf..7249d4054fedb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1300,6 +1300,7 @@ public void closeComplete(Object ctx) { unregisterTopicPolicyListener(); log.info("[{}] Topic closed", topic); + cancelFencedTopicMonitoringTask(); closeFuture.complete(null); }) .exceptionally(ex -> { @@ -2982,6 +2983,13 @@ public boolean isPersistent() { return true; } + private synchronized void cancelFencedTopicMonitoringTask() { + ScheduledFuture monitoringTask = this.fencedTopicMonitoringTask; + if (monitoringTask != null && !monitoringTask.isDone()) { + monitoringTask.cancel(false); + } + } + private synchronized void fence() { isFenced = true; ScheduledFuture monitoringTask = this.fencedTopicMonitoringTask; @@ -2996,10 +3004,7 @@ private synchronized void fence() { private synchronized void unfence() { isFenced = false; - ScheduledFuture monitoringTask = this.fencedTopicMonitoringTask; - if (monitoringTask != null && !monitoringTask.isDone()) { - monitoringTask.cancel(false); - } + cancelFencedTopicMonitoringTask(); } private void closeFencedTopicForcefully() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 067cbccf273b0..5cb945e436f6f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -2181,6 +2181,28 @@ public void testTopicFencingTimeout() throws Exception { assertTrue((boolean) isClosingOrDeletingField.get(topic)); } + @Test + public void testTopicCloseFencingTimeout() throws Exception { + pulsar.getConfiguration().setTopicFencingTimeoutSeconds(10); + Method fence = PersistentTopic.class.getDeclaredMethod("fence"); + fence.setAccessible(true); + Field fencedTopicMonitoringTaskField = PersistentTopic.class.getDeclaredField("fencedTopicMonitoringTask"); + fencedTopicMonitoringTaskField.setAccessible(true); + + // create topic + PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + + // fence topic to init fencedTopicMonitoringTask + fence.invoke(topic); + + // close topic + topic.close().get(); + assertFalse(brokerService.getTopicReference(successTopicName).isPresent()); + ScheduledFuture fencedTopicMonitoringTask = (ScheduledFuture) fencedTopicMonitoringTaskField.get(topic); + assertTrue(fencedTopicMonitoringTask.isDone()); + assertTrue(fencedTopicMonitoringTask.isCancelled()); + } + @Test public void testGetDurableSubscription() throws Exception { ManagedLedger mockLedger = mock(ManagedLedger.class); From 76e1f740e8658fc841bc2a6b4569981c029c4592 Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Wed, 20 Apr 2022 23:23:44 -0700 Subject: [PATCH 504/823] [Functions] Check executor null when closing the FileSource (#15247) (cherry picked from commit 06ba587fb92eff81785f8d463c85aaa1095292e9) --- .../java/org/apache/pulsar/io/file/FileSource.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSource.java b/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSource.java index bc09c978dd621..3a51736cc2a36 100644 --- a/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSource.java +++ b/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSource.java @@ -58,13 +58,15 @@ public void open(Map config, SourceContext sourceContext) throws @Override public void close() throws Exception { - executor.shutdown(); - try { - if (!executor.awaitTermination(800, TimeUnit.MILLISECONDS)) { + if (executor != null) { + executor.shutdown(); + try { + if (!executor.awaitTermination(800, TimeUnit.MILLISECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { executor.shutdownNow(); } - } catch (InterruptedException e) { - executor.shutdownNow(); } } } From f95b98f1edba68ab22b121f77f7af96183f40e72 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:58:01 +0800 Subject: [PATCH 505/823] [Fix][Broker] Fix race condition in `OpAddEntry` (#15233) (cherry picked from commit b083e9a72227a3360d1ec33b5f239d82f0804e65) --- .../org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 62edcd5ca9de7..d03d98a3f064c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3743,12 +3743,13 @@ private void checkAddTimeout() { } OpAddEntry opAddEntry = pendingAddEntries.peek(); if (opAddEntry != null) { + final long finalAddOpCount = opAddEntry.addOpCount; boolean isTimedOut = opAddEntry.lastInitTime != -1 && TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - opAddEntry.lastInitTime) >= timeoutSec; if (isTimedOut) { log.error("Failed to add entry for ledger {} in time-out {} sec", (opAddEntry.ledger != null ? opAddEntry.ledger.getId() : -1), timeoutSec); - opAddEntry.handleAddTimeoutFailure(opAddEntry.ledger, opAddEntry.addOpCount); + opAddEntry.handleAddTimeoutFailure(opAddEntry.ledger, finalAddOpCount); } } } From 91a9e0b79c88b0c889be200b8b182eaf3661bf44 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Thu, 21 Apr 2022 17:26:35 +0800 Subject: [PATCH 506/823] Pulsar SQL support for Decimal data type (#15153) (cherry picked from commit 6b004ed6a2554ab826a00aa2a177963de3c5f44b) --- .../decoder/avro/PulsarAvroColumnDecoder.java | 19 +++++++++++++++++- .../avro/PulsarAvroRowDecoderFactory.java | 10 +++++++++- .../json/PulsarJsonRowDecoderFactory.java | 6 ++++++ .../sql/presto/TestPulsarConnector.java | 8 +++++++- .../sql/presto/TestPulsarRecordCursor.java | 15 ++++++++++++++ .../presto/decoder/AbstractDecoderTester.java | 5 +++++ .../presto/decoder/DecoderTestMessage.java | 6 +++++- .../sql/presto/decoder/DecoderTestUtil.java | 20 +++++++++++++++++++ .../presto/decoder/avro/TestAvroDecoder.java | 11 ++++++++++ 9 files changed, 96 insertions(+), 4 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 3b7e293557923..fc122fc1fa4ef 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -41,6 +41,8 @@ import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.Decimals; import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.MapType; @@ -54,6 +56,7 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.List; @@ -142,7 +145,7 @@ private boolean isSupportedType(Type type) { } private boolean isSupportedPrimitive(Type type) { - return type instanceof VarcharType || SUPPORTED_PRIMITIVE_TYPES.contains(type); + return type instanceof VarcharType || type instanceof DecimalType || SUPPORTED_PRIMITIVE_TYPES.contains(type); } public FieldValueProvider decodeField(GenericRecord avroRecord) { @@ -208,6 +211,13 @@ public long getLong() { return floatToIntBits((Float) value); } + if (columnType instanceof DecimalType) { + ByteBuffer buffer = (ByteBuffer) value; + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + return new BigInteger(bytes).longValue(); + } + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); @@ -237,6 +247,13 @@ private static Slice getSlice(Object value, Type type, String columnName) { } } + // The returned Slice size must be equals to 18 Byte + if (type instanceof DecimalType) { + ByteBuffer buffer = (ByteBuffer) value; + BigInteger bigInteger = new BigInteger(buffer.array()); + return Decimals.encodeUnscaledValue(bigInteger); + } + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index 26c333ab56b82..7f9169e527364 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -34,6 +34,7 @@ import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.RealType; @@ -130,7 +131,14 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: - //TODO: support decimal logicalType + // When the precision <= 0, throw Exception. + // When the precision > 0 and <= 18, use ShortDecimalType. and mapping Long + // When the precision > 18 and <= 36, use LongDecimalType. and mapping Slice + // When the precision > 36, throw Exception. + if (logicalType instanceof LogicalTypes.Decimal) { + LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; + return DecimalType.createDecimalType(decimal.getPrecision(), decimal.getScale()); + } return VarbinaryType.VARBINARY; case INT: if (logicalType == LogicalTypes.timeMillis()) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 10a500ba3618e..843c2c7f8362f 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -128,6 +128,12 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: + // In the current implementation, since JsonSchema is generated by Avro, + // there may exist LogicalTypes.Decimal. + // Mapping decimalType with varcharType in JsonSchema. + if (logicalType instanceof LogicalTypes.Decimal) { + return createUnboundedVarcharType(); + } return VarbinaryType.VARBINARY; case INT: if (logicalType == LogicalTypes.timeMillis()) { diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 39d8ba8213cc8..fdfde36cb28e1 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -25,6 +25,7 @@ import io.prestosql.spi.connector.ConnectorContext; import io.prestosql.spi.predicate.TupleDomain; import io.prestosql.testing.TestingConnectorContext; +import java.math.BigDecimal; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; @@ -166,6 +167,8 @@ public enum TestEnum { public int time; @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"date\" }") public int date; + @org.apache.avro.reflect.AvroSchema("{ \"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }") + public BigDecimal decimal; public TestPulsarConnector.Bar bar; public TestEnum field7; } @@ -253,6 +256,7 @@ public static class Bar { fooFieldNames.add("date"); fooFieldNames.add("bar"); fooFieldNames.add("field7"); + fooFieldNames.add("decimal"); ConnectorContext prestoConnectorContext = new TestingConnectorContext(); @@ -313,6 +317,7 @@ public static class Bar { LocalDate epoch = LocalDate.ofEpochDay(0); return Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); }); + fooFunctions.put("decimal", integer -> BigDecimal.valueOf(1234, 2)); fooFunctions.put("bar.field1", integer -> integer % 3 == 0 ? null : integer + 1); fooFunctions.put("bar.field2", integer -> integer % 2 == 0 ? null : String.valueOf(integer + 2)); fooFunctions.put("bar.field3", integer -> integer + 3.0f); @@ -331,7 +336,6 @@ public static class Bar { * @param schemaInfo * @param handleKeyValueType * @param includeInternalColumn - * @param dispatchingRowDecoderFactory * @return */ protected static List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, @@ -393,6 +397,7 @@ private static List getTopicEntries(String topicSchemaName) { LocalDate localDate = LocalDate.now(); LocalDate epoch = LocalDate.ofEpochDay(0); foo.date = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); + foo.decimal= BigDecimal.valueOf(count, 2); MessageMetadata messageMetadata = new MessageMetadata() .setProducerName("test-producer").setSequenceId(i) @@ -609,6 +614,7 @@ public void run() { foo.timestamp = (long) fooFunctions.get("timestamp").apply(count); foo.time = (int) fooFunctions.get("time").apply(count); foo.date = (int) fooFunctions.get("date").apply(count); + foo.decimal = (BigDecimal) fooFunctions.get("decimal").apply(count); foo.bar = bar; foo.field7 = (Foo.TestEnum) fooFunctions.get("field7").apply(count); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index d60ff20522c3c..2ea2616b6f1d2 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -22,7 +22,11 @@ import io.airlift.log.Logger; import io.netty.buffer.ByteBuf; import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarcharType; +import java.math.BigDecimal; import lombok.Data; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -143,6 +147,17 @@ public void testTopics() throws Exception { }else if (fooColumnHandles.get(i).getName().equals("field7")) { assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), fooFunctions.get("field7").apply(count).toString().getBytes()); columnsSeen.add(fooColumnHandles.get(i).getName()); + }else if (fooColumnHandles.get(i).getName().equals("decimal")) { + Type type = fooColumnHandles.get(i).getType(); + // In JsonDecoder, decimal trans to varcharType + if (type instanceof VarcharType) { + assertEquals(new String(pulsarRecordCursor.getSlice(i).getBytes()), + fooFunctions.get("decimal").apply(count).toString()); + } else { + DecimalType decimalType = (DecimalType) fooColumnHandles.get(i).getType(); + assertEquals(BigDecimal.valueOf(pulsarRecordCursor.getLong(i), decimalType.getScale()), fooFunctions.get("decimal").apply(count)); + } + columnsSeen.add(fooColumnHandles.get(i).getName()); } else { if (PulsarInternalColumn.getInternalFieldsMap().containsKey(fooColumnHandles.get(i).getName())) { columnsSeen.add(fooColumnHandles.get(i).getName()); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java index cd4fcaf0d1e33..98b7d8b6f69e7 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java @@ -26,6 +26,7 @@ import io.prestosql.spi.connector.ConnectorContext; import io.prestosql.spi.type.Type; import io.prestosql.testing.TestingConnectorContext; +import java.math.BigDecimal; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; @@ -102,6 +103,10 @@ protected void checkValue(Map decodedRo decoderTestUtil.checkValue(decodedRow, handle, value); } + protected void checkValue(Map decodedRow, DecoderColumnHandle handle, BigDecimal value) { + decoderTestUtil.checkValue(decodedRow, handle, value); + } + protected Block getBlock(Map decodedRow, DecoderColumnHandle handle) { FieldValueProvider provider = decodedRow.get(handle); assertNotNull(provider); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java index 115f3691c00f8..da6d92e5158ac 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.sql.presto.decoder; +import java.math.BigDecimal; import lombok.Data; import java.util.List; @@ -45,6 +46,10 @@ public static enum TestEnum { public int dateField; public TestRow rowField; public TestEnum enumField; + @org.apache.avro.reflect.AvroSchema("{ \"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }") + public BigDecimal decimalField; + @org.apache.avro.reflect.AvroSchema("{ \"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 30, \"scale\": 2 }") + public BigDecimal longDecimalField; public List arrayField; public Map mapField; @@ -62,7 +67,6 @@ public static class NestedRow { public long longField; } - public static class CompositeRow { public String stringField; public List arrayField; diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java index 4c3c4a634472c..496a6f061bfca 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java @@ -23,11 +23,16 @@ import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.block.Block; import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.Decimals; import io.prestosql.spi.type.MapType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.Type; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Map; +import static io.prestosql.spi.type.UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH; import static io.prestosql.testing.TestingConnectorSession.SESSION; import static org.testng.Assert.*; @@ -113,6 +118,21 @@ public void checkValue(Map decodedRow, assertEquals(provider.getBoolean(), value); } + public void checkValue(Map decodedRow, DecoderColumnHandle handle, BigDecimal value) { + FieldValueProvider provider = decodedRow.get(handle); + DecimalType decimalType = (DecimalType) handle.getType(); + BigDecimal actualDecimal; + if (decimalType.getFixedSize() == UNSCALED_DECIMAL_128_SLICE_LENGTH) { + Slice slice = provider.getSlice(); + BigInteger bigInteger = Decimals.decodeUnscaledValue(slice); + actualDecimal = new BigDecimal(bigInteger, decimalType.getScale()); + } else { + actualDecimal = BigDecimal.valueOf(provider.getLong(), decimalType.getScale()); + } + assertNotNull(provider); + assertEquals(actualDecimal, value); + } + public void checkIsNull(Map decodedRow, DecoderColumnHandle handle) { FieldValueProvider provider = decodedRow.get(handle); assertNotNull(provider); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java index 1cfbbb4fce5e9..7b270c7995be0 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java @@ -25,11 +25,13 @@ import io.prestosql.spi.PrestoException; import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.StandardTypes; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.TypeSignatureParameter; import io.prestosql.spi.type.VarcharType; +import java.math.BigDecimal; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -87,6 +89,8 @@ public void testPrimitiveType() { message.longField = 222L; message.timestampField = System.currentTimeMillis(); message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_1; + message.decimalField = BigDecimal.valueOf(2233, 2); + message.longDecimalField = new BigDecimal("1234567891234567891234567891.23"); LocalTime now = LocalTime.now(ZoneId.systemDefault()); message.timeField = now.toSecondOfDay() * 1000; @@ -127,6 +131,13 @@ public void testPrimitiveType() { "enumField", VARCHAR, false, false, "enumField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); checkValue(decodedRow, enumFieldColumnHandle, message.enumField.toString()); + PulsarColumnHandle decimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "decimalField", DecimalType.createDecimalType(4, 2), false, false, "decimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, decimalFieldColumnHandle, message.decimalField); + + PulsarColumnHandle longDecimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "longDecimalField", DecimalType.createDecimalType(30, 2), false, false, "longDecimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, longDecimalFieldColumnHandle, message.longDecimalField); } @Test From ce7af7779268371dd4fdbd9635e449377350feb0 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Mon, 25 Apr 2022 22:17:52 +0800 Subject: [PATCH 507/823] [fix] [broker] Fix problem at RateLimiter#tryAcquire (#15306) (cherry picked from commit 84b65598481fd9bbb6e06e2deb335222a04b9c6b) --- .../pulsar/common/util/RateLimiter.java | 3 +-- .../pulsar/common/util/RateLimiterTest.java | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java index 20ca181c40080..8f02bcc0e5cf3 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java @@ -189,8 +189,7 @@ public synchronized boolean tryAcquire(long acquirePermit) { canAcquire = acquirePermit < 0 || acquiredPermits < this.permits; } else { // acquired-permits can't be larger than the rate - if (acquirePermit > this.permits) { - acquiredPermits = this.permits; + if (acquirePermit + acquiredPermits > this.permits) { return false; } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java index 788ab749390db..57090fcc7b7e0 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java @@ -133,6 +133,24 @@ public void testTryAcquire() { rate.close(); } + @Test + public void testTryAcquireMoreThanPermits() { + final long rateTimeMSec = 1000; + RateLimiter rate = RateLimiter.builder().permits(3).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) + .build(); + assertTrue(rate.tryAcquire(2)); + assertEquals(rate.getAvailablePermits(), 1); + + //try to acquire failed, not decrease availablePermits. + assertFalse(rate.tryAcquire(2)); + assertEquals(rate.getAvailablePermits(), 1); + + assertTrue(rate.tryAcquire(1)); + assertEquals(rate.getAvailablePermits(), 0); + + rate.close(); + } + @Test public void testMultipleTryAcquire() { final long rateTimeMSec = 1000; @@ -189,7 +207,7 @@ public void testDispatchRate() throws Exception { Thread.sleep(rateTimeMSec); // check after three rate-time: acquiredPermits is 0 - assertEquals(rate.getAvailablePermits() > 0, true); + assertTrue(rate.getAvailablePermits() > 0); rate.close(); } From 8094fe558028806f44fb84d1cdaaab4c9841089e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 27 Apr 2022 15:10:01 +0800 Subject: [PATCH 508/823] [C++] Wait until event loop terminates when closing the Client (#15316) * [C++] Wait until event loops terminates when closing the Client Fixes #13267 ### Motivation Unlike Java client, the `Client` of C++ client has a `shutdown` method that is responsible to execute the following steps: 1. Call `shutdown` on all internal producers and consumers 2. Close all connections in the pool 3. Close all executors of the executor providers. When an executor is closed, it call `io_service::stop()`, which makes the event loop (`io_service::run()`) in another thread return as soon as possible. However, there is no wait operation. If a client failed to create a producer or consumer, the `close` method will call `shutdown` and close all executors immediately and exits the application. In this case, the detached event loop thread might not exit ASAP, then valgrind will detect the memory leak. This memory leak can be avoided by sleeping for a while after `Client::close` returns or there are still other things to do after that. However, we should still adopt the semantics that after `Client::shutdown` returns, all event loop threads should be terminated. ### Modifications - Add a timeout parameter to the `close` method of `ExecutorService` and `ExecutorServiceProvider` as the max blocking timeout if it's non-negative. - Add a `TimeoutProcessor` helper class to update the left timeout after calling all methods that accept the timeout parameter. - Call `close` on all `ExecutorServiceProvider`s in `ClientImpl::shutdown` with 500ms timeout, which could be long enough. In addition, in `handleClose` method, call `shutdown` in another thread to avoid the deadlock. ### Verifying this change After applying this patch, the reproduce code in #13627 will pass the valgrind check. ``` ==3013== LEAK SUMMARY: ==3013== definitely lost: 0 bytes in 0 blocks ==3013== indirectly lost: 0 bytes in 0 blocks ==3013== possibly lost: 0 bytes in 0 blocks ``` (cherry picked from commit cd78f39a92521f3847b022580a6e66e651b5cb4b) --- pulsar-client-cpp/lib/ClientImpl.cc | 37 +++++++++++++--- pulsar-client-cpp/lib/ExecutorService.cc | 33 +++++++++++--- pulsar-client-cpp/lib/ExecutorService.h | 11 ++++- pulsar-client-cpp/lib/TimeUtils.h | 48 +++++++++++++++++++++ pulsar-client-cpp/tests/CustomLoggerTest.cc | 26 +++++++---- 5 files changed, 130 insertions(+), 25 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientImpl.cc b/pulsar-client-cpp/lib/ClientImpl.cc index 8cdaacc8b2523..6d9c4ed044ed4 100644 --- a/pulsar-client-cpp/lib/ClientImpl.cc +++ b/pulsar-client-cpp/lib/ClientImpl.cc @@ -26,6 +26,7 @@ #include "PartitionedConsumerImpl.h" #include "MultiTopicsConsumerImpl.h" #include "PatternMultiTopicsConsumerImpl.h" +#include "TimeUtils.h" #include #include #include @@ -34,6 +35,7 @@ #include #include #include +#include #ifdef USE_LOG4CXX #include "Log4CxxLogger.h" #endif @@ -534,13 +536,20 @@ void ClientImpl::handleClose(Result result, SharedInt numberOfOpenHandlers, Resu lock.unlock(); LOG_DEBUG("Shutting down producers and consumers for client"); - shutdown(); - if (callback) { - if (closingError != ResultOk) { - LOG_DEBUG("Problem in closing client, could not close one or more consumers or producers"); + // handleClose() is called in ExecutorService's event loop, while shutdown() tried to wait the event + // loop exits. So here we use another thread to call shutdown(). + auto self = shared_from_this(); + std::thread shutdownTask{[this, self, callback] { + shutdown(); + if (callback) { + if (closingError != ResultOk) { + LOG_DEBUG( + "Problem in closing client, could not close one or more consumers or producers"); + } + callback(closingError); } - callback(closingError); - } + }}; + shutdownTask.detach(); } } @@ -576,11 +585,25 @@ void ClientImpl::shutdown() { return; } LOG_DEBUG("ConnectionPool is closed"); - ioExecutorProvider_->close(); + + // 500ms as the timeout is long enough because ExecutorService::close calls io_service::stop() internally + // and waits until io_service::run() in another thread returns, which should be as soon as possible after + // stop() is called. + TimeoutProcessor timeoutProcessor{500}; + + timeoutProcessor.tik(); + ioExecutorProvider_->close(timeoutProcessor.getLeftTimeout()); + timeoutProcessor.tok(); LOG_DEBUG("ioExecutorProvider_ is closed"); + + timeoutProcessor.tik(); listenerExecutorProvider_->close(); + timeoutProcessor.tok(); LOG_DEBUG("listenerExecutorProvider_ is closed"); + + timeoutProcessor.tik(); partitionListenerExecutorProvider_->close(); + timeoutProcessor.tok(); LOG_DEBUG("partitionListenerExecutorProvider_ is closed"); } diff --git a/pulsar-client-cpp/lib/ExecutorService.cc b/pulsar-client-cpp/lib/ExecutorService.cc index 9cfbd82881d3e..b9b5ed464784d 100644 --- a/pulsar-client-cpp/lib/ExecutorService.cc +++ b/pulsar-client-cpp/lib/ExecutorService.cc @@ -21,6 +21,7 @@ #include #include #include +#include "TimeUtils.h" #include "LogUtils.h" DECLARE_LOG_OBJECT() @@ -29,7 +30,7 @@ namespace pulsar { ExecutorService::ExecutorService() {} -ExecutorService::~ExecutorService() { close(); } +ExecutorService::~ExecutorService() { close(0); } void ExecutorService::start() { auto self = shared_from_this(); @@ -37,11 +38,16 @@ void ExecutorService::start() { if (self->isClosed()) { return; } + LOG_INFO("Run io_service in a single thread"); boost::system::error_code ec; self->getIOService().run(ec); if (ec) { LOG_ERROR("Failed to run io_service: " << ec.message()); + } else { + LOG_INFO("Event loop of ExecutorService exits successfully"); } + self->ioServiceDone_ = true; + self->cond_.notify_all(); }}; t.detach(); } @@ -79,13 +85,23 @@ DeadlineTimerPtr ExecutorService::createDeadlineTimer() { return DeadlineTimerPtr(new boost::asio::deadline_timer(io_service_)); } -void ExecutorService::close() { +void ExecutorService::close(long timeoutMs) { bool expectedState = false; if (!closed_.compare_exchange_strong(expectedState, true)) { return; } + if (timeoutMs == 0) { // non-blocking + io_service_.stop(); + return; + } + std::unique_lock lock{mutex_}; io_service_.stop(); + if (timeoutMs > 0) { + cond_.wait_for(lock, std::chrono::milliseconds(timeoutMs), [this] { return ioServiceDone_.load(); }); + } else { // < 0 + cond_.wait(lock, [this] { return ioServiceDone_.load(); }); + } } void ExecutorService::postWork(std::function task) { io_service_.post(task); } @@ -106,14 +122,17 @@ ExecutorServicePtr ExecutorServiceProvider::get() { return executors_[idx]; } -void ExecutorServiceProvider::close() { +void ExecutorServiceProvider::close(long timeoutMs) { Lock lock(mutex_); - for (ExecutorList::iterator it = executors_.begin(); it != executors_.end(); ++it) { - if (*it != NULL) { - (*it)->close(); + TimeoutProcessor timeoutProcessor{timeoutMs}; + for (auto &&executor : executors_) { + timeoutProcessor.tik(); + if (executor) { + executor->close(timeoutProcessor.getLeftTimeout()); } - it->reset(); + timeoutProcessor.tok(); + executor.reset(); } } } // namespace pulsar diff --git a/pulsar-client-cpp/lib/ExecutorService.h b/pulsar-client-cpp/lib/ExecutorService.h index 6b0909194b7d9..e4cbb3ce62ef0 100644 --- a/pulsar-client-cpp/lib/ExecutorService.h +++ b/pulsar-client-cpp/lib/ExecutorService.h @@ -20,6 +20,8 @@ #define _PULSAR_EXECUTOR_SERVICE_HEADER_ #include +#include +#include #include #include #include @@ -50,7 +52,8 @@ class PULSAR_PUBLIC ExecutorService : public std::enable_shared_from_this task); - void close(); + // See TimeoutProcessor for the semantics of the parameter. + void close(long timeoutMs = 3000); IOService &getIOService() { return io_service_; } bool isClosed() const noexcept { return closed_; } @@ -68,6 +71,9 @@ class PULSAR_PUBLIC ExecutorService : public std::enable_shared_from_this ExecutorList; diff --git a/pulsar-client-cpp/lib/TimeUtils.h b/pulsar-client-cpp/lib/TimeUtils.h index 1da7d65923a9f..45157ae855b98 100644 --- a/pulsar-client-cpp/lib/TimeUtils.h +++ b/pulsar-client-cpp/lib/TimeUtils.h @@ -19,6 +19,8 @@ #pragma once #include +#include +#include #include @@ -33,4 +35,50 @@ class PULSAR_PUBLIC TimeUtils { static ptime now(); static int64_t currentTimeMillis(); }; + +// This class processes a timeout with the following semantics: +// > 0: wait at most the timeout until a blocking operation completes +// == 0: do not wait the blocking operation +// < 0: wait infinitely until a blocking operation completes. +// +// Here is a simple example usage: +// +// ```c++ +// // Wait at most 300 milliseconds +// TimeoutProcessor timeoutProcessor{300}; +// while (!allOperationsAreDone()) { +// timeoutProcessor.tik(); +// // This method may block for some time +// performBlockingOperation(timeoutProcessor.getLeftTimeout()); +// timeoutProcessor.tok(); +// } +// ``` +// +// The template argument is the same as std::chrono::duration. +template +class TimeoutProcessor { + public: + using Clock = std::chrono::high_resolution_clock; + + TimeoutProcessor(long timeout) : leftTimeout_(timeout) {} + + long getLeftTimeout() const noexcept { return leftTimeout_; } + + void tik() { before_ = Clock::now(); } + + void tok() { + if (leftTimeout_ > 0) { + leftTimeout_ -= std::chrono::duration_cast(Clock::now() - before_).count(); + if (leftTimeout_ <= 0) { + // The timeout exceeds, getLeftTimeout() will return 0 to indicate we should not wait more + leftTimeout_ = 0; + } + } + } + + private: + std::atomic_long leftTimeout_; + std::chrono::time_point before_; +}; + } // namespace pulsar diff --git a/pulsar-client-cpp/tests/CustomLoggerTest.cc b/pulsar-client-cpp/tests/CustomLoggerTest.cc index 0b4e76adcc4bb..bd80c312e3ba8 100644 --- a/pulsar-client-cpp/tests/CustomLoggerTest.cc +++ b/pulsar-client-cpp/tests/CustomLoggerTest.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include using namespace pulsar; @@ -28,35 +29,42 @@ static std::vector logLines; class MyTestLogger : public Logger { public: - MyTestLogger() = default; + MyTestLogger(const std::string &fileName) : fileName_(fileName) {} bool isEnabled(Level level) override { return true; } void log(Level level, int line, const std::string &message) override { std::stringstream ss; - ss << " " << level << ":" << line << " " << message << std::endl; + ss << std::this_thread::get_id() << " " << level << " " << fileName_ << ":" << line << " " << message + << std::endl; logLines.emplace_back(ss.str()); } + + private: + const std::string fileName_; }; class MyTestLoggerFactory : public LoggerFactory { public: - Logger *getLogger(const std::string &fileName) override { return logger; } - - private: - MyTestLogger *logger = new MyTestLogger; + Logger *getLogger(const std::string &fileName) override { return new MyTestLogger(fileName); } }; TEST(CustomLoggerTest, testCustomLogger) { // simulate new client created on a different thread (because logging factory is called once per thread) - auto testThread = std::thread([] { + std::atomic_int numLogLines{0}; + auto testThread = std::thread([&numLogLines] { ClientConfiguration clientConfig; auto customLogFactory = new MyTestLoggerFactory(); clientConfig.setLogger(customLogFactory); // reset to previous log factory Client client("pulsar://localhost:6650", clientConfig); client.close(); - ASSERT_EQ(logLines.size(), 7); + ASSERT_TRUE(logLines.size() > 0); + for (auto &&line : logLines) { + std::cout << line; + std::cout.flush(); + } + numLogLines = logLines.size(); LogUtils::resetLoggerFactory(); }); testThread.join(); @@ -65,7 +73,7 @@ TEST(CustomLoggerTest, testCustomLogger) { Client client("pulsar://localhost:6650", clientConfig); client.close(); // custom logger didn't get any new lines - ASSERT_EQ(logLines.size(), 7); + ASSERT_EQ(logLines.size(), numLogLines); } TEST(CustomLoggerTest, testConsoleLoggerFactory) { From a462731bbfb24c519d4a5edf163e6dd789039129 Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Wed, 27 Apr 2022 15:29:05 +0800 Subject: [PATCH 509/823] [fix][broker] fix resource group does not report usage (#15292) * fix resource group does not report usage * fix checkstyle * fix mistake Co-authored-by: wangjialing (cherry picked from commit 4560737bf9c0a8f419c37f6e2cb3a230dcfd4352) --- .../resourcegroup/ResourceQuotaCalculatorImpl.java | 4 ++-- .../resourcegroup/ResourceQuotaCalculatorImplTest.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java index ca83cae91c5ad..5dc50f2a25536 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java @@ -108,7 +108,7 @@ public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBy final float toleratedDriftPercentage = ResourceGroupService.UsageReportSuppressionTolerancePercentage; if (currentBytesUsed > 0) { long diff = abs(currentBytesUsed - lastReportedBytes); - float diffPercentage = (diff / currentBytesUsed) * 100; + float diffPercentage = (float) diff * 100 / lastReportedBytes; if (diffPercentage > toleratedDriftPercentage) { return true; } @@ -116,7 +116,7 @@ public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBy if (currentMessagesUsed > 0) { long diff = abs(currentMessagesUsed - lastReportedMessages); - float diffPercentage = (diff / currentMessagesUsed) * 100; + float diffPercentage = (float) diff * 100 / lastReportedMessages; if (diffPercentage > toleratedDriftPercentage) { return true; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java index 1a98838bb4525..af8615936ccfe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java @@ -112,5 +112,15 @@ public void testRQCalcGlobUsedZeroTest() throws PulsarAdminException { Assert.assertTrue(newQuota == config); } + @Test + public void testNeedToReportLocalUsage() { + // If the percentage change (increase or decrease) in usage is more than 5% for + // either bytes or messages, send a report. + Assert.assertFalse(rqCalc.needToReportLocalUsage(1040, 1000, 104, 100, System.currentTimeMillis())); + Assert.assertFalse(rqCalc.needToReportLocalUsage(950, 1000, 95, 100, System.currentTimeMillis())); + Assert.assertTrue(rqCalc.needToReportLocalUsage(1060, 1000, 106, 100, System.currentTimeMillis())); + Assert.assertTrue(rqCalc.needToReportLocalUsage(940, 1000, 94, 100, System.currentTimeMillis())); + } + private ResourceQuotaCalculatorImpl rqCalc; } \ No newline at end of file From b0eff93b8fdbee40caf269a430acbef617ed31fd Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 28 Apr 2022 11:06:15 +0800 Subject: [PATCH 510/823] [improve][broker] Use shrink map for message redelivery. (#15342) (cherry picked from commit 615f05af3e7c72d83b3fe24f64566ed58244ea5d) --- .../service/persistent/MessageRedeliveryController.java | 9 ++++++--- .../persistent/MessageRedeliveryControllerTest.java | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java index be143565c483f..c7f96fffcefc6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java @@ -26,8 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap.LongPair; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; import org.apache.pulsar.common.util.collections.ConcurrentSortedLongPairSet; import org.apache.pulsar.common.util.collections.LongPairSet; @@ -37,7 +37,10 @@ public class MessageRedeliveryController { public MessageRedeliveryController(boolean allowOutOfOrderDelivery) { this.messagesToRedeliver = new ConcurrentSortedLongPairSet(128, 2); - this.hashesToBeBlocked = allowOutOfOrderDelivery ? null : new ConcurrentLongLongPairHashMap(128, 2); + this.hashesToBeBlocked = allowOutOfOrderDelivery + ? null + : ConcurrentLongLongPairHashMap + .newBuilder().concurrencyLevel(2).expectedItems(128).autoShrink(true).build(); } public boolean add(long ledgerId, long entryId) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java index 9a785f6f95fd6..478677a25e42a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java @@ -30,7 +30,7 @@ import java.util.Set; import java.util.TreeSet; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; import org.apache.pulsar.common.util.collections.LongPairSet; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; From 38057e42840f8becb4ae893133469b8cc26ee10f Mon Sep 17 00:00:00 2001 From: wuxuanqicn <89442834+wuxuanqicn@users.noreply.github.com> Date: Thu, 28 Apr 2022 11:57:39 +0800 Subject: [PATCH 511/823] [fix][test]: fix flaky test of ManagedCursorMetricsTest.testManagedCursorMetrics (#9919) (#14720) Fixes #9919 ### Motivation we need make sure broker executed all ack command and updated metrics, then we can generate and check metric ### Modifications - enable AckReceipt - await until ack procedure complete(ACK and ACK_RESPONSE command) Co-authored-by: xuanqi.wu (cherry picked from commit be7057a1fd878111e85ec112bac8f0b72350e744) --- .../stats/ManagedCursorMetricsTest.java | 26 +++++++++++++++---- .../apache/pulsar/client/impl/ClientCnx.java | 1 + 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java index 5e20c09fed1dd..4648ae2fb8f4a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java @@ -18,22 +18,27 @@ */ package org.apache.pulsar.broker.stats; +import java.util.List; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.bookkeeper.client.PulsarMockLedgerHandle; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.stats.metrics.ManagedCursorMetrics; +import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.PulsarTestClient; import org.apache.pulsar.common.stats.Metrics; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.List; -import java.util.concurrent.TimeUnit; - @Test(groups = "broker") public class ManagedCursorMetricsTest extends MockedPulsarServiceBaseTest { @@ -49,6 +54,11 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected PulsarClient createNewPulsarClient(ClientBuilder clientBuilder) throws PulsarClientException { + return PulsarTestClient.create(clientBuilder); + } + @Test public void testManagedCursorMetrics() throws Exception { final String subName = "my-sub"; @@ -63,14 +73,18 @@ public void testManagedCursorMetrics() throws Exception { metricsList = metrics.generate(); Assert.assertTrue(metricsList.isEmpty()); - Consumer consumer = pulsarClient.newConsumer() + PulsarTestClient pulsarClient = (PulsarTestClient) this.pulsarClient; + @Cleanup + ConsumerImpl consumer = (ConsumerImpl) this.pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Shared) .ackTimeout(1, TimeUnit.SECONDS) .subscriptionName(subName) + .isAckReceiptEnabled(true) .subscribe(); - Producer producer = pulsarClient.newProducer() + @Cleanup + Producer producer = this.pulsarClient.newProducer() .topic(topicName) .create(); @@ -83,6 +97,8 @@ public void testManagedCursorMetrics() throws Exception { producer.send(message.getBytes()); consumer.acknowledge(consumer.receive().getMessageId()); } + + Awaitility.await().until(() -> pulsarClient.getConnection(topicName).get().getPendingRequests().size() == 0); metricsList = metrics.generate(); Assert.assertFalse(metricsList.isEmpty()); Assert.assertNotEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistLedgerSucceed"), 0L); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 00b8e4f7561dd..a8d1cf51c7147 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -109,6 +109,7 @@ public class ClientCnx extends PulsarHandler { protected final Authentication authentication; private State state; + @Getter private final ConcurrentLongHashMap> pendingRequests = ConcurrentLongHashMap.>newBuilder() .expectedItems(16) From 2fbdb1747186b4f319704178e1cd5d2685e7b00a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 28 Apr 2022 20:30:30 +0800 Subject: [PATCH 512/823] [improve][broker] Support shrink for ConcurrentSortedLongPairSet (#15354) (cherry picked from commit 24d4d76bb9e39010bae3f4cbd8ddba6422570b4e) --- .../MessageRedeliveryController.java | 2 +- .../collections/ConcurrentLongPairSet.java | 53 +++++++++++-------- .../ConcurrentSortedLongPairSet.java | 27 ++++++++-- .../common/util/collections/LongPairSet.java | 7 +++ .../ConcurrentSortedLongPairSetTest.java | 43 +++++++++++++++ 5 files changed, 105 insertions(+), 27 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java index c7f96fffcefc6..46fa1b2b050a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java @@ -36,7 +36,7 @@ public class MessageRedeliveryController { private final ConcurrentLongLongPairHashMap hashesToBeBlocked; public MessageRedeliveryController(boolean allowOutOfOrderDelivery) { - this.messagesToRedeliver = new ConcurrentSortedLongPairSet(128, 2); + this.messagesToRedeliver = new ConcurrentSortedLongPairSet(128, 2, true); this.hashesToBeBlocked = allowOutOfOrderDelivery ? null : ConcurrentLongLongPairHashMap diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java index 66ecaee4bfa5b..7b5e75813fa78 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java @@ -175,6 +175,7 @@ public long size() { return size; } + @Override public long capacity() { long capacity = 0; for (int i = 0; i < sections.length; i++) { @@ -447,20 +448,7 @@ private boolean remove(long item1, long item2, int hash) { bucket = (bucket + 2) & (table.length - 1); } } finally { - if (autoShrink && size < resizeThresholdBelow) { - try { - int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); - int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); - if (newCapacity < capacity && newResizeThresholdUp > size) { - // shrink the hashmap - rehash(newCapacity); - } - } finally { - unlockWrite(stamp); - } - } else { - unlockWrite(stamp); - } + tryShrinkThenUnlock(stamp); } } @@ -469,23 +457,42 @@ private int removeIf(LongPairPredicate filter) { int removedItems = 0; // Go through all the buckets for this section - for (int bucket = 0; bucket < table.length; bucket += 2) { - long storedItem1 = table[bucket]; - long storedItem2 = table[bucket + 1]; - - if (storedItem1 != DeletedItem && storedItem1 != EmptyItem) { - if (filter.test(storedItem1, storedItem2)) { - long h = hash(storedItem1, storedItem2); - if (remove(storedItem1, storedItem2, (int) h)) { + long stamp = writeLock(); + try { + for (int bucket = 0; bucket < table.length; bucket += 2) { + long storedItem1 = table[bucket]; + long storedItem2 = table[bucket + 1]; + if (storedItem1 != DeletedItem && storedItem1 != EmptyItem) { + if (filter.test(storedItem1, storedItem2)) { + SIZE_UPDATER.decrementAndGet(this); + cleanBucket(bucket); removedItems++; } } } + } finally { + tryShrinkThenUnlock(stamp); } - return removedItems; } + private void tryShrinkThenUnlock(long stamp) { + if (autoShrink && size < resizeThresholdBelow) { + try { + int newCapacity = alignToPowerOfTwo((int) (capacity / shrinkFactor)); + int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); + if (newCapacity < capacity && newResizeThresholdUp > size) { + // shrink the hashmap + rehash(newCapacity); + } + } finally { + unlockWrite(stamp); + } + } else { + unlockWrite(stamp); + } + } + private void cleanBucket(int bucket) { int nextInArray = (bucket + 2) & (table.length - 1); if (table[nextInArray] == EmptyItem) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java index e4cb668fc92d3..06efd0490d184 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java @@ -48,14 +48,15 @@ public class ConcurrentSortedLongPairSet implements LongPairSet { protected final NavigableMap longPairSets = new ConcurrentSkipListMap<>(); - private int expectedItems; - private int concurrencyLevel; + private final int expectedItems; + private final int concurrencyLevel; /** * If {@link #longPairSets} adds and removes the item-set frequently then it allocates and removes * {@link ConcurrentLongPairSet} for the same item multiple times which can lead to gc-puases. To avoid such * situation, avoid removing empty LogPairSet until it reaches max limit. */ - private int maxAllowedSetOnRemove; + private final int maxAllowedSetOnRemove; + private final boolean autoShrink; private static final int DEFAULT_MAX_ALLOWED_SET_ON_REMOVE = 10; public ConcurrentSortedLongPairSet() { @@ -70,10 +71,20 @@ public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel) { this(expectedItems, concurrencyLevel, DEFAULT_MAX_ALLOWED_SET_ON_REMOVE); } + public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, boolean autoShrink) { + this(expectedItems, concurrencyLevel, DEFAULT_MAX_ALLOWED_SET_ON_REMOVE, autoShrink); + } + public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, int maxAllowedSetOnRemove) { + this(expectedItems, concurrencyLevel, maxAllowedSetOnRemove, false); + } + + public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, int maxAllowedSetOnRemove, + boolean autoShrink) { this.expectedItems = expectedItems; this.concurrencyLevel = concurrencyLevel; this.maxAllowedSetOnRemove = maxAllowedSetOnRemove; + this.autoShrink = autoShrink; } @Override @@ -82,6 +93,7 @@ public boolean add(long item1, long item2) { (key) -> ConcurrentLongPairSet.newBuilder() .expectedItems(expectedItems) .concurrencyLevel(concurrencyLevel) + .autoShrink(autoShrink) .build()); return messagesToReplay.add(item1, item2); } @@ -194,6 +206,15 @@ public long size() { return size.get(); } + @Override + public long capacity() { + AtomicLong capacity = new AtomicLong(0); + longPairSets.forEach((item1, longPairSet) -> { + capacity.getAndAdd(longPairSet.capacity()); + }); + return capacity.get(); + } + @Override public boolean contains(long item1, long item2) { ConcurrentLongPairSet longPairSet = longPairSets.get(item1); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java index 32de7e4c232bd..f27b994f777d2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java @@ -107,6 +107,13 @@ public interface LongPairPredicate { */ long size(); + /** + * Returns capacity of the set. + * + * @return + */ + long capacity(); + /** * Checks if given (item1,item2) composite value exists into set. * diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java index fcb9884a795ad..62dfa21dc81c9 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; +import com.google.common.collect.ComparisonChain; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; @@ -181,6 +182,20 @@ public void testIfRemoval() { values = new ArrayList<>(set.items()); values.sort(null); assertEquals(values, Lists.newArrayList(new LongPair(6, 6), new LongPair(7, 7))); + + set = new ConcurrentSortedLongPairSet(128, 2, true); + set.add(2, 2); + set.add(1, 3); + set.add(3, 1); + set.add(2, 1); + set.add(3, 2); + set.add(1, 2); + set.add(1, 1); + removeItems = set.removeIf((ledgerId, entryId) -> { + return ComparisonChain.start().compare(ledgerId, 1).compare(entryId, 3) + .result() <= 0; + }); + assertEquals(removeItems, 3); } @Test @@ -245,4 +260,32 @@ public void testIsEmpty() { set.add(1, 1); assertFalse(set.isEmpty()); } + + @Test + public void testShrink() { + LongPairSet set = new ConcurrentSortedLongPairSet(2, 1, true); + set.add(0, 0); + assertTrue(set.capacity() == 4); + set.add(0, 1); + assertTrue(set.capacity() == 4); + set.add(1, 1); + assertTrue(set.capacity() == 8); + set.add(1, 2); + assertTrue(set.capacity() == 8); + set.add(1, 3); + set.add(1, 4); + set.add(1, 5); + assertTrue(set.capacity() == 12); + set.remove(1, 5); + // not shrink + assertTrue(set.capacity() == 12); + set.remove(1, 4); + // the internal map does not keep shrinking at every remove() operation + assertTrue(set.capacity() == 12); + set.remove(1, 3); + set.remove(1, 2); + set.remove(1, 1); + // shrink + assertTrue(set.capacity() == 8); + } } From 687911344a5477462cfcc905611b648c13521825 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 29 Apr 2022 17:24:15 +0300 Subject: [PATCH 513/823] [Proxy & Client] Configure Netty DNS resolver to match JDK DNS caching setting, share DNS resolver instance in Proxy (#15219) * Align Netty DNS resolver cache settings with Java DNS cache settings - Netty DNS resolver caches forever by default - this could cause problems with Kubernetes deployments * Share Netty DNSNameResolver in proxy * Remove overriding of ConnectionPool.close since it's not necessary * Address review comment: remove ProxyConnectionPool (cherry picked from commit 39b186269c6ccdfa94412a28cc3719a4338368ac) --- .../pulsar/client/impl/ConnectionPool.java | 26 ++++++- pulsar-common/pom.xml | 5 ++ .../common/util/netty/DnsResolverUtil.java | 75 +++++++++++++++++++ .../pulsar/proxy/server/ProxyConnection.java | 36 +++++---- .../proxy/server/ProxyConnectionPool.java | 60 --------------- .../pulsar/proxy/server/ProxyService.java | 3 + .../server/ServiceChannelInitializer.java | 2 +- 7 files changed, 126 insertions(+), 81 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java delete mode 100644 pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnectionPool.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 3df52eaed2106..59aaac0477b80 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -47,6 +48,7 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.netty.DnsResolverUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,13 +64,20 @@ public class ConnectionPool implements AutoCloseable { private final boolean isSniProxy; protected final DnsNameResolver dnsResolver; + private final boolean shouldCloseDnsResolver; public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { this(conf, eventLoopGroup, () -> new ClientCnx(conf, eventLoopGroup)); } public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, - Supplier clientCnxSupplier) throws PulsarClientException { + Supplier clientCnxSupplier) throws PulsarClientException { + this(conf, eventLoopGroup, clientCnxSupplier, Optional.empty()); + } + + public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + Supplier clientCnxSupplier, Optional dnsNameResolver) + throws PulsarClientException { this.eventLoopGroup = eventLoopGroup; this.clientConfig = conf; this.maxConnectionsPerHosts = conf.getConnectionsPerBroker(); @@ -92,8 +101,15 @@ public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGrou throw new PulsarClientException(e); } - this.dnsResolver = new DnsNameResolverBuilder(eventLoopGroup.next()).traceEnabled(true) - .channelType(EventLoopUtil.getDatagramChannelClass(eventLoopGroup)).build(); + this.shouldCloseDnsResolver = !dnsNameResolver.isPresent(); + this.dnsResolver = dnsNameResolver.orElseGet(() -> createDnsNameResolver(conf, eventLoopGroup)); + } + + private static DnsNameResolver createDnsNameResolver(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { + DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder(eventLoopGroup.next()) + .traceEnabled(true).channelType(EventLoopUtil.getDatagramChannelClass(eventLoopGroup)); + DnsResolverUtil.applyJdkDnsCacheSettings(dnsNameResolverBuilder); + return dnsNameResolverBuilder.build(); } private static final Random random = new Random(); @@ -315,7 +331,9 @@ public void releaseConnection(ClientCnx cnx) { @Override public void close() throws Exception { closeAllConnections(); - dnsResolver.close(); + if (shouldCloseDnsResolver) { + dnsResolver.close(); + } } private void cleanupConnection(InetSocketAddress address, int connectionKey, diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 42c823a7f8615..e8b5c59579090 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -72,6 +72,11 @@ netty-handler + + io.netty + netty-resolver-dns + + io.netty netty-transport-native-epoll diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java new file mode 100644 index 0000000000000..8b06dbf36eca3 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.netty; + +import io.netty.resolver.dns.DnsNameResolverBuilder; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DnsResolverUtil { + private static final int MIN_TTL = 0; + private static final int TTL; + private static final int NEGATIVE_TTL; + + // default TTL value when JDK setting is "forever" (-1) + private static final int DEFAULT_TTL = 60; + + // default negative TTL value when JDK setting is "forever" (-1) + private static final int DEFAULT_NEGATIVE_TTL = 10; + + static { + int ttl = DEFAULT_TTL; + int negativeTtl = DEFAULT_NEGATIVE_TTL; + try { + // use reflection to call sun.net.InetAddressCachePolicy's get and getNegative methods for getting + // effective JDK settings for DNS caching + Class inetAddressCachePolicyClass = Class.forName("sun.net.InetAddressCachePolicy"); + Method getTTLMethod = inetAddressCachePolicyClass.getMethod("get"); + ttl = (Integer) getTTLMethod.invoke(null); + Method getNegativeTTLMethod = inetAddressCachePolicyClass.getMethod("getNegative"); + negativeTtl = (Integer) getNegativeTTLMethod.invoke(null); + } catch (NoSuchMethodException | ClassNotFoundException | InvocationTargetException + | IllegalAccessException e) { + log.warn("Cannot get DNS TTL settings from sun.net.InetAddressCachePolicy class", e); + } + TTL = useDefaultTTLWhenSetToForever(ttl, DEFAULT_TTL); + NEGATIVE_TTL = useDefaultTTLWhenSetToForever(negativeTtl, DEFAULT_NEGATIVE_TTL); + } + + private static int useDefaultTTLWhenSetToForever(int ttl, int defaultTtl) { + return ttl < 0 ? defaultTtl : ttl; + } + + private DnsResolverUtil() { + // utility class with static methods, prevent instantiation + } + + /** + * Configure Netty's {@link DnsNameResolverBuilder}'s ttl and negativeTtl to match the JDK's DNS caching settings. + * If the JDK setting for TTL is forever (-1), the TTL will be set to 60 seconds. + * + * @param dnsNameResolverBuilder The Netty {@link DnsNameResolverBuilder} instance to apply the settings + */ + public static void applyJdkDnsCacheSettings(DnsNameResolverBuilder dnsNameResolverBuilder) { + dnsNameResolverBuilder.ttl(MIN_TTL, TTL); + dnsNameResolverBuilder.negativeTtl(NEGATIVE_TTL); + } +} diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 29060faeff25a..5d81c4b803a78 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -22,9 +22,11 @@ import io.netty.channel.ChannelFutureListener; import io.netty.handler.codec.haproxy.HAProxyMessage; +import io.netty.resolver.dns.DnsNameResolver; import java.net.SocketAddress; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -77,6 +79,7 @@ public class ProxyConnection extends PulsarHandler { private final AtomicLong requestIdGenerator = new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2)); private final ProxyService service; + private final DnsNameResolver dnsNameResolver; AuthenticationDataSource authenticationData; private State state; private final Supplier sslHandlerSupplier; @@ -125,9 +128,11 @@ ConnectionPool getConnectionPool() { return connectionPool; } - public ProxyConnection(ProxyService proxyService, Supplier sslHandlerSupplier) { + public ProxyConnection(ProxyService proxyService, Supplier sslHandlerSupplier, + DnsNameResolver dnsNameResolver) { super(30, TimeUnit.SECONDS); this.service = proxyService; + this.dnsNameResolver = dnsNameResolver; this.state = State.Init; this.sslHandlerSupplier = sslHandlerSupplier; this.brokerProxyValidator = service.getBrokerProxyValidator(); @@ -235,27 +240,26 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce } private synchronized void completeConnect(AuthData clientData) throws PulsarClientException { + Supplier clientCnxSupplier; if (service.getConfiguration().isAuthenticationEnabled()) { if (service.getConfiguration().isForwardAuthorizationCredentials()) { this.clientAuthData = clientData; this.clientAuthMethod = authMethod; } - if (this.connectionPool == null) { - this.connectionPool = new ProxyConnectionPool(clientConf, service.getWorkerGroup(), - () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, clientAuthData, - clientAuthMethod, protocolVersionToAdvertise)); - } else { - LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", - remoteAddress, state, clientAuthRole); - } + clientCnxSupplier = + () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, clientAuthData, + clientAuthMethod, protocolVersionToAdvertise); } else { - if (this.connectionPool == null) { - this.connectionPool = new ProxyConnectionPool(clientConf, service.getWorkerGroup(), - () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise)); - } else { - LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {}", - remoteAddress, state); - } + clientCnxSupplier = + () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise); + } + + if (this.connectionPool == null) { + this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(), + clientCnxSupplier, Optional.of(dnsNameResolver)); + } else { + LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", + remoteAddress, state, clientAuthRole); } LOG.info("[{}] complete connection, init proxy handler. authenticated with {} role {}, hasProxyToBrokerUrl: {}", diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnectionPool.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnectionPool.java deleted file mode 100644 index cd1b31d343455..0000000000000 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnectionPool.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.proxy.server; - -import java.io.IOException; -import java.util.concurrent.ExecutionException; -import java.util.function.Supplier; - -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.impl.ClientCnx; -import org.apache.pulsar.client.impl.ConnectionPool; -import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.EventLoopGroup; - -public class ProxyConnectionPool extends ConnectionPool { - public ProxyConnectionPool(ClientConfigurationData clientConfig, EventLoopGroup eventLoopGroup, - Supplier clientCnxSupplier) throws PulsarClientException { - super(clientConfig, eventLoopGroup, clientCnxSupplier); - } - - @Override - public void close() throws IOException { - log.info("Closing ProxyConnectionPool."); - pool.forEach((address, clientCnxPool) -> { - if (clientCnxPool != null) { - clientCnxPool.forEach((identifier, clientCnx) -> { - if (clientCnx != null && clientCnx.isDone()) { - try { - clientCnx.get().close(); - } catch (InterruptedException | ExecutionException e) { - log.error("Unable to close get client connection future.", e); - } - } - }); - } - }); - dnsResolver.close(); - } - - private static final Logger log = LoggerFactory.getLogger(ProxyConnectionPool.class); -} diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index abefe49f0597d..0f4ea152b50ee 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -58,6 +58,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.netty.DnsResolverUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -153,6 +154,8 @@ public ProxyService(ProxyConfiguration proxyConfig, DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder(workerGroup.next()) .channelType(EventLoopUtil.getDatagramChannelClass(workerGroup)); + DnsResolverUtil.applyJdkDnsCacheSettings(dnsNameResolverBuilder); + dnsNameResolver = dnsNameResolverBuilder.build(); brokerProxyValidator = new BrokerProxyValidator(dnsNameResolver.asAddressResolver(), diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 12abf871b5078..4f423c1f5d5b5 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -174,7 +174,7 @@ public SslHandler get() { } ch.pipeline().addLast("handler", - new ProxyConnection(proxyService, sslHandlerSupplier)); + new ProxyConnection(proxyService, sslHandlerSupplier, proxyService.getDnsNameResolver())); } } From 5a94b19fdfdb6f7a72409d38ff4b718713786dca Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 29 Apr 2022 14:59:32 +0300 Subject: [PATCH 514/823] [Proxy] Fix proxy connection leak when inbound connection closes while connecting is in progress (#15366) * [Proxy] Fix proxy connection leak when inbound connection closes while connecting is in progress * Add logging when execution is rejected (cherry picked from commit 4ae070c5b26be3bf92c007ca309bd4adbae5ce93) --- .../proxy/server/DirectProxyHandler.java | 19 ++-- .../pulsar/proxy/server/ProxyConnection.java | 103 ++++++++++++++---- 2 files changed, 92 insertions(+), 30 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index e04ac1e4ae83d..24802f60a3db0 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -80,9 +80,7 @@ public class DirectProxyHandler { private final ProxyService service; private final Runnable onHandshakeCompleteAction; - public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, String brokerHostAndPort, - InetSocketAddress targetBrokerAddress, int protocolVersion, - Supplier sslHandlerSupplier) { + public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) { this.service = service; this.authentication = proxyConnection.getClientAuthentication(); this.inboundChannel = proxyConnection.ctx().channel(); @@ -92,6 +90,10 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, this.clientAuthData = proxyConnection.clientAuthData; this.clientAuthMethod = proxyConnection.clientAuthMethod; this.onHandshakeCompleteAction = proxyConnection::cancelKeepAliveTask; + } + + public void connect(String brokerHostAndPort, InetSocketAddress targetBrokerAddress, + int protocolVersion, Supplier sslHandlerSupplier) { ProxyConfiguration config = service.getConfiguration(); // Start the connection attempt. @@ -208,6 +210,12 @@ private ByteBuf encodeProxyProtocolMessage(HAProxyMessage msg) { (byte) 'Y', }; + public void close() { + if (outboundChannel != null) { + outboundChannel.close(); + } + } + enum BackendState { Init, HandshakeCompleted } @@ -344,10 +352,7 @@ protected void handleConnected(CommandConnected connected) { onHandshakeCompleteAction.run(); startDirectProxying(connected); - int maxMessageSize = - connected.hasMaxMessageSize() ? connected.getMaxMessageSize() : Commands.INVALID_MAX_MESSAGE_SIZE; - inboundChannel.writeAndFlush(Commands.newConnected(connected.getProtocolVersion(), maxMessageSize)) - .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); + proxyConnection.brokerConnected(DirectProxyHandler.this, connected); } private void startDirectProxying(CommandConnected connected) { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 5d81c4b803a78..39870f62af848 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -19,14 +19,16 @@ package org.apache.pulsar.proxy.server; import static com.google.common.base.Preconditions.checkArgument; - +import static com.google.common.base.Preconditions.checkState; import io.netty.channel.ChannelFutureListener; import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.resolver.dns.DnsNameResolver; +import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -50,6 +52,7 @@ import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.api.proto.CommandAuthResponse; import org.apache.pulsar.common.api.proto.CommandConnect; +import org.apache.pulsar.common.api.proto.CommandConnected; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.api.proto.CommandLookupTopic; import org.apache.pulsar.common.api.proto.CommandGetSchema; @@ -114,6 +117,9 @@ enum State { // Follow redirects ProxyLookupRequests, + // Connecting to the broker + ProxyConnectingToBroker, + // If we are proxying a connection to a specific broker, we // are just forwarding data between the 2 connections, without // looking into it @@ -167,8 +173,8 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { public synchronized void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); - if (directProxyHandler != null && directProxyHandler.outboundChannel != null) { - directProxyHandler.outboundChannel.close(); + if (directProxyHandler != null) { + directProxyHandler.close(); directProxyHandler = null; } @@ -189,11 +195,22 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - state = State.Closing; super.exceptionCaught(ctx, cause); - LOG.warn("[{}] Got exception {} : {} {}", remoteAddress, cause.getClass().getSimpleName(), cause.getMessage(), + LOG.warn("[{}] Got exception {} : Message: {} State: {}", remoteAddress, cause.getClass().getSimpleName(), + cause.getMessage(), state, ClientCnx.isKnownException(cause) ? null : cause); - ctx.close(); + if (state != State.Closed) { + state = State.Closing; + } + if (ctx.channel().isOpen()) { + ctx.close(); + } else { + // close connection to broker if that is present + if (directProxyHandler != null) { + directProxyHandler.close(); + directProxyHandler = null; + } + } } @Override @@ -222,18 +239,26 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce break; case ProxyConnectionToBroker: - // Pass the buffer to the outbound connection and schedule next read - // only if we can write on the connection - ProxyService.opsCounter.inc(); - if (msg instanceof ByteBuf) { - int bytes = ((ByteBuf) msg).readableBytes(); - directProxyHandler.getInboundChannelRequestsRate().recordEvent(bytes); - ProxyService.bytesCounter.inc(bytes); + if (directProxyHandler != null) { + ProxyService.opsCounter.inc(); + if (msg instanceof ByteBuf) { + int bytes = ((ByteBuf) msg).readableBytes(); + directProxyHandler.getInboundChannelRequestsRate().recordEvent(bytes); + ProxyService.bytesCounter.inc(bytes); + } + directProxyHandler.outboundChannel.writeAndFlush(msg) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); + } else { + LOG.warn("Received message of type {} while connection to broker is missing in state {}. " + + "Dropping the input message (readable bytes={}).", msg.getClass(), state, + msg instanceof ByteBuf ? ((ByteBuf) msg).readableBytes() : -1); } - directProxyHandler.outboundChannel.writeAndFlush(msg) - .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; - + case ProxyConnectingToBroker: + LOG.warn("Received message of type {} while connecting to broker. " + + "Dropping the input message (readable bytes={}).", msg.getClass(), + msg instanceof ByteBuf ? ((ByteBuf) msg).readableBytes() : -1); + break; default: break; } @@ -279,14 +304,9 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie return; } + state = State.ProxyConnectingToBroker; brokerProxyValidator.resolveAndCheckTargetAddress(proxyToBrokerUrl) - .thenAcceptAsync(address -> { - // Client already knows which broker to connect. Let's open a - // connection there and just pass bytes in both directions - state = State.ProxyConnectionToBroker; - directProxyHandler = new DirectProxyHandler(service, this, proxyToBrokerUrl, address, - protocolVersionToAdvertise, sslHandlerSupplier); - }, ctx.executor()) + .thenAcceptAsync(this::connectToBroker, ctx.executor()) .exceptionally(throwable -> { if (throwable instanceof TargetAddressDeniedException || throwable.getCause() instanceof TargetAddressDeniedException) { @@ -319,6 +339,43 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie } } + private void handleBrokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) { + checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop"); + if (state == State.ProxyConnectingToBroker && ctx.channel().isOpen() && this.directProxyHandler == null) { + this.directProxyHandler = directProxyHandler; + state = State.ProxyConnectionToBroker; + int maxMessageSize = + connected.hasMaxMessageSize() ? connected.getMaxMessageSize() : Commands.INVALID_MAX_MESSAGE_SIZE; + ctx.writeAndFlush(Commands.newConnected(connected.getProtocolVersion(), maxMessageSize)) + .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); + } else { + LOG.warn("[{}] Channel is {}. ProxyConnection is in {}. " + + "Closing connection to broker '{}'.", + remoteAddress, ctx.channel().isOpen() ? "open" : "already closed", + state != State.ProxyConnectingToBroker ? "invalid state " + state : "state " + state, + proxyToBrokerUrl); + directProxyHandler.close(); + ctx.close(); + } + } + + private void connectToBroker(InetSocketAddress brokerAddress) { + checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop"); + DirectProxyHandler directProxyHandler = new DirectProxyHandler(service, this); + directProxyHandler.connect(proxyToBrokerUrl, brokerAddress, + protocolVersionToAdvertise, sslHandlerSupplier); + } + + public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) { + try { + final CommandConnected finalConnected = new CommandConnected().copyFrom(connected); + ctx.executor().submit(() -> handleBrokerConnected(directProxyHandler, finalConnected)); + } catch (RejectedExecutionException e) { + LOG.error("Event loop was already closed. Closing broker connection.", e); + directProxyHandler.close(); + } + } + // According to auth result, send newConnected or newAuthChallenge command. private void doAuthentication(AuthData clientData) throws Exception { AuthData brokerData = authState.authenticate(clientData); From 278531f2f923c1c0c31364c804e6650a0628dafa Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 29 Apr 2022 17:07:11 +0300 Subject: [PATCH 515/823] [Broker/Client] Close connection if a ping or pong message cannot be sent (#15382) * [Broker/Client] Close connection if a ping message cannot be sent - the connection should be closed if a ping message cannot be sent * Handle write errors for pong messages (cherry picked from commit 2ddef95f31ce37486f3f76b4d59730361a77bf6e) --- .../pulsar/common/protocol/PulsarHandler.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index 48e7ffa7b887b..6a0b7bb1fa5f4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -83,7 +83,14 @@ final protected void handlePing(CommandPing ping) { if (log.isDebugEnabled()) { log.debug("[{}] Replying back to ping message", ctx.channel()); } - ctx.writeAndFlush(Commands.newPong()); + ctx.writeAndFlush(Commands.newPong()) + .addListener(future -> { + if (!future.isSuccess()) { + log.warn("[{}] Forcing connection to close since cannot send a pong message.", + ctx.channel(), future.cause()); + ctx.close(); + } + }); } @Override @@ -110,7 +117,14 @@ private void handleKeepAliveTimeout() { log.debug("[{}] Sending ping message", ctx.channel()); } waitingForPingResponse = true; - ctx.writeAndFlush(Commands.newPing()); + ctx.writeAndFlush(Commands.newPing()) + .addListener(future -> { + if (!future.isSuccess()) { + log.warn("[{}] Forcing connection to close since cannot send a ping message.", + ctx.channel(), future.cause()); + ctx.close(); + } + }); } else { if (log.isDebugEnabled()) { log.debug("[{}] Peer doesn't support keep-alive", ctx.channel()); From f2bb45b7de8235732a172148481a06a181c5dfd3 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 8 Apr 2022 00:44:15 -0500 Subject: [PATCH 516/823] Use tlsCertRefreshCheckDurationSec instead of 0 for refresh value (#15075) (cherry picked from commit e398d7e412c21e47c7e5d48225088f12874ebd29) --- .../java/org/apache/pulsar/common/util/SecurityUtility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index d5a0c5a576722..4fe3a3756a1f0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -549,7 +549,7 @@ public static SslContextFactory createSslContextFactory(boolean tlsAllowInsecure SslContextFactory sslCtxFactory = null; if (autoRefresh) { sslCtxFactory = new SslContextFactoryWithAutoRefresh(tlsAllowInsecureConnection, tlsTrustCertsFilePath, - tlsCertificateFilePath, tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, 0); + tlsCertificateFilePath, tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, certRefreshInSec); } else { sslCtxFactory = new SslContextFactory(); SSLContext sslCtx = createSslContext(tlsAllowInsecureConnection, tlsTrustCertsFilePath, From c509e4b7982d1401c221346d86fff7beae54fa7d Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 20 Apr 2022 22:25:17 -0500 Subject: [PATCH 517/823] Add KeyStore support in WebSocket, Function Worker HTTPS Servers (#15084) * Add KeyStore support in WebSocket, Function Worker HTTPS Servers * Avoid leaking worker config password * Fix checkstyle * Replace broker with appropriate text in annotations * Update python script for new configs Co-authored-by: Lari Hotari We support configuring KeyStores for the broker and the proxy, but not the WebSocket or the Function Worker. By adding this support, users are able to provide KeyStores of type PCKS12 or JKS, which adds flexibility. Further, these KeyStores simplify support for additional algorithms because we rely on the TLS provider to load the KeyStore instead of loading keys ourselves. * Add `KeyStoreSSLContext`s to the function worker server * Add `KeyStoreSSLContext`s to the web socket server * Add configurations to the function worker, the web socket, and the proxy configuration files to simply configuration * Rely on `toString`, not `ObjectMapper`, when converting the `WorkerConfig` to a string so that we don't log the KeyStore password. (Add a test to verify this logic. Note that we don't want the `ObjectMapper` to ignore the field because we use mappers when converting configuration classes.) I manually verified that this change works in a minikube cluster. The underlying method named `KeyStoreSSLContext#createSslContextFactory` is already used and tested, so I don't believe we need additional testing on that component. This change adds a new way to configure TLS in the WebSocket and Function Worker HTTPS Servers. As such, it adds new configuration. This configuration is named in the same way that the broker and proxy configuration is named, so it is consistent. I've documented the new configuration in the appropriate configuration files. (cherry picked from commit a4103960a4eb7803af6fabd9fe54a3137f82aff0) --- conf/broker.conf | 2 + conf/functions_worker.yml | 40 ++++++++++ conf/proxy.conf | 27 +++++++ conf/websocket.conf | 37 +++++++++ docker/pulsar/scripts/gen-yml-from-env.py | 4 +- .../pulsar/broker/ServiceConfiguration.java | 2 +- .../pulsar/functions/worker/WorkerConfig.java | 75 ++++++++++++++++++- .../worker/WorkerApiV2ResourceConfigTest.java | 18 ++++- .../functions/worker/PulsarWorkerService.java | 10 +-- .../functions/worker/rest/WorkerServer.java | 33 ++++++-- .../pulsar/websocket/service/ProxyServer.java | 35 ++++++--- .../service/WebSocketProxyConfiguration.java | 65 +++++++++++++++- 12 files changed, 314 insertions(+), 34 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 849121f9fe8c9..cfd1e9ef5f96b 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -600,6 +600,8 @@ tlsRequireTrustedClientCertOnConnect=false tlsProvider= ### --- KeyStore TLS config variables --- ### +## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. + # Enable TLS with KeyStore type configuration in broker. tlsEnabledWithKeyStore=false diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index b235d600756f2..06204dfdcfbec 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -301,6 +301,46 @@ tlsAllowInsecureConnection: false tlsEnableHostnameVerification: false # Tls cert refresh duration in seconds (set 0 to check on every new connection) tlsCertRefreshCheckDurationSec: 300 +# Whether client certificates are required for TLS. Connections are rejected if the client +# certificate isn't trusted. +tlsRequireTrustedClientCertOnConnect: false + +### --- KeyStore TLS config variables --- ### +## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. + +# TLS Provider for KeyStore type +tlsProvider: + +# Enable TLS with KeyStore type configuration in function worker. +tlsEnabledWithKeyStore: false + +# TLS KeyStore type configuration in function worker: JKS, PKCS12 +tlsKeyStoreType: JKS + +# TLS KeyStore path in function worker +tlsKeyStore: + +# TLS KeyStore password for function worker +tlsKeyStorePassword: + +# TLS TrustStore type configuration in function worker: JKS, PKCS12 +tlsTrustStoreType: JKS + +# TLS TrustStore path in function worker +tlsTrustStore: + +# TLS TrustStore password in function worker, default value is empty password +tlsTrustStorePassword: + +# Specify the tls protocols the function worker's web service will use to negotiate during TLS handshake +# (a comma-separated list of protocol names). +# Examples:- [TLSv1.3, TLSv1.2] +webServiceTlsProtocols: + +# Specify the tls cipher the function worker will use to negotiate during TLS Handshake +# (a comma-separated list of ciphers). +# Examples:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] +webServiceTlsCiphers: ######################## # State Management diff --git a/conf/proxy.conf b/conf/proxy.conf index fbb009878e315..fbdcad2b2b3e0 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -68,6 +68,33 @@ webServicePort=8080 # Port to use to server HTTPS request webServicePortTls= +### --- KeyStore TLS config variables --- ### +## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. + +# TLS Provider for KeyStore type +tlsProvider= + +# Enable TLS with KeyStore type configuration in proxy. +tlsEnabledWithKeyStore=false + +# TLS KeyStore type configuration in proxy: JKS, PKCS12 +tlsKeyStoreType=JKS + +# TLS KeyStore path in proxy +tlsKeyStore= + +# TLS KeyStore password for proxy +tlsKeyStorePassword= + +# TLS TrustStore type configuration in proxy: JKS, PKCS12 +tlsTrustStoreType=JKS + +# TLS TrustStore path in proxy +tlsTrustStore= + +# TLS TrustStore password in proxy, default value is empty password +tlsTrustStorePassword= + # Path for the file used to determine the rotation status for the proxy instance when responding # to service discovery health checks statusFilePath= diff --git a/conf/websocket.conf b/conf/websocket.conf index 814e58753ef7b..c2a138e7b38fa 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -119,6 +119,43 @@ tlsRequireTrustedClientCertOnConnect=false # Tls cert refresh duration in seconds (set 0 to check on every new connection) tlsCertRefreshCheckDurationSec=300 +### --- KeyStore TLS config variables --- ### +## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. + +# TLS Provider for KeyStore type +tlsProvider= + +# Enable TLS with KeyStore type configuration in WebSocket. +tlsEnabledWithKeyStore=false + +# TLS KeyStore type configuration in WebSocket: JKS, PKCS12 +tlsKeyStoreType=JKS + +# TLS KeyStore path in WebSocket +tlsKeyStore= + +# TLS KeyStore password for WebSocket +tlsKeyStorePassword= + +# TLS TrustStore type configuration in WebSocket: JKS, PKCS12 +tlsTrustStoreType=JKS + +# TLS TrustStore path in WebSocket +tlsTrustStore= + +# TLS TrustStore password in WebSocket, default value is empty password +tlsTrustStorePassword= + +# Specify the tls protocols the proxy's web service will use to negotiate during TLS handshake +# (a comma-separated list of protocol names). +# Examples:- [TLSv1.3, TLSv1.2] +webServiceTlsProtocols= + +# Specify the tls cipher the proxy will use to negotiate during TLS Handshake +# (a comma-separated list of ciphers). +# Examples:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] +webServiceTlsCiphers= + ### --- Deprecated config variables --- ### # Deprecated. Use configurationStoreServers diff --git a/docker/pulsar/scripts/gen-yml-from-env.py b/docker/pulsar/scripts/gen-yml-from-env.py index 8aee68b5282db..779341d1269cc 100755 --- a/docker/pulsar/scripts/gen-yml-from-env.py +++ b/docker/pulsar/scripts/gen-yml-from-env.py @@ -47,7 +47,9 @@ 'proxyRoles', 'schemaRegistryCompatibilityCheckers', 'brokerClientTlsCiphers', - 'brokerClientTlsProtocols' + 'brokerClientTlsProtocols', + 'webServiceTlsCiphers', + 'webServiceTlsProtocols', ] PF_ENV_PREFIX = 'PF_' diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index dc310d5056fcc..250d7d41ffa79 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2245,7 +2245,7 @@ public class ServiceConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_KEYSTORE_TLS, - doc = "TLS Provider for Specify the SSL provider for the broker service: \n" + doc = "Specify the TLS provider for the broker service: \n" + "When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.\n" + "When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc." ) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index eb40ae3c27210..e51e5dfb7425f 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -36,9 +36,11 @@ import java.util.Map; import java.util.Properties; import java.util.Set; - +import java.util.TreeSet; import lombok.AccessLevel; import lombok.Getter; +import lombok.ToString; +import lombok.experimental.Accessors; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; import org.apache.pulsar.common.configuration.Category; @@ -47,7 +49,6 @@ import org.apache.pulsar.common.functions.Resources; import lombok.Data; -import lombok.experimental.Accessors; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.functions.auth.KubernetesSecretsTokenAuthProvider; import org.apache.pulsar.functions.instance.state.BKStateStoreProviderImpl; @@ -76,6 +77,8 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { @Category private static final String CATEGORY_SECURITY = "Common Security Settings (applied for both worker and client)"; @Category + private static final String CATEGORY_KEYSTORE_TLS = "KeyStoreTLS"; + @Category private static final String CATEGORY_WORKER_SECURITY = "Worker Security Settings"; @Category private static final String CATEGORY_CLIENT_SECURITY = "Security settings for clients talking to brokers"; @@ -381,6 +384,74 @@ public boolean isBrokerClientAuthenticationEnabled() { doc = "Tls cert refresh duration in seconds (set 0 to check on every new connection)" ) private long tlsCertRefreshCheckDurationSec = 300; + + /**** --- KeyStore TLS config variables. --- ****/ + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "Enable TLS with KeyStore type configuration in function worker" + ) + private boolean tlsEnabledWithKeyStore = false; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "Specify the TLS provider for the function worker service: \n" + + "When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.\n" + + "When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc." + ) + private String tlsProvider = null; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "TLS KeyStore type configuration in function worker: JKS, PKCS12" + ) + private String tlsKeyStoreType = "JKS"; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "TLS KeyStore path in function worker" + ) + private String tlsKeyStore = null; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "TLS KeyStore password for function worker" + ) + @ToString.Exclude + private String tlsKeyStorePassword = null; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "TLS TrustStore type configuration in function worker: JKS, PKCS12" + ) + private String tlsTrustStoreType = "JKS"; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "TLS TrustStore path in function worker" + ) + private String tlsTrustStore = null; + + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "TLS TrustStore password for function worker, null means empty password." + ) + @ToString.Exclude + private String tlsTrustStorePassword = null; + + @FieldContext( + category = CATEGORY_WORKER_SECURITY, + doc = "Specify the tls protocols the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLSv1.3, TLSv1.2]" + ) + private Set webServiceTlsProtocols = new TreeSet<>(); + + @FieldContext( + category = CATEGORY_WORKER_SECURITY, + doc = "Specify the tls cipher the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]" + ) + private Set webServiceTlsCiphers = new TreeSet<>(); + @FieldContext( category = CATEGORY_WORKER_SECURITY, doc = "Enforce authentication" diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/WorkerApiV2ResourceConfigTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/WorkerApiV2ResourceConfigTest.java index 6cfaea9e2427a..bf45dd958c823 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/WorkerApiV2ResourceConfigTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/WorkerApiV2ResourceConfigTest.java @@ -25,10 +25,11 @@ import static org.testng.Assert.assertTrue; import java.net.URL; +import java.util.Locale; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.pulsar.functions.auth.KubernetesSecretsTokenAuthProvider; import org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory; -import org.apache.pulsar.functions.worker.WorkerConfig; import org.testng.annotations.Test; /** @@ -121,4 +122,19 @@ public void testLoadResourceRestrictionsConfig() throws Exception { assertTrue(newK8SWc.isFunctionInstanceResourceChangeInLockStep()); } + + @Test + public void testPasswordsNotLeakedOnToString() throws Exception { + URL yamlUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); + WorkerConfig wc = WorkerConfig.load(yamlUrl.toURI().getPath()); + assertFalse(wc.toString().toLowerCase(Locale.ROOT).contains("password"), "Stringified config must not contain password"); + } + + @Test + public void testPasswordsPresentOnObjectMapping() throws Exception { + URL yamlUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); + WorkerConfig wc = WorkerConfig.load(yamlUrl.toURI().getPath()); + assertTrue((new ObjectMapper().writeValueAsString(wc)).toLowerCase(Locale.ROOT).contains("password"), + "ObjectMapper output must include passwords for proper serialization"); + } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index 0f5c402fbceed..ef92e85853b7b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -19,9 +19,6 @@ package org.apache.pulsar.functions.worker; import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import io.netty.util.concurrent.DefaultThreadFactory; @@ -399,12 +396,7 @@ public void start(AuthenticationService authenticationService, workerStatsManager.startupTimeStart(); log.info("/** Starting worker id={} **/", workerConfig.getWorkerId()); - - try { - log.info("Worker Configs: {}", new ObjectMapper().writeValueAsString(workerConfig)); - } catch (JsonProcessingException e) { - log.warn("Failed to print worker configs with error {}", e.getMessage(), e); - } + log.info("Worker Configs: {}", workerConfig); try { DistributedLogConfiguration dlogConf = WorkerUtils.getDlogConf(workerConfig); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java index e531084442466..9747f4b3da48e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java @@ -26,6 +26,7 @@ import org.apache.pulsar.broker.web.JettyRequestLogFactory; import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.functions.worker.rest.api.v2.WorkerApiV2Resource; @@ -122,13 +123,31 @@ private void init() { if (this.workerConfig.getTlsEnabled()) { try { - SslContextFactory sslCtxFactory = SecurityUtility.createSslContextFactory( - this.workerConfig.isTlsAllowInsecureConnection(), this.workerConfig.getTlsTrustCertsFilePath(), - this.workerConfig.getTlsCertificateFilePath(), this.workerConfig.getTlsKeyFilePath(), - this.workerConfig.isTlsRequireTrustedClientCertOnConnect(), - true, - this.workerConfig.getTlsCertRefreshCheckDurationSec()); - httpsConnector = new ServerConnector(server, 1, 1, sslCtxFactory); + SslContextFactory sslCtxFactory; + if (workerConfig.isTlsEnabledWithKeyStore()) { + sslCtxFactory = KeyStoreSSLContext.createSslContextFactory( + workerConfig.getTlsProvider(), + workerConfig.getTlsKeyStoreType(), + workerConfig.getTlsKeyStore(), + workerConfig.getTlsKeyStorePassword(), + workerConfig.isTlsAllowInsecureConnection(), + workerConfig.getTlsTrustStoreType(), + workerConfig.getTlsTrustStore(), + workerConfig.getTlsTrustStorePassword(), + workerConfig.isTlsRequireTrustedClientCertOnConnect(), + workerConfig.getTlsCertRefreshCheckDurationSec() + ); + } else { + sslCtxFactory = SecurityUtility.createSslContextFactory( + workerConfig.isTlsAllowInsecureConnection(), + workerConfig.getTlsTrustCertsFilePath(), + workerConfig.getTlsCertificateFilePath(), + workerConfig.getTlsKeyFilePath(), + workerConfig.isTlsRequireTrustedClientCertOnConnect(), + true, + workerConfig.getTlsCertRefreshCheckDurationSec()); + } + httpsConnector = new ServerConnector(server, sslCtxFactory); httpsConnector.setPort(this.workerConfig.getWorkerPortTls()); connectors.add(httpsConnector); } catch (Exception e) { diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java index 3179bb452a980..9cfed156c3f28 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java @@ -37,6 +37,7 @@ import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -76,15 +77,31 @@ public ProxyServer(WebSocketProxyConfiguration config) // TLS enabled connector if (config.getWebServicePortTls().isPresent()) { try { - SslContextFactory sslCtxFactory = SecurityUtility.createSslContextFactory( - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - true, - config.getTlsCertRefreshCheckDurationSec()); - connectorTls = new ServerConnector(server, -1, -1, sslCtxFactory); + SslContextFactory sslCtxFactory; + if (config.isTlsEnabledWithKeyStore()) { + sslCtxFactory = KeyStoreSSLContext.createSslContextFactory( + config.getTlsProvider(), + config.getTlsKeyStoreType(), + config.getTlsKeyStore(), + config.getTlsKeyStorePassword(), + config.isTlsAllowInsecureConnection(), + config.getTlsTrustStoreType(), + config.getTlsTrustStore(), + config.getTlsTrustStorePassword(), + config.isTlsRequireTrustedClientCertOnConnect(), + config.getTlsCertRefreshCheckDurationSec() + ); + } else { + sslCtxFactory = SecurityUtility.createSslContextFactory( + config.isTlsAllowInsecureConnection(), + config.getTlsTrustCertsFilePath(), + config.getTlsCertificateFilePath(), + config.getTlsKeyFilePath(), + config.isTlsRequireTrustedClientCertOnConnect(), + true, + config.getTlsCertRefreshCheckDurationSec()); + } + connectorTls = new ServerConnector(server, sslCtxFactory); connectorTls.setPort(config.getWebServicePortTls().get()); connectors.add(connectorTls); } catch (Exception e) { diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java index d37d14208fdac..2cd8eb23f7f44 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java @@ -21,16 +21,16 @@ import java.util.Optional; import java.util.Properties; import java.util.Set; - +import java.util.TreeSet; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; import org.apache.pulsar.common.configuration.FieldContext; import org.apache.pulsar.common.configuration.PulsarConfiguration; import com.google.common.collect.Sets; -import lombok.Getter; -import lombok.Setter; - @Getter @Setter public class WebSocketProxyConfiguration implements PulsarConfiguration { @@ -160,6 +160,63 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "TLS cert refresh duration (in seconds). 0 means checking every new connection.") private long tlsCertRefreshCheckDurationSec = 300; + /**** --- KeyStore TLS config variables. --- ****/ + @FieldContext( + doc = "Enable TLS with KeyStore type configuration for WebSocket" + ) + private boolean tlsEnabledWithKeyStore = false; + + @FieldContext( + doc = "Specify the TLS provider for the WebSocket service: \n" + + "When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.\n" + + "When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc." + ) + private String tlsProvider = null; + + @FieldContext( + doc = "TLS KeyStore type configuration in WebSocket: JKS, PKCS12" + ) + private String tlsKeyStoreType = "JKS"; + + @FieldContext( + doc = "TLS KeyStore path in WebSocket" + ) + private String tlsKeyStore = null; + + @FieldContext( + doc = "TLS KeyStore password for WebSocket" + ) + @ToString.Exclude + private String tlsKeyStorePassword = null; + + @FieldContext( + doc = "TLS TrustStore type configuration in WebSocket: JKS, PKCS12" + ) + private String tlsTrustStoreType = "JKS"; + + @FieldContext( + doc = "TLS TrustStore path in WebSocket" + ) + private String tlsTrustStore = null; + + @FieldContext( + doc = "TLS TrustStore password for WebSocket, null means empty password." + ) + @ToString.Exclude + private String tlsTrustStorePassword = null; + + @FieldContext( + doc = "Specify the tls protocols the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLSv1.3, TLSv1.2]" + ) + private Set webServiceTlsProtocols = new TreeSet<>(); + + @FieldContext( + doc = "Specify the tls cipher the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]" + ) + private Set webServiceTlsCiphers = new TreeSet<>(); + @FieldContext(doc = "Key-value properties. Types are all String") private Properties properties = new Properties(); } From a118ab953ede75d6ecfdf32aff95595f270486a1 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Sun, 1 May 2022 12:35:46 +0800 Subject: [PATCH 518/823] [improve][broker-web&websocket&proxy&function-worker] Full-support set ssl provider, ciphers and protocols (#13740) Fixes #13734 Pulsar doesn't set ssl provider, ciphers and protocols to the web, websocket and proxy service when `tlsEnabledWithKeyStore=false` - Add `org.apache.pulsar.jetty.tls` package in pulsar-broker-common for Jetty TLS support - Add a new `webServiceTlsProvider=Conscrypt` to broker and proxy config - Update `Conscrypt` as the `tlsProvider` value in websocket config In the old version, we implicitly use the `Conscrypt` provider, now we need to set it explicitly. (cherry picked from commit bf15e83a3f19b698c1492071792045e47824675c) --- conf/broker.conf | 3 + conf/functions_worker.yml | 6 +- conf/proxy.conf | 9 +- conf/standalone.conf | 3 + conf/websocket.conf | 8 +- .../pulsar/broker/ServiceConfiguration.java | 21 ++ .../org/apache/pulsar/jetty/package-info.java | 19 ++ .../jetty/tls/JettySslContextFactory.java | 116 +++++++++++ .../apache/pulsar/jetty/tls/package-info.java | 19 ++ .../jetty/tls/JettySslContextFactoryTest.java | 174 ++++++++++++++++ ...ettySslContextFactoryWithKeyStoreTest.java | 193 ++++++++++++++++++ .../test/resources/ssl/jetty_client_key.jks | Bin 0 -> 2679 bytes .../test/resources/ssl/jetty_client_trust.jks | Bin 0 -> 1207 bytes .../test/resources/ssl/jetty_server_key.jks | Bin 0 -> 2679 bytes .../test/resources/ssl/jetty_server_trust.jks | Bin 0 -> 1207 bytes .../src/test/resources/ssl/my-ca/ca.pem | 18 ++ .../test/resources/ssl/my-ca/client-ca.pem | 19 ++ .../test/resources/ssl/my-ca/client-key.pem | 28 +++ .../test/resources/ssl/my-ca/server-ca.pem | 19 ++ .../test/resources/ssl/my-ca/server-key.pem | 28 +++ .../apache/pulsar/broker/PulsarService.java | 2 + .../apache/pulsar/broker/web/WebService.java | 16 +- .../proxy/ProxyPublishConsumeTlsTest.java | 2 +- pulsar-common/pom.xml | 5 - .../common/util/DefaultSslContextBuilder.java | 23 ++- .../pulsar/common/util/SecurityUtility.java | 74 +++---- .../util/keystoretls/KeyStoreSSLContext.java | 57 +----- .../SslContextFactoryWithAutoRefresh.java | 66 ------ .../functions/worker/rest/WorkerServer.java | 16 +- .../proxy/server/AdminProxyHandler.java | 14 +- .../proxy/server/ProxyConfiguration.java | 21 ++ .../apache/pulsar/proxy/server/WebServer.java | 15 +- .../pulsar/websocket/service/ProxyServer.java | 13 +- .../service/WebSocketProxyConfiguration.java | 6 +- 34 files changed, 791 insertions(+), 222 deletions(-) create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/package-info.java create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/package-info.java create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java create mode 100644 pulsar-broker-common/src/test/resources/ssl/jetty_client_key.jks create mode 100644 pulsar-broker-common/src/test/resources/ssl/jetty_client_trust.jks create mode 100644 pulsar-broker-common/src/test/resources/ssl/jetty_server_key.jks create mode 100644 pulsar-broker-common/src/test/resources/ssl/jetty_server_trust.jks create mode 100644 pulsar-broker-common/src/test/resources/ssl/my-ca/ca.pem create mode 100644 pulsar-broker-common/src/test/resources/ssl/my-ca/client-ca.pem create mode 100644 pulsar-broker-common/src/test/resources/ssl/my-ca/client-key.pem create mode 100644 pulsar-broker-common/src/test/resources/ssl/my-ca/server-ca.pem create mode 100644 pulsar-broker-common/src/test/resources/ssl/my-ca/server-key.pem delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/SslContextFactoryWithAutoRefresh.java diff --git a/conf/broker.conf b/conf/broker.conf index cfd1e9ef5f96b..b67afa89c0bbe 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -599,6 +599,9 @@ tlsRequireTrustedClientCertOnConnect=false # When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc. tlsProvider= +# Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc. +webServiceTlsProvider=Conscrypt + ### --- KeyStore TLS config variables --- ### ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 06204dfdcfbec..0b228d26a58e2 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -305,11 +305,11 @@ tlsCertRefreshCheckDurationSec: 300 # certificate isn't trusted. tlsRequireTrustedClientCertOnConnect: false -### --- KeyStore TLS config variables --- ### +### --- TLS config variables --- ### ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. -# TLS Provider for KeyStore type -tlsProvider: +# Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc. +tlsProvider: Conscrypt # Enable TLS with KeyStore type configuration in function worker. tlsEnabledWithKeyStore: false diff --git a/conf/proxy.conf b/conf/proxy.conf index fbdcad2b2b3e0..4f98663afc175 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -68,12 +68,17 @@ webServicePort=8080 # Port to use to server HTTPS request webServicePortTls= -### --- KeyStore TLS config variables --- ### +### --- TLS config variables --- ### ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. -# TLS Provider for KeyStore type +# Specify the TLS provider for the broker service: +# When using TLS authentication with CACert, the valid value is either OPENSSL or JDK. +# When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc. tlsProvider= +# Specify the TLS provider for the web service, available values can be SunJSSE, Conscrypt and etc. +webServiceTlsProvider=Conscrypt + # Enable TLS with KeyStore type configuration in proxy. tlsEnabledWithKeyStore=false diff --git a/conf/standalone.conf b/conf/standalone.conf index 16312125a0254..01030d049d488 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -362,6 +362,9 @@ tlsRequireTrustedClientCertOnConnect=false # When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc. tlsProvider= +# Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc. +webServiceTlsProvider=Conscrypt + ### --- KeyStore TLS config variables --- ### # Enable TLS with KeyStore type configuration in broker. tlsEnabledWithKeyStore=false diff --git a/conf/websocket.conf b/conf/websocket.conf index c2a138e7b38fa..535fade4ea65a 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -96,6 +96,7 @@ brokerClientTrustCertsFilePath= anonymousUserRole= ### --- TLS --- ### +## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. # Deprecated - use webServicePortTls and brokerClientTlsEnabled instead tlsEnabled=false @@ -119,11 +120,8 @@ tlsRequireTrustedClientCertOnConnect=false # Tls cert refresh duration in seconds (set 0 to check on every new connection) tlsCertRefreshCheckDurationSec=300 -### --- KeyStore TLS config variables --- ### -## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. - -# TLS Provider for KeyStore type -tlsProvider= +# Specify the TLS provider for the WebSocket: SunJSSE, Conscrypt and etc. +tlsProvider=Conscrypt # Enable TLS with KeyStore type configuration in WebSocket. tlsEnabledWithKeyStore=false diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 250d7d41ffa79..5370971854570 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.Setter; @@ -144,6 +145,26 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private Optional webServicePortTls = Optional.empty(); + @FieldContext( + category = CATEGORY_SERVER, + doc = "Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc." + ) + private String webServiceTlsProvider = "Conscrypt"; + + @FieldContext( + category = CATEGORY_TLS, + doc = "Specify the tls protocols the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLSv1.3, TLSv1.2]" + ) + private Set webServiceTlsProtocols = new TreeSet<>(); + + @FieldContext( + category = CATEGORY_TLS, + doc = "Specify the tls cipher the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]" + ) + private Set webServiceTlsCiphers = new TreeSet<>(); + @FieldContext( category = CATEGORY_SERVER, doc = "Hostname or IP address the service binds on" diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/package-info.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/package-info.java new file mode 100644 index 0000000000000..f01bd9198f568 --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.jetty; diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java new file mode 100644 index 0000000000000..514fa4a07250e --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.jetty.tls; + +import java.util.Set; +import javax.net.ssl.SSLContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.DefaultSslContextBuilder; +import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.keystoretls.NetSslContextBuilder; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +@Slf4j +public class JettySslContextFactory { + static { + // DO NOT EDIT - Load Conscrypt provider + if (SecurityUtility.CONSCRYPT_PROVIDER != null) { + } + } + + public static SslContextFactory.Server createServerSslContextWithKeystore(String sslProviderString, + String keyStoreTypeString, + String keyStore, + String keyStorePassword, + boolean allowInsecureConnection, + String trustStoreTypeString, + String trustStore, + String trustStorePassword, + boolean requireTrustedClientCertOnConnect, + Set ciphers, + Set protocols, + long certRefreshInSec) { + NetSslContextBuilder sslCtxRefresher = new NetSslContextBuilder( + sslProviderString, + keyStoreTypeString, + keyStore, + keyStorePassword, + allowInsecureConnection, + trustStoreTypeString, + trustStore, + trustStorePassword, + requireTrustedClientCertOnConnect, + certRefreshInSec); + + return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher, + requireTrustedClientCertOnConnect, ciphers, protocols); + } + + public static SslContextFactory createServerSslContext(String sslProviderString, boolean tlsAllowInsecureConnection, + String tlsTrustCertsFilePath, + String tlsCertificateFilePath, + String tlsKeyFilePath, + boolean tlsRequireTrustedClientCertOnConnect, + Set ciphers, + Set protocols, + long certRefreshInSec) { + DefaultSslContextBuilder sslCtxRefresher = + new DefaultSslContextBuilder(tlsAllowInsecureConnection, tlsTrustCertsFilePath, tlsCertificateFilePath, + tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, certRefreshInSec, sslProviderString); + + return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher, + tlsRequireTrustedClientCertOnConnect, ciphers, protocols); + } + + private static class Server extends SslContextFactory.Server { + private final SslContextAutoRefreshBuilder sslCtxRefresher; + + public Server(String sslProviderString, SslContextAutoRefreshBuilder sslCtxRefresher, + boolean requireTrustedClientCertOnConnect, Set ciphers, Set protocols) { + super(); + this.sslCtxRefresher = sslCtxRefresher; + + if (ciphers != null && ciphers.size() > 0) { + this.setIncludeCipherSuites(ciphers.toArray(new String[0])); + } + + if (protocols != null && protocols.size() > 0) { + this.setIncludeProtocols(protocols.toArray(new String[0])); + } + + if (sslProviderString != null && !sslProviderString.equals("")) { + setProvider(sslProviderString); + } + + if (requireTrustedClientCertOnConnect) { + this.setNeedClientAuth(true); + this.setTrustAll(false); + } else { + this.setWantClientAuth(true); + this.setTrustAll(true); + } + } + + @Override + public SSLContext getSslContext() { + return sslCtxRefresher.get(); + } + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/package-info.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/package-info.java new file mode 100644 index 0000000000000..8978699ff7926 --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.jetty.tls; diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java new file mode 100644 index 0000000000000..c1816674880a7 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.jetty.tls; + +import com.google.common.io.Resources; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.pulsar.common.util.SecurityUtility; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.testng.annotations.Test; + +@Slf4j +public class JettySslContextFactoryTest { + + @Test + public void testJettyTlsServerTls() throws Exception { + Server server = new Server(); + List connectors = new ArrayList<>(); + SslContextFactory factory = JettySslContextFactory.createServerSslContext( + null, + false, + Resources.getResource("ssl/my-ca/ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-key.pem").getPath(), + true, + null, + null, + 600); + + ServerConnector connector = new ServerConnector(server, factory); + connector.setPort(0); + connectors.add(connector); + server.setConnectors(connectors.toArray(new ServerConnector[0])); + server.start(); + // client connect + HttpClientBuilder httpClientBuilder = HttpClients.custom(); + RegistryBuilder registryBuilder = RegistryBuilder.create(); + registryBuilder.register("https", + new SSLConnectionSocketFactory(getClientSslContext(), new NoopHostnameVerifier())); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); + httpClientBuilder.setConnectionManager(cm); + CloseableHttpClient httpClient = httpClientBuilder.build(); + HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); + httpClient.execute(httpGet); + httpClient.close(); + server.stop(); + } + + @Test(expectedExceptions = SSLHandshakeException.class) + public void testJettyTlsServerInvalidTlsProtocol() throws Exception { + Server server = new Server(); + List connectors = new ArrayList<>(); + SslContextFactory factory = JettySslContextFactory.createServerSslContext( + null, + false, + Resources.getResource("ssl/my-ca/ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-key.pem").getPath(), + true, + null, + new HashSet() { + { + this.add("TLSv1.3"); + } + }, + 600); + factory.setHostnameVerifier((s, sslSession) -> true); + ServerConnector connector = new ServerConnector(server, factory); + connector.setPort(0); + connectors.add(connector); + server.setConnectors(connectors.toArray(new ServerConnector[0])); + server.start(); + // client connect + HttpClientBuilder httpClientBuilder = HttpClients.custom(); + RegistryBuilder registryBuilder = RegistryBuilder.create(); + registryBuilder.register("https", new SSLConnectionSocketFactory(getClientSslContext(), + new String[]{"TLSv1.2"}, null, new NoopHostnameVerifier())); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); + httpClientBuilder.setConnectionManager(cm); + CloseableHttpClient httpClient = httpClientBuilder.build(); + HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); + httpClient.execute(httpGet); + httpClient.close(); + server.stop(); + } + + @Test(expectedExceptions = SSLHandshakeException.class) + public void testJettyTlsServerInvalidCipher() throws Exception { + Server server = new Server(); + List connectors = new ArrayList<>(); + SslContextFactory factory = JettySslContextFactory.createServerSslContext( + null, + false, + Resources.getResource("ssl/my-ca/ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), + Resources.getResource("ssl/my-ca/server-key.pem").getPath(), + true, + new HashSet() { + { + this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + } + }, + new HashSet() { + { + this.add("TLSv1.2"); + } + }, + 600); + + factory.setHostnameVerifier((s, sslSession) -> true); + ServerConnector connector = new ServerConnector(server, factory); + connector.setPort(0); + connectors.add(connector); + server.setConnectors(connectors.toArray(new ServerConnector[0])); + server.start(); + // client connect + HttpClientBuilder httpClientBuilder = HttpClients.custom(); + RegistryBuilder registryBuilder = RegistryBuilder.create(); + registryBuilder.register("https", new SSLConnectionSocketFactory(getClientSslContext(), + new String[]{"TLSv1.2"}, new String[]{"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"}, + new NoopHostnameVerifier())); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); + httpClientBuilder.setConnectionManager(cm); + CloseableHttpClient httpClient = httpClientBuilder.build(); + HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); + httpClient.execute(httpGet); + httpClient.close(); + server.stop(); + } + + private static SSLContext getClientSslContext() throws GeneralSecurityException, IOException { + return SecurityUtility.createSslContext( + false, + Resources.getResource("ssl/my-ca/ca.pem").getPath(), + Resources.getResource("ssl/my-ca/client-ca.pem").getPath(), + Resources.getResource("ssl/my-ca/client-key.pem").getPath(), + null + ); + } +} diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java new file mode 100644 index 0000000000000..292bd123fdfaf --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.jetty.tls; + +import com.google.common.io.Resources; +import java.io.FileInputStream; +import java.security.KeyStore; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.TrustManagerFactory; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.testng.annotations.Test; + +@Slf4j +public class JettySslContextFactoryWithKeyStoreTest { + + @Test + public void testJettyTlsServerTls() throws Exception { + Server server = new Server(); + List connectors = new ArrayList<>(); + SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, + "JKS", Resources.getResource("ssl/jetty_server_key.jks").getPath(), + "jetty_server_pwd", false, "JKS", + Resources.getResource("ssl/jetty_server_trust.jks").getPath(), + "jetty_server_pwd", true, null, + null, 600); + factory.setHostnameVerifier((s, sslSession) -> true); + ServerConnector connector = new ServerConnector(server, factory); + connector.setPort(0); + connectors.add(connector); + server.setConnectors(connectors.toArray(new ServerConnector[0])); + server.start(); + // client connect + HttpClientBuilder httpClientBuilder = HttpClients.custom(); + RegistryBuilder registryBuilder = RegistryBuilder.create(); + registryBuilder.register("https", + new SSLConnectionSocketFactory(getClientSslContext(), new NoopHostnameVerifier())); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); + httpClientBuilder.setConnectionManager(cm); + CloseableHttpClient httpClient = httpClientBuilder.build(); + HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); + httpClient.execute(httpGet); + httpClient.close(); + server.stop(); + } + + @Test(expectedExceptions = SSLHandshakeException.class) + public void testJettyTlsServerInvalidTlsProtocol() throws Exception { + Configurator.setRootLevel(Level.INFO); + Server server = new Server(); + List connectors = new ArrayList<>(); + SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, + "JKS", Resources.getResource("ssl/jetty_server_key.jks").getPath(), + "jetty_server_pwd", false, "JKS", + Resources.getResource("ssl/jetty_server_trust.jks").getPath(), + "jetty_server_pwd", true, null, + new HashSet() { + { + this.add("TLSv1.3"); + } + }, 600); + factory.setHostnameVerifier((s, sslSession) -> true); + ServerConnector connector = new ServerConnector(server, factory); + connector.setPort(0); + connectors.add(connector); + server.setConnectors(connectors.toArray(new ServerConnector[0])); + server.start(); + // client connect + HttpClientBuilder httpClientBuilder = HttpClients.custom(); + RegistryBuilder registryBuilder = RegistryBuilder.create(); + registryBuilder.register("https", new SSLConnectionSocketFactory(getClientSslContext(), + new String[]{"TLSv1.2"}, null, new NoopHostnameVerifier())); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); + httpClientBuilder.setConnectionManager(cm); + CloseableHttpClient httpClient = httpClientBuilder.build(); + HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); + httpClient.execute(httpGet); + httpClient.close(); + server.stop(); + } + + @Test(expectedExceptions = SSLHandshakeException.class) + public void testJettyTlsServerInvalidCipher() throws Exception { + Server server = new Server(); + List connectors = new ArrayList<>(); + SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, + "JKS", Resources.getResource("ssl/jetty_server_key.jks").getPath(), + "jetty_server_pwd", false, "JKS", + Resources.getResource("ssl/jetty_server_trust.jks").getPath(), + "jetty_server_pwd", true, new HashSet() { + { + this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + } + }, + new HashSet() { + { + this.add("TLSv1.2"); + } + }, 600); + factory.setHostnameVerifier((s, sslSession) -> true); + ServerConnector connector = new ServerConnector(server, factory); + connector.setPort(0); + connectors.add(connector); + server.setConnectors(connectors.toArray(new ServerConnector[0])); + server.start(); + // client connect + HttpClientBuilder httpClientBuilder = HttpClients.custom(); + RegistryBuilder registryBuilder = RegistryBuilder.create(); + registryBuilder.register("https", new SSLConnectionSocketFactory(getClientSslContext(), + new String[]{"TLSv1.2"}, new String[]{"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"}, + new NoopHostnameVerifier())); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); + httpClientBuilder.setConnectionManager(cm); + CloseableHttpClient httpClient = httpClientBuilder.build(); + HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); + httpClient.execute(httpGet); + httpClient.close(); + server.stop(); + } + + private static SSLContext getClientSslContext() { + return getSslContext(Resources.getResource("ssl/jetty_client_key.jks").getPath(), + "jetty_client_pwd", + Resources.getResource("ssl/jetty_client_trust.jks").getPath(), + "jetty_client_pwd"); + } + + private static SSLContext getSslContext(String keyStorePath, String keyStorePassword, + String trustStorePath, String trustStorePassword) { + try { + SSLContext sslContext = SSLContext.getInstance("TLS"); + // key store + KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + KeyStore keyStore = KeyStore.getInstance("JKS"); + try (FileInputStream inputStream = new FileInputStream(keyStorePath)) { + keyStore.load(inputStream, keyStorePassword.toCharArray()); + } + keyManagerFactory.init(keyStore, keyStorePassword.toCharArray()); + KeyManager[] keyManagers = keyManagerFactory.getKeyManagers(); + // trust store + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + KeyStore trustStore = KeyStore.getInstance("JKS"); + try (FileInputStream inputStream = new FileInputStream(trustStorePath)) { + trustStore.load(inputStream, trustStorePassword.toCharArray()); + } + trustManagerFactory.init(trustStore); + sslContext.init(keyManagers, trustManagerFactory.getTrustManagers(), new SecureRandom()); + return sslContext; + } catch (Exception e) { + log.error("load ssl context error ", e); + return null; + } + } + +} \ No newline at end of file diff --git a/pulsar-broker-common/src/test/resources/ssl/jetty_client_key.jks b/pulsar-broker-common/src/test/resources/ssl/jetty_client_key.jks new file mode 100644 index 0000000000000000000000000000000000000000..2b8ea64347ddc536209770871a3629a62e806b04 GIT binary patch literal 2679 zcma)+c{J1u8^>qH%$Q+FmaJK_T+7gpGDC*!%h)1Z*+RA{WQ)i;cTAa5A>lS-Np=(2 z)r2TS!ia1oyM`#tgmk@~^S~?o&*%GmzkhxKB2Od)%mom6&~T(G*^>O7 z56lfNB=Q`E5_!abVG)1`tNt$uMuQSzdB1S>ugQS({>O#l0YeIjFvDLE2L$}>fbjw6 z0G)qM6rctb?(Jm`w&pdWD`__usFO&%hhSMa1oE2z1S|`n;D~=GLXa>Ja1@U8CR>92 zxu9TGsDQ;l0;cFo?FM$EYqFP0BdOQ zOW2Hc$=;m$0C^bRSZl3fEO;l&2s5JHyEm2{!fO*0THA6&$|mJUHHYCPsG4s=^7)gK zoZ{8nam4&_g$GSLyevUJRY`AuovLjT-!LlGJ9OH*Q*vE&eWi_1BXry|Sz7F;a&0yA zQ(s4V(7f$I!CPzI8|jv2%W3y*B(QdGBF;%zQ}k>$8kM)zc3=}UXe;ogMXKv>q~(^a>#WN9j$ zG-M6PdmVnfS1SLFTJ(@^#o9f~WQUsmmgFJI?cbJ{;vy}>H%#ajU4P`!!+pmUbU8`e zhc2lW)bGcKOT2haTIPTOzhxV)+_cC>1Vo`9nAcsc65^0QDzQ|@euELH}pPzW6?DY_xNg`rMG>Pe*&e| zbsLj!VYL|VvOa1$xkA&hz{qAXQpXf$xVtOhp~;#``vI&F;~_Mn-SEZU+asl|p27S9 z8%VTwoB9@Sq`#On|93t!$qDqJ5@9tPDVo+D`PTE`a73gm(M7<0=ldMHSR$|y>r66g zB1|WBo%yMqQB4p^hv-LHUCshs@I)*gh*!U082}z4VBouwYw$ zR)kFQau3;4i#0zw*q#XpYajO}!NROW~6{VO4=o-!qZcmN&9_8n`8K=+f)IT6ijJJ}<;8?@-* z`f(s;zEDsu$ZfT9Z7U&}`99UKAk1N^>f&NYE}sPuOkF8Cv_+COa^kpinjaSHS4nDA zwxVY!Uhqt&-|LSj$BoLo(K964putKVh)TB;7?uyDOI!<{t@y${qHTp2u;L3ff8au~ z+xHlwtuII%xRhoWof=;EWU=PuxUF-OcQ(a;q`BhM6pb1Vi~~=A%pLD1`e8E=z4LahXJTapqc1TI{oLc(_$?W89?rL-9dJ@$x<(TCZTnw&#zVOvehNeC6 z^|DBg5MBRL@gpgx)yR2}OJ=dP+x9WsOk$Hkfbff(WF8GlF8<}F{i)mi`b$if^cxC& zL(>2d|GO@c!kQ>)9#9A<7IX>Z4~hlk{*gp9g`t8@gb;sOP0UFhO&ttYO9u-801@i* z_Y*g~kO+1Bh3p_;(643jj{^Q*C9U>nO9xX*|H7^Kpr`s@4N+EoKhpl6k~Z$cCx?C< zX~HV8M=umjI3^RhhN`6)+|T^nf#Z5}rZ*PbjWevmFrx}XmI@E5kJFk4refCI)79;W z`~wW%A_jJO3fij$k0?G6o78JDq$R0+9GB#IZ?zFB{;l+d!sc=C{Lp)wb=mTV2(~lf zd%5tyEB*mLF+>PnaTiKT)vU{Yy~d9S-c0z$#W{bf{NlE!G~Yxz#Po~y7Ydh8HO=Cs zufvxB<}FIfx>K}{-LdpGt@3p-lt>h+{Q+dCgud|ekicK(GU3HafzshO{U#PZ4Cfks zzNQ=z2JS2n2-Zs><-|?In^Xl~|9zS_W!AWNw}4-@>XJH%3xnsZ1oqfob&HCdS8mWa z{OX;PeBIg&zolM#V@!Hd8!Th4rCMFyq|y5NSF#KxMfot#y=*v^@QKhJuAe1_;h>eF zA9Hl{lkb7dt0Q+t6%rKLQAalVn$(`f`80@=;*@<$%Chx1;*2SW^my&Dfwasg*rr@w zW*=7_%jXmQbL+6nSUJ8~MRhLsGGUcMp2|xWO_#|g)ok?~&y9Oqww&71d}R@P?u5^q zFeLcI{rguAsbbnr&;M*0e%LR^9OxJhpnf`Os_C~?bj5i`d+kZz-P@wJ#E~G2TKTo| zn$Ee!){KtNG-OI^AdD`p8R-wPkAJ(;YjS_>(au8MwMP(e+@1!QENz zQT=$w6$jf1IhA$xtHuz7B4)2-RHZchN99rVxNY3IH1vfRPd$|0KW}ml@+&$Y+J~z- zA!O-Ta1H6IG({E1g}ETNP4ydS_u(M=TRd>zZ=mz7~; zzDlv@U%3_XP+8*XW-_uQp`I8SklWk)juwrxp9LV%@xfPpwRXeq>6Tzw-H&f7!lQad ztnI_YP52g?HtI{=-f7gC`a5ux6`?k3CU}-&)X6M9_MvwLgnBlCH~jI1-slxO^0b3c z9vyN=`IckGzR4LYR07qZtzYNFhv47Z;1(f_juXWTPH#MxLh;ipSB1%;XnnPO$&o$& z=HEs3jK6DlkSY!KYO!r0W}RAfDsaby`;60|+l&3X#4fY9SZaN|edJ%^8g_4*yKo9hB0UwLkk>X_cg1#}x>sw+RSQap z@)k;sbCS=yZzmQGYv}Haeb~B1Uye!{%9>}Fs2)=hYH|0SbQ5!#_PVIIUTDhk44UrG zo_+BJFaYELBpjy3&jpcyfuO>OwKJQG@jt literal 0 HcmV?d00001 diff --git a/pulsar-broker-common/src/test/resources/ssl/jetty_client_trust.jks b/pulsar-broker-common/src/test/resources/ssl/jetty_client_trust.jks new file mode 100644 index 0000000000000000000000000000000000000000..166a2e00fb371dd160b46c58c7fdda651b80476d GIT binary patch literal 1207 zcmV;o1W5ZZf&{Yy0Ru3C1Z)NgDuzgg_YDCD0ic2eT?B#zSulbGRWO1CQ3eSrhDe6@ z4FLxRpn?QKFoFa=0s#Opf&@1P2`Yw2hW8Bt2LUi<1_>&LNQUx*>HEztr3ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q5^M!Qal>5Fu9lG2=y3IQM{p1Hj5fkrUO$PsVgh_z>aVy6Owl z_B9+#ZC|>KfcUL|7_CY&aO4xIf|96gWjHehp;Y4!>sS@?EWjQ=d+XsZg^7dp^TeY- zzmjwZ9@gMR;!r&giVFFc?)qDf>!lsdCxaBa+rIK=D%ZxSF@q;=}sH zU`O19^Av45I@o%cH&)LRjP0uu45d#_5!XkhqG@7f7r*OlNP|c%>f_<5_}+};6V)Ye zo<&eUjqm%MmS6YE*Eo86+8~=D9_0!c34C{~kf;Ik0n2H34li!D_#G%HS5EXBPa4g1 zMZbWlwCjx`h<`-x=4tGu75Mxk3(#Z84s zDUF@87lhI{F91%P>^+1ypM3H?3ir^XTWH#SfMogz_CvvQTOondxHfV=Mc!=BF@qE1 z+wFg<&CWA^go4=Eq76dLZn3Y69?Q3+Us-dr-jn^MvD> zth`d6)$l^5<1*&malqV@mqlV3Aw%p<8c2hSEPo3oLUzYf*nhxH6!dw2KB-v(ZJF~j zgI~)+$p&SJ%m!Z`1=CSOu?$E(F|qkajvaE1toh*9V(7iR+j^&|mzeX{a)Ed7?+r?Q z>IpT!G`B9+XGP=uj+ZiF8k+HF2^-*-Ph^^l5{+)6_0oLnGnN!U*enujJW)y>3wQz2 z;f5ykk#&1Ld=Ooq6@lzy^K}rOV#U|&>&8T{xFg~Wj|Vr@Eo$U&!2IKra#PjkEK$)S_Fi%(FtbThGYxZF5Z`UpSYg?e2E)u!vTys ze;3jUqo#IR=b`1Pf;YU|nS!fyv?ss>JBZ&EA&9{AB-j1(#1-J8QE&}mmGv0I*j9+L ze@zWFisk;5)z#5mRS*q6Dv4u?poAuood3M$$9g4CmOT$O&@YwLl;|2%=|;o8+emn9ebOhnC?~#}$I^h@!r!!V#h5^OS=4h@@+Rx&ql41i_-s*j zNR#b7c`+Vl@obamJi@l89PjWCP^gsBpoZ_fZe5gs{MoW~Qe*y3wWxU=YUB_kDt!Tg zFkP!&rzfLU$z!l8PPH3s6n)}QGnb}TW4pQ4tBiBYKnSwgXOZ+Fv!V858yjfYFG4qK zc=86Fmswe&6(HIi;SGFvXHjAYapq$y7DoYcNW3>e4KzOF50RjU7hM@CGG!p;- literal 0 HcmV?d00001 diff --git a/pulsar-broker-common/src/test/resources/ssl/jetty_server_key.jks b/pulsar-broker-common/src/test/resources/ssl/jetty_server_key.jks new file mode 100644 index 0000000000000000000000000000000000000000..b6189b75c8ad01656e2d5e9102de21d82183b61f GIT binary patch literal 2679 zcma)+X*d*&7RP6pF$~#Fq~@K3!Jx5Z8)e_JXJ;5?DO<>vEsQZkVXWCT$@1EgY?Uly zA4{^NM7CmNDe4v3=6arcpZ9(4r+Yt~=RD{4KmYULeE6gB&>$utGa3(-W?_>e8WML; z0UU9%^P@acFxtCpe;S$+O7Xt$>GBF{!nScm12Mg=JFEX)#0qAoqY_3E@ zpa(MuC%W^VX`cow3oFs zo><#J99M7b-8cJe736RVi8p6?H;e0!c2%V{3MKW9k@>K{>ooT(Mq>S=E(e)=(vGU~ z7O1DD%$vW;Bv~wMwQXoGULh955gQEPQ6MZMs=qH@6* z5uS$`^Iw2p$PR7A#4+$f?gyZA@{10{Tw$IoCuQRkN_|w?L^TnkXPiHB>PiPU#@~5Y z!9mF1LG@fNpYL=2QUdU?`?19gmN(@u`&sKl1Knp_h3*+SJKr@yAU*o~5!%{BgT*sYXC1;_qRcBeNkSefY;KcqnX_c5C<#P9as7(m8& zr6jy+>7rW8w3jxFeSsYtc9KT#2lwF~QVzDb?MM=Y_4@9VG7b#L9%xeMN9a#Ih9(Nl zuaqPVyLu)|ioncWPmAZv@eFd4Y6Im?(xT{(XNB|5NH4qo>@^X~B33{6j+ zj3o!x8}PX~jjA(o`4!)!9j5CwOS#AAyjA|SH~xn$vu^d4(kw%gQ1^W1o*~;FOR|`? zP~|H7wlYqCQ8lBP;|8x4#(w6_=irHkEA_9pNo9p&DFVuZ+0psqzoX&R%!%#6eeAj= zP~4)>1XvVo$P0^&pbw?Oja-m=*j-|KAZ$KI=mEm77`QEJqL{raHVK;TQcHVE?MB*U`f}%s>8wj9#+AFw|u+6 zI#Z>9su`^hZOZN^zkcey#8J7%TXPCg{lEiZ_T6TbIyVI@IDc>YMcA`!R6~^!&%!VH zio01fD{z?hTCFLJlh1PRrOLs|#a+r%vb^bJhbUV4K+=iujYH&B7=hlrraC)j5_?Gd zXzgqnx)ky(pptTx3p!r^mtkITP$|AaKs zb0_Pdzc&>U7_obEXsIt2&-_-qA200c`=i=p@SQ8t;*`ZIIh>LIX#c?$yNnxGx-@CG9V!L7FJ4KSuGqX&B+6|BaW5Xv0*68j*#pS{@pz^mx^gY>19M-NN5 zrdu}_)JjON(q`$K^JzTBGCk{OU{XGDln_a=ZOJVYd-bwwdgM6`Xzk@vK0_bGO}5&f z-SY^}80&9+E6cvf4C|N6SgZZ)jFgHB`URl|)_g2t(QFvM4E%wr2k@tr=I)Yv3MDkj zO723hARR^mkM0}vU$|-?shl}5+{NKKz5Oir;wo-DkB>RB79(V7*$qdk9Xq)lpD||0 zX5wlNM^%&J$5&;GhCxo#+$ymdAF4nHrK$p*-|1&GQq1?KpO^nwJ+~?`cS(8qDOz59 zYQ1ob{l2g!Fr5J-8DB2^^Okl{;#SFWh`+sNXVajcdQ9dfY8KvR;nsnw$JJ@Xs8(&g zKiV3RqlNdu3&TGj1Sug??R$EGAzGOng-@KoJbojDry?5aN>~q;7s)v6(}4 zd75i6-ME&)pkQs#EsOfpHa9`ZY>YXM;OHU6<b~0Kfjx$$TqxRP9Q?laVx({SEw|p zsq>Yr49M9W-J9Z$2Ftt)w{0&O{Vod*@~smt_}FRP`sJ(ILS&W8ed8r*tMoXBF?ODy z?E+S1Yj(=r+0^BD!^bawE!w`X;m+t+-3Y+2P7IL1*FrdD9J`b59n+Qzg~>4fytnR7 zd_FG%;)6q&#__}pRBW_!=uGWOwKEb49x(9eJ z()s{(+pgj?)%NF+lJ3#n!k8a>kGF<>*e&j%B|2rN9Nu9^s?~(t<36b*!Z0=F$ZD{M zc;dmmOJ&^l9+c01mQrS6;DJsJ1)qpA_p4#>A+8$R>BAU)S~C}p=2qb)=dwVE3PgCy zWT$&(%A&8J&!gE`z$i{;rZZpwh`%^7JT?I}quq=3p8No(`+aSzJp}R3&tU&3V+y$Q VG@ehyINfadmm$iEi5W&LNQU1{2WbhzDFg0%tyClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q6%HmdJvi=!x)|)(Hp2!0dp61HeN(zLVt~!UK$+y>FT6hquWc zr{0$V-CjGyITpE#ztPRgd(`p)YFNC@C_yu^0N01Xe*q8KQ;UmCk%nnl+PypV^2qf+ zR5yLpp)DMAmu}*~$K;662hCCZ_{m5P2>T^RxB|+ZzD-qP4)!Vf&Fe=T+xF@JBdWoF z{S1eg(YE3~A(q!fiujgl`J!f@`vN?5e2(Wnr{@>fso0GN%?f~^he4CIysbR?m{@;T z)lx{Nv%{~>sw_e}Vu z;OD|xTR{ud@3X3lmr#6Ro};!0(F_X4b$)6B2CCVUvrbIJI%)!`k3jnD-^f9Hn&cc$Ef*nIU{&(ehk z4R22VpYJy+S=az>ufLVyHVq|?fZZWknJoD@*scGvYVB122|{OxC-%g;Spmd?ygmT& z1UuMmhg!~l@A_`k$I(#?v*m2MZNoK?deqnSWE-bKmMedv9-u*ob~U|IwIh`Ueg_MH zeXQjav8gD=;3htb*_c^RFSCP}^|s%9NRC#c)!eor%_kzk)yd}&&o)?oP{(!!!N~pu3E;* zY2Ww4*mK2JS_z11Dc^y@vC#}G7yiV?JYXeTBoq<&&RHuhLchZfDRcTptBPE03vJ(o zg<1-94BM}s)}9%Pf<9eXy6+H`36S!$HUV+2q-G3Ll#XAY>s2f$UhNZ&>62r_1S!<| z=y+ro10#ie1Fkq;i~FdCDE%T{rOjo-J9%3z?ah?$iQi?twj8Vg1Qlj?lMK93ubTbKujdc^9ACs zc%-%U0zjYXYlz>Txj2xl%&TWzVevX=IVS)B literal 0 HcmV?d00001 diff --git a/pulsar-broker-common/src/test/resources/ssl/my-ca/ca.pem b/pulsar-broker-common/src/test/resources/ssl/my-ca/ca.pem new file mode 100644 index 0000000000000..3d5a80e234784 --- /dev/null +++ b/pulsar-broker-common/src/test/resources/ssl/my-ca/ca.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC9DCCAdygAwIBAgIUNbNkV2+K2Hf4Q1V5gdAENZQiLokwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAxMGUHVsc2FyMCAXDTIyMDExNDA0MjgwMFoYDzIxMjIwMTE2 +MDQyODAwWjARMQ8wDQYDVQQDEwZQdWxzYXIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDBR2K5EKVziLqdsz78efEW4lOwKiJ32e97uxn1Z6oKgkgImpVP +Z9aoJB4EwSnDg+6FV2YULdWPm7C6W33tDmWRaU/Hlo/cOejnK8UmiMu/EyDpE2Wj +n0RimGmwOkBi2IWIcIzWMmPDZ9kZc65OUeEmwZedKRy62PQyfCeNU4OOHQn3PXjI +NbXJZD5TvBmn4SJn2RP9EgmIPaBAh/Mng045ZeHHLhwMKC8EOyHc2aB7AL6brymR +xzsiYWdcJn4mqqMvT82mVvhkgAMOcR4CXYF8eYnsG6ZbDHb13CawcvLVREJZk7AB +XZi9Rd5xczxHILM8rdkIZfunaG1X5hbih5wJAgMBAAGjQjBAMA4GA1UdDwEB/wQE +AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTCC1lYG+62cUPjNk9q4jCm +Ps65njANBgkqhkiG9w0BAQsFAAOCAQEAKV2Lpu5cH5EsG53EWsYxEKvuQZ0LTxCE +wCDf/NxJaQbzfv0tsbZatMge0vcZ/5r8tZZoOC+pGTwk6MaRbEFH8PmvlH1LIQvu +Y34/YQZOy8wBTWwaIfFMnYWc0iAFoFt2Lzuq+GOI+svTFp729Ae8r7UxY/f9Lioc +ttdGr7vA6PpcIMoEIPjVp+m41uL9IDfX8eOxg4gVlwtqpbHdTzMrOz0YY+3qH/WK +6Qffw4pwitzAEj2zCn2lvGC5cbpd13SAaqtB3xL/Aet0SS2r3g9qDo1RruQhXUng +06U/Hqtn5K1fNQv3pivi3Jg5z1DfJWHkH37luAoIlOZHRmPK6rhp/g== +-----END CERTIFICATE----- diff --git a/pulsar-broker-common/src/test/resources/ssl/my-ca/client-ca.pem b/pulsar-broker-common/src/test/resources/ssl/my-ca/client-ca.pem new file mode 100644 index 0000000000000..adcae3393ade1 --- /dev/null +++ b/pulsar-broker-common/src/test/resources/ssl/my-ca/client-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHDCCAgSgAwIBAgIUJJpmKX3DnbUwJ7tUhCt8MTiwz0owDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAxMGUHVsc2FyMCAXDTIyMDExNDA0MjgwMFoYDzIxMjExMjIx +MDQyODAwWjARMQ8wDQYDVQQDEwZQdWxzYXIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDZN+CNZ1i1WaXulbwSASOfXErWXhGV9DHqavPp3DohgQdundfS +648T/X80uWQlyxu4L4j0oc97jtzc1AyZFXj5nocVsveEO9aDjnYCc5NdBNJLQHgl +IO59fEpTd55NO24g9a8/sxgn0ADCenMlngk1Ou+2QJBONw7W12/WUSUg6ICe+b+x +qPzgApue16oGw9HxhPwa3oEvVZrEnFIWLjsSWtezhgFHMCH9/ngk0KlRyes/EZCz +ZgkO5mgii2fmNDg+yuWUfw7Q0x6BJskGIrxisJiJBRR1+DIvJqgqxJsNmeeEQrZK +YHBukj5RWDFOpOHgqFbPsv45sVKoLrGFrMnNAgMBAAGjajBoMA4GA1UdDwEB/wQE +AwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQW +BBSwkx93xjYP4I+dcFF3xS9NLesmFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJ +KoZIhvcNAQELBQADggEBAAK3ZF63w46pT76QIOeSM3ocUm6izvW/IrxLUESfgRC4 +gg0/5VfPiHHUe6orn15KuPXHe7xCUFqc2oFn5aIU1B/6iOPeNItvMJidU0a3UAiw +hFK9MSFgESNBiEnu1dE5tPcIIxTyCFQ/8loeY3dsdcNVoguH/2J9v/XcMMga46A1 +wudaaa1nb+ZYnXkRuyObKVJQN7EqC+4edinMOTPBbF9wtRMAMBRHXXENXb9zFthi +Dbdn4YvadYsNHxh5ar+hQn/HSPMuCUPY/uUqxtBagb6aS0YnSoUscSLs1Jizg5NX +d+QV8X/5E6W4xWnptUZwVxOemkdnr6A8MH1eQKKFZTM= +-----END CERTIFICATE----- diff --git a/pulsar-broker-common/src/test/resources/ssl/my-ca/client-key.pem b/pulsar-broker-common/src/test/resources/ssl/my-ca/client-key.pem new file mode 100644 index 0000000000000..5b08b151c8094 --- /dev/null +++ b/pulsar-broker-common/src/test/resources/ssl/my-ca/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDZN+CNZ1i1WaXu +lbwSASOfXErWXhGV9DHqavPp3DohgQdundfS648T/X80uWQlyxu4L4j0oc97jtzc +1AyZFXj5nocVsveEO9aDjnYCc5NdBNJLQHglIO59fEpTd55NO24g9a8/sxgn0ADC +enMlngk1Ou+2QJBONw7W12/WUSUg6ICe+b+xqPzgApue16oGw9HxhPwa3oEvVZrE +nFIWLjsSWtezhgFHMCH9/ngk0KlRyes/EZCzZgkO5mgii2fmNDg+yuWUfw7Q0x6B +JskGIrxisJiJBRR1+DIvJqgqxJsNmeeEQrZKYHBukj5RWDFOpOHgqFbPsv45sVKo +LrGFrMnNAgMBAAECggEATeVZ45uiFja16J9NuG8sJSPluoY1bD8L/3KnUcAmIImy +7powIXVT8+k+StwI6/ywThbN2FyGmVqcHZz1f5hRr8KH0uJBHOyQetEFxM9Jk1v9 +Rfsymq36mImP5erJnAyp66vvUrqY+P4Ap71duam4x5wBBqyUk1fvPGA5vPOQiwHs +TN9JHizGobY25fpigWKIMamyE7HWXEUzVdOo83ZiNx53ths+WcF/kqto2v5LtyfJ +HgoPocfZI8tRz9tfgc8zOkvyjsvgdd6rLhd0r2oExnyQBJdktGFpQZMGambU328u +NqcdJscjP/HWAHRzuSdOvCMOEn8E5GIjcWEnQqOmSQKBgQDcpb655/UdcVxrv2Ou +8juucDJMpf6i/UcmlXVXx+3zGSuQZcCC2fupe3JcxPdK7bo65YlC3OoRihggh2sS +cnFMNHMfyoE3G/doXIr3QyL9UAQt4yb+7Nz7jRXYcg4Ytv+FVS6BSzIDEK17v+es +GuWDM3JwtigtzYS4tRh7lgmuBwKBgQD8BXp7yIyVv657B8OJJSoeGataziFPhZux +WKoS3gq24169ZWXwLc+nwrdgvBNrRaHuX+cYh93RF9+2WZrRcRL41XqN938adasY +zPsfOJa9IOgUzQtGUMSe1/WqvHfcvqZCqYq4u/LSdf+I67woP4tCqqn4E928aIZb +6PjLH+dUiwKBgH1ntn7y1t1lEKIspPtJsaHzIqNttMvuKAJF7+t0Nkl0hM4NBt1Y +BzDMeLNBP0vW0YGn89uMs3xEgHH8hV52rO4i4UuwTMCFpJgsAM+H2NsgHz/1WrSI +6xANn9zk9h4V5CRjxYq2sjYLxI4RBBtNLiTjmKd24F8n78cLJl8XZ2kBAoGAGoHF +ATH1v2ZaxqvpYApdpK7UfAeEL2YBGyUVNkjOXbAKbec1Uo6u8ZkkSnNdo4G+Z2EE +4Gqh5PUa3YYNJ4w6D5v8eOQYJUNNDJ26p+z+xcOpRU7PqcSi+YYDW8LY5InU2NwW +MBnsj0BD8TXCI4WTcx6aI/KK9t8TiqU1Tb/8R8MCgYANVinOLz2enB+Qzu4o88W/ +witKHI3D9+z/uWjp0Q4rwmr3OL4FD9vZWvL4qwbDgpfLirJ4e3UVfN1/FoytAKlk +Kykf8oDWciCIdxStt/yUpgQv78IL3vM5d9B8Qb7KCRtJ0BIXGJ7Gle3xJeuduZLe ++F+hwI3Dpv5HPqa9o6ttJw== +-----END PRIVATE KEY----- diff --git a/pulsar-broker-common/src/test/resources/ssl/my-ca/server-ca.pem b/pulsar-broker-common/src/test/resources/ssl/my-ca/server-ca.pem new file mode 100644 index 0000000000000..df5f69298e258 --- /dev/null +++ b/pulsar-broker-common/src/test/resources/ssl/my-ca/server-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHDCCAgSgAwIBAgIUVQHD0/oi9Ca50HA7DFLYOO2wEzYwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAxMGUHVsc2FyMCAXDTIyMDExNDA0MjgwMFoYDzIxMjExMjIx +MDQyODAwWjARMQ8wDQYDVQQDEwZQdWxzYXIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDBcqDkMhjLd9ik//UQijqbajQP5t6dvVZNn9gODQrS9oB/URur +NzCcPWYPJZfEJlTkV8mlmgq4dBjwghpy5ALOGiERk55JPIN4cy01hQ6j7YSPFvMv +BjqZvm5dpGDNTr7GY7THegMM1wpk9EaUOm7tBOHtf6ZnANjSMcQM74RCSBt0Koqw +06CKVDCbgJ5NNE1LgwYeVQAwtQAhY8rqqQKJvCorFbq7OiisFBnz5pRBT6N4kMo1 +9LZo3Oe2F2w9eH9vacQ0NjSOCNXqal9Xl/Pwy9JgKKppwZ/3nCgRc+yfjrnkRz0f +b+llb2NpR5Ge+tNMakqelE8bDSw/5BPjRPftAgMBAAGjajBoMA4GA1UdDwEB/wQE +AwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMB0GA1UdDgQW +BBRXws5mmLbW+xOLflUyUZ0I0uN96zAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJ +KoZIhvcNAQELBQADggEBAKMklpYJIkp4icz9Ea5wWQiRXWb94lGdyCA833VHeGB2 +fKvNXj1d6lEiy26pOjhDmycroKelj70WqOsqVgi4xh4Y9sj6pwb8Q423Tu3qNO1k +qaScTar2DANSigNzqlSbLshPWQ2ZyDwkvZPuqPgHzOXekzbUGwxgCiySaQkl2mCS +mBaG3XnESwiMIKkLphEv0MAvTVaImbSRWYEQ4OECwcHXxx+14wK8NLcdDIHcSzki +8Eq24CxDOeL5QxciGMi5tylsdCpT+D/BXTKiu46yoRjXUsTLYL53yUZZIqQ3A4CV +enZ/vHhP0Ev9RcRigFTqrBm7EC3b2AUpvqgRMnPwQZo= +-----END CERTIFICATE----- diff --git a/pulsar-broker-common/src/test/resources/ssl/my-ca/server-key.pem b/pulsar-broker-common/src/test/resources/ssl/my-ca/server-key.pem new file mode 100644 index 0000000000000..a3f3a36b73c37 --- /dev/null +++ b/pulsar-broker-common/src/test/resources/ssl/my-ca/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBcqDkMhjLd9ik +//UQijqbajQP5t6dvVZNn9gODQrS9oB/URurNzCcPWYPJZfEJlTkV8mlmgq4dBjw +ghpy5ALOGiERk55JPIN4cy01hQ6j7YSPFvMvBjqZvm5dpGDNTr7GY7THegMM1wpk +9EaUOm7tBOHtf6ZnANjSMcQM74RCSBt0Koqw06CKVDCbgJ5NNE1LgwYeVQAwtQAh +Y8rqqQKJvCorFbq7OiisFBnz5pRBT6N4kMo19LZo3Oe2F2w9eH9vacQ0NjSOCNXq +al9Xl/Pwy9JgKKppwZ/3nCgRc+yfjrnkRz0fb+llb2NpR5Ge+tNMakqelE8bDSw/ +5BPjRPftAgMBAAECggEBAJm2JsgMUo1ihn/dbnIdFCKoCgRUs7FtYCVADOJlVKN7 +AXGpFi4/JV4Qn4cLnQNcXfovE2iF9VzJy4NYLgH60YvJUVtxC8Yv0lukUVkEiDST +p9A3MTa9YVUG7xVzZwPcPVTQpzYV6lSKjpTXUTm5EKk/RvJ7itKv5plmt9x7eYFb +/JwqXo1Z6C4gfIFR85LWmrCsNUK5T9oooLz88D6+ZH3+fWlr75RDff2kqdLshMTs +N0Ov7NXcRFeruFs/IPrgTxjBMeNa2LFdYVPeeQ41L4uOI49uVBAmSn1be+THvDoj +Do+6wTEF/h6/VLoOaIFZZdHlqd4is+xcEg8gwVkCn2ECgYEAxqVvGKc9qaqEVwBx +U5Ru9OFx0NqEBvkYZRbCg1REcMFd3lqFTHvHiF3pmCp0XgLJKYuy42618IJXhj6D +Y15/p9jX0025MpnH/AdwpO6x5pv6gb/JOMnHOnq8sI3R+V6TVsv1WZj0sOj94mF0 ++Od++bQkUnSlfE4X7v+cJfo/Q8UCgYEA+Uz1yOyI9Dv1dEdBMdBA8MTriYU0uJCV +dVKzL/uC9XyguVBWu1HX0MvEKyjPRycvLB7TuQqAFLgCtC8EEuPGBpWtyXOm9Jxw +ToCfUZFuBQeMuf4vZcFgJjiEKTdKBxrvjkhyIhPR6JAy0WUr8Ry+ZtqvmG5NOEz5 +ptm1tznYngkCgYEAlckeyV8p/uqF2biKu3QcamgoU0zB6yQfAfK0fySmasNTzZtC +EhbvsOLnhgbVMiI1ny8ol5fedtlBuAchOWeDKIQ40as0r3QHuQG/LY6S9Im+zeFY +kIqNwInWB+cYYkmvHe6zNXlBYLh+4BmOgzTDqPPtw4MTWXTlVSDGlFhrJeUCgYBX +7rlS4Xt9ChkNpoRsWZROWGbr3rw1zWmqND1X01Lh28+lDZ1J/RguYXET+BUEd+G/ +oi/zuKxsomrxuxOoxgZ3FBx0TgK5jORgDCYl0zIHPB57DBkTvx123cBf+Ux3LR0K +BqubMXp8mUATc6gIJ6dRCBmfnmhGT4BPRcM+mXy6YQKBgGEGH37VABus+Oi3g1bk +qEAaUI1asRLJIfbY2ImxEroLIQAbTFuIQUsZTKpT7jJZubjYvy1Fev0LU/n7Kv2w +7ym41z70ro5uxwUBfJjnF3RtgncNcftn4b3siNzvBfKEBuhegMeS5YAbBIwABUpR +4mVpm9BLOiX4yENIT6JdUQFc +-----END PRIVATE KEY----- diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 60cd75501d0dc..a777ced4f956f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1383,6 +1383,8 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { conf.getBrokerClientAuthenticationParameters()); if (conf.isBrokerClientTlsEnabled()) { + builder.tlsCiphers(config.getBrokerClientTlsCiphers()) + .tlsProtocols(config.getBrokerClientTlsProtocols()); if (conf.isBrokerClientTlsEnabledWithKeyStore()) { builder.useKeyStoreTls(true) .tlsTrustStoreType(conf.getBrokerClientTlsTrustStoreType()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java index e5e2ab1d21889..e80a5160cb54a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java @@ -30,8 +30,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -99,8 +98,8 @@ public WebService(PulsarService pulsar) throws PulsarServerException { SslContextFactory sslCtxFactory; ServiceConfiguration config = pulsar.getConfiguration(); if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = KeyStoreSSLContext.createSslContextFactory( - config.getTlsProvider(), + sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( + config.getWebServiceTlsProvider(), config.getTlsKeyStoreType(), config.getTlsKeyStore(), config.getTlsKeyStorePassword(), @@ -109,15 +108,20 @@ public WebService(PulsarService pulsar) throws PulsarServerException { config.getTlsTrustStore(), config.getTlsTrustStorePassword(), config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), + config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec() ); } else { - sslCtxFactory = SecurityUtility.createSslContextFactory( + sslCtxFactory = JettySslContextFactory.createServerSslContext( + config.getWebServiceTlsProvider(), config.isTlsAllowInsecureConnection(), config.getTlsTrustCertsFilePath(), config.getTlsCertificateFilePath(), config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), true, + config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), + config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec()); } httpsConnector = new PulsarServerConnector(server, 1, 1, sslCtxFactory); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java index 4c82615aa7fc9..b780fde760e33 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java @@ -106,7 +106,7 @@ public void socketTest() throws GeneralSecurityException { SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setSslContext(SecurityUtility - .createSslContext(false, SecurityUtility.loadCertificatesFromPemFile(TLS_TRUST_CERT_FILE_PATH))); + .createSslContext(false, SecurityUtility.loadCertificatesFromPemFile(TLS_TRUST_CERT_FILE_PATH), null)); WebSocketClient consumeClient = new WebSocketClient(sslContextFactory); SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket(); diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index e8b5c59579090..fd715f87e0015 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -136,11 +136,6 @@ netty-codec-haproxy - - org.eclipse.jetty - jetty-util - - org.apache.commons commons-lang3 diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java index b1e8a14ff95f6..2e67b02f90b1e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java @@ -18,11 +18,8 @@ */ package org.apache.pulsar.common.util; -import java.io.FileNotFoundException; -import java.io.IOException; import java.security.GeneralSecurityException; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLException; @SuppressWarnings("checkstyle:JavadocType") public class DefaultSslContextBuilder extends SslContextAutoRefreshBuilder { @@ -31,23 +28,37 @@ public class DefaultSslContextBuilder extends SslContextAutoRefreshBuilder webServicePortTls = Optional.empty(); + @FieldContext( + category = CATEGORY_KEYSTORE_TLS, + doc = "Specify the TLS provider for the web service, available values can be SunJSSE, Conscrypt and etc." + ) + private String webServiceTlsProvider = "Conscrypt"; + + @FieldContext( + category = CATEGORY_TLS, + doc = "Specify the tls protocols the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLSv1.3, TLSv1.2]" + ) + private Set webServiceTlsProtocols = new TreeSet<>(); + + @FieldContext( + category = CATEGORY_TLS, + doc = "Specify the tls cipher the proxy's web service will use to negotiate during TLS Handshake.\n\n" + + "Example:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]" + ) + private Set webServiceTlsCiphers = new TreeSet<>(); + @FieldContext( category = CATEGORY_SERVER, doc = "The directory where nar Extraction happens" diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index 8a9956ca493fb..b8004f916fd8c 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -37,8 +37,7 @@ import org.apache.pulsar.broker.web.RateLimitingFilter; import org.apache.pulsar.broker.web.JettyRequestLogFactory; import org.apache.pulsar.broker.web.WebExecutorThreadPool; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; @@ -100,8 +99,8 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication try { SslContextFactory sslCtxFactory; if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = KeyStoreSSLContext.createSslContextFactory( - config.getTlsProvider(), + sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( + config.getWebServiceTlsProvider(), config.getTlsKeyStoreType(), config.getTlsKeyStore(), config.getTlsKeyStorePassword(), @@ -110,16 +109,20 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication config.getTlsTrustStore(), config.getTlsTrustStorePassword(), config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), + config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec() ); } else { - sslCtxFactory = SecurityUtility.createSslContextFactory( + sslCtxFactory = JettySslContextFactory.createServerSslContext( + config.getWebServiceTlsProvider(), config.isTlsAllowInsecureConnection(), config.getTlsTrustCertsFilePath(), config.getTlsCertificateFilePath(), config.getTlsKeyFilePath(), config.isTlsRequireTrustedClientCertOnConnect(), - true, + config.getWebServiceTlsCiphers(), + config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec()); } connectorTls = new ServerConnector(server, 1, 1, sslCtxFactory); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java index 9cfed156c3f28..85b7e2626cec4 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java @@ -36,8 +36,7 @@ import org.apache.pulsar.broker.web.JettyRequestLogFactory; import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -79,7 +78,7 @@ public ProxyServer(WebSocketProxyConfiguration config) try { SslContextFactory sslCtxFactory; if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = KeyStoreSSLContext.createSslContextFactory( + sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( config.getTlsProvider(), config.getTlsKeyStoreType(), config.getTlsKeyStore(), @@ -89,16 +88,20 @@ public ProxyServer(WebSocketProxyConfiguration config) config.getTlsTrustStore(), config.getTlsTrustStorePassword(), config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), + config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec() ); } else { - sslCtxFactory = SecurityUtility.createSslContextFactory( + sslCtxFactory = JettySslContextFactory.createServerSslContext( + config.getTlsProvider(), config.isTlsAllowInsecureConnection(), config.getTlsTrustCertsFilePath(), config.getTlsCertificateFilePath(), config.getTlsKeyFilePath(), config.isTlsRequireTrustedClientCertOnConnect(), - true, + config.getWebServiceTlsCiphers(), + config.getWebServiceTlsProtocols(), config.getTlsCertRefreshCheckDurationSec()); } connectorTls = new ServerConnector(server, sslCtxFactory); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java index 2cd8eb23f7f44..deb8ab3e032b6 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java @@ -167,11 +167,9 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { private boolean tlsEnabledWithKeyStore = false; @FieldContext( - doc = "Specify the TLS provider for the WebSocket service: \n" - + "When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.\n" - + "When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc." + doc = "Specify the TLS provider for the WebSocket service: SunJSSE, Conscrypt and etc." ) - private String tlsProvider = null; + private String tlsProvider = "Conscrypt"; @FieldContext( doc = "TLS KeyStore type configuration in WebSocket: JKS, PKCS12" From 6d16025d8955819481f372b74d5b050277581704 Mon Sep 17 00:00:00 2001 From: ran Date: Thu, 5 May 2022 21:55:50 +0800 Subject: [PATCH 519/823] [branch-2.9] [fix][license] Fix presto license exclude trim java agent (#15453) --- src/check-binary-license | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/check-binary-license b/src/check-binary-license index 5e8f365178acf..afe380d5c8ffb 100755 --- a/src/check-binary-license +++ b/src/check-binary-license @@ -96,7 +96,7 @@ done if [ "$NO_PRESTO" -ne 1 ]; then # check pulsar sql jars - JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort) + JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | grep -v 'java-version-trim-agent' | sed 's!.*/!!' | sort) LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/lib\/presto\/LICENSE/') LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH") LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp') From 2762bbc71f5648a488b7dc29cb464183534d4e73 Mon Sep 17 00:00:00 2001 From: Matt-Esch Date: Mon, 9 May 2022 17:30:42 +0100 Subject: [PATCH 520/823] [fix][c++ client] avoid race condition causing double callback on close (#15508) * avoid race condition causing double callback on close * Update pulsar-client-cpp/lib/ClientImpl.cc Co-authored-by: Yunze Xu --- pulsar-client-cpp/lib/ClientImpl.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientImpl.cc b/pulsar-client-cpp/lib/ClientImpl.cc index 6d9c4ed044ed4..dd00e3b9d4539 100644 --- a/pulsar-client-cpp/lib/ClientImpl.cc +++ b/pulsar-client-cpp/lib/ClientImpl.cc @@ -532,8 +532,13 @@ void ClientImpl::handleClose(Result result, SharedInt numberOfOpenHandlers, Resu } if (*numberOfOpenHandlers == 0) { Lock lock(mutex_); - state_ = Closed; - lock.unlock(); + if (state_ == Closed) { + LOG_DEBUG("Client is already shutting down, possible race condition in handleClose"); + return; + } else { + state_ = Closed; + lock.unlock(); + } LOG_DEBUG("Shutting down producers and consumers for client"); // handleClose() is called in ExecutorService's event loop, while shutdown() tried to wait the event From b5b673679218706442cd51591f0efb2b33096afd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 10 May 2022 22:46:44 -0500 Subject: [PATCH 521/823] [WebSocket] Fix MultiTopicReader#getConsumer ClassCastException (#15534) ### Motivation This fixes an issue similar to the one solved in https://github.com/apache/pulsar/pull/14316. When the `reader` is a `MultiTopicReader`, the `getConsumer()` method currently throws a `ClassCastException`. ### Modifications * Update `MultiTopicReader#getConsumer` so that it safely casts the `reader`. * Update the `ReaderHandler` constructor to use the `getConsumer` method. ### Verifying this change I expanded existing tests to cover the scenario that would have previously failed. ### Does this pull request potentially affect one of the following parts: No, this is not a breaking change. (cherry picked from commit dd6d3720f0a1d10df9885592ff0e0f9481325f2a) --- .../apache/pulsar/websocket/ReaderHandler.java | 16 ++++++++++------ .../pulsar/websocket/ReaderHandlerTest.java | 4 ++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index 2b87802ed9ee6..56b419f51d880 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -104,13 +104,11 @@ public ReaderHandler(WebSocketService service, HttpServletRequest request, Servl } this.reader = builder.create(); - if (reader instanceof MultiTopicsReaderImpl) { - this.subscription = ((MultiTopicsReaderImpl) reader).getMultiTopicsConsumer().getSubscription(); - } else if (reader instanceof ReaderImpl) { - this.subscription = ((ReaderImpl) reader).getConsumer().getSubscription(); - } else { + Consumer consumer = getConsumer(); + if (consumer == null) { throw new IllegalArgumentException(String.format("Illegal Reader Type %s", reader.getClass())); } + this.subscription = consumer.getSubscription(); if (!this.service.addReader(this)) { log.warn("[{}:{}] Failed to add reader handler for topic {}", request.getRemoteAddr(), request.getRemotePort(), topic); @@ -271,7 +269,13 @@ public void close() throws IOException { } public Consumer getConsumer() { - return reader != null ? ((ReaderImpl) reader).getConsumer() : null; + if (reader instanceof MultiTopicsReaderImpl) { + return ((MultiTopicsReaderImpl) reader).getMultiTopicsConsumer(); + } else if (reader instanceof ReaderImpl) { + return ((ReaderImpl) reader).getConsumer(); + } else { + return null; + } } public String getSubscription() { diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java index 0d2a13d1a7496..7dfa8b6e3146b 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java @@ -74,6 +74,8 @@ public void testCreateReaderImp() throws IOException { ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); // verify success Assert.assertEquals(readerHandler.getSubscription(), subName); + // Verify consumer is returned + readerHandler.getConsumer(); } @Test @@ -102,6 +104,8 @@ public void testCreateMultipleTopicReaderImp() throws IOException { ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); // verify success Assert.assertEquals(readerHandler.getSubscription(), subName); + // Verify consumer is successfully returned + readerHandler.getConsumer(); } @Test From bfd53f4cbfce4880814389ff89ed85c21af224af Mon Sep 17 00:00:00 2001 From: ran Date: Wed, 11 May 2022 20:41:32 +0800 Subject: [PATCH 522/823] [fix][test] Adjust flaky test concurrent consume reconnect (#15544) (cherry picked from commit 7070397649bc41fcd5e9070695ab2790228aa9c2) --- .../client/api/SimpleProducerConsumerTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 8a47c6fbb62c7..24302a16f8e13 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -820,10 +820,11 @@ public void testSillyUser() { // This is to test that the flow control counter doesn't get corrupted while concurrent receives during // reconnections - @Test(dataProvider = "batch", groups = "quarantine") + @Test(timeOut = 100_000, dataProvider = "batch", groups = "quarantine") public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception { final int recvQueueSize = 100; final int numConsumersThreads = 10; + final int receiveTimeoutSeconds = 100; String subName = UUID.randomUUID().toString(); final Consumer consumer = pulsarClient.newConsumer() @@ -837,12 +838,11 @@ public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) for (int i = 0; i < numConsumersThreads; i++) { executor.submit((Callable) () -> { barrier.await(); - consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + consumer.receive(receiveTimeoutSeconds, TimeUnit.SECONDS); return null; }); } - - barrier.await(); + barrier.await(); // the last thread reach barrier, start consume messages // we restart the broker to reconnect restartBroker(); @@ -878,7 +878,7 @@ public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) return null; }); } - barrier.await(); + barrier.await(); // the last thread reach barrier, start consume messages Awaitility.await().untilAsserted(() -> { // The available permits should be 20 and num messages in the queue should be 80 @@ -908,7 +908,7 @@ public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) return null; }); } - barrier.await(); + barrier.await(); // the last thread reach barrier, start consume messages restartBroker(); From 4040977025da6b885b2cc2d0195f5589fceb7704 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 12 May 2022 21:06:46 -0700 Subject: [PATCH 523/823] [fix][broker] Fix deadlock in broker after race condition in topic creation failure (#15570) * Fix deadlock in broker after race condition in topic creation failure * Fixed checkstyle --- .../org/apache/pulsar/broker/service/BrokerService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index a9f9c7fe090c1..5dc5edf9f0849 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1360,9 +1360,9 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { if (topicFuture.isCompletedExceptionally()) { log.warn("{} future is already completed with failure {}, closing the topic", topic, FutureUtil.getException(topicFuture)); - persistentTopic.stopReplProducers().whenComplete((v, exception) -> { + persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> { topics.remove(topic, topicFuture); - }); + }, executor()); } else { addTopicToStatsMaps(topicName, persistentTopic); topicFuture.complete(Optional.of(persistentTopic)); @@ -1372,10 +1372,10 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { "Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex); - persistentTopic.stopReplProducers().whenComplete((v, exception) -> { + persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> { topics.remove(topic, topicFuture); topicFuture.completeExceptionally(ex); - }); + }, executor()); return null; }); From 2c7679279389a249eb846aa800aaca7c0b3674a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 13 May 2022 18:10:44 +0200 Subject: [PATCH 524/823] [improve] Upgrade BookKeeper to 4.14.5 (2.8, 2.9, 2.10 branches) (#15581) (cherry picked from commit a92a8010b05467dfd4be490e2d395241a1b04307) --- .../server/src/assemble/LICENSE.bin.txt | 50 +++++++++---------- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 24 ++++----- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c45a4e74d8278..fa3372b8b7bc9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -397,31 +397,31 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.14.4.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.4.jar - - org.apache.bookkeeper-bookkeeper-proto-4.14.4.jar - - org.apache.bookkeeper-bookkeeper-server-4.14.4.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.4.jar - - org.apache.bookkeeper-circe-checksum-4.14.4.jar - - org.apache.bookkeeper-cpu-affinity-4.14.4.jar - - org.apache.bookkeeper-statelib-4.14.4.jar - - org.apache.bookkeeper-stream-storage-api-4.14.4.jar - - org.apache.bookkeeper-stream-storage-common-4.14.4.jar - - org.apache.bookkeeper-stream-storage-java-client-4.14.4.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.14.4.jar - - org.apache.bookkeeper-stream-storage-proto-4.14.4.jar - - org.apache.bookkeeper-stream-storage-server-4.14.4.jar - - org.apache.bookkeeper-stream-storage-service-api-4.14.4.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.14.4.jar - - org.apache.bookkeeper.http-http-server-4.14.4.jar - - org.apache.bookkeeper.http-vertx-http-server-4.14.4.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.4.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.4.jar - - org.apache.distributedlog-distributedlog-common-4.14.4.jar - - org.apache.distributedlog-distributedlog-core-4.14.4-tests.jar - - org.apache.distributedlog-distributedlog-core-4.14.4.jar - - org.apache.distributedlog-distributedlog-protocol-4.14.4.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.4.jar + - org.apache.bookkeeper-bookkeeper-common-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-proto-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-server-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.5.jar + - org.apache.bookkeeper-circe-checksum-4.14.5.jar + - org.apache.bookkeeper-cpu-affinity-4.14.5.jar + - org.apache.bookkeeper-statelib-4.14.5.jar + - org.apache.bookkeeper-stream-storage-api-4.14.5.jar + - org.apache.bookkeeper-stream-storage-common-4.14.5.jar + - org.apache.bookkeeper-stream-storage-java-client-4.14.5.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.14.5.jar + - org.apache.bookkeeper-stream-storage-proto-4.14.5.jar + - org.apache.bookkeeper-stream-storage-server-4.14.5.jar + - org.apache.bookkeeper-stream-storage-service-api-4.14.5.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.14.5.jar + - org.apache.bookkeeper.http-http-server-4.14.5.jar + - org.apache.bookkeeper.http-vertx-http-server-4.14.5.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.5.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.5.jar + - org.apache.distributedlog-distributedlog-common-4.14.5.jar + - org.apache.distributedlog-distributedlog-core-4.14.5-tests.jar + - org.apache.distributedlog-distributedlog-core-4.14.5.jar + - org.apache.distributedlog-distributedlog-protocol-4.14.5.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.5.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.13.jar diff --git a/pom.xml b/pom.xml index a6cc5f8f8ea15..40e01a8613d92 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.14.4 + 4.14.5 3.6.3 1.5.0 1.1.7 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 9fb5b74e2f771..120c26ec7958a 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -420,18 +420,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.14.4.jar - - bookkeeper-common-allocator-4.14.4.jar - - bookkeeper-proto-4.14.4.jar - - bookkeeper-server-4.14.4.jar - - bookkeeper-stats-api-4.14.4.jar - - bookkeeper-tools-framework-4.14.4.jar - - circe-checksum-4.14.4.jar - - codahale-metrics-provider-4.14.4.jar - - cpu-affinity-4.14.4.jar - - http-server-4.14.4.jar - - prometheus-metrics-provider-4.14.4.jar - - codahale-metrics-provider-4.14.4.jar + - bookkeeper-common-4.14.5.jar + - bookkeeper-common-allocator-4.14.5.jar + - bookkeeper-proto-4.14.5.jar + - bookkeeper-server-4.14.5.jar + - bookkeeper-stats-api-4.14.5.jar + - bookkeeper-tools-framework-4.14.5.jar + - circe-checksum-4.14.5.jar + - codahale-metrics-provider-4.14.5.jar + - cpu-affinity-4.14.5.jar + - http-server-4.14.5.jar + - prometheus-metrics-provider-4.14.5.jar + - codahale-metrics-provider-4.14.5.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar From 8a13146dd812596303479cd3ceb792da96b2485e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 18 May 2022 15:06:06 +0300 Subject: [PATCH 525/823] Upgrade Netty to 4.1.77.Final and netty-tcnative to 2.0.52.Final (#15646) - release notes https://netty.io/news/2022/05/06/2-1-77-Final.html - improves Alpine / musl compatibility - could help issues such as #14534 #11415 #11224 #10798 - improves shading compatibility - fixes a bug related to the native epoll transport and epoll_pwait2 (cherry picked from commit a8045fc4d6c5a61ad5d288c9d68e91b028ee3879) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 50 +++++++++---------- pom.xml | 4 +- pulsar-sql/presto-distribution/LICENSE | 46 ++++++++--------- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index aef37e4055867..5e2454863e555 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -105,7 +105,7 @@ io.netty netty-common - 4.1.76.Final + 4.1.77.Final test diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index fa3372b8b7bc9..5808df85c688f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -352,31 +352,31 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.11.jar * Netty - - io.netty-netty-buffer-4.1.76.Final.jar - - io.netty-netty-codec-4.1.76.Final.jar - - io.netty-netty-codec-dns-4.1.76.Final.jar - - io.netty-netty-codec-http-4.1.76.Final.jar - - io.netty-netty-codec-http2-4.1.76.Final.jar - - io.netty-netty-codec-socks-4.1.76.Final.jar - - io.netty-netty-codec-haproxy-4.1.76.Final.jar - - io.netty-netty-common-4.1.76.Final.jar - - io.netty-netty-handler-4.1.76.Final.jar - - io.netty-netty-handler-proxy-4.1.76.Final.jar - - io.netty-netty-resolver-4.1.76.Final.jar - - io.netty-netty-resolver-dns-4.1.76.Final.jar - - io.netty-netty-transport-4.1.76.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.76.Final.jar - - io.netty-netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.76.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.76.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.51.Final.jar + - io.netty-netty-buffer-4.1.77.Final.jar + - io.netty-netty-codec-4.1.77.Final.jar + - io.netty-netty-codec-dns-4.1.77.Final.jar + - io.netty-netty-codec-http-4.1.77.Final.jar + - io.netty-netty-codec-http2-4.1.77.Final.jar + - io.netty-netty-codec-socks-4.1.77.Final.jar + - io.netty-netty-codec-haproxy-4.1.77.Final.jar + - io.netty-netty-common-4.1.77.Final.jar + - io.netty-netty-handler-4.1.77.Final.jar + - io.netty-netty-handler-proxy-4.1.77.Final.jar + - io.netty-netty-resolver-4.1.77.Final.jar + - io.netty-netty-resolver-dns-4.1.77.Final.jar + - io.netty-netty-transport-4.1.77.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.77.Final.jar + - io.netty-netty-transport-native-epoll-4.1.77.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.77.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.77.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.77.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.52.Final.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar diff --git a/pom.xml b/pom.xml index 40e01a8613d92..ddc9f69ada3af 100644 --- a/pom.xml +++ b/pom.xml @@ -109,8 +109,8 @@ flexible messaging model and an intuitive client API. 1.1.7 3.2.5 5.1.0 - 4.1.76.Final - 2.0.51.Final + 4.1.77.Final + 2.0.52.Final 9.4.43.v20210629 2.5.2 2.34 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 120c26ec7958a..d1e6166c7ba7d 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -233,30 +233,30 @@ The Apache Software License, Version 2.0 - commons-lang3-3.11.jar * Netty - netty-3.10.6.Final.jar - - netty-buffer-4.1.76.Final.jar - - netty-codec-4.1.76.Final.jar - - netty-codec-dns-4.1.76.Final.jar - - netty-codec-http-4.1.76.Final.jar - - netty-codec-haproxy-4.1.76.Final.jar - - netty-codec-socks-4.1.76.Final.jar - - netty-handler-proxy-4.1.76.Final.jar - - netty-common-4.1.76.Final.jar - - netty-handler-4.1.76.Final.jar + - netty-buffer-4.1.77.Final.jar + - netty-codec-4.1.77.Final.jar + - netty-codec-dns-4.1.77.Final.jar + - netty-codec-http-4.1.77.Final.jar + - netty-codec-haproxy-4.1.77.Final.jar + - netty-codec-socks-4.1.77.Final.jar + - netty-handler-proxy-4.1.77.Final.jar + - netty-common-4.1.77.Final.jar + - netty-handler-4.1.77.Final.jar - netty-reactive-streams-2.0.4.jar - - netty-resolver-4.1.76.Final.jar - - netty-resolver-dns-4.1.76.Final.jar - - netty-tcnative-boringssl-static-2.0.51.Final.jar - - netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.51.Final.jar - - netty-transport-4.1.76.Final.jar - - netty-transport-classes-epoll-4.1.76.Final.jar - - netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.76.Final.jar - - netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar + - netty-resolver-4.1.77.Final.jar + - netty-resolver-dns-4.1.77.Final.jar + - netty-tcnative-boringssl-static-2.0.52.Final.jar + - netty-tcnative-boringssl-static-2.0.52.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.52.Final.jar + - netty-transport-4.1.77.Final.jar + - netty-transport-classes-epoll-4.1.77.Final.jar + - netty-transport-native-epoll-4.1.77.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.77.Final.jar + - netty-transport-native-unix-common-4.1.77.Final-linux-x86_64.jar * Joda Time - joda-time-2.10.5.jar * Jetty From 9eeb885cafa4ed6e1a1e1037be06814f91dac46b Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 7 May 2022 20:24:15 +0800 Subject: [PATCH 526/823] [Fix][txn] Make transaction stats consistent at end txn (#15472) ### Motivation When the end transaction log is appended to the transaction log, the transaction ended. The transaction should be removed from the `txnMetaMap`. If transactionLog fails to delete the location, we only need to log it. ### Modification Not complete exceptionally, but only give a warn log. (cherry picked from commit 5e6580abc8aea515581f0d23964b46bb58e493f4) --- .../broker/transaction/TransactionTest.java | 24 +++++++++++++++ .../impl/MLTransactionMetadataStore.java | 29 +++++++++---------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 296f04177f8cb..f264289d6962f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -101,6 +101,7 @@ import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; import org.apache.pulsar.transaction.coordinator.TransactionRecoverTracker; import org.apache.pulsar.transaction.coordinator.TransactionTimeoutTracker; @@ -970,4 +971,27 @@ public void testPendingAckMarkDeletePosition() throws Exception { Integer.parseInt(lastConfirmedEntry[1]) - 2); }); } + + @Test + public void testConsistencyOfTransactionStatsAtEndTxn() throws Exception { + TransactionMetadataStore transactionMetadataStore = getPulsarServiceList().get(0) + .getTransactionMetadataStoreService() + .getStores() + .get(new TransactionCoordinatorID(0)); + + Field field = MLTransactionMetadataStore.class.getDeclaredField("transactionLog"); + field.setAccessible(true); + MLTransactionLogImpl transactionLog = (MLTransactionLogImpl) field.get(transactionMetadataStore); + Field field1 = MLTransactionLogImpl.class.getDeclaredField("cursor"); + field1.setAccessible(true); + ManagedCursorImpl managedCursor = (ManagedCursorImpl) field1.get(transactionLog); + managedCursor.close(); + + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + + transaction.commit().get(); + } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index f93de8b017552..685d57e664e6f 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -372,23 +372,20 @@ public CompletableFuture updateTxnStatus(TxnID txnID, TxnStatus newStatus, this.transactionTimeoutCount.increment(); } if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { - transactionLog.deletePosition(txnMetaListPair.getRight()).whenComplete((v, ex) -> { - if (ex != null) { - promise.completeExceptionally(ex); - return; - } - this.transactionMetadataStoreStats - .addTransactionExecutionLatencySample(System.currentTimeMillis() - - txnMetaListPair.getLeft().getOpenTimestamp()); - if (newStatus == TxnStatus.COMMITTED) { - committedTransactionCount.increment(); - } else { - abortedTransactionCount.increment(); - } - txnMetaMap.remove(txnID.getLeastSigBits()); - promise.complete(null); + this.transactionMetadataStoreStats + .addTransactionExecutionLatencySample(System.currentTimeMillis() + - txnMetaListPair.getLeft().getOpenTimestamp()); + if (newStatus == TxnStatus.COMMITTED) { + committedTransactionCount.increment(); + } else { + abortedTransactionCount.increment(); + } + txnMetaMap.remove(txnID.getLeastSigBits()); + transactionLog.deletePosition(txnMetaListPair.getRight()).exceptionally(ex -> { + log.warn("Failed to delete transaction log position " + + "at end transaction [{}]", txnID); + return null; }); - return; } promise.complete(null); } catch (InvalidTxnStatusException e) { From 67c57802ee3efe3295724f488cd014023370a333 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 13 May 2022 13:48:34 +0800 Subject: [PATCH 527/823] [Fix][Txn] Fix transaction PendingAck lowWaterMark (#15530) ### Motivation Now, PendingAckHandle use the ending transaction ID to append abort mark, but it is wrong. We should append abort mark for the first transaction in the individualAckOfTransaction after judgment. ### Modification Append abort mark for the first transaction in the individualAckOfTransaction after judgment. (cherry picked from commit 498cde9ad3dd62142d73e024ea424bd76726dfaa) --- .../pendingack/impl/PendingAckHandleImpl.java | 22 ++-- .../pendingack/PendingAckPersistentTest.java | 117 ++++++++++++++++++ 2 files changed, 124 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 6fafc686169d7..ab5a8ff14542a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -598,21 +598,13 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { if (firstTxn.getMostSigBits() == txnID.getMostSigBits() && firstTxn.getLeastSigBits() <= lowWaterMark) { - this.pendingAckStoreFuture.whenComplete((pendingAckStore, throwable) -> { - if (throwable == null) { - pendingAckStore.appendAbortMark(txnID, AckType.Individual).thenAccept(v -> { - synchronized (PendingAckHandleImpl.this) { - log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " - + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); - individualAckOfTransaction.remove(firstTxn); - handleLowWaterMark(txnID, lowWaterMark); - } - }).exceptionally(e -> { - log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " - + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); - return null; - }); - } + abortTxn(firstTxn, null, lowWaterMark).thenRun(() -> { + log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); + }).exceptionally(e -> { + log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); + return null; }); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index d8da6631499cc..196d9bbebaaa7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -23,7 +23,9 @@ import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -32,6 +34,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.TransactionTestBase; @@ -45,6 +48,8 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @@ -348,4 +353,116 @@ private void testDeleteTopicThenDeletePendingAckManagedLedger() throws Exception assertFalse(topics.contains(MLPendingAckStore.getTransactionPendingAckStoreSuffix(topic, subName2))); assertFalse(topics.contains(topic)); } + + @Test + public void testPendingAckLowWaterMarkRemoveFirstTxn() throws Exception { + String topic = TopicName.get(TopicDomain.persistent.toString(), + NamespaceName.get(NAMESPACE1), "test").toString(); + + String subName = "subName"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Failover) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + for (int i = 0; i < 5; i++) { + producer.newMessage().send(); + } + + Transaction transaction1 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + + Message message1 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message1.getMessageId(), transaction1); + transaction1.commit().get(); + + + Transaction transaction2 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + while (transaction1.getTxnID().getMostSigBits() != transaction2.getTxnID().getMostSigBits()) { + transaction2 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + } + + Transaction transaction3 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + while (transaction1.getTxnID().getMostSigBits() != transaction3.getTxnID().getMostSigBits()) { + transaction3 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + } + + Message message3 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message3.getMessageId(), transaction2); + transaction2.commit().get(); + + Message message2 = consumer.receive(5, TimeUnit.SECONDS); + + Field field = TransactionImpl.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(transaction1, TransactionImpl.State.OPEN); + + consumer.acknowledgeAsync(message2.getMessageId(), transaction1).get(); + Message message4 = consumer.receive(5, TimeUnit.SECONDS); + field.set(transaction2, TransactionImpl.State.OPEN); + consumer.acknowledgeAsync(message4.getMessageId(), transaction2).get(); + + Message message5 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message5.getMessageId(), transaction3); + transaction3.commit().get(); + + + PersistentTopic persistentTopic = + (PersistentTopic) getPulsarServiceList() + .get(0) + .getBrokerService() + .getTopic(topic, false) + .get() + .get(); + + PersistentSubscription persistentSubscription = persistentTopic.getSubscription(subName); + PendingAckHandleImpl pendingAckHandle = new PendingAckHandleImpl(persistentSubscription); + + Method method = PendingAckHandleImpl.class.getDeclaredMethod("initPendingAckStore"); + method.setAccessible(true); + method.invoke(pendingAckHandle); + + Field field1 = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field1.setAccessible(true); + CompletableFuture completableFuture = + (CompletableFuture) field1.get(pendingAckHandle); + + Awaitility.await().until(() -> { + completableFuture.get(); + return true; + }); + + Field field2 = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); + field2.setAccessible(true); + LinkedMap> individualAckOfTransaction = + (LinkedMap>) field2.get(pendingAckHandle); + + assertFalse(individualAckOfTransaction.containsKey(transaction1.getTxnID())); + assertFalse(individualAckOfTransaction.containsKey(transaction2.getTxnID())); + + } } From 0b400b2478daaedf1c463ca984cc6ace50d783b7 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 6 May 2022 17:00:35 +0800 Subject: [PATCH 528/823] Cherry pick #15418 #15530 #15472 --- .../transaction/buffer/impl/TopicTransactionBuffer.java | 7 +++++-- .../transaction/pendingack/impl/MLPendingAckStore.java | 6 ++++-- .../transaction/coordinator/impl/MLTransactionLogImpl.java | 6 ++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 40b327dde0ace..a348ccbb76440 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -698,6 +698,8 @@ static class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallbac private volatile boolean isReadable = true; + private static final int NUMBER_OF_PER_READ_ENTRY = 100; + private FillEntryQueueCallback(SpscArrayQueue entryQueue, ManagedCursor cursor, TopicTransactionBufferRecover recover) { this.entryQueue = entryQueue; @@ -705,10 +707,11 @@ private FillEntryQueueCallback(SpscArrayQueue entryQueue, ManagedCursor c this.recover = recover; } boolean fillQueue() { - if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { + if (entryQueue.size() + NUMBER_OF_PER_READ_ENTRY < entryQueue.capacity() + && outstandingReadsRequests.get() == 0) { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); - cursor.asyncReadEntries(100, this, System.nanoTime(), PositionImpl.latest); + cursor.asyncReadEntries(NUMBER_OF_PER_READ_ENTRY, this, System.nanoTime(), PositionImpl.latest); } else { if (entryQueue.size() == 0) { isReadable = false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index a53db3391baaf..bb8b961e36b88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -367,12 +367,14 @@ class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private volatile boolean isReadable = true; private final AtomicLong outstandingReadsRequests = new AtomicLong(0); + private static final int NUMBER_OF_PER_READ_ENTRY = 100; boolean fillQueue() { - if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { + if (entryQueue.size() + NUMBER_OF_PER_READ_ENTRY < entryQueue.capacity() + && outstandingReadsRequests.get() == 0) { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); - readAsync(100, this); + readAsync(NUMBER_OF_PER_READ_ENTRY, this); } } return isReadable; diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index f14784e70bbd6..1727f477be82e 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -237,12 +237,14 @@ class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private final AtomicLong outstandingReadsRequests = new AtomicLong(0); private volatile boolean isReadable = true; + private static final int NUMBER_OF_PER_READ_ENTRY = 100; boolean fillQueue() { - if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { + if (entryQueue.size() + NUMBER_OF_PER_READ_ENTRY < entryQueue.capacity() + && outstandingReadsRequests.get() == 0) { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); - readAsync(100, this); + readAsync(NUMBER_OF_PER_READ_ENTRY, this); return isReadable; } else { return false; From de799e0c0c00d9a051ecde614e22dfb4b9e25576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 19 Jan 2022 18:39:07 +0100 Subject: [PATCH 529/823] Remove --illegal-access errors resulting from Google Guice (upgrade to 5.0.1) (#13810) (cherry picked from commit b0c72596eb1aa67110fa307b33379a3fed420505) --- pom.xml | 3 ++- pulsar-io/data-generator/pom.xml | 12 ++++++++++++ tiered-storage/jcloud/pom.xml | 13 +++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ddc9f69ada3af..30594c27c8b5e 100644 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,8 @@ flexible messaging model and an intuitive client API. 1.11.774 1.10.2 2.10.5 - 2.3.0 + 2.4.0 + 5.0.1 3.8.11.2 8.0.11 42.2.25 diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 1a49c3eb32b6e..3170c8c45a635 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -49,6 +49,18 @@ jfairy 0.5.9 + + com.google.inject + guice + ${guice.version} + provided + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + provided + org.apache.avro diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 716aed18b98f0..06e4dd80e433a 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -99,6 +99,19 @@ ${jclouds.version} provided + + + com.google.inject + guice + ${guice.version} + provided + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + provided + javax.xml.bind From 51dd759c243aec49a818e82c87a71fba44c0f50c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 8 Feb 2022 16:48:06 +0100 Subject: [PATCH 530/823] Fix NoClassDefFoundError: com/google/inject/AbstractModule in pulsar-io/batch-data-generator and Jcloud offloader (#14150) (cherry picked from commit 25ccf45fcaa71d97cae26244a6b7dfca75a4a089) --- pulsar-io/data-generator/pom.xml | 4 ++-- tiered-storage/jcloud/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 3170c8c45a635..ecaaf64b77515 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -53,13 +53,13 @@ com.google.inject guice ${guice.version} - provided + runtime com.google.inject.extensions guice-assistedinject ${guice.version} - provided + runtime diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 06e4dd80e433a..0e17362ef1b12 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -104,13 +104,13 @@ com.google.inject guice ${guice.version} - provided + runtime com.google.inject.extensions guice-assistedinject ${guice.version} - provided + runtime From fc8cc1c90b21ef2fedb96ffe524ede49abbffa15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 17 Feb 2022 13:37:27 +0100 Subject: [PATCH 531/823] Remove --illegal-access errors resulting from Google Guice - Pulsar IO, Offloaders and Pulsar SQL - Bump Guice to 5.1.0 (#14300) * Remove --illegal-access errors resulting from Google Guice - Batch Data Generator connector * and jcloud-shaded * use dependencyManagement * fix pulsar-sql (cherry picked from commit 332eca8279bb145b3b272d93806f4c89f8a8923f) --- pom.xml | 14 +++++++++++++- pulsar-io/data-generator/pom.xml | 12 ------------ pulsar-sql/presto-distribution/LICENSE | 3 +-- pulsar-sql/presto-distribution/pom.xml | 11 ++++------- tiered-storage/jcloud/pom.xml | 13 ------------- 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 30594c27c8b5e..86735142faf89 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ flexible messaging model and an intuitive client API. 1.10.2 2.10.5 2.4.0 - 5.0.1 + 5.1.0 3.8.11.2 8.0.11 42.2.25 @@ -635,6 +635,18 @@ flexible messaging model and an intuitive client API. ${guava.version} + + com.google.inject + guice + ${guice.version} + + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + org.apache.commons commons-lang3 diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index ecaaf64b77515..1a49c3eb32b6e 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -49,18 +49,6 @@ jfairy 0.5.9 - - com.google.inject - guice - ${guice.version} - runtime - - - com.google.inject.extensions - guice-assistedinject - ${guice.version} - runtime - org.apache.avro diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d1e6166c7ba7d..411a7645fe488 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -225,8 +225,7 @@ The Apache Software License, Version 2.0 - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - failureaccess-1.0.1.jar * Google Guice - - guice-4.2.3.jar - - guice-multibindings-4.2.0.jar + - guice-5.1.0.jar * Apache Commons - commons-math3-3.6.1.jar - commons-compress-1.21.jar diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index b074172568578..cd6bec3020ba6 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -100,6 +100,10 @@ javax.activation activation + + com.google.inject.extensions + guice-multibindings + @@ -137,13 +141,6 @@ ${objectsize.version} - - - com.google.inject.extensions - guice-multibindings - ${guice.version} - - diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 0e17362ef1b12..716aed18b98f0 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -99,19 +99,6 @@ ${jclouds.version} provided - - - com.google.inject - guice - ${guice.version} - runtime - - - com.google.inject.extensions - guice-assistedinject - ${guice.version} - runtime - javax.xml.bind From 08ed2754c42cedfc42f51017467fd4c2699b00f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 20 May 2022 08:59:19 +0200 Subject: [PATCH 532/823] [improve][offloaders] Upgrade JClouds to 2.5.0 (#15649) (cherry picked from commit d1f4e193a5a2b9b6477ed002758622bbeb045162) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 86735142faf89..92de778abc963 100644 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,7 @@ flexible messaging model and an intuitive client API. 1.11.774 1.10.2 2.10.5 - 2.4.0 + 2.5.0 5.1.0 3.8.11.2 8.0.11 From cbbfe08eea0c7ed079e9a4bb3362bb97a4dd05cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 23 May 2022 13:30:38 +0200 Subject: [PATCH 533/823] [fix][security] Tiered storage: Upgrade Hadoop to 3.3.3 to get rid of CVE-2022-26612 (#15660) (cherry picked from commit d926582c14dfa7f98231289824b85d8f05e1d5f4) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 92de778abc963..4e05ec6072ed9 100644 --- a/pom.xml +++ b/pom.xml @@ -153,7 +153,7 @@ flexible messaging model and an intuitive client API. 42.2.25 0.3.2 2.6.0 - 3.3.0 + 3.3.3 7.9.1 332 2.13 From 95dd390815ce496b378bd187be4a8a859e9fa0ec Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 24 May 2022 14:22:02 +0800 Subject: [PATCH 534/823] [branch-2.9][Authorization] Optimize the logic of allowing namespace operation. (#15731) --- .../PulsarAuthorizationProvider.java | 125 ++++++++++-------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index cfb05ab1e2c39..d0884da7dc08f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -524,27 +524,43 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam String role, NamespaceOperation operation, AuthenticationDataSource authData) { - CompletableFuture isAuthorizedFuture; - switch (operation) { - case PACKAGES: - isAuthorizedFuture = allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.packages); - break; - case GET_TOPICS: - case UNSUBSCRIBE: - case CLEAR_BACKLOG: - isAuthorizedFuture = allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.consume); - break; - default: - isAuthorizedFuture = CompletableFuture.completedFuture(false); + if (log.isDebugEnabled()) { + log.debug("Check allowNamespaceOperationAsync [{}] on [{}].", operation.name(), namespaceName); } - CompletableFuture isTenantAdminFuture = validateTenantAdminAccess(namespaceName.getTenant(), role, authData); - return isTenantAdminFuture.thenCombine(isAuthorizedFuture, (isTenantAdmin, isAuthorized) -> { - if (log.isDebugEnabled()) { - log.debug("Verify if role {} is allowed to {} to topic {}: isTenantAdmin={}, isAuthorized={}", - role, operation, namespaceName, isTenantAdmin, isAuthorized); - } - return isTenantAdmin || isAuthorized; - }); + return validateTenantAdminAccess(namespaceName.getTenant(), role, authData) + .thenCompose(isSuperUserOrAdmin -> { + if (log.isDebugEnabled()) { + log.debug("Verify if role {} is allowed to {} to namespace {}: isSuperUserOrAdmin={}", + role, operation, namespaceName, isSuperUserOrAdmin); + } + if (isSuperUserOrAdmin) { + return CompletableFuture.completedFuture(true); + } else { + switch (operation) { + case PACKAGES: + return allowTheSpecifiedActionOpsAsync( + namespaceName, role, authData, AuthAction.packages); + case GET_TOPIC: + case GET_TOPICS: + case UNSUBSCRIBE: + case CLEAR_BACKLOG: + return allowTheSpecifiedActionOpsAsync( + namespaceName, role, authData, AuthAction.consume); + case CREATE_TOPIC: + case DELETE_TOPIC: + case ADD_BUNDLE: + case GET_BUNDLE: + case DELETE_BUNDLE: + case GRANT_PERMISSION: + case GET_PERMISSION: + case REVOKE_PERMISSION: + return CompletableFuture.completedFuture(false); + default: + return FutureUtil.failedFuture(new IllegalStateException( + "NamespaceOperation [" + operation.name() + "] is not supported.")); + } + } + }); } @Override @@ -561,42 +577,8 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, String role, TopicOperation operation, AuthenticationDataSource authData) { - log.debug("Check allowTopicOperationAsync [" + operation.name() + "] on [" + topicName.toString() + "]."); - - CompletableFuture isAuthorizedFuture; - - switch (operation) { - case LOOKUP: - case GET_STATS: - case GET_METADATA: - isAuthorizedFuture = canLookupAsync(topicName, role, authData); - break; - case PRODUCE: - isAuthorizedFuture = canProduceAsync(topicName, role, authData); - break; - case GET_SUBSCRIPTIONS: - case CONSUME: - case SUBSCRIBE: - case UNSUBSCRIBE: - case SKIP: - case EXPIRE_MESSAGES: - case PEEK_MESSAGES: - case RESET_CURSOR: - case GET_BACKLOG_SIZE: - case SET_REPLICATED_SUBSCRIPTION_STATUS: - isAuthorizedFuture = canConsumeAsync(topicName, role, authData, authData.getSubscription()); - break; - case TERMINATE: - case COMPACT: - case OFFLOAD: - case UNLOAD: - case ADD_BUNDLE_RANGE: - case GET_BUNDLE_RANGE: - case DELETE_BUNDLE_RANGE: - return validateTenantAdminAccess(topicName.getTenant(), role, authData); - default: - return FutureUtil.failedFuture( - new IllegalStateException("TopicOperation [" + operation.name() + "] is not supported.")); + if (log.isDebugEnabled()) { + log.debug("Check allowTopicOperationAsync [{}] on [{}].", operation.name(), topicName); } return validateTenantAdminAccess(topicName.getTenant(), role, authData) @@ -608,7 +590,36 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, if (isSuperUserOrAdmin) { return CompletableFuture.completedFuture(true); } else { - return isAuthorizedFuture; + switch (operation) { + case LOOKUP: + case GET_STATS: + case GET_METADATA: + return canLookupAsync(topicName, role, authData); + case PRODUCE: + return canProduceAsync(topicName, role, authData); + case GET_SUBSCRIPTIONS: + case CONSUME: + case SUBSCRIBE: + case UNSUBSCRIBE: + case SKIP: + case EXPIRE_MESSAGES: + case PEEK_MESSAGES: + case RESET_CURSOR: + case GET_BACKLOG_SIZE: + case SET_REPLICATED_SUBSCRIPTION_STATUS: + return canConsumeAsync(topicName, role, authData, authData.getSubscription()); + case TERMINATE: + case COMPACT: + case OFFLOAD: + case UNLOAD: + case ADD_BUNDLE_RANGE: + case GET_BUNDLE_RANGE: + case DELETE_BUNDLE_RANGE: + return CompletableFuture.completedFuture(false); + default: + return FutureUtil.failedFuture(new IllegalStateException( + "TopicOperation [" + operation.name() + "] is not supported.")); + } } }); } From 55be557d7a65b88ad767e8be4a8e2dc89399e562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 10 Mar 2022 04:49:57 +0100 Subject: [PATCH 535/823] [owasp] add suppressions for Kotlin stdlib CVE-2022-24329 (#14629) (cherry picked from commit 4910519eb5c20249d982cab40813af3e870e4f90) --- src/owasp-dependency-check-suppressions.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 838e142b65d55..0f567bc37db70 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -42,6 +42,14 @@ .* + + + 461367948840adbb0839c51d91ed74ef4a9ccb52 + CVE-2022-24329 + + Date: Thu, 17 Mar 2022 04:23:56 +0100 Subject: [PATCH 536/823] [owasp] add suppression for Kotlin stdlib CVE-2022-24329 - part 2 (#14715) (cherry picked from commit 828d057a0d75fe250e07a98d546843e988b1a34e) (cherry picked from commit cc4178b510fbfd1a835507d801b133791bbbdb35) --- src/owasp-dependency-check-suppressions.xml | 32 +++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 0f567bc37db70..90698c0843529 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -42,12 +42,34 @@ .* + - - 461367948840adbb0839c51d91ed74ef4a9ccb52 - CVE-2022-24329 + + ef50bfa2c0491a11dcc35d9822edbfd6170e1ea2 + cpe:/a:jetbrains:kotlin + + + + 3546900a3ebff0c43f31190baf87a9220e37b7ea + CVE-2022-24329 + + + + 3302f9ec8a5c1ed220781dbd37770072549bd333 + CVE-2022-24329 + + + + 461367948840adbb0839c51d91ed74ef4a9ccb52 + CVE-2022-24329 From 959fd0960e3fd8cb9f848cc1e0bb1b821a9d54e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 19 May 2022 10:26:05 +0200 Subject: [PATCH 537/823] [fix][owasp] Fix false positive google-http-client-gson-1.41.0.jar (#15651) (cherry picked from commit cd0d4299f403505c0713270439d2c46d376de450) --- src/owasp-dependency-check-false-positives.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 7b945a2bbc9ec..4984db5762bfd 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -59,4 +59,13 @@ ^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$ cpe:/a:netty:netty + + + + + 1a754a5dd672218a2ac667d7ff2b28df7a5a240e + CVE-2022-25647 + \ No newline at end of file From 7134705ef42d8e5d099ebd1810e57d44bbad4a0d Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 29 Mar 2022 01:45:14 +0800 Subject: [PATCH 538/823] [C++] Fix send callback might not be invoked in key based batching (#14898) * [C++] Fix send callback might not be invoked in key based batching ### Motivation When C++ client enables key based batching, there is a chance that the send callback is not invoked. See https://github.com/apache/pulsar/blob/32df93f693bfdf42953bd728a12ecdea1796dcc8/pulsar-client-cpp/lib/ProducerImpl.cc#L272-L275 If a batch container has multiple batches, only one batch could be processed during `closeAsync`. Even worse, the semaphores of other batches won't be released. ### Modifications - Add a `clearPendingBatches` method to clear all pending batches and process them. Then call this method in `closeAsync` and `getPendingCallbacksWhenFailed`. - Add a test `testCloseBeforeSend` to verify when a producer has multiple pending batches, all callbacks can be invoked in `closeAsync`. * Add processAndClear() to batch message container (cherry picked from commit f3295ff0b14526de27791493d4c45cf814ef3654) --- .../lib/BatchMessageContainerBase.h | 26 ++++++++++ pulsar-client-cpp/lib/ProducerImpl.cc | 47 +++++-------------- .../tests/KeyBasedBatchingTest.cc | 32 ++++++++++++- 3 files changed, 69 insertions(+), 36 deletions(-) diff --git a/pulsar-client-cpp/lib/BatchMessageContainerBase.h b/pulsar-client-cpp/lib/BatchMessageContainerBase.h index 8a32d8e9dca8a..71eef5fab6287 100644 --- a/pulsar-client-cpp/lib/BatchMessageContainerBase.h +++ b/pulsar-client-cpp/lib/BatchMessageContainerBase.h @@ -112,6 +112,9 @@ class BatchMessageContainerBase : public boost::noncopyable { bool hasEnoughSpace(const Message& msg) const noexcept; bool isEmpty() const noexcept; + void processAndClear(std::function opSendMsgCallback, + FlushCallback flushCallback); + protected: // references to ProducerImpl's fields const std::string& topicName_; @@ -157,6 +160,29 @@ inline void BatchMessageContainerBase::resetStats() { sizeInBytes_ = 0; } +inline void BatchMessageContainerBase::processAndClear( + std::function opSendMsgCallback, FlushCallback flushCallback) { + if (isEmpty()) { + if (flushCallback) { + flushCallback(ResultOk); + } + } else { + const auto numBatches = getNumBatches(); + if (numBatches == 1) { + OpSendMsg opSendMsg; + Result result = createOpSendMsg(opSendMsg, flushCallback); + opSendMsgCallback(result, opSendMsg); + } else if (numBatches > 1) { + std::vector opSendMsgs; + std::vector results = createOpSendMsgs(opSendMsgs, flushCallback); + for (size_t i = 0; i < results.size(); i++) { + opSendMsgCallback(results[i], opSendMsgs[i]); + } + } // else numBatches is 0, do nothing + } + clear(); +} + inline std::ostream& operator<<(std::ostream& os, const BatchMessageContainerBase& container) { container.serialize(os); return os; diff --git a/pulsar-client-cpp/lib/ProducerImpl.cc b/pulsar-client-cpp/lib/ProducerImpl.cc index e9812d46054f7..e15d388ef64e2 100644 --- a/pulsar-client-cpp/lib/ProducerImpl.cc +++ b/pulsar-client-cpp/lib/ProducerImpl.cc @@ -268,13 +268,14 @@ std::shared_ptr ProducerImpl::getPendingCallback } if (batchMessageContainer_) { - OpSendMsg opSendMsg; - if (batchMessageContainer_->createOpSendMsg(opSendMsg) == ResultOk) { - callbacks->opSendMsgs.emplace_back(opSendMsg); - } - - releaseSemaphoreForSendOp(opSendMsg); - batchMessageContainer_->clear(); + batchMessageContainer_->processAndClear( + [this, &callbacks](Result result, const OpSendMsg& opSendMsg) { + if (result == ResultOk) { + callbacks->opSendMsgs.emplace_back(opSendMsg); + } + releaseSemaphoreForSendOp(opSendMsg); + }, + nullptr); } pendingMessagesQueue_.clear(); @@ -507,15 +508,8 @@ PendingFailures ProducerImpl::batchMessageAndSend(const FlushCallback& flushCall LOG_DEBUG("batchMessageAndSend " << *batchMessageContainer_); batchTimer_->cancel(); - if (PULSAR_UNLIKELY(batchMessageContainer_->isEmpty())) { - if (flushCallback) { - flushCallback(ResultOk); - } - } else { - const size_t numBatches = batchMessageContainer_->getNumBatches(); - if (numBatches == 1) { - OpSendMsg opSendMsg; - Result result = batchMessageContainer_->createOpSendMsg(opSendMsg, flushCallback); + batchMessageContainer_->processAndClear( + [this, &failures](Result result, const OpSendMsg& opSendMsg) { if (result == ResultOk) { sendMessage(opSendMsg); } else { @@ -525,25 +519,8 @@ PendingFailures ProducerImpl::batchMessageAndSend(const FlushCallback& flushCall releaseSemaphoreForSendOp(opSendMsg); failures.add(std::bind(opSendMsg.sendCallback_, result, MessageId{})); } - } else if (numBatches > 1) { - std::vector opSendMsgs; - std::vector results = batchMessageContainer_->createOpSendMsgs(opSendMsgs, flushCallback); - for (size_t i = 0; i < results.size(); i++) { - if (results[i] == ResultOk) { - sendMessage(opSendMsgs[i]); - } else { - // A spot has been reserved for this batch, but the batch failed to be pushed to the - // queue, so we need to release the spot manually - LOG_ERROR("batchMessageAndSend | Failed to createOpSendMsgs[" << i - << "]: " << results[i]); - releaseSemaphoreForSendOp(opSendMsgs[i]); - failures.add(std::bind(opSendMsgs[i].sendCallback_, results[i], MessageId{})); - } - } - } // else numBatches is 0, do nothing - } - - batchMessageContainer_->clear(); + }, + flushCallback); return failures; } diff --git a/pulsar-client-cpp/tests/KeyBasedBatchingTest.cc b/pulsar-client-cpp/tests/KeyBasedBatchingTest.cc index 3bec21ac3d738..fcb558a7dadb9 100644 --- a/pulsar-client-cpp/tests/KeyBasedBatchingTest.cc +++ b/pulsar-client-cpp/tests/KeyBasedBatchingTest.cc @@ -41,7 +41,6 @@ class KeyBasedBatchingTest : public ::testing::Test { void TearDown() override { client_.close(); } - void setTopicName(const std::string& topicName) { topicName_ = topicName; } void initTopicName(const std::string& testName) { topicName_ = "KeyBasedBatchingTest-" + testName + "-" + std::to_string(time(nullptr)); } @@ -179,3 +178,34 @@ TEST_F(KeyBasedBatchingTest, testSingleBatch) { ASSERT_EQ(ResultTimeout, consumer_.receive(msg, 3000)); ASSERT_EQ(numMessageSent.load(), numMessages); } + +TEST_F(KeyBasedBatchingTest, testCloseBeforeSend) { + initTopicName("CloseBeforeSend"); + // Any asynchronous send won't be completed unless `close()` or `flush()` is triggered + initProducer(createDefaultProducerConfig().setBatchingMaxMessages(static_cast(-1))); + + std::mutex mtx; + std::vector results; + auto saveResult = [&mtx, &results](Result result) { + std::lock_guard lock(mtx); + results.emplace_back(result); + }; + auto sendAsync = [saveResult, this](const std::string& key, const std::string& value) { + producer_.sendAsync(MessageBuilder().setOrderingKey(key).setContent(value).build(), + [saveResult](Result result, const MessageId& id) { saveResult(result); }); + }; + + constexpr int numKeys = 10; + for (int i = 0; i < numKeys; i++) { + sendAsync("key-" + std::to_string(i), "value"); + } + + ASSERT_EQ(ResultOk, producer_.close()); + + // After close() completed, all callbacks should have failed with ResultAlreadyClosed + std::lock_guard lock(mtx); + ASSERT_EQ(results.size(), numKeys); + for (int i = 0; i < numKeys; i++) { + ASSERT_EQ(results[i], ResultAlreadyClosed) << " results[" << i << "] is " << results[i]; + } +} From efed983c3e8f3ab9ee69ac069203bcdb8c8affc2 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 22 Apr 2022 23:48:42 +0800 Subject: [PATCH 539/823] [C++] Remove the flaky and meaningless tests (#15271) Fixes #13849 Fixes #14848 ### Motivation #11570 adds a `testSendAsyncCloseAsyncConcurrentlyWithLazyProducers` for the case that some `sendAsync` calls that are invoked after `closeAsync` is called in another thread must complete with `ResultAlreadyClosed`. It's flaky because the synchronization between two threads is not strict. This test uses `sendStartLatch` for the order of `sendAsync` and `closeAsync`: ``` sendAsync 0,1,...,9 -> sendStartLatch is done -> closeAsync ``` However, it cannot guarantee the rest `sendAsync` calls happen after `closeAsync` is called. If so, all `sendAsync` calls will complete with `ResultOk`. On the other hand, this test is meaningless because it requires strict synchronization between two threads so there is no need to run `sendAsync` and `closeAsync` in two threads. The verification of this test is also wrong, see https://github.com/apache/pulsar/issues/13849#issuecomment-1079098248. When `closeAsync` is called, the previous `sendAsync` calls might not complete, so all `sendAsync` will complete with `ResultAlreadyClosed`, not only those called after `closeAsync`. In addition, this PR also tries to fix the flaky `testReferenceCount`, which assumes too strictly. ### Modifications - Remove `testSendAsyncCloseAsyncConcurrentlyWithLazyProducers` - Only check the reference count is greater than 0 instead of equal to 1 (cherry picked from commit eeea9ca1f6eeef1248b7fe8f36be30be835d2480) --- pulsar-client-cpp/tests/ClientTest.cc | 2 +- pulsar-client-cpp/tests/ProducerTest.cc | 83 ------------------------- 2 files changed, 1 insertion(+), 84 deletions(-) diff --git a/pulsar-client-cpp/tests/ClientTest.cc b/pulsar-client-cpp/tests/ClientTest.cc index 1ba0164ad87b7..364e170f8969c 100644 --- a/pulsar-client-cpp/tests/ClientTest.cc +++ b/pulsar-client-cpp/tests/ClientTest.cc @@ -211,7 +211,7 @@ TEST(ClientTest, testReferenceCount) { LOG_INFO("Reference count of the reader's underlying consumer: " << consumers[1].use_count()); readerWeakPtr = PulsarFriend::getReaderImplWeakPtr(reader); - ASSERT_EQ(readerWeakPtr.use_count(), 1); + ASSERT_TRUE(readerWeakPtr.use_count() > 0); LOG_INFO("Reference count of the reader: " << readerWeakPtr.use_count()); } diff --git a/pulsar-client-cpp/tests/ProducerTest.cc b/pulsar-client-cpp/tests/ProducerTest.cc index 258811fcdaf33..9ddca1f704294 100644 --- a/pulsar-client-cpp/tests/ProducerTest.cc +++ b/pulsar-client-cpp/tests/ProducerTest.cc @@ -159,89 +159,6 @@ TEST(ProducerTest, testSendAsyncAfterCloseAsyncWithLazyProducers) { ASSERT_EQ(ResultOk, result); } -TEST(ProducerTest, testSendAsyncCloseAsyncConcurrentlyWithLazyProducers) { - // run sendAsync and closeAsync concurrently and verify that all sendAsync callbacks are called - // and that messages sent after closeAsync is invoked receive ResultAlreadyClosed. - for (int run = 0; run < 20; run++) { - LOG_INFO("Start of run " << run); - Client client(serviceUrl); - const std::string partitionedTopic = - "testProducerIsConnectedPartitioned-" + std::to_string(time(nullptr)); - - int res = makePutRequest( - adminUrl + "admin/v2/persistent/public/default/" + partitionedTopic + "/partitions", "10"); - ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; - - ProducerConfiguration producerConfiguration; - producerConfiguration.setLazyStartPartitionedProducers(true); - producerConfiguration.setPartitionsRoutingMode(ProducerConfiguration::UseSinglePartition); - producerConfiguration.setBatchingEnabled(true); - Producer producer; - ASSERT_EQ(ResultOk, client.createProducer(partitionedTopic, producerConfiguration, producer)); - - int sendCount = 100; - std::vector> promises(sendCount); - Promise promiseClose; - - // only call closeAsync once at least 10 messages have been sent - Latch sendStartLatch(10); - Latch closeLatch(1); - int closedAt = 0; - - std::thread t1([&]() { - for (int i = 0; i < sendCount; i++) { - sendStartLatch.countdown(); - Message msg = MessageBuilder().setContent("test").build(); - - if (closeLatch.getCount() == 0 && closedAt == 0) { - closedAt = i; - LOG_INFO("closedAt set to " << closedAt) - } - - producer.sendAsync(msg, WaitForCallbackValue(promises[i])); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - }); - - std::thread t2([&]() { - sendStartLatch.wait(std::chrono::milliseconds(1000)); - LOG_INFO("Closing"); - producer.closeAsync(WaitForCallback(promiseClose)); - LOG_INFO("Close called"); - closeLatch.countdown(); - Result result; - promiseClose.getFuture().get(result); - ASSERT_EQ(ResultOk, result); - LOG_INFO("Closed"); - }); - - t1.join(); - t2.join(); - - // make sure that all messages after the moment when closeAsync was invoked - // return AlreadyClosed - for (int i = 0; i < sendCount; i++) { - LOG_DEBUG("Checking " << i) - - // whether a message was sent successfully or not, it's callback - // must have been invoked - ASSERT_EQ(true, promises[i].isComplete()); - MessageId mi; - Result res = promises[i].getFuture().get(mi); - LOG_DEBUG("Result is " << res); - - // for the messages sent after closeAsync was invoked, they - // should all return ResultAlreadyClosed - if (i >= closedAt) { - ASSERT_EQ(ResultAlreadyClosed, res); - } - } - - client.close(); - LOG_INFO("End of run " << run); - } -} - TEST(ProducerTest, testBacklogQuotasExceeded) { std::string ns = "public/test-backlog-quotas"; std::string topic = ns + "/testBacklogQuotasExceeded" + std::to_string(time(nullptr)); From f6a7408b5ce58fceef599407f958db3f5f559412 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Wed, 20 Oct 2021 15:58:20 +0800 Subject: [PATCH 540/823] [pulsar-java-client] Auto-recovery after exception like out of direct memory (#12170) (cherry picked from commit ee62763ae4fe749b24857876ea0d8a34021042de) --- .../impl/BatchMessageContainerImpl.java | 34 ++++++--- .../impl/BatchMessageContainerImplTest.java | 75 +++++++++++++++++++ 2 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 9f81e03e5af8a..cea567e8d687e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -66,18 +66,28 @@ public boolean add(MessageImpl msg, SendCallback callback) { } if (++numMessagesInBatch == 1) { - // some properties are common amongst the different messages in the batch, hence we just pick it up from - // the first message - messageMetadata.setSequenceId(msg.getSequenceId()); - lowestSequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); - this.firstCallback = callback; - batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT - .buffer(Math.min(maxBatchSize, ClientCnx.getMaxMessageSize())); - if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { - currentTxnidMostBits = msg.getMessageBuilder().getTxnidMostBits(); - } - if (msg.getMessageBuilder().hasTxnidLeastBits() && currentTxnidLeastBits == -1) { - currentTxnidLeastBits = msg.getMessageBuilder().getTxnidLeastBits(); + try { + // some properties are common amongst the different messages in the batch, hence we just pick it up from + // the first message + messageMetadata.setSequenceId(msg.getSequenceId()); + lowestSequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); + this.firstCallback = callback; + batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT + .buffer(Math.min(maxBatchSize, ClientCnx.getMaxMessageSize())); + if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { + currentTxnidMostBits = msg.getMessageBuilder().getTxnidMostBits(); + } + if (msg.getMessageBuilder().hasTxnidLeastBits() && currentTxnidLeastBits == -1) { + currentTxnidLeastBits = msg.getMessageBuilder().getTxnidLeastBits(); + } + } catch (Throwable e) { + log.error("construct first message failed, exception is ", e); + if (batchedMessageMetadataAndPayload != null) { + // if payload has been allocated release it + batchedMessageMetadataAndPayload.release(); + } + discard(new PulsarClientException(e)); + return false; } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java new file mode 100644 index 0000000000000..3d554871141e3 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorBuilderImpl; +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorImpl; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.testng.IObjectFactory; +import org.testng.annotations.ObjectFactory; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +@PrepareForTest({ByteBufAllocatorImpl.class, ByteBufAllocatorBuilderImpl.class}) +@PowerMockIgnore({"javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*"}) +public class BatchMessageContainerImplTest { + + @ObjectFactory + public IObjectFactory getObjectFactory() { + return new org.powermock.modules.testng.PowerMockObjectFactory(); + } + + @Test + public void recoveryAfterOom() throws Exception { + final ByteBufAllocatorImpl mockAllocator = PowerMockito.mock(ByteBufAllocatorImpl.class); + PowerMockito.whenNew(ByteBufAllocatorImpl.class).withAnyArguments().thenReturn(mockAllocator); + PowerMockito.when(mockAllocator.buffer(Mockito.anyInt(), Mockito.anyInt())).thenThrow(new OutOfMemoryError("test")).thenReturn(null); + final ProducerImpl producer = Mockito.mock(ProducerImpl.class); + final ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); + producerConfigurationData.setCompressionType(CompressionType.NONE); + Mockito.when(producer.getConfiguration()).thenReturn(producerConfigurationData); + final BatchMessageContainerImpl batchMessageContainer = new BatchMessageContainerImpl(); + batchMessageContainer.setProducer(producer); + MessageMetadata messageMetadata1 = new MessageMetadata(); + messageMetadata1.setSequenceId(1L); + messageMetadata1.setProducerName("producer1"); + messageMetadata1.setPublishTime(System.currentTimeMillis()); + ByteBuffer payload1 = ByteBuffer.wrap("payload1".getBytes(StandardCharsets.UTF_8)); + final MessageImpl message1 = MessageImpl.create(messageMetadata1, payload1, Schema.BYTES, null); + batchMessageContainer.add(message1, null); + MessageMetadata messageMetadata2 = new MessageMetadata(); + messageMetadata2.setSequenceId(1L); + messageMetadata2.setProducerName("producer1"); + messageMetadata2.setPublishTime(System.currentTimeMillis()); + ByteBuffer payload2 = ByteBuffer.wrap("payload2".getBytes(StandardCharsets.UTF_8)); + final MessageImpl message2 = MessageImpl.create(messageMetadata2, payload2, Schema.BYTES, null); + // after oom, our add can self-healing, won't throw exception + batchMessageContainer.add(message2, null); + } + +} From dc861c46b5b57693f0f4e132a70ba9153096775e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 13 May 2022 20:18:42 +0800 Subject: [PATCH 541/823] [Java Client] Fix wrong behavior of deduplication for key based batching (#15413) ### Motivation Currently message deduplication doesn't work well for key based batching. First, the key based batch container doesn't update the `lastSequenceIdPushed`. So a batch could contain both duplicated and not duplicated messages. Second, when `createOpSendMsgs` is called, the `OpSendMsg` objects are sorted by the lowest sequence ids, and the highest sequence id is not set. If a batch contains sequence id 0,1,2, then the message with sequence id 1 or 2 won't be dropped. ### Modifications - Refactor the key based batch container that the `BatchMessageContainerImpl` is reused instead of maintaining a `KeyedBatch` class. - When `createOpSendMsgs` is called, clear the highest sequence id field and configure the sequence id field with the highest sequence id to fix the second issue described before. - Add `testKeyBasedBatchingOrder` to show and verify the current behavior. - Add test for key based batching into `testProducerDeduplicationWithDiscontinuousSequenceId` to verify `lastSlastSequenceIdPushed` is updated correctly. (cherry picked from commit a77333705ffb352da39767d53975353bb4f8864e) --- .../client/api/ClientDeduplicationTest.java | 98 +++++++- .../impl/BatchMessageContainerImpl.java | 28 ++- .../impl/BatchMessageKeyBasedContainer.java | 211 ++++-------------- 3 files changed, 155 insertions(+), 182 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java index 304bb6eaaa02d..52017444a2b76 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java @@ -20,19 +20,37 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Slf4j @Test(groups = "flaky") public class ClientDeduplicationTest extends ProducerConsumerBase { + @DataProvider + public static Object[][] batchingTypes() { + return new Object[][] { + { BatcherBuilder.DEFAULT }, + { BatcherBuilder.KEY_BASED } + }; + } + @BeforeClass @Override protected void setup() throws Exception { @@ -46,7 +64,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - @Test + @Test(priority = -1) public void testNamespaceDeduplicationApi() throws Exception { final String namespace = "my-property/my-ns"; assertNull(admin.namespaces().getDeduplicationStatus(namespace)); @@ -174,9 +192,10 @@ public void testProducerDeduplication() throws Exception { producer.close(); } - @Test(timeOut = 30000) - public void testProducerDeduplicationWithDiscontinuousSequenceId() throws Exception { - String topic = "persistent://my-property/my-ns/testProducerDeduplicationWithDiscontinuousSequenceId"; + @Test(timeOut = 30000, dataProvider = "batchingTypes") + public void testProducerDeduplicationWithDiscontinuousSequenceId(BatcherBuilder batcherBuilder) throws Exception { + String topic = "persistent://my-property/my-ns/testProducerDeduplicationWithDiscontinuousSequenceId-" + + System.currentTimeMillis(); admin.namespaces().setDeduplicationStatus("my-property/my-ns", true); // Set infinite timeout @@ -185,7 +204,9 @@ public void testProducerDeduplicationWithDiscontinuousSequenceId() throws Except .topic(topic) .producerName("my-producer-name") .enableBatching(true) + .batcherBuilder(batcherBuilder) .batchingMaxMessages(10) + .batchingMaxPublishDelay(1L, TimeUnit.HOURS) .sendTimeout(0, TimeUnit.SECONDS); Producer producer = producerBuilder.create(); @@ -208,7 +229,8 @@ public void testProducerDeduplicationWithDiscontinuousSequenceId() throws Except producer.flush(); for (int i = 0; i < 4; i++) { - Message msg = consumer.receive(); + Message msg = consumer.receive(3, TimeUnit.SECONDS); + assertNotNull(msg); assertEquals(new String(msg.getData()), "my-message-" + i); consumer.acknowledge(msg); } @@ -284,4 +306,68 @@ public void testProducerDeduplicationNonBatchAsync() throws Exception { producer.close(); } + + @Test(timeOut = 30000) + public void testKeyBasedBatchingOrder() throws Exception { + final String topic = "persistent://my-property/my-ns/test-key-based-batching-order"; + admin.namespaces().setDeduplicationStatus("my-property/my-ns", true); + + final Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscribe(); + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .batcherBuilder(BatcherBuilder.KEY_BASED) + .batchingMaxMessages(100) + .batchingMaxBytes(1024 * 1024 * 5) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + // | key | sequence id list | + // | :-- | :--------------- | + // | A | 0, 3, 4 | + // | B | 1, 2 | + final List> sendFutures = new ArrayList<>(); + sendFutures.add(producer.newMessage().key("A").value("msg-0").sequenceId(0L).sendAsync()); + sendFutures.add(producer.newMessage().key("B").value("msg-1").sequenceId(1L).sendAsync()); + sendFutures.add(producer.newMessage().key("B").value("msg-2").sequenceId(2L).sendAsync()); + sendFutures.add(producer.newMessage().key("A").value("msg-3").sequenceId(3L).sendAsync()); + sendFutures.add(producer.newMessage().key("A").value("msg-4").sequenceId(4L).sendAsync()); + // The message order is expected to be [1, 2, 0, 3, 4]. The sequence ids are not ordered strictly, but: + // 1. The sequence ids for a given key are ordered. + // 2. The highest sequence ids of batches are ordered. + producer.flush(); + + FutureUtil.waitForAll(sendFutures); + final List sendMessageIds = sendFutures.stream().map(CompletableFuture::join) + .collect(Collectors.toList()); + for (int i = 0; i < sendMessageIds.size(); i++) { + log.info("Send msg-{} to {}", i, sendMessageIds.get(i)); + } + + final List sequenceIdList = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + final Message msg = consumer.receive(3, TimeUnit.SECONDS); + if (msg == null) { + break; + } + log.info("Received {}, key: {}, seq id: {}, msg id: {}", + msg.getValue(), msg.getKey(), msg.getSequenceId(), msg.getMessageId()); + assertNotNull(msg); + sequenceIdList.add(msg.getSequenceId()); + } + assertEquals(sequenceIdList, Arrays.asList(1L, 2L, 0L, 3L, 4L)); + + for (int i = 0; i < 5; i++) { + // Currently sending a duplicated message won't throw an exception. Instead, an invalid result is returned. + final MessageId messageId = producer.newMessage().value("msg").sequenceId(i).send(); + assertTrue(messageId instanceof BatchMessageIdImpl); + final BatchMessageIdImpl messageIdImpl = (BatchMessageIdImpl) messageId; + assertEquals(messageIdImpl.getLedgerId(), -1L); + assertEquals(messageIdImpl.getEntryId(), -1L); + } + + consumer.close(); + producer.close(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index cea567e8d687e..996875a7131fa 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -19,13 +19,13 @@ package org.apache.pulsar.client.impl; import com.google.common.collect.Lists; - import io.netty.buffer.ByteBuf; - +import io.netty.util.ReferenceCountUtil; import java.io.IOException; import java.util.Arrays; import java.util.List; - +import lombok.Getter; +import lombok.Setter; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; @@ -49,7 +49,11 @@ class BatchMessageContainerImpl extends AbstractBatchMessageContainer { private MessageMetadata messageMetadata = new MessageMetadata(); // sequence id for this batch which will be persisted as a single entry by broker + @Getter + @Setter private long lowestSequenceId = -1L; + @Getter + @Setter private long highestSequenceId = -1L; private ByteBuf batchedMessageMetadataAndPayload; private List> messages = Lists.newArrayList(); @@ -57,6 +61,14 @@ class BatchMessageContainerImpl extends AbstractBatchMessageContainer { // keep track of callbacks for individual messages being published in a batch protected SendCallback firstCallback; + public BatchMessageContainerImpl() { + } + + public BatchMessageContainerImpl(ProducerImpl producer) { + this(); + setProducer(producer); + } + @Override public boolean add(MessageImpl msg, SendCallback callback) { @@ -82,10 +94,6 @@ public boolean add(MessageImpl msg, SendCallback callback) { } } catch (Throwable e) { log.error("construct first message failed, exception is ", e); - if (batchedMessageMetadataAndPayload != null) { - // if payload has been allocated release it - batchedMessageMetadataAndPayload.release(); - } discard(new PulsarClientException(e)); return false; } @@ -104,7 +112,6 @@ public boolean add(MessageImpl msg, SendCallback callback) { } highestSequenceId = msg.getSequenceId(); ProducerImpl.LAST_SEQ_ID_PUSHED_UPDATER.getAndUpdate(producer, prev -> Math.max(prev, msg.getSequenceId())); - return isBatchFull(); } @@ -172,6 +179,10 @@ public void discard(Exception ex) { if (firstCallback != null) { firstCallback.sendComplete(ex); } + if (batchedMessageMetadataAndPayload != null) { + ReferenceCountUtil.safeRelease(batchedMessageMetadataAndPayload); + batchedMessageMetadataAndPayload = null; + } } catch (Throwable t) { log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, producerName, lowestSequenceId, t); @@ -193,6 +204,7 @@ public OpSendMsg createOpSendMsg() throws IOException { return null; } messageMetadata.setNumMessagesInBatch(numMessagesInBatch); + messageMetadata.setSequenceId(lowestSequenceId); messageMetadata.setHighestSequenceId(highestSequenceId); if (currentTxnidMostBits != -1) { messageMetadata.setTxnidMostBits(currentTxnidMostBits); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index 505ca75743c53..77990eeeacbf1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -18,27 +18,12 @@ */ package org.apache.pulsar.client.impl; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; - -import io.netty.buffer.ByteBuf; -import io.netty.util.ReferenceCountUtil; - import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; - -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; -import org.apache.pulsar.common.api.proto.CompressionType; -import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.compression.CompressionCodec; -import org.apache.pulsar.common.protocol.ByteBufPair; -import org.apache.pulsar.common.protocol.Commands; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +38,7 @@ */ class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { - private Map batches = new HashMap<>(); + private final Map batches = new HashMap<>(); @Override public boolean add(MessageImpl msg, SendCallback callback) { @@ -61,29 +46,16 @@ public boolean add(MessageImpl msg, SendCallback callback) { log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, producerName, numMessagesInBatch); } - numMessagesInBatch++; - currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); String key = getKey(msg); - KeyedBatch part = batches.get(key); - if (part == null) { - part = new KeyedBatch(); - part.addMsg(msg, callback); - part.compressionType = compressionType; - part.compressor = compressor; - part.maxBatchSize = maxBatchSize; - part.topicName = topicName; - part.producerName = producerName; - batches.putIfAbsent(key, part); - - if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { - currentTxnidMostBits = msg.getMessageBuilder().getTxnidMostBits(); - } - if (msg.getMessageBuilder().hasTxnidLeastBits() && currentTxnidLeastBits == -1) { - currentTxnidLeastBits = msg.getMessageBuilder().getTxnidLeastBits(); - } - - } else { - part.addMsg(msg, callback); + final BatchMessageContainerImpl batchMessageContainer = batches.computeIfAbsent(key, + __ -> new BatchMessageContainerImpl(producer)); + batchMessageContainer.add(msg, callback); + // The `add` method fails iff the container is empty, i.e. the `msg` is the first message to add, while `msg` + // was failed to add. In this case, `clear` method will be called and the batch container is empty and there is + // no need to update the stats. + if (!batchMessageContainer.isEmpty()) { + numMessagesInBatch++; + currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); } return isBatchFull(); } @@ -92,7 +64,7 @@ public boolean add(MessageImpl msg, SendCallback callback) { public void clear() { numMessagesInBatch = 0; currentBatchSizeBytes = 0; - batches = new HashMap<>(); + batches.clear(); currentTxnidMostBits = -1L; currentTxnidLeastBits = -1L; } @@ -104,13 +76,7 @@ public boolean isEmpty() { @Override public void discard(Exception ex) { - try { - // Need to protect ourselves from any exception being thrown in the future handler from the application - batches.forEach((k, v) -> v.firstCallback.sendComplete(ex)); - } catch (Throwable t) { - log.warn("[{}] [{}] Got exception while completing the callback", topicName, producerName, t); - } - batches.forEach((k, v) -> ReferenceCountUtil.safeRelease(v.batchedMessageMetadataAndPayload)); + batches.forEach((k, v) -> v.discard(ex)); clear(); } @@ -119,64 +85,45 @@ public boolean isMultiBatches() { return true; } - private ProducerImpl.OpSendMsg createOpSendMsg(KeyedBatch keyedBatch) throws IOException { - ByteBuf encryptedPayload = producer.encryptMessage(keyedBatch.messageMetadata, keyedBatch.getCompressedBatchMetadataAndPayload()); - if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { - keyedBatch.discard(new PulsarClientException.InvalidMessageException( - "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); - return null; - } - - final int numMessagesInBatch = keyedBatch.messages.size(); - long currentBatchSizeBytes = 0; - for (MessageImpl message : keyedBatch.messages) { - currentBatchSizeBytes += message.getDataBuffer().readableBytes(); - } - keyedBatch.messageMetadata.setNumMessagesInBatch(numMessagesInBatch); - if (currentTxnidMostBits != -1) { - keyedBatch.messageMetadata.setTxnidMostBits(currentTxnidMostBits); - } - if (currentTxnidLeastBits != -1) { - keyedBatch.messageMetadata.setTxnidLeastBits(currentTxnidLeastBits); - } - ByteBufPair cmd = producer.sendMessage(producer.producerId, keyedBatch.sequenceId, numMessagesInBatch, - keyedBatch.messageMetadata, encryptedPayload); - - ProducerImpl.OpSendMsg op = ProducerImpl.OpSendMsg.create(keyedBatch.messages, cmd, keyedBatch.sequenceId, keyedBatch.firstCallback); - - op.setNumMessagesInBatch(numMessagesInBatch); - op.setBatchSizeByte(currentBatchSizeBytes); - return op; - } - @Override public List createOpSendMsgs() throws IOException { - List result = new ArrayList<>(); - List list = new ArrayList<>(batches.values()); - list.sort(((o1, o2) -> ComparisonChain.start() - .compare(o1.sequenceId, o2.sequenceId) - .result())); - for (KeyedBatch keyedBatch : list) { - ProducerImpl.OpSendMsg op = createOpSendMsg(keyedBatch); - if (op != null) { - result.add(op); + try { + // In key based batching, the sequence ids might not be ordered, for example, + // | key | sequence id list | + // | :-- | :--------------- | + // | A | 0, 3, 4 | + // | B | 1, 2 | + // The message order should be 1, 2, 0, 3, 4 so that a message with a sequence id <= 4 should be dropped. + // However, for a MessageMetadata with both `sequence_id` and `highest_sequence_id` fields, the broker will + // expect a strict order so that the batch of key "A" (0, 3, 4) will be dropped. + // Therefore, we should update the `sequence_id` field to the highest sequence id and remove the + // `highest_sequence_id` field to allow the weak order. + batches.values().forEach(batchMessageContainer -> { + batchMessageContainer.setLowestSequenceId(batchMessageContainer.getHighestSequenceId()); + }); + return batches.values().stream().sorted((o1, o2) -> + (int) (o1.getLowestSequenceId() - o2.getLowestSequenceId()) + ).map(batchMessageContainer -> { + try { + return batchMessageContainer.createOpSendMsg(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + }).collect(Collectors.toList()); + } catch (IllegalStateException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } else { + throw e; } } - return result; } @Override public boolean hasSameSchema(MessageImpl msg) { String key = getKey(msg); - KeyedBatch part = batches.get(key); - if (part == null || part.messages.isEmpty()) { - return true; - } - if (!part.messageMetadata.hasSchemaVersion()) { - return msg.getSchemaVersion() == null; - } - return Arrays.equals(msg.getSchemaVersion(), - part.messageMetadata.getSchemaVersion()); + BatchMessageContainerImpl batchMessageContainer = batches.get(key); + return batchMessageContainer == null || batchMessageContainer.hasSameSchema(msg); } private String getKey(MessageImpl msg) { @@ -186,78 +133,6 @@ private String getKey(MessageImpl msg) { return msg.getKey(); } - private static class KeyedBatch { - private final MessageMetadata messageMetadata = new MessageMetadata(); - // sequence id for this batch which will be persisted as a single entry by broker - private long sequenceId = -1; - private ByteBuf batchedMessageMetadataAndPayload; - private List> messages = Lists.newArrayList(); - private SendCallback previousCallback = null; - private CompressionType compressionType; - private CompressionCodec compressor; - private int maxBatchSize; - private String topicName; - private String producerName; - - // keep track of callbacks for individual messages being published in a batch - private SendCallback firstCallback; - - private ByteBuf getCompressedBatchMetadataAndPayload() { - for (MessageImpl msg : messages) { - batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msg.getMessageBuilder(), - msg.getDataBuffer(), batchedMessageMetadataAndPayload); - } - int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); - ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); - batchedMessageMetadataAndPayload.release(); - if (compressionType != CompressionType.NONE) { - messageMetadata.setCompression(compressionType); - messageMetadata.setUncompressedSize(uncompressedSize); - } - - // Update the current max batch size using the uncompressed size, which is what we need in any case to - // accumulate the batch content - maxBatchSize = Math.max(maxBatchSize, uncompressedSize); - return compressedPayload; - } - - private void addMsg(MessageImpl msg, SendCallback callback) { - if (messages.size() == 0) { - sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); - batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT - .buffer(Math.min(maxBatchSize, ClientCnx.getMaxMessageSize())); - firstCallback = callback; - } - if (previousCallback != null) { - previousCallback.addCallback(msg, callback); - } - previousCallback = callback; - messages.add(msg); - } - - public void discard(Exception ex) { - try { - // Need to protect ourselves from any exception being thrown in the future handler from the application - if (firstCallback != null) { - firstCallback.sendComplete(ex); - } - } catch (Throwable t) { - log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, producerName, - sequenceId, t); - } - clear(); - } - - public void clear() { - messages = Lists.newArrayList(); - firstCallback = null; - previousCallback = null; - messageMetadata.clear(); - sequenceId = -1; - batchedMessageMetadataAndPayload = null; - } - } - private static final Logger log = LoggerFactory.getLogger(BatchMessageKeyBasedContainer.class); } From 4f65f65c6fe6f7e97e83a13d038c6dbf28c3f8c6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 19 May 2022 11:49:25 +0800 Subject: [PATCH 542/823] [Java Client] Fix messages sent by producers without schema cannot be decoded (#15622) ### Motivation When I tried to consume a topic via a consumer with Avro schema while the topic was produced by a producer without schema, the consumption failed. It's because `MultiVersionSchemaInfoProvider#getSchemaByVersion` doesn't check if `schemaVersion` is an empty byte array. If yes, a `BytesSchemaVersion` of an empty array will be passed to `cache.get` and then passed to `loadSchema`. https://github.com/apache/pulsar/blob/f90ef9c6ad88c4f94ce1fcc682bbf3f3189cbf2a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProvider.java#L94-L98 However, `LookupService#getSchema` cannot accept an empty byte array as the version, so `loadSchema` failed. The root cause is that the schema version was set unexpectly when messages were sent by a producer without schema. At broker side, the returned schema version is never null. If the schema version was an empty array, then it means the message doesn't have schema. However, at Java client side, the empty byte array is treated as an existing schema and the schema version field will be set. When consumer receives the message, it will try to load schema whose version is an empty array. ### Modifications - When a producer receives a response whose schema version is an empty byte array, just ignore it. - Make `MesasgeImpl#getSchemaVersion` return null if the schema version is an empty byte array so that the consumer can consume messages produced by older version producers without schema. And return the internal schema for `getRegetReaderSchema` when `getSchemaVersion` returns null. - Fix the existing tests. Since producer without schema won't set the `schema_version` field after this patch, some tests that rely on the precise stats should be modified. - Add `testConsumeAvroMessagesWithoutSchema` to cover the case that messages without schema are compatible with the schema. This patch also modifies the existing behavior when `schemaValidationEnforced` is false and messages are produced by a producer without schema and consumed by a consumer with schema. 1. If the message is incompatible with the schema - Before: `getSchemaVersion` returns an empty array and `getValue` fails with `SerializationException`: > org.apache.commons.lang3.SerializationException: Failed at fetching schema info for EMPTY - After: `getSchemaVersion` returns `null` and `getValue` fails with `SchemaSerializationException`. 2. Otherwise (the message is compatible with the schema) - Before: `getSchemaVersion` returns an empty array and `getValue` fails with `SerializationException`. - After: `getSchemaVersion` returns `null` and `getValue` returns the correctly decoded object. (cherry picked from commit ecd275dc21f33483a649e5b872990771257b1d45) --- .../broker/admin/PersistentTopicsTest.java | 2 +- .../RGUsageMTAggrWaitForAllMsgsTest.java | 7 ++- .../broker/stats/PrometheusMetricsTest.java | 2 +- .../pulsar/client/api/SimpleSchemaTest.java | 52 +++++++++++++++++-- .../org/apache/pulsar/schema/SchemaTest.java | 17 +++--- .../pulsar/client/impl/MessageImpl.java | 26 ++++++++-- .../pulsar/client/impl/ProducerImpl.java | 11 ++-- .../pulsar/client/impl/ProducerResponse.java | 14 ++++- 8 files changed, 105 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 78f3d2e7a209f..2baf5b0f4692c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -737,7 +737,7 @@ public void testGetBacklogSizeByMessageId() throws Exception{ completableFuture = batchProducer.sendAsync("a".getBytes()); } completableFuture.get(); - Assert.assertEquals(Optional.ofNullable(admin.topics().getBacklogSizeByMessageId(topicName + "-partition-0", MessageId.earliest)), Optional.of(350L)); + Assert.assertEquals(Optional.ofNullable(admin.topics().getBacklogSizeByMessageId(topicName + "-partition-0", MessageId.earliest)), Optional.of(320L)); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java index fda8693dd8478..ce3f033a3b0f1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java @@ -556,7 +556,7 @@ private void verfyRGProdConsStats(String[] topicStrings, log.debug("verfyProdConsStats: topicStatsMap has {} entries", topicStatsMap.size()); - // Pulsar runtime adds some additional bytes in the exchanges: a 45-byte per-message + // Pulsar runtime adds some additional bytes in the exchanges: a 42-byte per-message // metadata of some kind, plus more as the number of messages increases. // Hence the ">=" assertion with ExpectedNumBytesSent/Received in the following checks. final int ExpectedNumBytesSent = sentNumBytes + PER_MESSAGE_METADATA_OHEAD * sentNumMsgs; @@ -787,9 +787,8 @@ private void verifyRGMetrics(String[] topicStrings, } private static final Logger log = LoggerFactory.getLogger(RGUsageMTAggrWaitForAllMsgsTest.class); - - // Empirically, there appears to be a 45-byte overhead for metadata, imposed by Pulsar runtime. - private static final int PER_MESSAGE_METADATA_OHEAD = 45; + // Empirically, there appears to be a 42-byte overhead for metadata, imposed by Pulsar runtime. + private static final int PER_MESSAGE_METADATA_OHEAD = 42; private static final int PUBLISH_INTERVAL_SECS = 10; private static final int NUM_PRODUCERS = 4; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 9f098935efabf..3a7d23b06e7aa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1217,7 +1217,7 @@ public void testCompaction() throws Exception { assertEquals(cm.get(0).value, 10); cm = (List) metrics.get("pulsar_compaction_compacted_entries_size"); assertEquals(cm.size(), 1); - assertEquals(cm.get(0).value, 870); + assertEquals(cm.get(0).value, 840); pulsarClient.close(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index fd8036eaf9e3a..983a7f341e06d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -49,7 +49,6 @@ import org.apache.pulsar.client.impl.schema.reader.AvroReader; import org.apache.pulsar.client.impl.schema.writer.AvroWriter; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; @@ -62,6 +61,7 @@ import org.testng.annotations.Test; import java.io.ByteArrayInputStream; +import java.io.EOFException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -305,7 +305,13 @@ public void newProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Ex + " if SchemaValidationEnabled is enabled"); } Message msg3 = c.receive(); - Assert.assertEquals(msg3.getSchemaVersion(), SchemaVersion.Empty.bytes()); + assertNull(msg3.getSchemaVersion()); + try { + msg3.getValue(); + fail("Schema should be incompatible"); + } catch (SchemaSerializationException e) { + assertTrue(e.getCause() instanceof EOFException); + } } catch (PulsarClientException e) { if (schemaValidationEnforced) { Assert.assertTrue(e instanceof IncompatibleSchemaException); @@ -366,7 +372,13 @@ public void newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema() + " if SchemaValidationEnabled is enabled"); } Message msg3 = c.receive(); - Assert.assertEquals(msg3.getSchemaVersion(), SchemaVersion.Empty.bytes()); + assertNull(msg3.getSchemaVersion()); + try { + msg3.getValue(); + fail("Schema should be incompatible"); + } catch (SchemaSerializationException e) { + assertTrue(e.getCause() instanceof EOFException); + } } catch (PulsarClientException e) { if (schemaValidationEnforced) { Assert.assertTrue(e instanceof IncompatibleSchemaException); @@ -1253,4 +1265,38 @@ public void testAutoKeyValueConsumeGenericObjectNullValues(KeyValueEncodingType } } + + @Test + public void testConsumeAvroMessagesWithoutSchema() throws Exception { + if (schemaValidationEnforced) { + return; + } + final String topic = "test-consume-avro-messages-without-schema-" + UUID.randomUUID(); + final Schema schema = Schema.AVRO(V1Data.class); + final Consumer consumer = pulsarClient.newConsumer(schema) + .topic(topic) + .subscriptionName("sub") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + final Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + + final int numMessages = 5; + for (int i = 0; i < numMessages; i++) { + producer.send(schema.encode(new V1Data(i))); + } + + for (int i = 0; i < numMessages; i++) { + final Message msg = consumer.receive(3, TimeUnit.SECONDS); + assertNotNull(msg); + log.info("Received {} from {}", msg.getValue().i, topic); + assertEquals(msg.getValue().i, i); + assertEquals(msg.getReaderSchema().orElse(Schema.BYTES).getSchemaInfo(), schema.getSchemaInfo()); + consumer.acknowledge(msg); + } + + producer.close(); + consumer.close(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index c45888f3858e0..682d6a52b7897 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -29,7 +29,6 @@ import static org.testng.Assert.fail; import static org.testng.internal.junit.ArrayAsserts.assertArrayEquals; -import com.google.common.base.Throwables; import lombok.EqualsAndHashCode; import org.apache.avro.Schema.Parser; import com.fasterxml.jackson.databind.JsonNode; @@ -1071,7 +1070,7 @@ public void testAvroSchemaWithHttpLookup() throws Exception { stopBroker(); isTcpLookup = false; setup(); - testEmptySchema(); + testIncompatibleSchema(); } @Test @@ -1079,10 +1078,10 @@ public void testAvroSchemaWithTcpLookup() throws Exception { stopBroker(); isTcpLookup = true; setup(); - testEmptySchema(); + testIncompatibleSchema(); } - private void testEmptySchema() throws Exception { + private void testIncompatibleSchema() throws Exception { final String namespace = "test-namespace-" + randomName(16); String ns = PUBLIC_TENANT + "/" + namespace; admin.namespaces().createNamespace(ns, Sets.newHashSet(CLUSTER_NAME)); @@ -1116,12 +1115,14 @@ private void testEmptySchema() throws Exception { producer.send("test".getBytes(StandardCharsets.UTF_8)); Message message1 = consumer.receive(); Assert.assertEquals(test, message1.getValue()); + Message message2 = consumer.receive(); try { - Message message2 = consumer.receive(); message2.getValue(); - } catch (Throwable ex) { - Assert.assertTrue(Throwables.getRootCause(ex) instanceof SchemaSerializationException); - Assert.assertEquals(Throwables.getRootCause(ex).getMessage(),"Empty schema version"); + } catch (SchemaSerializationException e) { + final String schemaString = + new String(Schema.AVRO(User.class).getSchemaInfo().getSchema(), StandardCharsets.UTF_8); + Assert.assertTrue(e.getMessage().contains(schemaString)); + Assert.assertTrue(e.getMessage().contains("payload (4 bytes)")); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java index 67c176cfe63bf..fd00475de71f3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Collections; import java.util.List; @@ -42,6 +43,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.impl.schema.AbstractSchema; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; @@ -391,12 +393,14 @@ public Optional> getReaderSchema() { if (schema == null) { return Optional.empty(); } + byte[] schemaVersion = getSchemaVersion(); + if (schemaVersion == null) { + return Optional.of(schema); + } if (schema instanceof AutoConsumeSchema) { - byte[] schemaVersion = getSchemaVersion(); return Optional.of(((AutoConsumeSchema) schema) .atSchemaVersion(schemaVersion)); } else if (schema instanceof AbstractSchema) { - byte[] schemaVersion = getSchemaVersion(); return Optional.of(((AbstractSchema) schema) .atSchemaVersion(schemaVersion)); } else { @@ -404,10 +408,13 @@ public Optional> getReaderSchema() { } } + // For messages produced by older version producers without schema, the schema version is an empty byte array + // rather than null. @Override public byte[] getSchemaVersion() { if (msgMetadata.hasSchemaVersion()) { - return msgMetadata.getSchemaVersion(); + byte[] schemaVersion = msgMetadata.getSchemaVersion(); + return (schemaVersion.length == 0) ? null : schemaVersion; } else { return null; } @@ -472,8 +479,19 @@ private KeyValueSchemaImpl getKeyValueSchema() { } } - private T decode(byte[] schemaVersion) { + try { + return decodeBySchema(schemaVersion); + } catch (ArrayIndexOutOfBoundsException e) { + // It usually means the message was produced without schema check while the message is not compatible with + // the current schema. Therefore, convert it to SchemaSerializationException with a better description. + final int payloadSize = payload.readableBytes(); + throw new SchemaSerializationException("payload (" + payloadSize + " bytes) cannot be decoded with schema " + + new String(schema.getSchemaInfo().getSchema(), StandardCharsets.UTF_8)); + } + } + + private T decodeBySchema(byte[] schemaVersion) { T value = poolMessage ? schema.decode(payload.nioBuffer(), schemaVersion) : null; if (value != null) { return value; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 2637c4953046f..d5d1d6e73825b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -675,9 +675,14 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call } } else { log.info("[{}] [{}] GetOrCreateSchema succeed", topic, producerName); - SchemaHash schemaHash = SchemaHash.of(msg.getSchemaInternal()); - schemaCache.putIfAbsent(schemaHash, v); - msg.getMessageBuilder().setSchemaVersion(v); + // In broker, if schema version is an empty byte array, it means the topic doesn't have schema. In this + // case, we should not cache the schema version so that the schema version of the message metadata will + // be null, instead of an empty array. + if (v.length != 0) { + SchemaHash schemaHash = SchemaHash.of(msg.getSchemaInternal()); + schemaCache.putIfAbsent(schemaHash, v); + msg.getMessageBuilder().setSchemaVersion(v); + } msg.setSchemaState(MessageImpl.SchemaState.Ready); } cnx.ctx().channel().eventLoop().execute(() -> { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerResponse.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerResponse.java index 36b47f2b6d62f..2c9cfa74d1aee 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerResponse.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerResponse.java @@ -21,9 +21,9 @@ import java.util.Optional; import lombok.AllArgsConstructor; -import lombok.Data; +import lombok.Getter; -@Data +@Getter @AllArgsConstructor public class ProducerResponse { private String producerName; @@ -31,4 +31,14 @@ public class ProducerResponse { private byte[] schemaVersion; private Optional topicEpoch; + + // Shadow the default getter generated by lombok. In broker, if the schema version is an empty byte array, it means + // the topic doesn't have schema. + public byte[] getSchemaVersion() { + if (schemaVersion != null && schemaVersion.length != 0) { + return schemaVersion; + } else { + return null; + } + } } From 7ca2fc3d25e0716c330aa8dc67422074a8ad0bc8 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 24 May 2022 21:30:12 +0800 Subject: [PATCH 543/823] [branch-2.9][fix][broker] Fix `` license issue. (#15758) --- pulsar-sql/presto-distribution/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index cd6bec3020ba6..2785c979c7a5e 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -38,7 +38,6 @@ 0.170 2.6 0.0.12 - 4.2.0 2.13.2 From 698bf60780d4fcfd628fb136f67312ed72a07396 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 24 May 2022 21:30:37 +0800 Subject: [PATCH 544/823] [branch-2.9][fix][deploy]: fix the pid occupied check when use pulsar-daemon start or stop process. (#15745) --- bin/pulsar-daemon | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon index 8bbc62806be3b..3bd82acb946a7 100755 --- a/bin/pulsar-daemon +++ b/bin/pulsar-daemon @@ -143,7 +143,7 @@ mkdir -p "$PULSAR_LOG_DIR" case $startStop in (start) if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then + if ps -p `cat $pid` > /dev/null 2>&1; then echo $command running as process `cat $pid`. Stop it first. exit 1 fi @@ -165,7 +165,7 @@ case $startStop in (stop) if [ -f $pid ]; then TARGET_PID=$(cat $pid) - if kill -0 $TARGET_PID > /dev/null 2>&1; then + if ps -p $TARGET_PID > /dev/null 2>&1; then echo "stopping $command" kill $TARGET_PID @@ -186,7 +186,7 @@ case $startStop in echo "Shutdown completed." fi - if kill -0 $TARGET_PID > /dev/null 2>&1; then + if ps -p $TARGET_PID > /dev/null 2>&1; then fileName=$location/$command.out $JAVA_HOME/bin/jstack $TARGET_PID > $fileName echo "Thread dumps are taken for analysis at $fileName" From f7058ba58bef010c5eb9776c0856bbdc231fda24 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 25 May 2022 07:26:14 +0800 Subject: [PATCH 545/823] [branch-2.9][enh][monitor]: add metrics for pulsar web service thread pool (#15741) --- .../PrometheusMetricsGenerator.java | 4 +- .../pulsar/broker/web/WebExecutorStats.java | 100 ++++++++++++++++++ .../apache/pulsar/broker/web/WebService.java | 4 + .../broker/stats/PrometheusMetricsTest.java | 6 +- .../pulsar/broker/web/WebServiceTest.java | 47 ++++++-- 5 files changed, 151 insertions(+), 10 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index 9d5e1c77c69a4..cd6afd1535dec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -242,7 +242,9 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String for (int i = 0; i < metricFamily.samples.size(); i++) { Sample sample = metricFamily.samples.get(i); stream.write(sample.name); - stream.write("{cluster=\"").write(cluster).write('"'); + if (!sample.labelNames.contains("cluster")) { + stream.write("{cluster=\"").write(cluster).write('"'); + } for (int j = 0; j < sample.labelNames.size(); j++) { String labelValue = sample.labelValues.get(j); if (labelValue != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java new file mode 100644 index 0000000000000..45f3a1e562b26 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.web; + +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Gauge; +import java.util.concurrent.atomic.AtomicBoolean; + +public class WebExecutorStats implements AutoCloseable { + private static final AtomicBoolean CLOSED = new AtomicBoolean(false); + + private final Gauge maxThreads; + private final Gauge minThreads; + private final Gauge idleThreads; + private final Gauge activeThreads; + private final Gauge currentThreads; + private final WebExecutorThreadPool executor; + + private static volatile WebExecutorStats instance; + + static synchronized WebExecutorStats getStats(WebExecutorThreadPool executor) { + if (null == instance) { + instance = new WebExecutorStats(executor); + } + + return instance; + } + + private WebExecutorStats(WebExecutorThreadPool executor) { + this.executor = executor; + + this.maxThreads = Gauge.build("pulsar_web_executor_max_threads", "-").create() + .setChild(new Gauge.Child() { + public double get() { + return WebExecutorStats.this.executor.getMaxThreads(); + } + }) + .register(); + + this.minThreads = Gauge.build("pulsar_web_executor_min_threads", "-").create() + .setChild(new Gauge.Child() { + public double get() { + return WebExecutorStats.this.executor.getMinThreads(); + } + }) + .register(); + + this.idleThreads = Gauge.build("pulsar_web_executor_idle_threads", "-").create() + .setChild(new Gauge.Child() { + public double get() { + return WebExecutorStats.this.executor.getIdleThreads(); + } + }) + .register(); + + this.activeThreads = Gauge.build("pulsar_web_executor_active_threads", "-").create() + .setChild(new Gauge.Child() { + public double get() { + return WebExecutorStats.this.executor.getThreads() + - WebExecutorStats.this.executor.getIdleThreads(); + } + }) + .register(); + + this.currentThreads = Gauge.build("pulsar_web_executor_current_threads", "-").create() + .setChild(new Gauge.Child() { + public double get() { + return WebExecutorStats.this.executor.getThreads(); + } + }) + .register(); + } + + @Override + public void close() throws Exception { + if (CLOSED.compareAndSet(false, true)) { + CollectorRegistry.defaultRegistry.unregister(this.activeThreads); + CollectorRegistry.defaultRegistry.unregister(this.maxThreads); + CollectorRegistry.defaultRegistry.unregister(this.minThreads); + CollectorRegistry.defaultRegistry.unregister(this.idleThreads); + CollectorRegistry.defaultRegistry.unregister(this.currentThreads); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java index e80a5160cb54a..f2542745bfa8d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java @@ -65,6 +65,8 @@ public class WebService implements AutoCloseable { private final PulsarService pulsar; private final Server server; private final List handlers; + + private final WebExecutorStats executorStats; private final WebExecutorThreadPool webServiceExecutor; public final int maxConcurrentRequests; @@ -78,6 +80,7 @@ public WebService(PulsarService pulsar) throws PulsarServerException { this.webServiceExecutor = new WebExecutorThreadPool( pulsar.getConfiguration().getNumHttpServerThreads(), "pulsar-web"); + this.executorStats = WebExecutorStats.getStats(webServiceExecutor); this.server = new Server(webServiceExecutor); this.maxConcurrentRequests = pulsar.getConfiguration().getMaxConcurrentHttpRequests(); List connectors = new ArrayList<>(); @@ -275,6 +278,7 @@ public void close() throws PulsarServerException { jettyStatisticsCollector = null; } webServiceExecutor.join(); + this.executorStats.close(); log.info("Web service closed"); } catch (Exception e) { throw new PulsarServerException(e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 3a7d23b06e7aa..b4094e488f363 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1317,9 +1317,9 @@ public static Multimap parseMetrics(String metrics) { return parsed; } - static class Metric { - Map tags = new TreeMap<>(); - double value; + public static class Metric { + public Map tags = new TreeMap<>(); + public double value; @Override public String toString() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java index b8f3ac472c8fc..23a3db916925b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java @@ -24,12 +24,14 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.common.io.CharStreams; import com.google.common.io.Closeables; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -38,12 +40,7 @@ import java.security.PrivateKey; import java.security.SecureRandom; import java.security.cert.Certificate; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.CompletableFuture; import javax.net.ssl.HttpsURLConnection; @@ -59,6 +56,8 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.stats.PrometheusMetricsTest; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; @@ -99,6 +98,42 @@ public class WebServiceTest { private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt"; private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key"; + @Test + public void testWebExecutorMetrics() throws Exception { + setupEnv(true, "1.0", true, false, false, false, -1, false); + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + String metricsStr = statsOut.toString(); + Multimap metrics = PrometheusMetricsTest.parseMetrics(metricsStr); + + Collection maxThreads = metrics.get("pulsar_web_executor_max_threads"); + Collection minThreads = metrics.get("pulsar_web_executor_min_threads"); + Collection activeThreads = metrics.get("pulsar_web_executor_active_threads"); + Collection idleThreads = metrics.get("pulsar_web_executor_idle_threads"); + Collection currentThreads = metrics.get("pulsar_web_executor_current_threads"); + + for (PrometheusMetricsTest.Metric metric : maxThreads) { + Assert.assertNotNull(metric.tags.get("cluster")); + Assert.assertTrue(metric.value > 0); + } + for (PrometheusMetricsTest.Metric metric : minThreads) { + Assert.assertNotNull(metric.tags.get("cluster")); + Assert.assertTrue(metric.value > 0); + } + for (PrometheusMetricsTest.Metric metric : activeThreads) { + Assert.assertNotNull(metric.tags.get("cluster")); + Assert.assertTrue(metric.value >= 0); + } + for (PrometheusMetricsTest.Metric metric : idleThreads) { + Assert.assertNotNull(metric.tags.get("cluster")); + Assert.assertTrue(metric.value >= 0); + } + for (PrometheusMetricsTest.Metric metric : currentThreads) { + Assert.assertNotNull(metric.tags.get("cluster")); + Assert.assertTrue(metric.value > 0); + } + } + /** * Test that the {@WebService} class properly passes the allowUnversionedClients value. We do this by setting * allowUnversionedClients to true, then making a request with no version, which should go through. From 6ded18081101fd4bce8ce488f646c58b643f3ba7 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 25 May 2022 09:05:21 +0800 Subject: [PATCH 546/823] [branch-2.9] [Authorization] Role with namespace produce authz can also get topics. (#15740) --- .../PulsarAuthorizationProvider.java | 35 +++++++++++++++++++ .../AuthorizationProducerConsumerTest.java | 5 +++ 2 files changed, 40 insertions(+) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index d0884da7dc08f..9aea1261cf216 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -542,6 +542,7 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam namespaceName, role, authData, AuthAction.packages); case GET_TOPIC: case GET_TOPICS: + return allowConsumeOrProduceOpsAsync(namespaceName, role, authData); case UNSUBSCRIBE: case CLEAR_BACKLOG: return allowTheSpecifiedActionOpsAsync( @@ -563,6 +564,40 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam }); } + private CompletableFuture allowConsumeOrProduceOpsAsync(NamespaceName namespaceName, + String role, + AuthenticationDataSource authenticationData) { + CompletableFuture finalResult = new CompletableFuture<>(); + allowTheSpecifiedActionOpsAsync(namespaceName, role, authenticationData, AuthAction.consume) + .whenComplete((consumeAuthorized, e) -> { + if (e == null) { + if (consumeAuthorized) { + finalResult.complete(consumeAuthorized); + return; + } + } else { + if (log.isDebugEnabled()) { + log.debug("Namespace [{}] Role [{}] exception occurred while trying to check Consume " + + "permission. {}", namespaceName, role, e.getCause()); + } + } + allowTheSpecifiedActionOpsAsync(namespaceName, role, authenticationData, AuthAction.produce) + .whenComplete((produceAuthorized, ex) -> { + if (ex == null) { + finalResult.complete(produceAuthorized); + } else { + if (log.isDebugEnabled()) { + log.debug("Namespace [{}] Role [{}] exception occurred while trying to check " + + "Produce permission. {}", namespaceName, role, ex.getCause()); + } + finalResult.completeExceptionally(ex.getCause()); + } + }); + }); + + return finalResult; + } + @Override public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceName namespaceName, PolicyName policy, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 62aa429436d80..dcfb16c92de61 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -432,6 +432,11 @@ public void testClearBacklogPermission() throws Exception { assertEquals(sub1Admin.topics().getStats(topicName + "-partition-0").getSubscriptions() .get(subscriptionName).getMsgBacklog(), 0); + superAdmin.namespaces().revokePermissionsOnNamespace(namespace, subscriptionRole); + superAdmin.namespaces().grantPermissionOnNamespace(namespace, subscriptionRole, + Sets.newHashSet(AuthAction.produce)); + assertEquals(sub1Admin.topics().getPartitionedTopicList(namespace), + Lists.newArrayList(topicName)); log.info("-- Exiting {} test --", methodName); } From dfbc09f5eee0a2d4217663cd9cb56126a0b7283f Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 25 May 2022 09:11:26 +0800 Subject: [PATCH 547/823] [branch-2.9] [Tests] Fix flaky WrongTypeOfReturnValue Mockito misuse issue in broker tests. (#15738) --- .../mledger/impl/OffloadPrefixReadTest.java | 4 +- .../admin/AdminApiGetLastMessageIdTest.java | 2 +- .../apache/pulsar/broker/admin/AdminTest.java | 16 +++--- .../pulsar/broker/admin/NamespacesTest.java | 6 +- .../broker/admin/PersistentTopicsTest.java | 8 ++- .../broker/admin/ResourceGroupsTest.java | 2 +- .../auth/MockedPulsarServiceBaseTest.java | 6 +- .../LeaderElectionServiceTest.java | 6 +- .../SimpleLoadManagerImplTest.java | 5 +- .../lookup/http/HttpTopicLookupv2Test.java | 8 +-- .../lookup/http/v2/TopicLookupTest.java | 4 +- .../OwnerShipForCurrentServerTestBase.java | 8 ++- ...sistentDispatcherFailoverConsumerTest.java | 15 ++--- .../PersistentTopicConcurrentTest.java | 11 ++-- .../broker/service/PersistentTopicTest.java | 57 ++++++++++--------- .../pulsar/broker/service/ServerCnxTest.java | 20 ++++--- .../persistent/MessageDuplicationTest.java | 11 +++- .../PersistentSubscriptionTest.java | 7 ++- .../transaction/TransactionTestBase.java | 6 +- .../TransactionMetaStoreTestBase.java | 4 +- .../pulsar/broker/web/WebServiceTest.java | 3 +- .../api/ClientDeduplicationFailureTest.java | 2 +- .../api/DispatcherBlockConsumerTest.java | 4 +- .../impl/BrokerClientIntegrationTest.java | 6 +- .../client/impl/ConnectionPoolTest.java | 10 ++-- .../worker/PulsarFunctionE2ESecurityTest.java | 2 +- .../worker/PulsarFunctionLocalRunTest.java | 2 +- .../worker/PulsarFunctionPublishTest.java | 2 +- .../worker/PulsarWorkerAssignmentTest.java | 2 +- .../pulsar/io/AbstractPulsarE2ETest.java | 2 +- .../pulsar/io/PulsarFunctionAdminTest.java | 2 +- .../pulsar/io/PulsarFunctionTlsTest.java | 2 +- .../proxy/ProxyAuthenticationTest.java | 3 +- .../proxy/ProxyAuthorizationTest.java | 3 +- .../proxy/ProxyConfigurationTest.java | 3 +- .../proxy/ProxyPublishConsumeTest.java | 4 +- .../proxy/ProxyPublishConsumeTlsTest.java | 3 +- .../ProxyPublishConsumeWithoutZKTest.java | 3 +- .../proxy/v1/V1_ProxyAuthenticationTest.java | 5 +- .../pulsar/client/api/MessageRouterTest.java | 4 +- .../worker/FunctionRuntimeManagerTest.java | 4 +- .../sql/presto/TestPulsarConnector.java | 4 +- .../sql/presto/TestPulsarRecordCursor.java | 2 +- .../presto/decoder/AbstractDecoderTester.java | 2 +- 44 files changed, 160 insertions(+), 125 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java index 426a00db25ec8..e761b04b79b3e 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java @@ -66,7 +66,7 @@ public class OffloadPrefixReadTest extends MockedBookKeeperTestCase { @Test public void testOffloadRead() throws Exception { - MockLedgerOffloader offloader = spy(new MockLedgerOffloader()); + MockLedgerOffloader offloader = spy(MockLedgerOffloader.class); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); @@ -122,7 +122,7 @@ public void testOffloadRead() throws Exception { @Test public void testBookkeeperFirstOffloadRead() throws Exception { - MockLedgerOffloader offloader = spy(new MockLedgerOffloader()); + MockLedgerOffloader offloader = spy(MockLedgerOffloader.class); MockClock clock = new MockClock(); offloader.getOffloadPolicies() .setManagedLedgerOffloadedReadPriority(OffloadedReadPriority.BOOKKEEPER_FIRST); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java index cba0686383157..ea0b5c2a155c4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java @@ -81,7 +81,7 @@ protected void setup() throws Exception { new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet("test"))); admin.namespaces().createNamespace("prop/ns-abc"); admin.namespaces().setNamespaceReplicationClusters("prop/ns-abc", Sets.newHashSet("test")); - persistentTopics = spy(new PersistentTopics()); + persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); persistentTopics.setPulsar(pulsar); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 82e5b90b54f14..a2f04927de928 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -130,17 +130,17 @@ public void setup() throws Exception { conf.setClusterName(configClusterName); super.internalSetup(); - clusters = spy(new Clusters()); + clusters = spy(Clusters.class); clusters.setPulsar(pulsar); doReturn("test").when(clusters).clientAppId(); doNothing().when(clusters).validateSuperUserAccess(); - properties = spy(new Properties()); + properties = spy(Properties.class); properties.setPulsar(pulsar); doReturn("test").when(properties).clientAppId(); doNothing().when(properties).validateSuperUserAccess(); - namespaces = spy(new Namespaces()); + namespaces = spy(Namespaces.class); namespaces.setServletContext(new MockServletContext()); namespaces.setPulsar(pulsar); doReturn("test").when(namespaces).clientAppId(); @@ -149,7 +149,7 @@ public void setup() throws Exception { doNothing().when(namespaces).validateAdminAccessForTenant("other-tenant"); doNothing().when(namespaces).validateAdminAccessForTenant("new-property"); - brokers = spy(new Brokers()); + brokers = spy(Brokers.class); brokers.setPulsar(pulsar); doReturn("test").when(brokers).clientAppId(); doNothing().when(brokers).validateSuperUserAccess(); @@ -157,7 +157,7 @@ public void setup() throws Exception { uriField = PulsarWebResource.class.getDeclaredField("uri"); uriField.setAccessible(true); - persistentTopics = spy(new PersistentTopics()); + persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); persistentTopics.setPulsar(pulsar); doReturn("test").when(persistentTopics).clientAppId(); @@ -167,11 +167,11 @@ public void setup() throws Exception { doNothing().when(persistentTopics).validateAdminAccessForTenant("other-tenant"); doNothing().when(persistentTopics).validateAdminAccessForTenant("prop-xyz"); - resourceQuotas = spy(new ResourceQuotas()); + resourceQuotas = spy(ResourceQuotas.class); resourceQuotas.setServletContext(new MockServletContext()); resourceQuotas.setPulsar(pulsar); - brokerStats = spy(new BrokerStats()); + brokerStats = spy(BrokerStats.class); brokerStats.setServletContext(new MockServletContext()); brokerStats.setPulsar(pulsar); @@ -180,7 +180,7 @@ public void setup() throws Exception { doReturn("test").when(persistentTopics).clientAppId(); doReturn(mock(AuthenticationDataHttps.class)).when(persistentTopics).clientAuthData(); - schemasResource = spy(new SchemasResource(mockClock)); + schemasResource = spy(SchemasResource.class); schemasResource.setServletContext(new MockServletContext()); schemasResource.setPulsar(pulsar); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 16ae98f0b43cc..69bede376c802 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -155,7 +155,7 @@ public void setup() throws Exception { conf.setClusterName(testLocalCluster); super.internalSetup(); - namespaces = spy(new Namespaces()); + namespaces = spy(Namespaces.class); namespaces.setServletContext(new MockServletContext()); namespaces.setPulsar(pulsar); doReturn(false).when(namespaces).isRequestHttps(); @@ -1089,7 +1089,7 @@ public void testValidateTopicOwnership() throws Exception { ownership.setAccessible(true); ownership.set(pulsar.getNamespaceService(), MockOwnershipCache); TopicName topicName = TopicName.get(testNs.getPersistentTopicName("my-topic")); - PersistentTopics topics = spy(new PersistentTopics()); + PersistentTopics topics = spy(PersistentTopics.class); topics.setServletContext(new MockServletContext()); topics.setPulsar(pulsar); doReturn(false).when(topics).isRequestHttps(); @@ -1253,7 +1253,7 @@ public void testSubscribeRate() throws Exception { admin.tenants().deleteTenant("my-tenants"); } - class MockLedgerOffloader implements LedgerOffloader { + public static class MockLedgerOffloader implements LedgerOffloader { ConcurrentHashMap offloads = new ConcurrentHashMap(); ConcurrentHashMap deletes = new ConcurrentHashMap(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 2baf5b0f4692c..4591aebc67df3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; @@ -116,7 +117,7 @@ public void initPersistentTopics() throws Exception { @BeforeMethod protected void setup() throws Exception { super.internalSetup(); - persistentTopics = spy(new PersistentTopics()); + persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); persistentTopics.setPulsar(pulsar); doReturn(false).when(persistentTopics).isRequestHttps(); @@ -126,7 +127,7 @@ protected void setup() throws Exception { doNothing().when(persistentTopics).validateAdminAccessForTenant(this.testTenant); doReturn(mock(AuthenticationDataHttps.class)).when(persistentTopics).clientAuthData(); - nonPersistentTopic = spy(new NonPersistentTopics()); + nonPersistentTopic = spy(NonPersistentTopics.class); nonPersistentTopic.setServletContext(new MockServletContext()); nonPersistentTopic.setPulsar(pulsar); namespaceResources = mock(NamespaceResources.class); @@ -139,7 +140,8 @@ protected void setup() throws Exception { PulsarResources resources = spy(new PulsarResources(pulsar.getLocalMetadataStore(), pulsar.getConfigurationMetadataStore())); - doReturn(spy(new TopicResources(pulsar.getLocalMetadataStore()))).when(resources).getTopicResources(); + doReturn(spyWithClassAndConstructorArgs(TopicResources.class, + pulsar.getLocalMetadataStore())).when(resources).getTopicResources(); Whitebox.setInternalState(pulsar, "pulsarResources", resources); admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://broker-use.com:8080").build()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java index b6510a18995bf..4910f6798bae4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java @@ -52,7 +52,7 @@ public class ResourceGroupsTest extends MockedPulsarServiceBaseTest { @Override protected void setup() throws Exception { super.internalSetup(); - resourcegroups = spy(new ResourceGroups()); + resourcegroups = spy(ResourceGroups.class); resourcegroups.setServletContext(new MockServletContext()); resourcegroups.setPulsar(pulsar); doReturn(false).when(resourcegroups).isRequestHttps(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index c50b477b5d313..c731d310d9941 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.auth; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -327,7 +328,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { doReturn(createLocalMetadataStore()).when(pulsar).createLocalMetadataStore(); doReturn(createConfigurationMetadataStore()).when(pulsar).createConfigurationMetadataStore(); - Supplier namespaceServiceSupplier = () -> spy(new NamespaceService(pulsar)); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); doReturn(sameThreadOrderedSafeExecutor).when(pulsar).getOrderedExecutor(); @@ -391,7 +393,7 @@ public static MockZooKeeper createMockZooKeeperGlobal() { } public static NonClosableMockBookKeeper createMockBookKeeper(OrderedExecutor executor) throws Exception { - return spy(new NonClosableMockBookKeeper(executor)); + return spyWithClassAndConstructorArgs(NonClosableMockBookKeeper.class, executor); } // Prevent the MockBookKeeper instance from being closed when the broker is restarted within a test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java index d252b1cf326f4..ec19480ae06fa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java @@ -44,6 +44,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; + @Slf4j @Test(groups = "broker") public class LeaderElectionServiceTest { @@ -75,7 +77,7 @@ public void anErrorShouldBeThrowBeforeLeaderElected() throws PulsarServerExcepti config.setAdvertisedAddress("localhost"); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); @Cleanup - PulsarService pulsar = Mockito.spy(new MockPulsarService(config)); + PulsarService pulsar = spyWithClassAndConstructorArgs(MockPulsarService.class, config); pulsar.start(); // mock pulsar.getLeaderElectionService() in a thread safe way @@ -135,7 +137,7 @@ private void checkLookupException(String tenant, String namespace, PulsarClient } } - private static class MockPulsarService extends PulsarService { + public static class MockPulsarService extends PulsarService { public MockPulsarService(ServiceConfiguration config) { super(config); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index 059191b787636..d08699998e7b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -108,7 +109,7 @@ void setup() throws Exception { bkEnsemble.start(); // Start broker 1 - ServiceConfiguration config1 = spy(new ServiceConfiguration()); + ServiceConfiguration config1 = spy(ServiceConfiguration.class); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); @@ -336,7 +337,7 @@ public void testLoadReportParsing() throws Exception { @Test(enabled = true) public void testDoLoadShedding() throws Exception { - SimpleLoadManagerImpl loadManager = spy(new SimpleLoadManagerImpl(pulsar1)); + SimpleLoadManagerImpl loadManager = spyWithClassAndConstructorArgs(SimpleLoadManagerImpl.class, pulsar1); PulsarResourceDescription rd = new PulsarResourceDescription(); rd.put("memory", new ResourceUsage(1024, 4096)); rd.put("cpu", new ResourceUsage(10, 100)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java index b65b084ccdc35..47db73d39f799 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java @@ -73,7 +73,7 @@ public void setUp() throws Exception { pulsar = mock(PulsarService.class); ns = mock(NamespaceService.class); auth = mock(AuthorizationService.class); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); clusters = new TreeSet<>(); clusters.add("use"); @@ -102,7 +102,7 @@ public void setUp() throws Exception { @Test public void crossColoLookup() throws Exception { - TopicLookup destLookup = spy(new TopicLookup()); + TopicLookup destLookup = spy(TopicLookup.class); doReturn(false).when(destLookup).isRequestHttps(); destLookup.setPulsar(pulsar); doReturn("null").when(destLookup).clientAppId(); @@ -132,7 +132,7 @@ public void testNotEnoughLookupPermits() throws Exception { BrokerService brokerService = pulsar.getBrokerService(); doReturn(new Semaphore(0)).when(brokerService).getLookupRequestSemaphore(); - TopicLookup destLookup = spy(new TopicLookup()); + TopicLookup destLookup = spy(TopicLookup.class); doReturn(false).when(destLookup).isRequestHttps(); destLookup.setPulsar(pulsar); doReturn("null").when(destLookup).clientAppId(); @@ -170,7 +170,7 @@ public void testValidateReplicationSettingsOnNamespace() throws Exception { // doReturn(Optional.of(policies2)).when(policiesCache) // .get(AdminResource.path(POLICIES, property, cluster, ns2)); - TopicLookup destLookup = spy(new TopicLookup()); + TopicLookup destLookup = spy(TopicLookup.class); doReturn(false).when(destLookup).isRequestHttps(); destLookup.setPulsar(pulsar); doReturn("null").when(destLookup).clientAppId(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/v2/TopicLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/v2/TopicLookupTest.java index 317b320a9a4d4..21e7abded4789 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/v2/TopicLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/v2/TopicLookupTest.java @@ -46,7 +46,7 @@ public class TopicLookupTest extends PulsarWebResourceTest { @Override protected ResourceConfig configure() { - resource = spy(new TestableTopicLookup()); + resource = spy(TestableTopicLookup.class); return new ResourceConfig().register(resource); } @@ -70,7 +70,7 @@ public void testListenerName() { assertEquals(resource.actualListenerName, "query"); } - private static class TestableTopicLookup extends TopicLookup { + public static class TestableTopicLookup extends TopicLookup { private String actualListenerName; @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java index 9a8021db28c79..a6604d9639736 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.namespace; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; @@ -118,7 +119,7 @@ protected void startBroker() throws Exception { conf.setWebServicePortTls(Optional.of(0)); serviceConfigurationList.add(conf); - PulsarService pulsar = spy(new PulsarService(conf)); + PulsarService pulsar = spyWithClassAndConstructorArgs(PulsarService.class, conf); setupBrokerMocks(pulsar); pulsar.start(); @@ -133,7 +134,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { MockZooKeeperSession mockZooKeeperSession = MockZooKeeperSession.newInstance(mockZooKeeper); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createLocalMetadataStore(); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createConfigurationMetadataStore(); - Supplier namespaceServiceSupplier = () -> spy(new NamespaceService(pulsar)); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); SameThreadOrderedSafeExecutor executor = new SameThreadOrderedSafeExecutor(); @@ -157,7 +159,7 @@ public static MockZooKeeper createMockZooKeeper() throws Exception { } public static NonClosableMockBookKeeper createMockBookKeeper(OrderedExecutor executor) throws Exception { - return spy(new NonClosableMockBookKeeper(executor)); + return spyWithClassAndConstructorArgs(NonClosableMockBookKeeper.class, executor); } // Prevent the MockBookKeeper instance from being closed when the broker is restarted within a test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index b886adee15ef4..663a1023543d4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.matches; import static org.mockito.ArgumentMatchers.same; @@ -122,9 +123,9 @@ public class PersistentDispatcherFailoverConsumerTest { @BeforeMethod public void setup() throws Exception { executor = OrderedExecutor.newBuilder().numThreads(1).name("persistent-dispatcher-failover-test").build(); - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); - pulsar = spy(new PulsarService(svcConfig)); + pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(svcConfig).when(pulsar).getConfiguration(); mlFactoryMock = mock(ManagedLedgerFactory.class); @@ -141,7 +142,7 @@ public void setup() throws Exception { PulsarResources pulsarResources = new PulsarResources(store, store); doReturn(pulsarResources).when(pulsar).getPulsarResources(); - brokerService = spy(new BrokerService(pulsar, eventLoopGroup)); + brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); doReturn(brokerService).when(pulsar).getBrokerService(); consumerChanges = new LinkedBlockingQueue<>(); @@ -167,9 +168,7 @@ public void setup() throws Exception { return null; }).when(channelCtx).writeAndFlush(any(), any()); - serverCnx = mock(ServerCnx.class, withSettings() - .useConstructor(pulsar) - .defaultAnswer(CALLS_REAL_METHODS)); + serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); doReturn(true).when(serverCnx).isActive(); doReturn(true).when(serverCnx).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress(); @@ -178,9 +177,7 @@ public void setup() throws Exception { doReturn(new PulsarCommandSenderImpl(null, serverCnx)) .when(serverCnx).getCommandSender(); - serverCnxWithOldVersion = mock(ServerCnx.class, withSettings() - .useConstructor(pulsar) - .defaultAnswer(CALLS_REAL_METHODS)); + serverCnxWithOldVersion = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); doReturn(true).when(serverCnxWithOldVersion).isActive(); doReturn(true).when(serverCnxWithOldVersion).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java index 78c7e255739a8..9bd469a955b95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; @@ -83,10 +84,10 @@ public class PersistentTopicConcurrentTest extends MockedBookKeeperTestCase { @BeforeMethod public void setup(Method m) throws Exception { super.setUp(m); - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); @Cleanup - PulsarService pulsar = spy(new PulsarService(svcConfig)); + PulsarService pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(svcConfig).when(pulsar).getConfiguration(); @Cleanup(value = "shutdownGracefully") @@ -100,12 +101,10 @@ public void setup(Method m) throws Exception { mlFactoryMock = factory; doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory(); - brokerService = spy(new BrokerService(pulsar, eventLoopGroup)); + brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); doReturn(brokerService).when(pulsar).getBrokerService(); - serverCnx = mock(ServerCnx.class, withSettings() - .useConstructor(pulsar) - .defaultAnswer(CALLS_REAL_METHODS)); + serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); doReturn(true).when(serverCnx).isActive(); NamespaceService nsSvc = mock(NamespaceService.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 5cb945e436f6f..260042513996e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; @@ -175,10 +176,10 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase { public void setup() throws Exception { eventLoopGroup = new NioEventLoopGroup(); executor = OrderedExecutor.newBuilder().numThreads(1).build(); - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setAdvertisedAddress("localhost"); svcConfig.setBrokerShutdownTimeoutMs(0L); - pulsar = spy(new PulsarService(svcConfig)); + pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(svcConfig).when(pulsar).getConfiguration(); doReturn(mock(Compactor.class)).when(pulsar).getCompactor(); @@ -199,20 +200,18 @@ public void setup() throws Exception { doReturn(executor).when(pulsar).getOrderedExecutor(); store = new ZKMetadataStore(mockZk); - PulsarResources pulsarResources = spy(new PulsarResources(store, store)); - NamespaceResources nsr = spy(new NamespaceResources(store, store, 30)); + PulsarResources pulsarResources = spyWithClassAndConstructorArgs(PulsarResources.class, store, store); + NamespaceResources nsr = spyWithClassAndConstructorArgs(NamespaceResources.class, store, store, 30); doReturn(nsr).when(pulsarResources).getNamespaceResources(); doReturn(pulsarResources).when(pulsar).getPulsarResources(); doReturn(store).when(pulsar).getLocalMetadataStore(); doReturn(store).when(pulsar).getConfigurationMetadataStore(); - brokerService = spy(new BrokerService(pulsar, eventLoopGroup)); + brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); doReturn(brokerService).when(pulsar).getBrokerService(); - serverCnx = mock(ServerCnx.class, withSettings() - .useConstructor(pulsar) - .defaultAnswer(CALLS_REAL_METHODS)); + serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); doReturn(true).when(serverCnx).isActive(); doReturn(true).when(serverCnx).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress(); @@ -354,7 +353,8 @@ public void setMetadataFromEntryData(ByteBuf entryData) { @Test public void testDispatcherMultiConsumerReadFailed() throws Exception { - PersistentTopic topic = spy(new PersistentTopic(successTopicName, ledgerMock, brokerService)); + PersistentTopic topic = spyWithClassAndConstructorArgs(PersistentTopic.class, + successTopicName, ledgerMock, brokerService); ManagedCursor cursor = mock(ManagedCursor.class); when(cursor.getName()).thenReturn("cursor"); PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor, null); @@ -364,7 +364,8 @@ public void testDispatcherMultiConsumerReadFailed() throws Exception { @Test public void testDispatcherSingleConsumerReadFailed() throws Exception { - PersistentTopic topic = spy(new PersistentTopic(successTopicName, ledgerMock, brokerService)); + PersistentTopic topic = + spyWithClassAndConstructorArgs(PersistentTopic.class, successTopicName, ledgerMock, brokerService); ManagedCursor cursor = mock(ManagedCursor.class); when(cursor.getName()).thenReturn("cursor"); PersistentDispatcherSingleActiveConsumer dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, @@ -561,7 +562,7 @@ private void testMaxProducers() throws Exception { @Test public void testMaxProducersForBroker() throws Exception { // set max clients - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(2).when(svcConfig).getMaxProducersPerTopic(); doReturn(svcConfig).when(pulsar).getConfiguration(); testMaxProducers(); @@ -569,7 +570,7 @@ public void testMaxProducersForBroker() throws Exception { @Test public void testMaxProducersForNamespace() throws Exception { - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(svcConfig).when(pulsar).getConfiguration(); // set max clients Policies policies = new Policies(); @@ -605,7 +606,7 @@ private Producer getMockedProducerWithSpecificAddress(Topic topic, long producer @Test public void testMaxSameAddressProducers() throws Exception { // set max clients - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(2).when(svcConfig).getMaxSameAddressProducersPerTopic(); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -902,7 +903,7 @@ private void testMaxConsumersShared() throws Exception { @Test public void testMaxConsumersSharedForBroker() throws Exception { // set max clients - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(2).when(svcConfig).getMaxConsumersPerSubscription(); doReturn(3).when(svcConfig).getMaxConsumersPerTopic(); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -912,7 +913,7 @@ public void testMaxConsumersSharedForBroker() throws Exception { @Test public void testMaxConsumersSharedForNamespace() throws Exception { - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(svcConfig).when(pulsar).getConfiguration(); // set max clients @@ -1005,7 +1006,7 @@ private void testMaxConsumersFailover() throws Exception { @Test public void testMaxConsumersFailoverForBroker() throws Exception { // set max clients - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(2).when(svcConfig).getMaxConsumersPerSubscription(); doReturn(3).when(svcConfig).getMaxConsumersPerTopic(); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -1015,7 +1016,7 @@ public void testMaxConsumersFailoverForBroker() throws Exception { @Test public void testMaxConsumersFailoverForNamespace() throws Exception { - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(svcConfig).when(pulsar).getConfiguration(); // set max clients @@ -1037,9 +1038,7 @@ private Consumer getMockedConsumerWithSpecificAddress(Topic topic, Subscription final String consumerNameBase = "consumer"; final String role = "appid1"; - ServerCnx cnx = mock(ServerCnx.class, withSettings() - .useConstructor(pulsar) - .defaultAnswer(CALLS_REAL_METHODS)); + ServerCnx cnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); doReturn(true).when(cnx).isActive(); doReturn(true).when(cnx).isWritable(); doReturn(new InetSocketAddress(address, 1234)).when(cnx).clientAddress(); @@ -1053,7 +1052,7 @@ private Consumer getMockedConsumerWithSpecificAddress(Topic topic, Subscription @Test public void testMaxSameAddressConsumers() throws Exception { // set max clients - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(2).when(svcConfig).getMaxSameAddressConsumersPerTopic(); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -2106,13 +2105,19 @@ public void testCheckInactiveSubscriptions() throws Exception { .concurrencyLevel(1) .build(); // This subscription is connected by consumer. - PersistentSubscription nonDeletableSubscription1 = spy(new PersistentSubscription(topic, "nonDeletableSubscription1", cursorMock, false)); + PersistentSubscription nonDeletableSubscription1 = + spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, + "nonDeletableSubscription1", cursorMock, false); subscriptions.put(nonDeletableSubscription1.getName(), nonDeletableSubscription1); // This subscription is not connected by consumer. - PersistentSubscription deletableSubscription1 = spy(new PersistentSubscription(topic, "deletableSubscription1", cursorMock, false)); + PersistentSubscription deletableSubscription1 = + spyWithClassAndConstructorArgs(PersistentSubscription.class, + topic, "deletableSubscription1", cursorMock, false); subscriptions.put(deletableSubscription1.getName(), deletableSubscription1); // This subscription is replicated. - PersistentSubscription nonDeletableSubscription2 = spy(new PersistentSubscription(topic, "nonDeletableSubscription2", cursorMock, true)); + PersistentSubscription nonDeletableSubscription2 = + spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, + "nonDeletableSubscription2", cursorMock, true); subscriptions.put(nonDeletableSubscription2.getName(), nonDeletableSubscription2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -2131,7 +2136,7 @@ public void testCheckInactiveSubscriptions() throws Exception { NamespaceName ns = TopicName.get(successTopicName).getNamespaceObject(); doReturn(Optional.of(new Policies())).when(nsr).getPolicies(ns); - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(5).when(svcConfig).getSubscriptionExpirationTimeMinutes(); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -2146,7 +2151,7 @@ public void testCheckInactiveSubscriptions() throws Exception { @Test public void testTopicFencingTimeout() throws Exception { - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); doReturn(svcConfig).when(pulsar).getConfiguration(); PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 8881ce90812a0..1a5efb19f3eed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; @@ -152,9 +153,9 @@ public class ServerCnxTest { public void setup() throws Exception { eventLoopGroup = new NioEventLoopGroup(); executor = OrderedExecutor.newBuilder().numThreads(1).build(); - svcConfig = spy(new ServiceConfiguration()); + svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); - pulsar = spy(new PulsarService(svcConfig)); + pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(new DefaultSchemaRegistryService()).when(pulsar).getSchemaRegistryService(); svcConfig.setKeepAliveIntervalSeconds(inSec(1, TimeUnit.SECONDS)); @@ -176,14 +177,14 @@ public void setup() throws Exception { doReturn(store).when(pulsar).getLocalMetadataStore(); doReturn(store).when(pulsar).getConfigurationMetadataStore(); - brokerService = spy(new BrokerService(pulsar, eventLoopGroup)); + brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); BrokerInterceptor interceptor = mock(BrokerInterceptor.class); doReturn(interceptor).when(brokerService).getInterceptor(); doReturn(brokerService).when(pulsar).getBrokerService(); doReturn(executor).when(pulsar).getOrderedExecutor(); - PulsarResources pulsarResources = spy(new PulsarResources(store, store)); - namespaceResources = spy(new NamespaceResources(store, store, 30)); + PulsarResources pulsarResources = spyWithClassAndConstructorArgs(PulsarResources.class, store, store); + namespaceResources = spyWithClassAndConstructorArgs(NamespaceResources.class, store, store, 30); doReturn(namespaceResources).when(pulsarResources).getNamespaceResources(); doReturn(pulsarResources).when(pulsar).getPulsarResources(); @@ -502,7 +503,8 @@ public void testProducerCommandWithAuthorizationPositive() throws Exception { @Test(timeOut = 30000) public void testNonExistentTopic() throws Exception { - AuthorizationService authorizationService = spy(new AuthorizationService(svcConfig, pulsar.getPulsarResources())); + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); doReturn(authorizationService).when(brokerService).getAuthorizationService(); doReturn(true).when(brokerService).isAuthorizationEnabled(); svcConfig.setAuthorizationEnabled(true); @@ -535,7 +537,8 @@ public void testNonExistentTopic() throws Exception { @Test(timeOut = 30000) public void testClusterAccess() throws Exception { svcConfig.setAuthorizationEnabled(true); - AuthorizationService authorizationService = spy(new AuthorizationService(svcConfig, pulsar.getPulsarResources())); + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); Field providerField = AuthorizationService.class.getDeclaredField("provider"); providerField.setAccessible(true); PulsarAuthorizationProvider authorizationProvider = spy(new PulsarAuthorizationProvider(svcConfig, @@ -566,7 +569,8 @@ public void testClusterAccess() throws Exception { @Test(timeOut = 30000) public void testNonExistentTopicSuperUserAccess() throws Exception { - AuthorizationService authorizationService = spy(new AuthorizationService(svcConfig, pulsar.getPulsarResources())); + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); doReturn(authorizationService).when(brokerService).getAuthorizationService(); doReturn(true).when(brokerService).isAuthorizationEnabled(); Field providerField = AuthorizationService.class.getDeclaredField("provider"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index 4dc7f7f42320d..117c9dd15828b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -37,6 +37,7 @@ import java.lang.reflect.Field; import java.util.Map; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -71,7 +72,8 @@ public void testIsDuplicate() { doReturn(serviceConfiguration).when(pulsarService).getConfiguration(); PersistentTopic persistentTopic = mock(PersistentTopic.class); ManagedLedger managedLedger = mock(ManagedLedger.class); - MessageDeduplication messageDeduplication = spy(new MessageDeduplication(pulsarService, persistentTopic, managedLedger)); + MessageDeduplication messageDeduplication = + spyWithClassAndConstructorArgs(MessageDeduplication.class, pulsarService, persistentTopic, managedLedger); doReturn(true).when(messageDeduplication).isEnabled(); String producerName1 = "producer1"; @@ -164,7 +166,8 @@ public void testInactiveProducerRemove() throws Exception { serviceConfiguration.setBrokerDeduplicationProducerInactivityTimeoutMinutes(1); doReturn(serviceConfiguration).when(pulsarService).getConfiguration(); - MessageDeduplication messageDeduplication = spy(new MessageDeduplication(pulsarService, topic, managedLedger)); + MessageDeduplication messageDeduplication = + spyWithClassAndConstructorArgs(MessageDeduplication.class, pulsarService, topic, managedLedger); doReturn(true).when(messageDeduplication).isEnabled(); Topic.PublishContext publishContext = mock(Topic.PublishContext.class); @@ -238,7 +241,9 @@ public void testIsDuplicateWithFailure() { doReturn(eventLoopGroup).when(brokerService).executor(); doReturn(pulsarService).when(brokerService).pulsar(); - PersistentTopic persistentTopic = spy(new PersistentTopic("topic-1", brokerService, managedLedger, messageDeduplication)); + PersistentTopic persistentTopic = + spyWithClassAndConstructorArgs(PersistentTopic.class, "topic-1", + brokerService, managedLedger, messageDeduplication); String producerName1 = "producer1"; ByteBuf byteBuf1 = getMessage(producerName1, 0); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index b72fe76f14caa..b9304cb5fb8ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.any; @@ -109,10 +110,10 @@ public void setup() throws Exception { executor = OrderedExecutor.newBuilder().numThreads(1).name("persistent-subscription-test").build(); eventLoopGroup = new NioEventLoopGroup(); - ServiceConfiguration svcConfig = spy(new ServiceConfiguration()); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); svcConfig.setTransactionCoordinatorEnabled(true); - pulsarMock = spy(new PulsarService(svcConfig)); + pulsarMock = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); PulsarResources pulsarResources = mock(PulsarResources.class); doReturn(pulsarResources).when(pulsarMock).getPulsarResources(); NamespaceResources namespaceResources = mock(NamespaceResources.class); @@ -182,7 +183,7 @@ public CompletableFuture checkInitializedBefore(PersistentSubscription doReturn(store).when(pulsarMock).getLocalMetadataStore(); doReturn(store).when(pulsarMock).getConfigurationMetadataStore(); - brokerMock = spy(new BrokerService(pulsarMock, eventLoopGroup)); + brokerMock = spyWithClassAndConstructorArgs(BrokerService.class, pulsarMock, eventLoopGroup); doNothing().when(brokerMock).unloadNamespaceBundlesGracefully(); doReturn(brokerMock).when(pulsarMock).getBrokerService(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 7cae6ca3ec30b..2829aa9f9131d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.transaction; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; @@ -171,7 +172,7 @@ protected void startBroker() throws Exception { conf.setTopicLevelPoliciesEnabled(true); serviceConfigurationList.add(conf); - PulsarService pulsar = spy(new PulsarService(conf)); + PulsarService pulsar = spyWithClassAndConstructorArgs(PulsarService.class, conf); setupBrokerMocks(pulsar); pulsar.start(); @@ -187,7 +188,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { MockZooKeeperSession mockZooKeeperSession = MockZooKeeperSession.newInstance(mockZooKeeper); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createLocalMetadataStore(); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createConfigurationMetadataStore(); - Supplier namespaceServiceSupplier = () -> spy(new NamespaceService(pulsar)); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); SameThreadOrderedSafeExecutor executor = new SameThreadOrderedSafeExecutor(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java index 04579078ad4af..b012dfa87d29a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java @@ -34,6 +34,8 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; + public abstract class TransactionMetaStoreTestBase extends TestRetrySupport { private static final Logger log = LoggerFactory.getLogger(TransactionMetaStoreTestBase.class); @@ -76,7 +78,7 @@ protected final void setup() throws Exception { config.setTransactionCoordinatorEnabled(true); configurations[i] = config; - pulsarServices[i] = Mockito.spy(new PulsarService(config)); + pulsarServices[i] = spyWithClassAndConstructorArgs(PulsarService.class, config); pulsarServices[i].start(); pulsarAdmins[i] = PulsarAdmin.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java index 23a3db916925b..4173d5152ea35 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.web; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; @@ -424,7 +425,7 @@ private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowU config.setHttpRequestsLimitEnabled(true); config.setHttpRequestsMaxPerSecond(rateLimit); } - pulsar = spy(new PulsarService(config)); + pulsar = spyWithClassAndConstructorArgs(PulsarService.class, config); // mock zk MockZooKeeper mockZooKeeper = MockedPulsarServiceBaseTest.createMockZooKeeper(); ZooKeeperClientFactory mockZooKeeperClientFactory = new ZooKeeperClientFactory() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java index a2b4b5c3dbf53..2ddb9e8c8a35a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java @@ -81,7 +81,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); config.setWebServicePort(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java index 0f21e211d92ce..867ea1592ebde 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.api; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; @@ -630,7 +631,8 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw // if broker unload bundle gracefully then cursor metadata recovered from zk else from ledger if (unloadBundleGracefully) { // set clean namespace which will not let broker unload bundle gracefully: stop broker - Supplier namespaceServiceSupplier = () -> spy(new NamespaceService(pulsar)); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); } stopBroker(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java index 2be6067c7b7fc..911a23b1bb8cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.UUID.randomUUID; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.any; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; @@ -822,8 +823,9 @@ public void testAvroSchemaProducerConsumerWithSpecifiedReaderAndWriter() throws public void testJsonSchemaProducerConsumerWithSpecifiedReaderAndWriter() throws PulsarClientException { final String topicName = "persistent://my-property/my-ns/my-topic1"; ObjectMapper mapper = new ObjectMapper(); - SchemaReader reader = Mockito.spy(new JacksonJsonReader<>(mapper, TestMessageObject.class)); - SchemaWriter writer = Mockito.spy(new JacksonJsonWriter<>(mapper)); + SchemaReader reader = + spyWithClassAndConstructorArgs(JacksonJsonReader.class, mapper, TestMessageObject.class); + SchemaWriter writer = spyWithClassAndConstructorArgs(JacksonJsonWriter.class, mapper); SchemaDefinition schemaDefinition = new SchemaDefinitionBuilderImpl() .withPojo(TestMessageObject.class) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index 1e97550322b67..235bd7167a5e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -36,6 +36,8 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.IntStream; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; + @Test(groups = "broker-impl") public class ConnectionPoolTest extends MockedPulsarServiceBaseTest { @@ -58,7 +60,7 @@ protected void cleanup() throws Exception { public void testSingleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = Mockito.spy(new ConnectionPool(conf, eventLoop)); + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); @@ -78,7 +80,7 @@ public void testDoubleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = Mockito.spy(new ConnectionPool(conf, eventLoop)); + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); @@ -101,7 +103,7 @@ public void testNoConnectionPool() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(0); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); - ConnectionPool pool = Mockito.spy(new ConnectionPool(conf, eventLoop)); + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved("127.0.0.1", pulsar.getBrokerListenPort().get()); @@ -123,7 +125,7 @@ public void testEnableConnectionPool() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(5); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); - ConnectionPool pool = Mockito.spy(new ConnectionPool(conf, eventLoop)); + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved("127.0.0.1", pulsar.getBrokerListenPort().get()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java index e7173a23e39b5..c399cd8b73419 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java @@ -125,7 +125,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); Set superUsers = Sets.newHashSet(ADMIN_SUBJECT); config.setSuperUserRoles(superUsers); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index 13ac623aa1de5..c48de5257b381 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -193,7 +193,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName(CLUSTER); Set superUsers = Sets.newHashSet("superUser", "admin"); config.setSuperUserRoles(superUsers); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java index 9f5e525d6b715..d985241e2903d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java @@ -119,7 +119,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); Set superUsers = Sets.newHashSet("superUser", "admin"); config.setSuperUserRoles(superUsers); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java index 019ca0bd577f4..1b66099c14c55 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java @@ -89,7 +89,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); final Set superUsers = Sets.newHashSet("superUser", "admin"); config.setSuperUserRoles(superUsers); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java index 1df4be1c71d0d..93cdd79ad01f4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java @@ -114,7 +114,7 @@ public void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); Set superUsers = Sets.newHashSet("superUser", "admin"); config.setSuperUserRoles(superUsers); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java index ddb1fb3d736d3..ffab0b78a2700 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java @@ -96,7 +96,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setClusterName("use"); Set superUsers = Sets.newHashSet("superUser", "admin"); config.setSuperUserRoles(superUsers); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java index 5867bb78404d6..15ee27dc3a56c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java @@ -105,7 +105,7 @@ void setup(Method method) throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); - config = spy(new ServiceConfiguration()); + config = spy(ServiceConfiguration.class); config.setBrokerShutdownTimeoutMs(0L); config.setClusterName("use"); Set superUsers = Sets.newHashSet("superUser", "admin"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java index 5741a5eb0e648..b848fa76d5498 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.websocket.proxy; import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.ArgumentMatchers.anyString; @@ -86,7 +87,7 @@ public void setup() throws Exception { config.setAnonymousUserRole("anonymousUser"); } - service = spy(new WebSocketService(config)); + service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java index 78f33706d5355..a2758b72a4e0f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.websocket.proxy; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -68,7 +69,7 @@ protected void setup() throws Exception { config.setClusterName("c1"); config.setWebServicePort(Optional.of(0)); config.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); - service = spy(new WebSocketService(config)); + service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); service.start(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java index 3848d6ecc1dc6..ec4937bdd2149 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.websocket.proxy; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -65,7 +66,7 @@ public Object[][] setProxyConfig() { public void configTest(int numIoThreads, int connectionsPerBroker) throws Exception { config.setWebSocketNumIoThreads(numIoThreads); config.setWebSocketConnectionsPerBroker(connectionsPerBroker); - WebSocketService service = spy(new WebSocketService(config)); + WebSocketService service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); service.start(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java index b12f670b8b22e..1e74cdae78751 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java @@ -19,10 +19,10 @@ package org.apache.pulsar.websocket.proxy; import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -104,7 +104,7 @@ public void setup() throws Exception { config.setWebServicePort(Optional.of(0)); config.setClusterName("test"); config.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); - service = spy(new WebSocketService(config)); + service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java index b780fde760e33..cdc2eb58d9a09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.websocket.proxy; import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -76,7 +77,7 @@ public void setup() throws Exception { config.setBrokerClientAuthenticationParameters("tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + ",tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setConfigurationStoreServers(GLOBAL_DUMMY_VALUE); - service = spy(new WebSocketService(config)); + service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java index 485f23bdeb165..5baaacd52d9c2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.websocket.proxy; import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -63,7 +64,7 @@ public void setup() throws Exception { config.setClusterName("test"); config.setServiceUrl(pulsar.getSafeWebServiceAddress()); config.setServiceUrlTls(pulsar.getWebServiceAddressTls()); - service = spy(new WebSocketService(config)); + service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeper)).when(service).createMetadataStore(anyString(), anyInt()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java index d315a10c46a28..03227e9587d13 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java @@ -19,11 +19,10 @@ package org.apache.pulsar.websocket.proxy.v1; import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - import com.google.common.collect.Sets; import java.net.URI; @@ -88,7 +87,7 @@ public void setup() throws Exception { config.setAnonymousUserRole("anonymousUser"); } - service = spy(new WebSocketService(config)); + service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/api/MessageRouterTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/api/MessageRouterTest.java index 48f5816e2d2d5..0c29ecb7fef8f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/api/MessageRouterTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/api/MessageRouterTest.java @@ -32,7 +32,7 @@ */ public class MessageRouterTest { - private static class TestMessageRouter implements MessageRouter { + public static class TestMessageRouter implements MessageRouter { @Override public int choosePartition(Message msg) { @@ -43,7 +43,7 @@ public int choosePartition(Message msg) { @SuppressWarnings("deprecation") @Test public void testChoosePartition() { - MessageRouter router = spy(new TestMessageRouter()); + MessageRouter router = spy(TestMessageRouter.class); Message mockedMsg = mock(Message.class); TopicMetadata mockedMetadata = mock(TopicMetadata.class); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index 79871db87840d..be832236aacc2 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -946,7 +946,7 @@ public void testFunctionRuntimeFactoryConfigsBackwardsCompatibility() throws Exc WorkerConfig workerConfig = new WorkerConfig(); workerConfig.setKubernetesContainerFactory(kubernetesContainerFactory); - KubernetesRuntimeFactory mockedKubernetesRuntimeFactory = spy(new KubernetesRuntimeFactory()); + KubernetesRuntimeFactory mockedKubernetesRuntimeFactory = spy(KubernetesRuntimeFactory.class); doNothing().when(mockedKubernetesRuntimeFactory).initialize( any(WorkerConfig.class), any(AuthenticationConfig.class), @@ -1112,7 +1112,7 @@ public void testKubernetesFunctionInstancesRestart() throws Exception { WorkerConfig.KubernetesContainerFactory kubernetesContainerFactory = new WorkerConfig.KubernetesContainerFactory(); workerConfig.setKubernetesContainerFactory(kubernetesContainerFactory); - KubernetesRuntimeFactory mockedKubernetesRuntimeFactory = spy(new KubernetesRuntimeFactory()); + KubernetesRuntimeFactory mockedKubernetesRuntimeFactory = spy(KubernetesRuntimeFactory.class); doNothing().when(mockedKubernetesRuntimeFactory).initialize( any(WorkerConfig.class), any(AuthenticationConfig.class), diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index fdfde36cb28e1..7db32f591482f 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -361,7 +361,7 @@ protected static List getColumnColumnHandles(TopicName topic public static PulsarMetadata mockColumnMetadata() { ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - PulsarConnectorConfig pulsarConnectorConfig = spy(new PulsarConnectorConfig()); + PulsarConnectorConfig pulsarConnectorConfig = spy(PulsarConnectorConfig.class); pulsarConnectorConfig.setMaxEntryReadBatchSize(1); pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); @@ -451,7 +451,7 @@ protected static List getPartitionedTopics(String ns) { @BeforeMethod public void setup() throws Exception { - this.pulsarConnectorConfig = spy(new PulsarConnectorConfig()); + this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class); this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1); this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 2ea2616b6f1d2..880c2fb585bbb 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -461,7 +461,7 @@ public void testGetSchemaInfo() throws Exception { PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class); Schemas schemas = Mockito.mock(Schemas.class); Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas); - PulsarConnectorConfig connectorConfig = spy(new PulsarConnectorConfig()); + PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class); Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin); PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor( new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java index 98b7d8b6f69e7..e5ceb321aaec5 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java @@ -63,7 +63,7 @@ public abstract class AbstractDecoderTester { protected void init() { ConnectorContext prestoConnectorContext = new TestingConnectorContext(); this.decoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); - this.pulsarConnectorConfig = spy(new PulsarConnectorConfig()); + this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class); this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1); this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); From cf341857aad0e3b7534b6ae0901398cfd8efd062 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 25 May 2022 09:43:52 +0800 Subject: [PATCH 548/823] [branch-2.9][improve][client] Avoid timer task run before previous subscribe complete. (#15747) --- .../impl/PatternMultiTopicsConsumerImpl.java | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java index 2f946af712bfb..114cb274bc3f8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java @@ -24,14 +24,15 @@ import com.google.common.collect.Lists; import io.netty.util.Timeout; import io.netty.util.TimerTask; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; @@ -81,40 +82,36 @@ public void run(Timeout timeout) throws Exception { return; } - CompletableFuture recheckFuture = new CompletableFuture<>(); - List> futures = Lists.newArrayListWithExpectedSize(2); - - client.getLookup().getTopicsUnderNamespace(namespaceName, subscriptionMode).thenAccept(topics -> { - if (log.isDebugEnabled()) { - log.debug("Get topics under namespace {}, topics.size: {}", namespaceName.toString(), topics.size()); - topics.forEach(topicName -> - log.debug("Get topics under namespace {}, topic: {}", namespaceName.toString(), topicName)); - } - - List newTopics = PulsarClientImpl.topicsPatternFilter(topics, topicsPattern); - List oldTopics = Lists.newArrayList(); - oldTopics.addAll(getPartitionedTopics()); - getPartitions().forEach(p -> { - TopicName t = TopicName.get(p); - if (!t.isPartitioned() || !oldTopics.contains(t.getPartitionedTopicName())) { - oldTopics.add(p); - } - }); - - futures.add(topicsChangeListener.onTopicsAdded(topicsListsMinus(newTopics, oldTopics))); - futures.add(topicsChangeListener.onTopicsRemoved(topicsListsMinus(oldTopics, newTopics))); - FutureUtil.waitForAll(futures) - .thenAccept(finalFuture -> recheckFuture.complete(null)) - .exceptionally(ex -> { + client.getLookup().getTopicsUnderNamespace(namespaceName, subscriptionMode) + .thenCompose(topics -> { + if (log.isDebugEnabled()) { + log.debug("Get topics under namespace {}, topics.size: {}", + namespaceName.toString(), topics.size()); + topics.forEach(topicName -> + log.debug("Get topics under namespace {}, topic: {}", + namespaceName.toString(), topicName)); + } + final List newTopics = PulsarClientImpl.topicsPatternFilter(topics, topicsPattern); + final List oldTopics = new ArrayList<>(getPartitionedTopics()); + for (String partition : getPartitions()) { + TopicName topicName = TopicName.get(partition); + if (!topicName.isPartitioned() || !oldTopics.contains(topicName.getPartitionedTopicName())) { + oldTopics.add(partition); + } + } + final List> listenersCallback = new ArrayList<>(2); + listenersCallback.add(topicsChangeListener.onTopicsAdded(topicsListsMinus(newTopics, oldTopics))); + listenersCallback.add(topicsChangeListener.onTopicsRemoved(topicsListsMinus(oldTopics, newTopics))); + return FutureUtil.waitForAll(Collections.unmodifiableList(listenersCallback)); + }).exceptionally(ex -> { log.warn("[{}] Failed to recheck topics change: {}", topic, ex.getMessage()); - recheckFuture.completeExceptionally(ex); return null; + }).thenAccept(__ -> { + // schedule the next re-check task + this.recheckPatternTimeout = client.timer() + .newTimeout(PatternMultiTopicsConsumerImpl.this, + Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); }); - }); - - // schedule the next re-check task - this.recheckPatternTimeout = client.timer().newTimeout(PatternMultiTopicsConsumerImpl.this, - Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); } public Pattern getPattern() { From ea435d8dd3cf753485e0a9cea06e0685287eaa3d Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Fri, 1 Apr 2022 00:42:21 +0800 Subject: [PATCH 549/823] [CI] Upgrade zlib version to 1.2.12 (#14964) Co-authored-by: Zike Yang (cherry picked from commit 8b2b988654887cdb9d123698d5237381704f89c3) --- pulsar-client-cpp/docker/alpine/Dockerfile | 8 ++++---- pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 | 8 ++++---- pulsar-client-cpp/docker/manylinux1/Dockerfile | 8 ++++---- pulsar-client-cpp/docker/manylinux2014/Dockerfile | 2 +- pulsar-client-cpp/pkg/deb/Dockerfile | 8 ++++---- pulsar-client-cpp/pkg/licenses/LICENSE-zlib.txt | 4 ++-- pulsar-client-cpp/pkg/rpm/Dockerfile | 8 ++++---- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pulsar-client-cpp/docker/alpine/Dockerfile b/pulsar-client-cpp/docker/alpine/Dockerfile index ef77284242cc7..12d1e2f9f9742 100644 --- a/pulsar-client-cpp/docker/alpine/Dockerfile +++ b/pulsar-client-cpp/docker/alpine/Dockerfile @@ -43,12 +43,12 @@ RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/ rm -rf /boost_1_72_0.tar.gz /boost_1_72_0 # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \ - tar xfz zlib-1.2.11.tar.gz && \ - cd zlib-1.2.11 && \ +RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ + tar xfz zlib-1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ make -j4 && make install && \ - rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11 + rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ diff --git a/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 b/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 index 0a9fbb40711b9..862785e901566 100644 --- a/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 +++ b/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 @@ -43,12 +43,12 @@ RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/ rm -rf /boost_1_72_0.tar.gz /boost_1_72_0 # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \ - tar xfz zlib-1.2.11.tar.gz && \ - cd zlib-1.2.11 && \ +RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ + tar xfz zlib-1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ make -j4 && make install && \ - rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11 + rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ diff --git a/pulsar-client-cpp/docker/manylinux1/Dockerfile b/pulsar-client-cpp/docker/manylinux1/Dockerfile index 0df92401a8f64..502c6f6fe774d 100644 --- a/pulsar-client-cpp/docker/manylinux1/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux1/Dockerfile @@ -46,12 +46,12 @@ RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \ #################################### # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \ - tar xvfz zlib-1.2.11.tar.gz && \ - cd zlib-1.2.11 && \ +RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ + tar xvfz zlib-1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ make && make install && \ - rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11 + rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ diff --git a/pulsar-client-cpp/docker/manylinux2014/Dockerfile b/pulsar-client-cpp/docker/manylinux2014/Dockerfile index 5bc1b0315ed61..4d1cfeef92ba3 100644 --- a/pulsar-client-cpp/docker/manylinux2014/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux2014/Dockerfile @@ -52,7 +52,7 @@ RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ tar xvfz zlib-1.2.12.tar.gz && \ cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ + make && make install && \ rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 # Compile OpenSSL diff --git a/pulsar-client-cpp/pkg/deb/Dockerfile b/pulsar-client-cpp/pkg/deb/Dockerfile index 77c3f470c068f..1925827b56ac4 100644 --- a/pulsar-client-cpp/pkg/deb/Dockerfile +++ b/pulsar-client-cpp/pkg/deb/Dockerfile @@ -49,12 +49,12 @@ RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/prot rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0 # ZLib -RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.11.tar.gz && \ - tar xvfz v1.2.11.tar.gz && \ - cd zlib-1.2.11 && \ +RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ + tar xvfz v1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ make && make install && \ - rm -rf /v1.2.11.tar.gz /zlib-1.2.11 + rm -rf /v1.2.12.tar.gz /zlib-1.2.12 # Zstandard RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ diff --git a/pulsar-client-cpp/pkg/licenses/LICENSE-zlib.txt b/pulsar-client-cpp/pkg/licenses/LICENSE-zlib.txt index cf25d04b7e1d1..f1f93cd6a6af5 100644 --- a/pulsar-client-cpp/pkg/licenses/LICENSE-zlib.txt +++ b/pulsar-client-cpp/pkg/licenses/LICENSE-zlib.txt @@ -1,7 +1,7 @@ zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.11, January 15th, 2017 + version 1.2.12, March 27th, 2022 - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/pulsar-client-cpp/pkg/rpm/Dockerfile b/pulsar-client-cpp/pkg/rpm/Dockerfile index c2406c5c7d32a..e83290dccca74 100644 --- a/pulsar-client-cpp/pkg/rpm/Dockerfile +++ b/pulsar-client-cpp/pkg/rpm/Dockerfile @@ -49,12 +49,12 @@ RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/prot rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0 # ZLib -RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.11.tar.gz && \ - tar xvfz v1.2.11.tar.gz && \ - cd zlib-1.2.11 && \ +RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ + tar xvfz v1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ CFLAGS="-fPIC -O3" ./configure && \ make && make install && \ - rm -rf /v1.2.11.tar.gz /zlib-1.2.11 + rm -rf /v1.2.12.tar.gz /zlib-1.2.12 # Zstandard RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ From e1b1c34d1fd2a80a189c0db5e86d2888a8c718a9 Mon Sep 17 00:00:00 2001 From: wuxuanqicn <89442834+wuxuanqicn@users.noreply.github.com> Date: Fri, 22 Apr 2022 01:46:22 +0800 Subject: [PATCH 550/823] [Flaky-test] BatchSourceExecutorTest.testLifeCycle (#10870) (#14717) ### Motivation event will be dropped at BatchSourceExecutor#triggerDiscover when discoverInProgress is true https://github.com/apache/pulsar/blob/f0d166f36e1fbd4df1e20ae2ccc7fcae822c17b4/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/batch/BatchSourceExecutor.java#L161-L182 ### Modifications await util task submitted in BatchSourceExecutor#triggerDiscover execute completed and discoverInProgress update to false Co-authored-by: xuanqi.wu (cherry picked from commit be13b2503a5bb47d672587c38408436c840f349e) --- pulsar-functions/instance/pom.xml | 6 ++++++ .../source/batch/BatchSourceExecutorTest.java | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index efbb8fb6229c4..d5d37d325562b 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -194,6 +194,12 @@ ${prometheus-jmx.version} + + org.awaitility + awaitility + test + + diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/batch/BatchSourceExecutorTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/batch/BatchSourceExecutorTest.java index ff15a10ebbe86..6715b74624b51 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/batch/BatchSourceExecutorTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/batch/BatchSourceExecutorTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.functions.source.batch; +import static org.awaitility.Awaitility.await; +import static org.testng.Assert.fail; import com.google.gson.Gson; import lombok.Getter; import org.apache.pulsar.client.api.ConsumerBuilder; @@ -44,7 +46,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Consumer; -import static org.testng.Assert.fail; /** * Unit tests for {@link org.apache.pulsar.functions.source.batch.BatchSourceExecutor} @@ -368,6 +369,8 @@ public void testLifeCycle() throws Exception { } Assert.assertEquals(testBatchSource.getRecordCount(), 6); Assert.assertEquals(testBatchSource.getDiscoverCount(), 1); + + awaitDiscoverNotInProgress(); triggerQueue.put("trigger"); completedQueue.take(); Assert.assertTrue(testBatchSource.getDiscoverCount() == 2); @@ -387,6 +390,8 @@ public void testPushLifeCycle() throws Exception { } Assert.assertEquals(testBatchPushSource.getRecordCount(), 5); Assert.assertEquals(testBatchPushSource.getDiscoverCount(), 1); + + awaitDiscoverNotInProgress(); triggerQueue.put("trigger"); completedQueue.take(); Assert.assertEquals(testBatchPushSource.getDiscoverCount(), 2); @@ -406,4 +411,8 @@ public void testDiscoveryPhaseError() throws Exception { fail("should have thrown an exception"); } + private void awaitDiscoverNotInProgress() { + await().until(() -> !batchSourceExecutor.discoverInProgress); + } + } \ No newline at end of file From a9e799af105cdd4a3785c5108fbc656b3da9cd36 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Thu, 28 Apr 2022 15:06:11 +0800 Subject: [PATCH 551/823] [fix][broker] Fix MessageDeduplication#inactiveProducers may not be persistence correctly (#15206) ### Motivation #15204 In the current implementation, When the first time execute `purgeInactiveProducers`, Although the produces does not expire, it removed directly from the collection(464 line). The will result in these producers never being remove. https://github.com/apache/pulsar/blob/9861dfb1208c4b6b8a1f17ef026e9af71c3e784c/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java#L454-L472 ### Modifications 1. It is removed from the collection only when the producer is inactive. 2. Take a snapshot after each removal of an inactive producer. When `managedLedger.getLastConfirmedEntry` equals `managedCursor.getMarkDeletedPosition()`, The`deduplication-snapshot-monitor` thread does not trigger a snapshot. The persistence these producers only the next time a message is produced, The can be confusing for users. ``` PositionImpl position = (PositionImpl) managedLedger.getLastConfirmedEntry(); if (position == null) { return; } PositionImpl markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); if (markDeletedPosition != null && position.compareTo(markDeletedPosition) <= 0) { return; } ``` (cherry picked from commit 8e1ca487c1026510fee264d65a34067ac427ee9d) --- .../persistent/MessageDeduplication.java | 13 ++++++--- .../persistent/MessageDuplicationTest.java | 29 ++++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index bafe93a5ffe0b..55d35201fe5be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -36,6 +36,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Topic.PublishContext; @@ -406,7 +407,7 @@ public void resetHighestSequenceIdPushed() { } } - private void takeSnapshot(PositionImpl position) { + private void takeSnapshot(Position position) { if (log.isDebugEnabled()) { log.debug("[{}] Taking snapshot of sequence ids map", topic.getName()); } @@ -417,7 +418,7 @@ private void takeSnapshot(PositionImpl position) { } }); - managedCursor.asyncMarkDelete(position, snapshot, new MarkDeleteCallback() { + getManagedCursor().asyncMarkDelete(position, snapshot, new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { if (log.isDebugEnabled()) { @@ -475,19 +476,23 @@ public synchronized void purgeInactiveProducers() { .toMillis(pulsar.getConfiguration().getBrokerDeduplicationProducerInactivityTimeoutMinutes()); Iterator> mapIterator = inactiveProducers.entrySet().iterator(); + boolean hasInactive = false; while (mapIterator.hasNext()) { java.util.Map.Entry entry = mapIterator.next(); String producerName = entry.getKey(); long lastActiveTimestamp = entry.getValue(); - mapIterator.remove(); - if (lastActiveTimestamp < minimumActiveTimestamp) { log.info("[{}] Purging dedup information for producer {}", topic.getName(), producerName); + mapIterator.remove(); highestSequencedPushed.remove(producerName); highestSequencedPersisted.remove(producerName); + hasInactive = true; } } + if (hasInactive) { + takeSnapshot(getManagedCursor().getMarkDeletedPosition()); + } } public long getLastPublishedSequenceId(String producerName) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index 117c9dd15828b..76caccadcc96a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -51,7 +51,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; +import org.apache.bookkeeper.mledger.ManagedCursor; @Slf4j @Test(groups = "broker") @@ -170,11 +170,14 @@ public void testInactiveProducerRemove() throws Exception { spyWithClassAndConstructorArgs(MessageDeduplication.class, pulsarService, topic, managedLedger); doReturn(true).when(messageDeduplication).isEnabled(); + ManagedCursor managedCursor = mock(ManagedCursor.class); + doReturn(managedCursor).when(messageDeduplication).getManagedCursor(); + Topic.PublishContext publishContext = mock(Topic.PublishContext.class); Field field = MessageDeduplication.class.getDeclaredField("inactiveProducers"); field.setAccessible(true); - Map map = (Map) field.get(messageDeduplication); + Map inactiveProducers = (Map) field.get(messageDeduplication); String producerName1 = "test1"; when(publishContext.getHighestSequenceId()).thenReturn(2L); @@ -190,18 +193,23 @@ public void testInactiveProducerRemove() throws Exception { when(publishContext.getProducerName()).thenReturn(producerName3); messageDeduplication.isDuplicate(publishContext, null); + // All 3 are added to the inactiveProducers list messageDeduplication.producerRemoved(producerName1); - assertTrue(map.containsKey(producerName1)); - messageDeduplication.producerAdded(producerName1); - assertFalse(map.containsKey(producerName1)); + messageDeduplication.producerRemoved(producerName2); + messageDeduplication.producerRemoved(producerName3); + + // Try first purgeInactive, all producer not inactive. messageDeduplication.purgeInactiveProducers(); + assertEquals(inactiveProducers.size(), 3); + + // Modify the inactive time of produce2 and produce3 // messageDeduplication.purgeInactiveProducers() will remove producer2 and producer3 - map.put(producerName2, System.currentTimeMillis() - 70000); - map.put(producerName3, System.currentTimeMillis() - 70000); + inactiveProducers.put(producerName2, System.currentTimeMillis() - 70000); + inactiveProducers.put(producerName3, System.currentTimeMillis() - 70000); + // Try second purgeInactive, produce2 and produce3 is inactive. messageDeduplication.purgeInactiveProducers(); - assertFalse(map.containsKey(producerName2)); - assertFalse(map.containsKey(producerName3)); - + assertFalse(inactiveProducers.containsKey(producerName2)); + assertFalse(inactiveProducers.containsKey(producerName3)); field = MessageDeduplication.class.getDeclaredField("highestSequencedPushed"); field.setAccessible(true); ConcurrentOpenHashMap highestSequencedPushed = (ConcurrentOpenHashMap) field.get(messageDeduplication); @@ -209,7 +217,6 @@ public void testInactiveProducerRemove() throws Exception { assertEquals((long) highestSequencedPushed.get(producerName1), 2L); assertFalse(highestSequencedPushed.containsKey(producerName2)); assertFalse(highestSequencedPushed.containsKey(producerName3)); - } @Test From ab62742aee564c31a670bde6952b2d5d085f7240 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 30 Apr 2022 04:57:42 +0300 Subject: [PATCH 552/823] [Broker] Fix typo in enum name and handle closing of the channel properly since writeAndFlush is asynchronous (#15384) (cherry picked from commit cd3816aa351ba8a1f0e9876eefe019b7f0d282d8) --- .../broker/service/ConnectionController.java | 16 ++++++++-------- .../apache/pulsar/broker/service/ServerCnx.java | 15 ++++++++------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConnectionController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConnectionController.java index 51540e179be19..65c3a6c4f2a8b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConnectionController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConnectionController.java @@ -36,7 +36,7 @@ public interface ConnectionController { * @param remoteAddress * @return */ - Sate increaseConnection(SocketAddress remoteAddress); + State increaseConnection(SocketAddress remoteAddress); /** * Decrease the number of connections counter. @@ -44,7 +44,7 @@ public interface ConnectionController { */ void decreaseConnection(SocketAddress remoteAddress); - enum Sate { + enum State { OK, REACH_MAX_CONNECTION_PER_IP, REACH_MAX_CONNECTION; } @@ -68,13 +68,13 @@ public DefaultConnectionController(ServiceConfiguration configuration) { } @Override - public Sate increaseConnection(SocketAddress remoteAddress) { + public State increaseConnection(SocketAddress remoteAddress) { if (!maxConnectionsLimitEnabled && !maxConnectionsLimitPerIpEnabled) { - return Sate.OK; + return State.OK; } if (!(remoteAddress instanceof InetSocketAddress) || !isLegalIpAddress(((InetSocketAddress) remoteAddress).getHostString())) { - return Sate.OK; + return State.OK; } lock.lock(); try { @@ -88,20 +88,20 @@ public Sate increaseConnection(SocketAddress remoteAddress) { if (maxConnectionsLimitEnabled && totalConnectionNum > maxConnections) { log.info("Reject connect request from {}, because reached the maximum number of connections {}", remoteAddress, totalConnectionNum); - return Sate.REACH_MAX_CONNECTION; + return State.REACH_MAX_CONNECTION; } if (maxConnectionsLimitPerIpEnabled && CONNECTIONS.get(ip).getValue() > maxConnectionPerIp) { log.info("Reject connect request from {}, because reached the maximum number " + "of connections per Ip {}", remoteAddress, CONNECTIONS.get(ip).getValue()); - return Sate.REACH_MAX_CONNECTION_PER_IP; + return State.REACH_MAX_CONNECTION_PER_IP; } } catch (Exception e) { log.error("increase connection failed", e); } finally { lock.unlock(); } - return Sate.OK; + return State.OK; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d8a51f60d65b1..03a6a80aed3ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -27,6 +27,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOption; @@ -270,13 +271,13 @@ public ServerCnx(PulsarService pulsar, String listenerName) { @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { super.channelActive(ctx); - ConnectionController.Sate sate = connectionController.increaseConnection(remoteAddress); - if (!sate.equals(ConnectionController.Sate.OK)) { - ctx.channel().writeAndFlush(Commands.newError(-1, ServerError.NotAllowedError, - sate.equals(ConnectionController.Sate.REACH_MAX_CONNECTION) - ? "Reached the maximum number of connections" - : "Reached the maximum number of connections on address" + remoteAddress)); - ctx.channel().close(); + ConnectionController.State state = connectionController.increaseConnection(remoteAddress); + if (!state.equals(ConnectionController.State.OK)) { + ctx.writeAndFlush(Commands.newError(-1, ServerError.NotAllowedError, + state.equals(ConnectionController.State.REACH_MAX_CONNECTION) + ? "Reached the maximum number of connections" + : "Reached the maximum number of connections on address" + remoteAddress)) + .addListener(ChannelFutureListener.CLOSE); return; } log.info("New connection from {}", remoteAddress); From 5773ddf9b87cb3ccbf453f2fe4e0ef94784e6311 Mon Sep 17 00:00:00 2001 From: AlvaroStream <102966649+alvarostream@users.noreply.github.com> Date: Sat, 30 Apr 2022 12:24:44 +0800 Subject: [PATCH 553/823] [Improve][admin|client] AsyncHttpConnector doesn't use the system properties configured (#15307) (cherry picked from commit ebf2487ab2bfb35362ffdffc520b1652426f49f1) --- .../org/apache/pulsar/client/api/BrokerServiceLookupTest.java | 1 + .../pulsar/client/admin/internal/http/AsyncHttpConnector.java | 1 + .../src/main/java/org/apache/pulsar/client/impl/HttpClient.java | 1 + .../pulsar/client/impl/auth/oauth2/protocol/TokenClient.java | 1 + 4 files changed, 4 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 6628cf64c6a93..670d184e5fc36 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -882,6 +882,7 @@ public void onThrowable(Throwable t) { private AsyncHttpClient getHttpClient(String version) { DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); + confBuilder.setUseProxyProperties(true); confBuilder.setFollowRedirect(true); confBuilder.setUserAgent(version); confBuilder.setKeepAliveStrategy(new DefaultKeepAliveStrategy() { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java index 1f302f6586cd5..95ea0717b97e8 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java @@ -96,6 +96,7 @@ public AsyncHttpConnector(int connectTimeoutMs, int readTimeoutMs, int requestTimeoutMs, int autoCertRefreshTimeSeconds, ClientConfigurationData conf) { DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); + confBuilder.setUseProxyProperties(true); confBuilder.setFollowRedirect(true); confBuilder.setRequestTimeout(conf.getRequestTimeoutMs()); confBuilder.setConnectTimeout(connectTimeoutMs); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index 2a7e434cd3885..285a7202c7206 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -72,6 +72,7 @@ protected HttpClient(ClientConfigurationData conf, EventLoopGroup eventLoopGroup this.serviceNameResolver.updateServiceUrl(conf.getServiceUrl()); DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); + confBuilder.setUseProxyProperties(true); confBuilder.setFollowRedirect(true); confBuilder.setMaxRedirects(conf.getMaxLookupRedirects()); confBuilder.setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_IN_SECONDS * 1000); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java index c2b97793e3512..1028da5f3e6b3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java @@ -54,6 +54,7 @@ public TokenClient(URL tokenUrl) { TokenClient(URL tokenUrl, AsyncHttpClient httpClient) { if (httpClient == null) { DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); + confBuilder.setUseProxyProperties(true); confBuilder.setFollowRedirect(true); confBuilder.setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_IN_SECONDS * 1000); confBuilder.setReadTimeout(DEFAULT_READ_TIMEOUT_IN_SECONDS * 1000); From cf035aef29f5341dbe0062a4252be086b5a9083d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 3 May 2022 10:59:19 +0800 Subject: [PATCH 554/823] [Proxy/Client] Fix DNS server denial-of-service issue when DNS entry expires (#15403) (cherry picked from commit 40d71691dab2a09d3457f8fa638b19ebc2e28dd7) --- .../client/impl/ConnectionPoolTest.java | 37 +++++++------ .../pulsar/client/impl/ConnectionPool.java | 53 ++++++++++--------- .../pulsar/proxy/server/ProxyConnection.java | 16 +++--- .../pulsar/proxy/server/ProxyService.java | 12 ++--- .../server/ServiceChannelInitializer.java | 2 +- 5 files changed, 61 insertions(+), 59 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index 235bd7167a5e3..30583bb64cda3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -21,6 +21,10 @@ import com.google.common.collect.Lists; import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.DefaultThreadFactory; +import java.net.InetSocketAddress; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.IntStream; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.util.netty.EventLoopUtil; @@ -30,11 +34,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.stream.IntStream; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; @@ -42,12 +41,14 @@ public class ConnectionPoolTest extends MockedPulsarServiceBaseTest { String serviceUrl; + int brokerPort; @BeforeClass @Override protected void setup() throws Exception { super.internalSetup(); - serviceUrl = "pulsar://non-existing-dns-name:" + pulsar.getBrokerListenPort().get(); + brokerPort = pulsar.getBrokerListenPort().get(); + serviceUrl = "pulsar://non-existing-dns-name:" + brokerPort; } @AfterClass(alwaysRun = true) @@ -64,9 +65,11 @@ public void testSingleIpAddress() throws Exception { conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); - List result = Lists.newArrayList(); - result.add(InetAddress.getByName("127.0.0.1")); - Mockito.when(pool.resolveName("non-existing-dns-name")).thenReturn(CompletableFuture.completedFuture(result)); + List result = Lists.newArrayList(); + result.add(new InetSocketAddress("127.0.0.1", brokerPort)); + Mockito.when(pool.resolveName(InetSocketAddress.createUnresolved("non-existing-dns-name", + brokerPort))) + .thenReturn(CompletableFuture.completedFuture(result)); client.newProducer().topic("persistent://sample/standalone/ns/my-topic").create(); @@ -76,20 +79,20 @@ public void testSingleIpAddress() throws Exception { @Test public void testDoubleIpAddress() throws Exception { - String serviceUrl = "pulsar://non-existing-dns-name:" + pulsar.getBrokerListenPort().get(); - ClientConfigurationData conf = new ClientConfigurationData(); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); - List result = Lists.newArrayList(); + List result = Lists.newArrayList(); // Add a non existent IP to the response to check that we're trying the 2nd address as well - result.add(InetAddress.getByName("127.0.0.99")); - result.add(InetAddress.getByName("127.0.0.1")); - Mockito.when(pool.resolveName("non-existing-dns-name")).thenReturn(CompletableFuture.completedFuture(result)); + result.add(new InetSocketAddress("127.0.0.99", brokerPort)); + result.add(new InetSocketAddress("127.0.0.1", brokerPort)); + Mockito.when(pool.resolveName(InetSocketAddress.createUnresolved("non-existing-dns-name", + brokerPort))) + .thenReturn(CompletableFuture.completedFuture(result)); // Create producer should succeed by trying the 2nd IP client.newProducer().topic("persistent://sample/standalone/ns/my-topic").create(); @@ -106,7 +109,7 @@ public void testNoConnectionPool() throws Exception { ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = - InetSocketAddress.createUnresolved("127.0.0.1", pulsar.getBrokerListenPort().get()); + InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); IntStream.range(1, 5).forEach(i -> { pool.getConnection(brokerAddress).thenAccept(cnx -> { Assert.assertTrue(cnx.channel().isActive()); @@ -128,7 +131,7 @@ public void testEnableConnectionPool() throws Exception { ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = - InetSocketAddress.createUnresolved("127.0.0.1", pulsar.getBrokerListenPort().get()); + InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); IntStream.range(1, 10).forEach(i -> { pool.getConnection(brokerAddress).thenAccept(cnx -> { Assert.assertTrue(cnx.channel().isActive()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 59aaac0477b80..15517e45ba3d7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -26,10 +26,10 @@ import io.netty.channel.ChannelException; import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; -import io.netty.resolver.dns.DnsNameResolver; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.dns.DnsAddressResolverGroup; import io.netty.resolver.dns.DnsNameResolverBuilder; import io.netty.util.concurrent.Future; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -63,7 +63,7 @@ public class ConnectionPool implements AutoCloseable { private final int maxConnectionsPerHosts; private final boolean isSniProxy; - protected final DnsNameResolver dnsResolver; + protected final AddressResolver addressResolver; private final boolean shouldCloseDnsResolver; public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { @@ -76,7 +76,8 @@ public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGrou } public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, - Supplier clientCnxSupplier, Optional dnsNameResolver) + Supplier clientCnxSupplier, + Optional> addressResolver) throws PulsarClientException { this.eventLoopGroup = eventLoopGroup; this.clientConfig = conf; @@ -101,15 +102,19 @@ public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGrou throw new PulsarClientException(e); } - this.shouldCloseDnsResolver = !dnsNameResolver.isPresent(); - this.dnsResolver = dnsNameResolver.orElseGet(() -> createDnsNameResolver(conf, eventLoopGroup)); + this.shouldCloseDnsResolver = !addressResolver.isPresent(); + this.addressResolver = addressResolver.orElseGet(() -> createAddressResolver(conf, eventLoopGroup)); } - private static DnsNameResolver createDnsNameResolver(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { - DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder(eventLoopGroup.next()) + private static AddressResolver createAddressResolver(ClientConfigurationData conf, + EventLoopGroup eventLoopGroup) { + DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder() .traceEnabled(true).channelType(EventLoopUtil.getDatagramChannelClass(eventLoopGroup)); DnsResolverUtil.applyJdkDnsCacheSettings(dnsNameResolverBuilder); - return dnsNameResolverBuilder.build(); + // use DnsAddressResolverGroup to create the AddressResolver since it contains a solution + // to prevent cache stampede / thundering herds problem when a DNS entry expires while the system + // is under high load + return new DnsAddressResolverGroup(dnsNameResolverBuilder).getResolver(eventLoopGroup.next()); } private static final Random random = new Random(); @@ -234,19 +239,17 @@ private CompletableFuture createConnection(InetSocketAddress logicalA * Resolve DNS asynchronously and attempt to connect to any IP address returned by DNS server. */ private CompletableFuture createConnection(InetSocketAddress unresolvedAddress) { - int port; - CompletableFuture> resolvedAddress; + CompletableFuture> resolvedAddress; try { if (isSniProxy) { URI proxyURI = new URI(clientConfig.getProxyServiceUrl()); - port = proxyURI.getPort(); - resolvedAddress = resolveName(proxyURI.getHost()); + resolvedAddress = + resolveName(InetSocketAddress.createUnresolved(proxyURI.getHost(), proxyURI.getPort())); } else { - port = unresolvedAddress.getPort(); - resolvedAddress = resolveName(unresolvedAddress.getHostString()); + resolvedAddress = resolveName(unresolvedAddress); } return resolvedAddress.thenCompose( - inetAddresses -> connectToResolvedAddresses(inetAddresses.iterator(), port, + inetAddresses -> connectToResolvedAddresses(inetAddresses.iterator(), isSniProxy ? unresolvedAddress : null)); } catch (URISyntaxException e) { log.error("Invalid Proxy url {}", clientConfig.getProxyServiceUrl(), e); @@ -259,18 +262,17 @@ private CompletableFuture createConnection(InetSocketAddress unresolved * Try to connect to a sequence of IP addresses until a successful connection can be made, or fail if no * address is working. */ - private CompletableFuture connectToResolvedAddresses(Iterator unresolvedAddresses, - int port, + private CompletableFuture connectToResolvedAddresses(Iterator unresolvedAddresses, InetSocketAddress sniHost) { CompletableFuture future = new CompletableFuture<>(); // Successfully connected to server - connectToAddress(unresolvedAddresses.next(), port, sniHost) + connectToAddress(unresolvedAddresses.next(), sniHost) .thenAccept(future::complete) .exceptionally(exception -> { if (unresolvedAddresses.hasNext()) { // Try next IP address - connectToResolvedAddresses(unresolvedAddresses, port, sniHost).thenAccept(future::complete) + connectToResolvedAddresses(unresolvedAddresses, sniHost).thenAccept(future::complete) .exceptionally(ex -> { // This is already unwinding the recursive call future.completeExceptionally(ex); @@ -286,9 +288,9 @@ private CompletableFuture connectToResolvedAddresses(Iterator> resolveName(String hostname) { - CompletableFuture> future = new CompletableFuture<>(); - dnsResolver.resolveAll(hostname).addListener((Future> resolveFuture) -> { + CompletableFuture> resolveName(InetSocketAddress unresolvedAddress) { + CompletableFuture> future = new CompletableFuture<>(); + addressResolver.resolveAll(unresolvedAddress).addListener((Future> resolveFuture) -> { if (resolveFuture.isSuccess()) { future.complete(resolveFuture.get()); } else { @@ -301,8 +303,7 @@ CompletableFuture> resolveName(String hostname) { /** * Attempt to establish a TCP connection to an already resolved single IP address. */ - private CompletableFuture connectToAddress(InetAddress ipAddress, int port, InetSocketAddress sniHost) { - InetSocketAddress remoteAddress = new InetSocketAddress(ipAddress, port); + private CompletableFuture connectToAddress(InetSocketAddress remoteAddress, InetSocketAddress sniHost) { if (clientConfig.isUseTls()) { return toCompletableFuture(bootstrap.register()) .thenCompose(channel -> channelInitializerHandler @@ -332,7 +333,7 @@ public void releaseConnection(ClientCnx cnx) { public void close() throws Exception { closeAllConnections(); if (shouldCloseDnsResolver) { - dnsResolver.close(); + addressResolver.close(); } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 39870f62af848..eeabced97b065 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -22,7 +22,8 @@ import static com.google.common.base.Preconditions.checkState; import io.netty.channel.ChannelFutureListener; import io.netty.handler.codec.haproxy.HAProxyMessage; -import io.netty.resolver.dns.DnsNameResolver; +import io.netty.handler.ssl.SslHandler; +import io.netty.resolver.dns.DnsAddressResolverGroup; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Collections; @@ -41,7 +42,6 @@ import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; import org.apache.pulsar.client.api.Authentication; -import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.ConnectionPool; @@ -66,9 +66,6 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.ssl.SslHandler; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.FutureListener; import lombok.Getter; /** @@ -82,7 +79,7 @@ public class ProxyConnection extends PulsarHandler { private final AtomicLong requestIdGenerator = new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2)); private final ProxyService service; - private final DnsNameResolver dnsNameResolver; + private final DnsAddressResolverGroup dnsAddressResolverGroup; AuthenticationDataSource authenticationData; private State state; private final Supplier sslHandlerSupplier; @@ -135,10 +132,10 @@ ConnectionPool getConnectionPool() { } public ProxyConnection(ProxyService proxyService, Supplier sslHandlerSupplier, - DnsNameResolver dnsNameResolver) { + DnsAddressResolverGroup dnsAddressResolverGroup) { super(30, TimeUnit.SECONDS); this.service = proxyService; - this.dnsNameResolver = dnsNameResolver; + this.dnsAddressResolverGroup = dnsAddressResolverGroup; this.state = State.Init; this.sslHandlerSupplier = sslHandlerSupplier; this.brokerProxyValidator = service.getBrokerProxyValidator(); @@ -281,7 +278,8 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie if (this.connectionPool == null) { this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(), - clientCnxSupplier, Optional.of(dnsNameResolver)); + clientCnxSupplier, + Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next()))); } else { LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", remoteAddress, state, clientAuthRole); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index 0f4ea152b50ee..10e122e794d3c 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -29,7 +29,7 @@ import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; -import io.netty.resolver.dns.DnsNameResolver; +import io.netty.resolver.dns.DnsAddressResolverGroup; import io.netty.resolver.dns.DnsNameResolverBuilder; import io.netty.util.concurrent.DefaultThreadFactory; import io.prometheus.client.Counter; @@ -75,7 +75,7 @@ public class ProxyService implements Closeable { private final ProxyConfiguration proxyConfig; private final Authentication proxyClientAuthentication; @Getter - private final DnsNameResolver dnsNameResolver; + private final DnsAddressResolverGroup dnsAddressResolverGroup; @Getter private final BrokerProxyValidator brokerProxyValidator; private String serviceUrl; @@ -152,13 +152,13 @@ public ProxyService(ProxyConfiguration proxyConfig, false, workersThreadFactory); this.authenticationService = authenticationService; - DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder(workerGroup.next()) + DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder() .channelType(EventLoopUtil.getDatagramChannelClass(workerGroup)); DnsResolverUtil.applyJdkDnsCacheSettings(dnsNameResolverBuilder); - dnsNameResolver = dnsNameResolverBuilder.build(); + dnsAddressResolverGroup = new DnsAddressResolverGroup(dnsNameResolverBuilder); - brokerProxyValidator = new BrokerProxyValidator(dnsNameResolver.asAddressResolver(), + brokerProxyValidator = new BrokerProxyValidator(dnsAddressResolverGroup.getResolver(workerGroup.next()), proxyConfig.getBrokerProxyAllowedHostNames(), proxyConfig.getBrokerProxyAllowedIPAddresses(), proxyConfig.getBrokerProxyAllowedTargetPorts()); @@ -291,7 +291,7 @@ public BrokerDiscoveryProvider getDiscoveryProvider() { } public void close() throws IOException { - dnsNameResolver.close(); + dnsAddressResolverGroup.close(); if (discoveryProvider != null) { discoveryProvider.close(); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 4f423c1f5d5b5..2ce2a93819f10 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -174,7 +174,7 @@ public SslHandler get() { } ch.pipeline().addLast("handler", - new ProxyConnection(proxyService, sslHandlerSupplier, proxyService.getDnsNameResolver())); + new ProxyConnection(proxyService, sslHandlerSupplier, proxyService.getDnsAddressResolverGroup())); } } From d0585050964fabc8173d406c6c1d4ca51c4798e2 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 5 May 2022 18:37:38 +0800 Subject: [PATCH 555/823] [improve][java-client] Add pending messages information while print the producer stats (#15440) (cherry picked from commit fbe650ce72462e97ca1ba8f9dcb41ab1b7ce47bd) --- .../pulsar/client/impl/ProducerStatsRecorderImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java index 6b435d683032b..180d53e4949a7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java @@ -163,14 +163,15 @@ protected void updateStats() { log.info("[{}] [{}] Pending messages: {} --- Publish throughput: {} msg/s --- {} Mbit/s --- " + "Latency: med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - max: {} ms --- " - + "Ack received rate: {} ack/s --- Failed messages: {}", producer.getTopic(), - producer.getProducerName(), producer.getPendingQueueSize(), + + "Ack received rate: {} ack/s --- Failed messages: {} --- Pending messages: {}", + producer.getTopic(), producer.getProducerName(), producer.getPendingQueueSize(), THROUGHPUT_FORMAT.format(sendMsgsRate), THROUGHPUT_FORMAT.format(sendBytesRate / 1024 / 1024 * 8), DEC.format(latencyPctValues[0]), DEC.format(latencyPctValues[2]), DEC.format(latencyPctValues[3]), DEC.format(latencyPctValues[4]), DEC.format(latencyPctValues[5]), - THROUGHPUT_FORMAT.format(currentNumAcksReceived / elapsed), currentNumSendFailedMsgs); + THROUGHPUT_FORMAT.format(currentNumAcksReceived / elapsed), currentNumSendFailedMsgs, + producer.getPendingQueueSize()); } } From 51c89f8f1f7ce3be455346d7e930e5b3f08d88ac Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 6 May 2022 21:40:51 +0800 Subject: [PATCH 556/823] [Improve][doc] Add config of IO and acceptor threads in proxy (#15340) * Add config of IO and acceptor threads in proxy * Update doc * Update site2/docs/reference-configuration.md Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> * Update site2/docs/reference-configuration.md Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> (cherry picked from commit da3f017240662fe9498dcc3d0f8513c02a740bb8) --- conf/proxy.conf | 6 ++++++ .../org/apache/pulsar/proxy/server/ProxyConfiguration.java | 4 ++-- site2/docs/reference-configuration.md | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/conf/proxy.conf b/conf/proxy.conf index 4f98663afc175..77129ccc71dc0 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -68,6 +68,12 @@ webServicePort=8080 # Port to use to server HTTPS request webServicePortTls= +# Number of threads used for Netty IO. Default is set to `2 * Runtime.getRuntime().availableProcessors()` +numIOThreads= + +# Number of threads used for Netty Acceptor. Default is set to `1` +numAcceptorThreads= + ### --- TLS config variables --- ### ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration. diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index e440133aa409d..d616bbd1ce423 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -569,14 +569,14 @@ public class ProxyConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_SERVER, - doc = "Number of threads to use for Netty IO." + doc = "Number of threads used for Netty IO." + " Default is set to `2 * Runtime.getRuntime().availableProcessors()`" ) private int numIOThreads = 2 * Runtime.getRuntime().availableProcessors(); @FieldContext( category = CATEGORY_SERVER, - doc = "Number of threads to use for Netty Acceptor." + doc = "Number of threads used for Netty Acceptor." + " Default is set to `1`" ) private int numAcceptorThreads = 1; diff --git a/site2/docs/reference-configuration.md b/site2/docs/reference-configuration.md index 6fe8565f9e9c6..f8d1e87f890b2 100644 --- a/site2/docs/reference-configuration.md +++ b/site2/docs/reference-configuration.md @@ -732,6 +732,8 @@ The [Pulsar proxy](concepts-architecture-overview.md#pulsar-proxy) can be config |tokenAudienceClaim| The token audience "claim" name, e.g. "aud". It is used to get the audience from token. If it is not set, the audience is not verified. || | tokenAudience | The token audience stands for this broker. The field `tokenAudienceClaim` of a valid token need contains this parameter.| | |haProxyProtocolEnabled | Enable or disable the [HAProxy](http://www.haproxy.org/) protocol. |false| +| numIOThreads | Number of threads used for Netty IO. | 2 * Runtime.getRuntime().availableProcessors() | +| numAcceptorThreads | Number of threads used for Netty Acceptor. | 1 | ## ZooKeeper From 77c333ca47ec9d1a01c5f4448cad309a60e3f208 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sat, 7 May 2022 15:35:04 +0800 Subject: [PATCH 557/823] [improve][client] improve logic when ACK grouping tracker checks duplicated message id (#15465) (cherry picked from commit f6faeecc819de880eb4a93e4bb359bebbc0bc855) --- .../impl/PersistentAcknowledgmentsGroupingTracker.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index e737a2a6f161d..dc311521083b5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -117,10 +117,7 @@ public PersistentAcknowledgmentsGroupingTracker(ConsumerImpl consumer, Consum @Override public boolean isDuplicate(@NonNull MessageId messageId) { final MessageId messageIdOfLastAck = lastCumulativeAck.messageId; - if (messageIdOfLastAck == null) { - return false; - } - if (messageId.compareTo(messageIdOfLastAck) <= 0) { + if (messageIdOfLastAck != null && messageId.compareTo(messageIdOfLastAck) <= 0) { // Already included in a cumulative ack return true; } else { From a861242dc57439be30def69d5a9a1821510ce7dd Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Sun, 8 May 2022 23:08:45 +0800 Subject: [PATCH 558/823] Support handling single role and non-jwt-token in MultiRolesTokenAuthorizationProvider (#14857) ### Motivation Currently, `MultiRolesTokenAuthorizationProvider` doesn't support handling the single string type role. It will return the empty role in that case. This PR adds support for handling the string-type role. This PR also adds support for handling the non-jwt-token. ### Modifications * Add support for handling the string-type role * Add support for handling the non-jwt-token ### Verifying this change This change is already covered by existing tests, such as *testMultiRolesAuthzWithSingleRole*. (cherry picked from commit 8bf6785c0803d314465b2d9156df6ca5bbb3c644) --- .../MultiRolesTokenAuthorizationProvider.java | 8 +- ...tiRolesTokenAuthorizationProviderTest.java | 101 ++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index 8a91d7f697158..c508ccbd5b419 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -59,7 +59,7 @@ public class MultiRolesTokenAuthorizationProvider extends PulsarAuthorizationPro // The token's claim that corresponds to the "role" string static final String CONF_TOKEN_AUTH_CLAIM = "tokenAuthClaim"; - private JwtParser parser; + private final JwtParser parser; private String roleClaim; public MultiRolesTokenAuthorizationProvider() { @@ -107,11 +107,15 @@ private List getRoles(AuthenticationDataSource authData) { return Collections.emptyList(); String[] splitToken = token.split("\\."); + if (splitToken.length < 2) { + log.warn("Unable to extract additional roles from JWT token"); + return Collections.emptyList(); + } String unsignedToken = splitToken[0] + "." + splitToken[1] + "."; Jwt jwt = parser.parseClaimsJwt(unsignedToken); try { - Collections.singletonList(jwt.getBody().get(roleClaim, String.class)); + return Collections.singletonList(jwt.getBody().get(roleClaim, String.class)); } catch (RequiredTypeException requiredTypeException) { try { List list = jwt.getBody().get(roleClaim, List.class); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java index edd0baa42ae25..078e2aad07aca 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java @@ -18,10 +18,14 @@ */ package org.apache.pulsar.broker.authorization; +import static org.mockito.Mockito.mock; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; +import java.util.Properties; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.resources.PulsarResources; import org.junit.Assert; import org.testng.annotations.Test; @@ -96,4 +100,101 @@ public String getHttpHeader(String name) { Assert.assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); } + + @Test + public void testMultiRolesAuthzWithSingleRole() throws Exception { + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + String testRole = "test-role"; + String token = Jwts.builder().claim("sub", testRole).signWith(secretKey).compact(); + + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + + AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + Assert.assertTrue(provider.authorize(ads, role -> { + if (role.equals(testRole)) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }).get()); + } + + @Test + public void testMultiRolesNotFailNonJWT() throws Exception { + String token = "a-non-jwt-token"; + + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + + AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + Assert.assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); + } + + @Test + public void testMultiRolesAuthzWithCustomRolesClaims() throws Exception { + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + String testRole = "test-role"; + String customRolesClaims = "role"; + String token = Jwts.builder().claim(customRolesClaims, new String[]{testRole}).signWith(secretKey).compact(); + + Properties properties = new Properties(); + properties.setProperty("tokenSettingPrefix", "prefix_"); + properties.setProperty("prefix_tokenAuthClaim", customRolesClaims); + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setProperties(properties); + + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + provider.initialize(conf, mock(PulsarResources.class)); + + AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + Assert.assertTrue(provider.authorize(ads, role -> { + if (role.equals(testRole)) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }).get()); + } } From 6bf9b3130e8ad44a6de6019a7cbbda33d692a6c2 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 9 May 2022 07:39:31 +0800 Subject: [PATCH 559/823] [security] Remove sensitive msg from consumer/producer stats log (#15483) ### Motivation Currently, we are print password field to consumer/producer stats log ### Modification - add missed `@JsonIgnore` on field and getMethod - delete unused `withoutAttribute` call (cherry picked from commit 8b2f3dd095f365fdb22c71078d5a3e0bf6cc9626) --- .../impl/ConsumerStatsRecorderImpl.java | 2 +- .../impl/ProducerStatsRecorderImpl.java | 2 +- .../impl/conf/ClientConfigurationData.java | 9 +++ .../conf/ClientConfigurationDataTest.java | 57 +++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java index fb61a9a8fa371..4fde45bd3b454 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java @@ -115,7 +115,7 @@ private void init(ConsumerConfigurationData conf) { try { log.info("Starting Pulsar consumer status recorder with config: {}", w.writeValueAsString(conf)); - log.info("Pulsar client config: {}", w.withoutAttribute("authentication").writeValueAsString(pulsarClient.getConfiguration())); + log.info("Pulsar client config: {}", w.writeValueAsString(pulsarClient.getConfiguration())); } catch (IOException e) { log.error("Failed to dump config info", e); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java index 180d53e4949a7..3acefa312807d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java @@ -99,7 +99,7 @@ private void init(ProducerConfigurationData conf) { try { log.info("Starting Pulsar producer perf with config: {}", w.writeValueAsString(conf)); - log.info("Pulsar client config: {}", w.withoutAttribute("authentication").writeValueAsString(pulsarClient.getConfiguration())); + log.info("Pulsar client config: {}", w.writeValueAsString(pulsarClient.getConfiguration())); } catch (IOException e) { log.error("Failed to dump config info", e); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 9765cc484cbf4..093e3e198839b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -30,6 +30,7 @@ import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.ProxyProtocol; @@ -60,6 +61,7 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "The implementation class of ServiceUrlProvider used to generate ServiceUrl." ) @JsonIgnore + @Getter(onMethod_ = @JsonIgnore) private transient ServiceUrlProvider serviceUrlProvider; @ApiModelProperty( @@ -254,6 +256,9 @@ public class ClientConfigurationData implements Serializable, Cloneable { name = "tlsTrustStorePassword", value = "Password of TLS TrustStore." ) + @Secret + @JsonIgnore + @Getter(onMethod_ = @JsonIgnore) private String tlsTrustStorePassword = null; @ApiModelProperty( @@ -312,8 +317,11 @@ public class ClientConfigurationData implements Serializable, Cloneable { name = "socks5ProxyUsername", value = "Password of SOCKS5 proxy." ) + @Secret + @JsonIgnore private String socks5ProxyPassword; + @JsonIgnore public Authentication getAuthentication() { if (authentication == null) { this.authentication = AuthenticationDisabled.INSTANCE; @@ -369,6 +377,7 @@ public String getSocks5ProxyUsername() { return Objects.nonNull(socks5ProxyUsername) ? socks5ProxyUsername : System.getProperty("socks5Proxy.username"); } + @JsonIgnore public String getSocks5ProxyPassword() { return Objects.nonNull(socks5ProxyPassword) ? socks5ProxyPassword : System.getProperty("socks5Proxy.password"); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java new file mode 100644 index 0000000000000..b5c30c9a7c6c4 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.impl.conf; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.testng.Assert; +import org.testng.annotations.Test; + +/** + * Unit test {@link ClientConfigurationData}. + */ +public class ClientConfigurationDataTest { + + private final ObjectWriter w; + + { + ObjectMapper m = new ObjectMapper(); + m.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + w = m.writer(); + } + + + @Test + public void testDoNotPrintSensitiveInfo() throws JsonProcessingException { + ClientConfigurationData clientConfigurationData = new ClientConfigurationData(); + clientConfigurationData.setTlsTrustStorePassword("xxxx"); + clientConfigurationData.setSocks5ProxyPassword("yyyy"); + clientConfigurationData.setAuthentication(new AuthenticationToken("zzzz")); + String s = w.writeValueAsString(clientConfigurationData); + Assert.assertFalse(s.contains("Password")); + Assert.assertFalse(s.contains("xxxx")); + Assert.assertFalse(s.contains("yyyy")); + Assert.assertFalse(s.contains("zzzz")); + } + +} From 6fd31d101bad252b512925ba733d8c636aa6307d Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Tue, 10 May 2022 16:03:45 +0800 Subject: [PATCH 560/823] [fix][txn] Topic transaction buffer recover don't close reader when throw RuntimeException (#15361) Fixes: https://github.com/apache/pulsar/issues/14878 ### Motivation clear unuse reader in topicTransactionBufferSnapshot topic When reader decode the Snapshot will throw RuntimeException not PulsarClientException We should catch the Exception then close the reader and topic ``` "java.util.concurrent.CompletionException: com.google.common.util.concurrent.UncheckedExecutionException: org.apache.commons.lang3.SerializationException: Failed at fetching schema info for 0 at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?] at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?] at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:704) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer.lambda$checkIfTBRecoverCompletely$3(TopicTransactionBuffer.java:232) ~[org.apache-pulsar-broker-2.9.2.jar:2.9.2] at java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:986) ~[?:?] at java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:970) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer$1.recoverExceptionally(TopicTransactionBuffer.java:196) ~[org.apache-pulsar-broker-2.9.2.jar:2.9.2] at org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer$TopicTransactionBufferRecover.lambda$null$1(TopicTransactionBuffer.java:647) ~[org.apache-pulsar-broker-2.9.2.jar:2.9.2] at java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:986) [?:?] at java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:970) [?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) [?:?] at java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:610) [?:?] at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:722) [?:?] at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478) [?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.74.Final.jar:4.1.74.Final] at java.lang.Thread.run(Thread.java:829) [?:?] Caused by: com.google.common.util.concurrent.UncheckedExecutionException: org.apache.commons.lang3.SerializationException: Failed at fetching schema info for 0 at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2051) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache.get(LocalCache.java:3951) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3974) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4935) ~[com.google.guava-guava-30.1-jre.jar:?] at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader.getSchemaReader(AbstractMultiVersionReader.java:83) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader.read(AbstractMultiVersionReader.java:90) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.AbstractStructSchema.decode(AbstractStructSchema.java:67) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.MessageImpl.decode(MessageImpl.java:484) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.MessageImpl.getValue(MessageImpl.java:462) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer$TopicTransactionBufferRecover.lambda$null$0(TopicTransactionBuffer.java:583) ~[org.apache-pulsar-broker-2.9.2.jar:2.9.2] at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:714) ~[?:?] ... 8 more Caused by: org.apache.commons.lang3.SerializationException: Failed at fetching schema info for 0 at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader.getSchemaInfoByVersion(AbstractMultiVersionReader.java:129) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.reader.MultiVersionAvroReader.loadReader(MultiVersionAvroReader.java:47) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader$1.load(AbstractMultiVersionReader.java:52) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader$1.load(AbstractMultiVersionReader.java:49) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3529) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2278) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2155) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2045) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache.get(LocalCache.java:3951) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3974) ~[com.google.guava-guava-30.1-jre.jar:?] at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4935) ~[com.google.guava-guava-30.1-jre.jar:?] at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader.getSchemaReader(AbstractMultiVersionReader.java:83) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.reader.AbstractMultiVersionReader.read(AbstractMultiVersionReader.java:90) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.schema.AbstractStructSchema.decode(AbstractStructSchema.java:67) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.MessageImpl.decode(MessageImpl.java:484) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.client.impl.MessageImpl.getValue(MessageImpl.java:462) ~[org.apache-pulsar-client-original-2.9.2.jar:2.9.2] at org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer$TopicTransactionBufferRecover.lambda$null$0(TopicTransactionBuffer.java:583) ~[org.apache-pulsar-broker-2.9.2.jar:2.9.2] at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:714) ~[?:?] ... 8 more ``` ### Modifications catch Exception then close the topic and reader (cherry picked from commit 0c58810d29838a161481f03c14990d0eb021a185) --- .../buffer/impl/TopicTransactionBuffer.java | 8 ++++---- .../TopicTransactionBufferRecoverTest.java | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index a348ccbb76440..c889e0069781c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -595,10 +595,10 @@ public void run() { callBack.noNeedToRecover(); return; } - } catch (PulsarClientException pulsarClientException) { - log.error("[{}]Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), pulsarClientException); - callBack.recoverExceptionally(pulsarClientException); + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + callBack.recoverExceptionally(ex); closeReader(reader); return; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index fe724dd2be7ae..dddda0f962db6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -457,7 +457,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, @Test(timeOut=30000) - public void testTransactionBufferRecoverThrowPulsarClientException() throws Exception { + public void testTransactionBufferRecoverThrowException() throws Exception { String topic = NAMESPACE1 + "/testTransactionBufferRecoverThrowPulsarClientException"; @Cleanup Producer producer = pulsarClient @@ -491,7 +491,14 @@ public void testTransactionBufferRecoverThrowPulsarClientException() throws Exce field.setAccessible(true); TransactionBufferSnapshotService transactionBufferSnapshotServiceOriginal = (TransactionBufferSnapshotService) field.get(getPulsarServiceList().get(0)); - // mock reader can't read snapshot fail + // mock reader can't read snapshot fail throw RuntimeException + doThrow(new RuntimeException("test")).when(reader).hasMoreEvents(); + // check reader close topic + checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, + transactionBufferSnapshotService, originalTopic, field, producer); + doReturn(true).when(reader).hasMoreEvents(); + + // mock reader can't read snapshot fail throw PulsarClientException doThrow(new PulsarClientException("test")).when(reader).hasMoreEvents(); // check reader close topic checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, From 42353c04151f06f26bf87629b04c94ce4610fe7a Mon Sep 17 00:00:00 2001 From: Xiaoyu Hou Date: Wed, 11 May 2022 20:20:28 +0800 Subject: [PATCH 561/823] [fix][broker]Close publishLimiter when disable it (#15520) (cherry picked from commit e8c971a2f15d9fe79eb88f92c022216a0ca57f73) --- .../java/org/apache/pulsar/broker/service/AbstractTopic.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index dedca9cced414..ba4756647a55f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -956,6 +956,9 @@ protected void updatePublishDispatcher(PublishRate publishRate) { } } else { log.info("Disabling publish throttling for {}", this.topic); + if (topicPublishRateLimiter != null) { + topicPublishRateLimiter.close(); + } this.topicPublishRateLimiter = PublishRateLimiter.DISABLED_RATE_LIMITER; enableProducerReadForPublishRateLimiting(); } From 662150b548f7a0798f8d193e1dc7252ba8f342be Mon Sep 17 00:00:00 2001 From: mattison chao Date: Wed, 25 May 2022 15:03:04 +0800 Subject: [PATCH 562/823] [PIP-163][Txn]Add lowWaterMark check before appending entry to TB (#15424) Master Issue: [#15423](https://github.com/apache/pulsar/issues/15423) Details can be found at https://github.com/apache/pulsar/issues/15423. (cherry picked from commit 15d6907153007ffbf94a351a19df31763b0c6d5a) --- .../service/persistent/PersistentTopic.java | 8 ++-- .../buffer/impl/TopicTransactionBuffer.java | 24 +++++++++++ .../buffer/TransactionLowWaterMarkTest.java | 40 +++++++++++++++++++ .../apache/pulsar/client/impl/ClientCnx.java | 2 +- .../pulsar/client/impl/ProducerImpl.java | 6 +-- 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 7249d4054fedb..6b1063ec49a54 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3044,8 +3044,7 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon return; } if (isExceedMaximumMessageSize(headersAndPayload.readableBytes())) { - publishContext.completed(new NotAllowedException("Exceed maximum message size") - , -1, -1); + publishContext.completed(new NotAllowedException("Exceed maximum message size"), -1, -1); decrementPendingWriteOpsAndCheck(); return; } @@ -3066,7 +3065,10 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon }) .exceptionally(throwable -> { throwable = throwable.getCause(); - if (!(throwable instanceof ManagedLedgerException)) { + if (throwable instanceof NotAllowedException) { + publishContext.completed((NotAllowedException) throwable, -1, -1); + return null; + } else if (!(throwable instanceof ManagedLedgerException)) { throwable = new ManagedLedgerException(throwable); } addFailed((ManagedLedgerException) throwable, publishContext); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index c889e0069781c..2ddac0862942f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import lombok.SneakyThrows; @@ -95,6 +96,8 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final CompletableFuture transactionBufferFuture = new CompletableFuture<>(); + private final ConcurrentHashMap lowWaterMarks = new ConcurrentHashMap<>(); + public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; @@ -240,6 +243,13 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) @Override public CompletableFuture appendBufferToTxn(TxnID txnId, long sequenceId, ByteBuf buffer) { CompletableFuture completableFuture = new CompletableFuture<>(); + Long lowWaterMark = lowWaterMarks.get(txnId.getMostSigBits()); + if (lowWaterMark != null && lowWaterMark >= txnId.getLeastSigBits()) { + completableFuture.completeExceptionally(new BrokerServiceException + .NotAllowedException("Transaction [" + txnId + "] has been ended. " + + "Please use a new transaction to send message.")); + return completableFuture; + } topic.getManagedLedger().asyncAddEntry(buffer, new AsyncCallbacks.AddEntryCallback() { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { @@ -275,6 +285,13 @@ public CompletableFuture openTransactionBufferReader(Tx @Override public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { + lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { + if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { + return lowWaterMark; + } else { + return oldLowWaterMark; + } + }); if (log.isDebugEnabled()) { log.debug("Transaction {} commit on topic {}.", txnID.toString(), topic.getName()); } @@ -315,6 +332,13 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { @Override public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { + lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { + if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { + return lowWaterMark; + } else { + return oldLowWaterMark; + } + }); if (log.isDebugEnabled()) { log.debug("Transaction {} abort on topic {}.", txnID.toString(), topic.getName()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java index 873509ff6bf97..ba0659892b452 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java @@ -42,6 +42,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; @@ -287,4 +288,43 @@ public void testPendingAckLowWaterMark() throws Exception { fail(); } } + + @Test + public void testTBLowWaterMarkEndToEnd() throws Exception { + Transaction txn1 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + Transaction txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + while (txn2.getTxnID().getMostSigBits() != txn1.getTxnID().getMostSigBits()) { + txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + } + + @Cleanup + Producer producer = pulsarClient + .newProducer() + .topic(TOPIC) + .sendTimeout(0, TimeUnit.SECONDS) + .enableBatching(false) + .create(); + + producer.newMessage(txn1).send(); + producer.newMessage(txn2).send(); + + txn1.commit().get(); + txn2.commit().get(); + + Field field = TransactionImpl.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(txn1, TransactionImpl.State.OPEN); + try { + producer.newMessage(txn1).send(); + fail(); + } catch (PulsarClientException.NotAllowedException ignore) { + // no-op + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index a8d1cf51c7147..3b71f6a62228d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -698,7 +698,7 @@ protected void handleSendError(CommandSendError sendError) { producers.get(producerId).terminated(this); break; case NotAllowedError: - producers.get(producerId).recoverNotAllowedError(sequenceId); + producers.get(producerId).recoverNotAllowedError(sequenceId, sendError.getMessage()); break; default: diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index d5d1d6e73825b..d944bf3acbb72 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1146,16 +1146,14 @@ protected synchronized void recoverChecksumError(ClientCnx cnx, long sequenceId) resendMessages(cnx, this.connectionHandler.getEpoch()); } - protected synchronized void recoverNotAllowedError(long sequenceId) { + protected synchronized void recoverNotAllowedError(long sequenceId, String errorMsg) { OpSendMsg op = pendingMessages.peek(); if(op != null && sequenceId == getHighestSequenceId(op)){ pendingMessages.remove(); releaseSemaphoreForSendOp(op); try { op.sendComplete( - new PulsarClientException.NotAllowedException( - format("The size of the message which is produced by producer %s to the topic " + - "%s is not allowed", producerName, topic))); + new PulsarClientException.NotAllowedException(errorMsg)); } catch (Throwable t) { log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topic, producerName, sequenceId, t); From 6f0af8b6b0122789a9c5bf1101206867e87c2d42 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 13 May 2022 10:36:33 +0800 Subject: [PATCH 563/823] Fix http produce msg redirect issue. (#15551) Master Issue: #15546 ### Motivation When lookup the topic ownership using REST produce, the redirect URI is incorrect, because : ``` uri.getPath(false); //Get the path of the current request relative to the base URI as a string. ``` So the redirect URI does not contain the base path: ``` URI redirectURI = new URI(String.format("%s%s", redirectAddresses.get(0), uri.getPath(false))) ``` (cherry picked from commit 7f976da1b51cd868ec49b5ab43259fea4d48c8e9) --- .../org/apache/pulsar/broker/rest/TopicsBase.java | 11 ++++++++--- .../org/apache/pulsar/broker/admin/TopicsTest.java | 8 ++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 770d77794d5df..86e8956d950d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -21,7 +21,7 @@ import io.netty.buffer.ByteBuf; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; +import java.net.URL; import java.nio.ByteBuffer; import java.sql.Time; import java.sql.Timestamp; @@ -41,6 +41,7 @@ import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import javax.ws.rs.core.UriBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumReader; @@ -378,10 +379,14 @@ private void processLookUpResult(List redirectAddresses, AsyncResponse log.debug("Redirect rest produce request for topic {} from {} to {}.", topicName, pulsar().getWebServiceAddress(), redirectAddresses.get(0)); } - URI redirectURI = new URI(String.format("%s%s", redirectAddresses.get(0), uri.getPath(false))); + URL redirectAddress = new URL(redirectAddresses.get(0)); + URI redirectURI = UriBuilder.fromUri(uri.getRequestUri()) + .host(redirectAddress.getHost()) + .port(redirectAddress.getPort()) + .build(); asyncResponse.resume(Response.temporaryRedirect(redirectURI).build()); future.complete(true); - } catch (URISyntaxException | NullPointerException e) { + } catch (Exception e) { if (log.isDebugEnabled()) { log.error("Error in preparing redirect url with rest produce message request for topic {}: {}", topicName, e.getMessage(), e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java index 7b77b1a74f20c..6466c9495c16e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java @@ -77,6 +77,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.io.ByteArrayOutputStream; +import java.net.URI; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -313,13 +314,13 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { @Test public void testLookUpWithRedirect() throws Exception { String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName; - String requestPath = "/admin/v3/topics/my-tenant/my-namespace/my-topic"; + URI requestPath = URI.create(pulsar.getWebServiceAddress() + "/topics/my-tenant/my-namespace/my-topic"); //create topic on one broker admin.topics().createNonPartitionedTopic(topicName); PulsarService pulsar2 = startBroker(getDefaultConf()); doReturn(false).when(topics).isRequestHttps(); UriInfo uriInfo = mock(UriInfo.class); - doReturn(requestPath).when(uriInfo).getPath(anyBoolean()); + doReturn(requestPath).when(uriInfo).getRequestUri(); Whitebox.setInternalState(topics, "uri", uriInfo); //do produce on another broker topics.setPulsar(pulsar2); @@ -336,8 +337,7 @@ public void testLookUpWithRedirect() throws Exception { // Verify got redirect response Assert.assertEquals(responseCaptor.getValue().getStatusInfo(), Response.Status.TEMPORARY_REDIRECT); // Verify URI point to address of broker the topic was created on - Assert.assertEquals(responseCaptor.getValue().getLocation().toString(), - pulsar.getWebServiceAddress() + requestPath); + Assert.assertEquals(responseCaptor.getValue().getLocation().toString(), requestPath.toString()); } @Test From 6295ee8a6ff666df8af42c6e34d43092baafd31c Mon Sep 17 00:00:00 2001 From: mattison chao Date: Wed, 25 May 2022 15:43:06 +0800 Subject: [PATCH 564/823] [PIP-153][optimize][txn] Optimize metadataPositions in MLPendingAckStore (#15137) Master Issue: https://github.com/apache/pulsar/issues/15073 Reduce the memory occupied by metadataPositions and avoid OOM Regularly store a small amount of data according to certain rules, the detailed implementation can be found in [PIP153](https://github.com/apache/pulsar/issues/15073) (cherry picked from commit ebca19b522fd9f4496689ca7d32ede345d28511a) --- conf/broker.conf | 6 + .../pulsar/broker/ServiceConfiguration.java | 11 +- .../pendingack/impl/MLPendingAckStore.java | 170 +++++++++--------- .../impl/MLPendingAckStoreProvider.java | 11 +- .../transaction/util/LogIndexLagBackoff.java | 49 +++++ .../broker/transaction/util/package-info.java | 22 +++ .../broker/transaction/TransactionTest.java | 4 +- .../pendingack/PendingAckMetadataTest.java | 2 +- .../pendingack/PendingAckPersistentTest.java | 105 +++++++++++ .../pulsar/utils/LogIndexLagBackOffTest.java | 55 ++++++ 10 files changed, 345 insertions(+), 90 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/LogIndexLagBackoff.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/package-info.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/utils/LogIndexLagBackOffTest.java diff --git a/conf/broker.conf b/conf/broker.conf index b67afa89c0bbe..3d11ce2538ad7 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1328,6 +1328,12 @@ transactionBufferSnapshotMinTimeInMillis=5000 # The max concurrent requests for transaction buffer client, default is 1000 transactionBufferClientMaxConcurrentRequests=1000 +# MLPendingAckStore maintains a ConcurrentSkipListMap pendingAckLogIndex, +# It stores the position in pendingAckStore as its value and saves a position used to determine +# whether the previous data can be cleaned up as a key. +# transactionPendingAckLogIndexMinLag is used to configure the minimum lag between indexes +transactionPendingAckLogIndexMinLag=500 + ### --- Packages management service configuration variables (begin) --- ### # Enable the packages management service or not diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 5370971854570..dc5640f2315fd 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2257,7 +2257,16 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private long transactionBufferClientOperationTimeoutInMills = 3000L; - /**** --- KeyStore TLS config variables --- ****/ + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "MLPendingAckStore maintain a ConcurrentSkipListMap pendingAckLogIndex`," + + "it store the position in pendingAckStore as value and save a position used to determine" + + "whether the previous data can be cleaned up as a key." + + "transactionPendingAckLogIndexMinLag is used to configure the minimum lag between indexes" + ) + private long transactionPendingAckLogIndexMinLag = 500L; + + /**** --- KeyStore TLS config variables. --- ****/ @FieldContext( category = CATEGORY_KEYSTORE_TLS, doc = "Enable TLS with KeyStore type configuration in broker" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index bb8b961e36b88..5fc210ca74165 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -42,6 +42,7 @@ import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckOp; +import org.apache.pulsar.broker.transaction.util.LogIndexLagBackoff; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.CommandAck.AckType; @@ -72,33 +73,40 @@ public class MLPendingAckStore implements PendingAckStore { private PositionImpl currentLoadPosition; + private final AtomicLong currentIndexLag = new AtomicLong(0); + private volatile long maxIndexLag; + + protected PositionImpl maxAckPosition = PositionImpl.EARLIEST; + private final LogIndexLagBackoff logIndexBackoff; + /** * The map is for pending ack store clear useless data. *

    - * When ack message append to pending ack store, it will store the position which is persistent as key. + * key:the largest ack position of origin topic, corresponds to the value position. *

    - * When ack message append to pending ack store, it will store the position which is the max position of this - * ack by the original topic as value. + * value:the position persistent by pendingAck log. *

    - * It will judge the position with the max sub cursor position whether smaller than the subCursor mark + * It will judge the position with the max sub cursor position (key) whether smaller than the subCursor mark * delete position. *

    - * If the max position is smaller than the subCursor mark delete position, the log cursor will mark delete - * the position. + * If the max position (key) is smaller than the subCursor mark delete position, + * the log cursor will mark delete the position before log position (value). */ - private final ConcurrentSkipListMap metadataPositions; + private final ConcurrentSkipListMap pendingAckLogIndex; private final ManagedCursor subManagedCursor; public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, - ManagedCursor subManagedCursor) { + ManagedCursor subManagedCursor, long transactionPendingAckLogIndexMinLag) { this.managedLedger = managedLedger; this.cursor = cursor; this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition(); this.entryQueue = new SpscArrayQueue<>(2000); this.lastConfirmedEntry = (PositionImpl) managedLedger.getLastConfirmedEntry(); - this.metadataPositions = new ConcurrentSkipListMap<>(); + this.pendingAckLogIndex = new ConcurrentSkipListMap<>(); this.subManagedCursor = subManagedCursor; + this.logIndexBackoff = new LogIndexLagBackoff(transactionPendingAckLogIndexMinLag, Long.MAX_VALUE, 1); + this.maxIndexLag = logIndexBackoff.next(0); } @Override @@ -219,62 +227,12 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { log.debug("[{}][{}] MLPendingAckStore message append success at {} txnId: {}, operation : {}", managedLedger.getName(), ctx, position, txnID, pendingAckMetadataEntry.getPendingAckOp()); } - // store the persistent position in to memory - if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT - && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) { - Optional optional = pendingAckMetadataEntry.getPendingAckMetadatasList() - .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(), - o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result()); - optional.ifPresent(pendingAckMetadata -> - metadataPositions.compute((PositionImpl) position, (thisPosition, otherPosition) -> { - PositionImpl nowPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(), - pendingAckMetadata.getEntryId()); - if (otherPosition == null) { - return nowPosition; - } else { - return nowPosition.compareTo(otherPosition) > 0 ? nowPosition : otherPosition; - } - })); - } - + currentIndexLag.incrementAndGet(); + handleMetadataEntry((PositionImpl) position, pendingAckMetadataEntry); buf.release(); completableFuture.complete(null); - if (!metadataPositions.isEmpty()) { - PositionImpl deletePosition = null; - while (!metadataPositions.isEmpty() - && metadataPositions.firstKey() != null - && subManagedCursor.getPersistentMarkDeletedPosition() != null - && metadataPositions.firstEntry().getValue() - .compareTo((PositionImpl) subManagedCursor.getPersistentMarkDeletedPosition()) <= 0) { - deletePosition = metadataPositions.firstKey(); - metadataPositions.remove(metadataPositions.firstKey()); - } - - if (deletePosition != null) { - PositionImpl finalDeletePosition = deletePosition; - cursor.asyncMarkDelete(deletePosition, - new AsyncCallbacks.MarkDeleteCallback() { - @Override - public void markDeleteComplete(Object ctx) { - if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store mark delete position : " - + "[{}] success", managedLedger.getName(), - finalDeletePosition); - } - } - - @Override - public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { - if (log.isDebugEnabled()) { - log.error("[{}] Transaction pending ack store mark delete position : " - + "[{}] fail!", managedLedger.getName(), - finalDeletePosition, exception); - } - } - }, null); - } - } + clearUselessLogData(); } @Override @@ -292,6 +250,68 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { return completableFuture; } + private void handleMetadataEntry(PositionImpl logPosition, PendingAckMetadataEntry pendingAckMetadataEntry) { + // store the persistent position in to memory + // store the max position of this entry retain + if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT + && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) { + Optional optional = pendingAckMetadataEntry.getPendingAckMetadatasList() + .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(), + o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result()); + + optional.ifPresent(pendingAckMetadata -> { + PositionImpl nowPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + + if (nowPosition.compareTo(maxAckPosition) > 0) { + maxAckPosition = nowPosition; + } + if (currentIndexLag.get() >= maxIndexLag) { + pendingAckLogIndex.compute(maxAckPosition, + (thisPosition, otherPosition) -> logPosition); + maxIndexLag = logIndexBackoff.next(pendingAckLogIndex.size()); + currentIndexLag.set(0); + } + }); + } + } + + private void clearUselessLogData() { + if (!pendingAckLogIndex.isEmpty()) { + PositionImpl deletePosition = null; + while (!pendingAckLogIndex.isEmpty() + && pendingAckLogIndex.firstKey() != null + && subManagedCursor.getPersistentMarkDeletedPosition() != null + && pendingAckLogIndex.firstEntry().getKey() + .compareTo((PositionImpl) subManagedCursor.getPersistentMarkDeletedPosition()) <= 0) { + deletePosition = pendingAckLogIndex.remove(pendingAckLogIndex.firstKey()); + } + + if (deletePosition != null) { + maxIndexLag = logIndexBackoff.next(pendingAckLogIndex.size()); + PositionImpl finalDeletePosition = deletePosition; + cursor.asyncMarkDelete(deletePosition, + new AsyncCallbacks.MarkDeleteCallback() { + @Override + public void markDeleteComplete(Object ctx) { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store mark delete position : " + + "[{}] success", managedLedger.getName(), + finalDeletePosition); + } + } + + @Override + public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}] Transaction pending ack store mark delete position : " + + "[{}] fail!", managedLedger.getName(), + finalDeletePosition, exception); + } + }, null); + } + } + } + class PendingAckReplay implements Runnable { private final FillEntryQueueCallback fillEntryQueueCallback; @@ -319,30 +339,12 @@ public void run() { currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.parseFrom(buffer, buffer.readableBytes()); - // store the persistent position in to memory - // store the max position of this entry retain - if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT - && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) { - Optional optional = pendingAckMetadataEntry.getPendingAckMetadatasList() - .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(), - o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result()); - - optional.ifPresent(pendingAckMetadata -> - metadataPositions.compute(PositionImpl.get(entry.getLedgerId(), entry.getEntryId()), - (thisPosition, otherPosition) -> { - PositionImpl nowPosition = PositionImpl - .get(pendingAckMetadata.getLedgerId(), - pendingAckMetadata.getEntryId()); - if (otherPosition == null) { - return nowPosition; - } else { - return nowPosition.compareTo(otherPosition) > 0 ? nowPosition - : otherPosition; - } - })); - } + currentIndexLag.incrementAndGet(); + handleMetadataEntry(new PositionImpl(entry.getLedgerId(), entry.getEntryId()), + pendingAckMetadataEntry); pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry); entry.release(); + clearUselessLogData(); } else { try { Thread.sleep(1); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index 5417caec3d7cd..6b84d6e329a3e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -74,9 +74,14 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() { @Override public void openCursorComplete(ManagedCursor cursor, Object ctx) { - pendingAckStoreFuture - .complete(new MLPendingAckStore(ledger, cursor, - subscription.getCursor())); + pendingAckStoreFuture.complete(new MLPendingAckStore(ledger, + cursor, + subscription.getCursor(), + originPersistentTopic + .getBrokerService() + .getPulsar() + .getConfiguration() + .getTransactionPendingAckLogIndexMinLag())); if (log.isDebugEnabled()) { log.debug("{},{} open MLPendingAckStore cursor success", originPersistentTopic.getName(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/LogIndexLagBackoff.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/LogIndexLagBackoff.java new file mode 100644 index 0000000000000..145381814ba61 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/LogIndexLagBackoff.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.util; + +import static com.google.common.base.Preconditions.checkArgument; +import lombok.Getter; + +public class LogIndexLagBackoff { + + @Getter + private final long minLag; + @Getter + private final long maxLag; + @Getter + private final double exponent; + + public LogIndexLagBackoff(long minLag, long maxLag, double exponent) { + checkArgument(minLag > 0, "min lag must be > 0"); + checkArgument(maxLag >= minLag, "maxLag should be >= minLag"); + checkArgument(exponent > 0, "exponent must be > 0"); + this.minLag = minLag; + this.maxLag = maxLag; + this.exponent = exponent; + } + + + public long next(int indexCount) { + if (indexCount <= 0) { + return minLag; + } + return (long) Math.min(this.maxLag, minLag * Math.pow(indexCount, exponent)); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/package-info.java new file mode 100644 index 0000000000000..58cb1c24b19e6 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/util/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Implementation of a transaction tools. + */ +package org.apache.pulsar.broker.transaction.util; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index f264289d6962f..0b59eda45234c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -620,7 +620,7 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class); doReturn(CompletableFuture.completedFuture( - new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null))) + new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null, 500))) .when(pendingAckStoreProvider).newPendingAckStore(any()); doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any()); @@ -901,6 +901,8 @@ public void testAutoCreateSchemaForTransactionSnapshot() throws Exception { @Test public void testPendingAckMarkDeletePosition() throws Exception { + getPulsarServiceList().get(0).getConfig().setTransactionPendingAckLogIndexMinLag(1); + getPulsarServiceList().get(0).getConfiguration().setManagedLedgerDefaultMarkDeleteRateLimit(5); String topic = NAMESPACE1 + "/test1"; @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java index c99eee6246371..14dbcdb8897f6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java @@ -72,7 +72,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { ManagedCursor cursor = completableFuture.get().openCursor("test"); ManagedCursor subCursor = completableFuture.get().openCursor("test"); MLPendingAckStore pendingAckStore = - new MLPendingAckStore(completableFuture.get(), cursor, subCursor); + new MLPendingAckStore(completableFuture.get(), cursor, subCursor, 500); Field field = MLPendingAckStore.class.getDeclaredField("managedLedger"); field.setAccessible(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index 196d9bbebaaa7..a64707f1ae889 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.Cleanup; @@ -48,6 +49,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.naming.NamespaceName; @@ -57,6 +59,7 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -200,6 +203,8 @@ public void individualPendingAckReplayTest() throws Exception { @Test public void cumulativePendingAckReplayTest() throws Exception { int messageCount = 1000; + getPulsarServiceList().get(0).getConfig().setTransactionPendingAckLogIndexMinLag(4 * messageCount + 2); + getPulsarServiceList().get(0).getConfiguration().setManagedLedgerDefaultMarkDeleteRateLimit(10); String subName = "cumulative-test"; @Cleanup @@ -354,6 +359,106 @@ private void testDeleteTopicThenDeletePendingAckManagedLedger() throws Exception assertFalse(topics.contains(topic)); } + @Test + public void testDeleteUselessLogDataWhenSubCursorMoved() throws Exception { + getPulsarServiceList().get(0).getConfig().setTransactionPendingAckLogIndexMinLag(5); + getPulsarServiceList().get(0).getConfiguration().setManagedLedgerDefaultMarkDeleteRateLimit(5); + String subName = "test-log-delete"; + String topic = TopicName.get(TopicDomain.persistent.toString(), + NamespaceName.get(NAMESPACE1), "test-log-delete").toString(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .enableBatching(false) + .create(); + + for (int i = 0; i < 20; i++) { + producer.newMessage().send(); + } + // init + Message message = consumer.receive(5, TimeUnit.SECONDS); + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + consumer.acknowledgeAsync(message.getMessageId(), transaction).get(); + + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService().getTopic(topic, false).get().get(); + + PersistentSubscription persistentSubscription = persistentTopic.getSubscription(subName); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription); + Field field1 = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field1.setAccessible(true); + PendingAckStore pendingAckStore = ((CompletableFuture) field1.get(pendingAckHandle)).get(); + + Field field3 = MLPendingAckStore.class.getDeclaredField("pendingAckLogIndex"); + Field field4 = MLPendingAckStore.class.getDeclaredField("maxIndexLag"); + + field3.setAccessible(true); + field4.setAccessible(true); + + ConcurrentSkipListMap pendingAckLogIndex = + (ConcurrentSkipListMap) field3.get(pendingAckStore); + long maxIndexLag = (long) field4.get(pendingAckStore); + Assert.assertEquals(pendingAckLogIndex.size(), 0); + Assert.assertEquals(maxIndexLag, 5); + transaction.commit().get(); + + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(persistentSubscription.getCursor().getPersistentMarkDeletedPosition().getEntryId(), + ((MessageIdImpl)message.getMessageId()).getEntryId())); + // 7 more acks. Will find that there are still only two records in the map. + Transaction transaction1 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + Message message0 = null; + //remove previous index + for (int i = 0; i < 4; i++) { + message0 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message0.getMessageId(), transaction1).get(); + } + Assert.assertEquals(pendingAckLogIndex.size(), 1); + maxIndexLag = (long) field4.get(pendingAckStore); + Assert.assertEquals(maxIndexLag, 5); + //add new index + for (int i = 0; i < 9; i++) { + message0= consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message0.getMessageId(), transaction1).get(); + } + + Assert.assertEquals(pendingAckLogIndex.size(), 2); + maxIndexLag = (long) field4.get(pendingAckStore); + Assert.assertEquals(maxIndexLag, 10); + + transaction1.commit().get(); + Message message1 = message0; + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(persistentSubscription.getCursor().getPersistentMarkDeletedPosition().getEntryId(), + ((MessageIdImpl)message1.getMessageId()).getEntryId())); + + Transaction transaction2 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + Message message2 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message2.getMessageId(), transaction2).get(); + + Assert.assertEquals(pendingAckLogIndex.size(), 0); + maxIndexLag = (long) field4.get(pendingAckStore); + Assert.assertEquals(maxIndexLag, 5); + } + @Test public void testPendingAckLowWaterMarkRemoveFirstTxn() throws Exception { String topic = TopicName.get(TopicDomain.persistent.toString(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/LogIndexLagBackOffTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/LogIndexLagBackOffTest.java new file mode 100644 index 0000000000000..8d4f2c356a692 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/LogIndexLagBackOffTest.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import org.apache.pulsar.broker.transaction.util.LogIndexLagBackoff; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test(groups = "utils") +public class LogIndexLagBackOffTest { + @Test + public void testGenerateNextLogIndexLag() { + LogIndexLagBackoff logIndexLagBackoff = new LogIndexLagBackoff(1, 10, 1); + Assert.assertEquals(logIndexLagBackoff.next(0), 1); + Assert.assertEquals(logIndexLagBackoff.next(6), 6); + + Assert.assertEquals(logIndexLagBackoff.next(77), 10); + + logIndexLagBackoff = new LogIndexLagBackoff(1, 10, 2); + Assert.assertEquals(logIndexLagBackoff.next(3), 9); + + try { + new LogIndexLagBackoff(-1, 2, 3); + } catch (IllegalArgumentException e) { + Assert.assertEquals(e.getMessage(), "min lag must be > 0"); + } + try { + new LogIndexLagBackoff(2, 1, 3); + } catch (IllegalArgumentException e) { + Assert.assertEquals(e.getMessage(), "maxLag should be >= minLag"); + } + try { + new LogIndexLagBackoff(1, 1, 0.2); + } catch (IllegalArgumentException e) { + Assert.assertEquals(e.getMessage(), "exponent must be > 0"); + } + + } +} From 7dcca2aaf8d8cf78544f9373687504a9c0190a11 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 16 May 2022 10:07:47 +0800 Subject: [PATCH 565/823] Use dispatchRateLimiterLock to update dispatchRateLimiter. (#15601) ### Motivation https://github.com/apache/pulsar/blob/58c82a71beb7506e422def391af532945be2b7a7/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java#L377-L399 The object lock may change when execute at line-382, and cause the lock to become useless. So use `dispatchRateLimiterLock` to synchronize. (cherry picked from commit ff4e6000f2d58eff930178ae0c02ef9c5fffb47c) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 6b1063ec49a54..5505c940af65d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -184,6 +184,7 @@ public class PersistentTopic extends AbstractTopic public boolean msgChunkPublished; private Optional dispatchRateLimiter = Optional.empty(); + private final Object dispatchRateLimiterLock = new Object(); private Optional subscribeRateLimiter = Optional.empty(); public volatile long delayedDeliveryTickTimeMillis = 1000; private final long backloggedCursorThresholdEntries; @@ -376,7 +377,7 @@ public CompletableFuture initialize() { } private void initializeRateLimiterIfNeeded(Optional policies) { - synchronized (dispatchRateLimiter) { + synchronized (dispatchRateLimiterLock) { // dispatch rate limiter for topic if (!dispatchRateLimiter.isPresent() && DispatchRateLimiter .isDispatchRateNeeded(brokerService, policies, topic, Type.TOPIC)) { @@ -3203,7 +3204,7 @@ private Optional getNamespacePolicies() { } private void initializeTopicDispatchRateLimiterIfNeeded(TopicPolicies policies) { - synchronized (dispatchRateLimiter) { + synchronized (dispatchRateLimiterLock) { if (!dispatchRateLimiter.isPresent() && policies.getDispatchRate() != null) { this.dispatchRateLimiter = Optional.of(new DispatchRateLimiter(this, Type.TOPIC)); } From f569fa4730717878bea748af42f28c800706e4e2 Mon Sep 17 00:00:00 2001 From: fengyubiao <9947090@qq.com> Date: Wed, 18 May 2022 18:41:08 +0800 Subject: [PATCH 566/823] Fix potential to add duplicated consumer (#15051) ### Motivation It's because of this issue https://github.com/apache/pulsar/issues/13787. Then diving into the codes, I find that if the client tries to subscribe multiple times over a short period of time, it is possible to have more than one consumer at the same dispatcher. just like below: ``` for ( long requestId = 1; i < 5; i++ ){ ByteBuf request1 = Commands.newSubscribe(topic, subscription, consumerId, requestId , getSubType(), priorityLevel, consumerName, isDurable, startMessageIdData, metadata, readCompacted, conf.isReplicateSubscriptionState(), InitialPosition.valueOf(subscriptionInitialPosition.getValue()), startMessageRollbackDuration, si, createTopicIfDoesNotExist, conf.getKeySharedPolicy(), // Use the current epoch to subscribe. conf.getSubscriptionProperties(), CONSUMER_EPOCH.get(this)); cnx.sendRequestWithId(request1, requestId).thenRun(() -> {}); } ``` The root cause is below snippet: https://github.com/apache/pulsar/blob/c2c05c49aff1ebc7b2b7a1d5bd547c33211e4479/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java#L994-L1021 If the consumer1 comes and is not done, then the same consumer2(it's the same with consumer1) comes, it may remove the prior consumer1(line 1015), but consumer1 add to subscription success in the end, Then the same cusumer3 comes, and it succeed, and will cause the same consumer to add duplicated. The right way to remove consumer (line 1015) is when the `existingConsumerFuture` is completedExceptionally. Even though the Java client couldn't occur the above behavior, other clients may not. So it's better to handle `subscribe` correctly on the broker side. ### Modifications Modify the process execution sequence to improve stability (cherry picked from commit 7bf495aca70798257d79d370f33c3870f31815a9) --- .../pulsar/broker/service/ServerCnx.java | 45 +++--- .../pulsar/broker/service/ServerCnxTest.java | 147 ++++++++++++++++++ 2 files changed, 174 insertions(+), 18 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 03a6a80aed3ce..3c420fe43b365 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -982,32 +982,31 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { consumerFuture); if (existingConsumerFuture != null) { - if (existingConsumerFuture.isDone() && !existingConsumerFuture.isCompletedExceptionally()) { - Consumer consumer = existingConsumerFuture.getNow(null); - log.info("[{}] Consumer with the same id is already created:" - + " consumerId={}, consumer={}", - remoteAddress, consumerId, consumer); - commandSender.sendSuccessResponse(requestId); - return null; - } else { + if (!existingConsumerFuture.isDone()){ // There was an early request to create a consumer with same consumerId. This can happen // when // client timeout is lower the broker timeouts. We need to wait until the previous // consumer // creation request either complete or fails. log.warn("[{}][{}][{}] Consumer with id is already present on the connection," - + " consumerId={}", remoteAddress, topicName, subscriptionName, consumerId); - ServerError error = null; - if (!existingConsumerFuture.isDone()) { - error = ServerError.ServiceNotReady; - } else { - error = getErrorCode(existingConsumerFuture); - consumers.remove(consumerId, existingConsumerFuture); - } - commandSender.sendErrorResponse(requestId, error, + + " consumerId={}", remoteAddress, topicName, subscriptionName, consumerId); + commandSender.sendErrorResponse(requestId, ServerError.ServiceNotReady, "Consumer is already present on the connection"); - return null; + } else if (existingConsumerFuture.isCompletedExceptionally()){ + ServerError error = getErrorCodeWithErrorLog(existingConsumerFuture, true, + String.format("Consumer subscribe failure. remoteAddress: %s, subscription: %s", + remoteAddress, subscriptionName)); + consumers.remove(consumerId, existingConsumerFuture); + commandSender.sendErrorResponse(requestId, error, + "Consumer that failed is already present on the connection"); + } else { + Consumer consumer = existingConsumerFuture.getNow(null); + log.info("[{}] Consumer with the same id is already created:" + + " consumerId={}, consumer={}", + remoteAddress, consumerId, consumer); + commandSender.sendSuccessResponse(requestId); } + return null; } boolean createTopicIfDoesNotExist = forceTopicCreation @@ -2597,6 +2596,11 @@ public void cancelPublishBufferLimiting() { } private ServerError getErrorCode(CompletableFuture future) { + return getErrorCodeWithErrorLog(future, false, null); + } + + private ServerError getErrorCodeWithErrorLog(CompletableFuture future, boolean logIfError, + String errorMessageIfLog) { ServerError error = ServerError.UnknownError; try { future.getNow(null); @@ -2604,6 +2608,11 @@ private ServerError getErrorCode(CompletableFuture future) { if (e.getCause() instanceof BrokerServiceException) { error = BrokerServiceException.getClientErrorCode(e.getCause()); } + if (logIfError){ + String finalErrorMessage = StringUtils.isNotBlank(errorMessageIfLog) + ? errorMessageIfLog : "Unknown Error"; + log.error(finalErrorMessage, e); + } } return error; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 1a5efb19f3eed..d2dac88efe852 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -51,6 +51,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; import java.util.function.Supplier; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; @@ -107,11 +109,14 @@ import org.apache.pulsar.common.protocol.Commands.ChecksumType; import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.zookeeper.ZooKeeperDataCache; import org.apache.zookeeper.ZooKeeper; import org.awaitility.Awaitility; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -1807,4 +1812,146 @@ public void testTopicIsNotReady() throws Exception { channel.finish(); } + + @Test + public void testNeverDelayConsumerFutureWhenNotFail() throws Exception{ + // Mock ServerCnx.field: consumers + ConcurrentLongHashMap.Builder mapBuilder = Mockito.mock(ConcurrentLongHashMap.Builder.class); + Mockito.when(mapBuilder.expectedItems(Mockito.anyInt())).thenReturn(mapBuilder); + Mockito.when(mapBuilder.concurrencyLevel(Mockito.anyInt())).thenReturn(mapBuilder); + ConcurrentLongHashMap consumers = Mockito.mock(ConcurrentLongHashMap.class); + Mockito.when(mapBuilder.build()).thenReturn(consumers); + ArgumentCaptor ignoreArgumentCaptor = ArgumentCaptor.forClass(Long.class); + final ArgumentCaptor deleteTimesMark = ArgumentCaptor.forClass(CompletableFuture.class); + Mockito.when(consumers.remove(ignoreArgumentCaptor.capture())).thenReturn(true); + Mockito.when(consumers.remove(ignoreArgumentCaptor.capture(), deleteTimesMark.capture())).thenReturn(true); + // case1: exists existingConsumerFuture, already complete or delay done after execute 'isDone()' many times + // case2: exists existingConsumerFuture, delay complete after execute 'isDone()' many times + // Why is the design so complicated, see: https://github.com/apache/pulsar/pull/15051 + // Try a delay of 3 stages. The simulation is successful after repeated judgments. + for(AtomicInteger futureWillDoneAfterDelayTimes = new AtomicInteger(1); + futureWillDoneAfterDelayTimes.intValue() <= 3; + futureWillDoneAfterDelayTimes.incrementAndGet()){ + final AtomicInteger futureCallTimes = new AtomicInteger(); + final Consumer mockConsumer = Mockito.mock(Consumer.class); + CompletableFuture existingConsumerFuture = new CompletableFuture(){ + + private boolean complete; + + // delay complete after execute 'isDone()' many times + @Override + public boolean isDone() { + if (complete) { + return true; + } + int executeIsDoneCommandTimes = futureCallTimes.incrementAndGet(); + return executeIsDoneCommandTimes >= futureWillDoneAfterDelayTimes.intValue(); + } + + // if trig "getNow()", then complete + @Override + public Consumer get(){ + complete = true; + return mockConsumer; + } + + // if trig "get()", then complete + @Override + public Consumer get(long timeout, TimeUnit unit){ + complete = true; + return mockConsumer; + } + + // if trig "get()", then complete + @Override + public Consumer getNow(Consumer ifAbsent){ + complete = true; + return mockConsumer; + } + + // never fail + public boolean isCompletedExceptionally(){ + return false; + } + }; + Mockito.when(consumers.putIfAbsent(Mockito.anyLong(), Mockito.any())).thenReturn(existingConsumerFuture); + // do test: delay complete after execute 'isDone()' many times + // Why is the design so complicated, see: https://github.com/apache/pulsar/pull/15051 + try (MockedStatic theMock = Mockito.mockStatic(ConcurrentLongHashMap.class)) { + // Inject consumers to ServerCnx + theMock.when(ConcurrentLongHashMap::newBuilder).thenReturn(mapBuilder); + // reset channels( serverChannel, clientChannel ) + resetChannel(); + setChannelConnected(); + // auth check disable + doReturn(false).when(brokerService).isAuthenticationEnabled(); + doReturn(false).when(brokerService).isAuthorizationEnabled(); + // do subscribe + ByteBuf clientCommand = Commands.newSubscribe(successTopicName, // + successSubName, 1 /* consumer id */, 1 /* request id */, SubType.Exclusive, 0, + "test" /* consumer name */, 0 /* avoid reseting cursor */); + channel.writeInbound(clientCommand); + Object responseObj = getResponse(); + Predicate responseAssert = obj -> { + if (responseObj instanceof CommandSuccess) { + return true; + } + if (responseObj instanceof CommandError) { + CommandError commandError = (CommandError) responseObj; + return ServerError.ServiceNotReady == commandError.getError(); + } + return false; + }; + // assert no consumer-delete event occur + assertFalse(deleteTimesMark.getAllValues().contains(existingConsumerFuture)); + // assert without another error occur + assertTrue(responseAssert.test(responseAssert)); + // Server will not close the connection + assertTrue(channel.isOpen()); + channel.finish(); + } + } + // case3: exists existingConsumerFuture, already complete and exception + CompletableFuture existingConsumerFuture = Mockito.mock(CompletableFuture.class); + Mockito.when(consumers.putIfAbsent(Mockito.anyLong(), Mockito.any())).thenReturn(existingConsumerFuture); + // make consumerFuture delay finish + Mockito.when(existingConsumerFuture.isDone()).thenReturn(true); + // when sync get return, future will return success value. + Mockito.when(existingConsumerFuture.get()).thenThrow(new NullPointerException()); + Mockito.when(existingConsumerFuture.get(Mockito.anyLong(), Mockito.any())). + thenThrow(new NullPointerException()); + Mockito.when(existingConsumerFuture.isCompletedExceptionally()).thenReturn(true); + Mockito.when(existingConsumerFuture.getNow(Mockito.any())).thenThrow(new NullPointerException()); + try (MockedStatic theMock = Mockito.mockStatic(ConcurrentLongHashMap.class)) { + // Inject consumers to ServerCnx + theMock.when(ConcurrentLongHashMap::newBuilder).thenReturn(mapBuilder); + // reset channels( serverChannel, clientChannel ) + resetChannel(); + setChannelConnected(); + // auth check disable + doReturn(false).when(brokerService).isAuthenticationEnabled(); + doReturn(false).when(brokerService).isAuthorizationEnabled(); + // do subscribe + ByteBuf clientCommand = Commands.newSubscribe(successTopicName, // + successSubName, 1 /* consumer id */, 1 /* request id */, SubType.Exclusive, 0, + "test" /* consumer name */, 0 /* avoid reseting cursor */); + channel.writeInbound(clientCommand); + Object responseObj = getResponse(); + Predicate responseAssert = obj -> { + if (responseObj instanceof CommandError) { + CommandError commandError = (CommandError) responseObj; + return ServerError.ServiceNotReady != commandError.getError(); + } + return false; + }; + // assert error response + assertTrue(responseAssert.test(responseAssert)); + // assert consumer-delete event occur + assertEquals(1L, + deleteTimesMark.getAllValues().stream().filter(f -> f == existingConsumerFuture).count()); + // Server will not close the connection + assertTrue(channel.isOpen()); + channel.finish(); + } + } } From 0813a1b40f80b350eb0daa24fd4a3880e3bfb92e Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 19 May 2022 22:07:44 +0800 Subject: [PATCH 567/823] [fix][broker] fix MetadataStoreException$NotFoundException while doing topic lookup (#15633) (cherry picked from commit 70551a6f6be05826b90295d3da5915613e53fa46) --- .../broker/loadbalance/ResourceUnit.java | 5 +++++ .../impl/ModularLoadManagerWrapper.java | 14 ++++++++++-- .../loadbalance/impl/SimpleResourceUnit.java | 22 +++++++++++++++++-- .../broker/namespace/NamespaceService.java | 16 +++++++++----- .../loadbalance/AdvertisedListenersTest.java | 18 +++++++++------ 5 files changed, 59 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java index 1afde4e365713..51becdb7f77cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java @@ -22,9 +22,14 @@ ResourceUnit represents any machine/unit which has resources that broker can use to serve its service units */ public interface ResourceUnit extends Comparable { + + String PROPERTY_KEY_BROKER_ZNODE_NAME = "__advertised_addr"; + String getResourceId(); ResourceDescription getAvailableResource(); boolean canFit(ResourceDescription resourceDescription); + + Object getProperty(String key); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java index a138fe397b020..f2d61d2dcd1e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.impl; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import org.apache.pulsar.broker.PulsarServerException; @@ -64,8 +65,12 @@ public LoadManagerReport generateLoadReport() { @Override public Optional getLeastLoaded(final ServiceUnitId serviceUnit) { Optional leastLoadedBroker = loadManager.selectBrokerForAssignment(serviceUnit); - return leastLoadedBroker.map(s -> new SimpleResourceUnit(getBrokerWebServiceUrl(s), - new PulsarResourceDescription())); + return leastLoadedBroker.map(s -> { + String webServiceUrl = getBrokerWebServiceUrl(s); + String brokerZnodeName = getBrokerZnodeName(s, webServiceUrl); + return new SimpleResourceUnit(webServiceUrl, + new PulsarResourceDescription(), Map.of(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName)); + }); } private String getBrokerWebServiceUrl(String broker) { @@ -77,6 +82,11 @@ private String getBrokerWebServiceUrl(String broker) { return String.format("http://%s", broker); } + private String getBrokerZnodeName(String broker, String webServiceUrl) { + String scheme = webServiceUrl.substring(0, webServiceUrl.indexOf("://")); + return String.format("%s://%s", scheme, broker); + } + @Override public List getLoadBalancingMetrics() { return loadManager.getLoadBalancingMetrics(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceUnit.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceUnit.java index 62f9b3e94a588..863a75dff4235 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceUnit.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceUnit.java @@ -19,19 +19,32 @@ package org.apache.pulsar.broker.loadbalance.impl; import com.google.common.base.MoreObjects; +import java.util.Collections; +import java.util.Map; import org.apache.pulsar.broker.loadbalance.ResourceDescription; import org.apache.pulsar.broker.loadbalance.ResourceUnit; public class SimpleResourceUnit implements ResourceUnit { - private String resourceId; - private ResourceDescription resourceDescription; + private final String resourceId; + private final ResourceDescription resourceDescription; + + private final Map properties; public SimpleResourceUnit(String resourceId, ResourceDescription resourceDescription) { this.resourceId = resourceId; this.resourceDescription = resourceDescription; + this.properties = Collections.emptyMap(); + } + + public SimpleResourceUnit(String resourceId, ResourceDescription resourceDescription, + Map properties) { + this.resourceId = resourceId; + this.resourceDescription = resourceDescription; + this.properties = properties == null ? Collections.emptyMap() : properties; } + @Override public String getResourceId() { // TODO Auto-generated method stub @@ -50,6 +63,11 @@ public boolean canFit(ResourceDescription resourceDescription) { return this.resourceDescription.compareTo(resourceDescription) > 0; } + @Override + public Object getProperty(String key) { + return properties.get(key); + } + @Override public int compareTo(ResourceUnit o) { return resourceId.compareTo(o.getResourceId()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 25d935d835505..58c39ac914338 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -46,6 +46,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -456,6 +457,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, return; } String candidateBroker = null; + String candidateBrokerAdvertisedAddr = null; LeaderElectionService les = pulsar.getLeaderElectionService(); if (les == null) { @@ -516,7 +518,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } } if (makeLoadManagerDecisionOnThisBroker) { - Optional availableBroker = getLeastLoadedFromLoadManager(bundle); + Optional> availableBroker = getLeastLoadedFromLoadManager(bundle); if (!availableBroker.isPresent()) { LOG.warn("Load manager didn't return any available broker. " + "Returning empty result to lookup. NamespaceBundle[{}]", @@ -524,7 +526,8 @@ private void searchForCandidateBroker(NamespaceBundle bundle, lookupFuture.complete(Optional.empty()); return; } - candidateBroker = availableBroker.get(); + candidateBroker = availableBroker.get().getLeft(); + candidateBrokerAdvertisedAddr = availableBroker.get().getRight(); authoritativeRedirect = true; } else { // forward to leader broker to make assignment @@ -595,7 +598,8 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } // Now setting the redirect url - createLookupResult(candidateBroker, authoritativeRedirect, options.getAdvertisedListenerName()) + createLookupResult(candidateBrokerAdvertisedAddr == null ? candidateBroker + : candidateBrokerAdvertisedAddr, authoritativeRedirect, options.getAdvertisedListenerName()) .thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))) .exceptionally(ex -> { lookupFuture.completeExceptionally(ex); @@ -690,7 +694,7 @@ private Set getAvailableBrokers() { * @return * @throws Exception */ - private Optional getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit) throws Exception { + private Optional> getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit) throws Exception { Optional leastLoadedBroker = loadManager.get().getLeastLoaded(serviceUnit); if (!leastLoadedBroker.isPresent()) { LOG.warn("No broker is available for {}", serviceUnit); @@ -698,12 +702,14 @@ private Optional getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit } String lookupAddress = leastLoadedBroker.get().getResourceId(); + String advertisedAddr = (String) leastLoadedBroker.get() + .getProperty(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME); if (LOG.isDebugEnabled()) { LOG.debug("{} : redirecting to the least loaded broker, lookup address={}", pulsar.getSafeWebServiceAddress(), lookupAddress); } - return Optional.of(lookupAddress); + return Optional.of(Pair.of(lookupAddress, advertisedAddr)); } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java index 6ff49674e2e77..489efa5755ba8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.loadbalance; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + import java.net.URI; import java.util.Optional; import lombok.Cleanup; @@ -69,14 +71,14 @@ private void updateConfig(ServiceConfiguration conf, String advertisedAddress) { int httpsPort = PortManager.nextFreePort(); // Use invalid domain name as identifier and instead make sure the advertised listeners work as intended - this.conf.setAdvertisedAddress(advertisedAddress); - this.conf.setAdvertisedListeners( + conf.setAdvertisedAddress(advertisedAddress); + conf.setAdvertisedListeners( "public:pulsar://localhost:" + pulsarPort + ",public_http:http://localhost:" + httpPort + ",public_https:https://localhost:" + httpsPort); - this.conf.setBrokerServicePort(Optional.of(pulsarPort)); - this.conf.setWebServicePort(Optional.of(httpPort)); - this.conf.setWebServicePortTls(Optional.of(httpsPort)); + conf.setBrokerServicePort(Optional.of(pulsarPort)); + conf.setWebServicePort(Optional.of(httpPort)); + conf.setWebServicePortTls(Optional.of(httpsPort)); } @Test @@ -85,6 +87,7 @@ public void testLookup() throws Exception { new HttpGet(pulsar.getWebServiceAddress() + "/lookup/v2/topic/persistent/public/default/my-topic"); request.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); request.addHeader(HttpHeaders.ACCEPT, "application/json"); + final String topic = "my-topic"; @Cleanup CloseableHttpClient httpClient = HttpClients.createDefault(); @@ -104,14 +107,15 @@ public void testLookup() throws Exception { // Produce data @Cleanup Producer p = pulsarClient.newProducer(Schema.STRING) - .topic("my-topic") + .topic(topic) .create(); p.send("hello"); // Verify we can get the correct HTTP redirect to the advertised listener for (PulsarAdmin a : getAllAdmins()) { - TopicStats s = a.topics().getStats("my-topic"); + TopicStats s = a.topics().getStats(topic); + assertNotNull(a.lookups().lookupTopic(topic)); assertEquals(s.getPublishers().size(), 1); } } From 3b4c1a7f635dfc763ec2d7a89b8176a904d03d05 Mon Sep 17 00:00:00 2001 From: mattison chao Date: Wed, 25 May 2022 16:05:45 +0800 Subject: [PATCH 568/823] [fix][broker] Fix NPE when set `AutoTopicCreationOverride` (#15653) (cherry picked from commit e2afcf0a7ee90908fa647656624aacb9fd93249c) --- .../pulsar/broker/admin/impl/NamespacesBase.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index df8fd1c4e610b..7b235c17602a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -94,6 +94,8 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.TenantOperation; +import org.apache.pulsar.common.policies.data.TopicHashPositions; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.ValidateResult; import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; @@ -810,9 +812,11 @@ protected void internalSetAutoTopicCreation(AsyncResponse asyncResponse, "Invalid configuration for autoTopicCreationOverride. the detail is " + validateResult.getErrorInfo()); } - if (maxPartitions > 0 && autoTopicCreationOverride.getDefaultNumPartitions() > maxPartitions) { - throw new RestException(Status.NOT_ACCEPTABLE, - "Number of partitions should be less than or equal to " + maxPartitions); + if (Objects.equals(autoTopicCreationOverride.getTopicType(), TopicType.PARTITIONED.toString())) { + if (maxPartitions > 0 && autoTopicCreationOverride.getDefaultNumPartitions() > maxPartitions) { + throw new RestException(Status.NOT_ACCEPTABLE, + "Number of partitions should be less than or equal to " + maxPartitions); + } } } // Force to read the data s.t. the watch to the cache content is setup. From 9ad8f0057feead8499eb780dfbcb544cbc86c069 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 23 May 2022 09:51:27 +0800 Subject: [PATCH 569/823] [fix][broker-common] expose configurationMetadataStore and localMetadataStore (#15661) (cherry picked from commit e7dff4b5136f55b62ea1fa79073c8b5236564a33) --- .../org/apache/pulsar/broker/resources/PulsarResources.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java index a33d446ed8293..a01d57817eed6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java @@ -50,8 +50,9 @@ public class PulsarResources { private final BookieResources bookieResources; @Getter private final TopicResources topicResources; - + @Getter private final Optional localMetadataStore; + @Getter private final Optional configurationMetadataStore; public PulsarResources(MetadataStore localMetadataStore, MetadataStore configurationMetadataStore) { From c34a5741fdaa64fd3f1f95add480a69bd482c419 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 24 May 2022 12:15:25 +0800 Subject: [PATCH 570/823] [fix][ML]Fix NPE when put value to `RangeCache`. (#15707) ### Motivation When `ReferenceCounted` object overrides the method `deallocate` to make the `getLength` value equal null will cause NPE because the `RangeCache#put` method is not thread-safe. (The process of describing this abstraction is not very clear, please refer to the code modification :) Pulsar implementation may throw an exception to make `OpAddEntry` fail abnormal and fence the topic. relative code as below: https://github.com/apache/pulsar/blob/defeec0e84a63ea865f3a2790bc61b66a02254c5/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java#L211-L217 **Exception screenshot:** ```java java.lang.NullPointerException: Cannot invoke "String.length()" because "value.s" is null at org.apache.bookkeeper.mledger.util.RangeCacheTest.lambda$testInParallel$6(RangeCacheTest.java:279) at org.apache.bookkeeper.mledger.util.RangeCache.put(RangeCache.java:77) at org.apache.bookkeeper.mledger.util.RangeCacheTest.testInParallel(RangeCacheTest.java:283) at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104) at java.base/java.lang.reflect.Method.invoke(Method.java:577) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:132) at org.testng.internal.TestInvoker.invokeMethod(TestInvoker.java:599) at org.testng.internal.TestInvoker.invokeTestMethod(TestInvoker.java:174) at org.testng.internal.MethodRunner.runInSequence(MethodRunner.java:46) at org.testng.internal.TestInvoker$MethodInvocationAgent.invoke(TestInvoker.java:822) at org.testng.internal.TestInvoker.invokeTestMethods(TestInvoker.java:147) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:146) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:128) at java.base/java.util.ArrayList.forEach(ArrayList.java:1511) at org.testng.TestRunner.privateRun(TestRunner.java:764) at org.testng.TestRunner.run(TestRunner.java:585) at org.testng.SuiteRunner.runTest(SuiteRunner.java:384) at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:378) at org.testng.SuiteRunner.privateRun(SuiteRunner.java:337) at org.testng.SuiteRunner.run(SuiteRunner.java:286) at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:53) at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:96) at org.testng.TestNG.runSuitesSequentially(TestNG.java:1218) at org.testng.TestNG.runSuitesLocally(TestNG.java:1140) at org.testng.TestNG.runSuites(TestNG.java:1069) at org.testng.TestNG.run(TestNG.java:1037) at com.intellij.rt.testng.IDEARemoteTestNG.run(IDEARemoteTestNG.java:66) at com.intellij.rt.testng.RemoteTestNGStarter.main(RemoteTestNGStarter.java:109) ``` ### Modifications - Make the `RangeCache#put` method to thread-safe. (cherry picked from commit b155d84c2ee397fe8003f452f04ae6cedf229b5c) --- .../bookkeeper/mledger/util/RangeCache.java | 12 ++++---- .../mledger/util/RangeCacheTest.java | 29 +++++++++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java index a5786ad867034..4a77ac91dcaf0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java @@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.tuple.Pair; /** @@ -74,12 +75,13 @@ public RangeCache(Weighter weighter, TimestampExtractor timestampE * @return whether the entry was inserted in the cache */ public boolean put(Key key, Value value) { - if (entries.putIfAbsent(key, value) == null) { + MutableBoolean flag = new MutableBoolean(); + entries.computeIfAbsent(key, (k) -> { size.addAndGet(weighter.getSize(value)); - return true; - } else { - return false; - } + flag.setValue(true); + return value; + }); + return flag.booleanValue(); } public Value get(Key key) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java index 95896d24f35f2..f31aa4a74f9d1 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java @@ -29,11 +29,15 @@ import io.netty.util.ReferenceCounted; import org.apache.commons.lang3.tuple.Pair; import org.testng.annotations.Test; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public class RangeCacheTest { class RefString extends AbstractReferenceCounted implements ReferenceCounted { - final String s; + String s; RefString(String s) { super(); @@ -43,7 +47,7 @@ class RefString extends AbstractReferenceCounted implements ReferenceCounted { @Override protected void deallocate() { - // no-op + s = null; } @Override @@ -122,6 +126,7 @@ public void customWeighter() { assertEquals(cache.getNumberOfEntries(), 2); } + @Test public void customTimeExtraction() { RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> x.s.length()); @@ -268,4 +273,24 @@ public void evictions() { assertEquals((long) res.getRight(), 10); assertEquals(cache.getSize(), 90); } + + @Test + public void testInParallel() { + RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + executor.scheduleWithFixedDelay(cache::clear, 10, 10, TimeUnit.MILLISECONDS); + for (int i = 0; i < 1000; i++) { + cache.put(UUID.randomUUID().toString(), new RefString("zero")); + } + executor.shutdown(); + } + + @Test + public void testPutSameObj() { + RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); + RefString s0 = new RefString("zero"); + assertEquals(s0.refCnt(), 1); + assertTrue(cache.put(0, s0)); + assertFalse(cache.put(0, s0)); + } } From 9955b0af970150a3b2cda1172458ad73ebb38580 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 3 May 2022 22:49:43 +0300 Subject: [PATCH 571/823] [Proxy] Remove unnecessary blocking DNS lookup in LookupProxyHandler (#15415) * [Proxy] Remove unnecessary blocking DNS lookup in LookupProxyHandler * Use existing code pattern for creating address (cherry picked from commit 7373a51690d728475d47846bfbcca4fa64f2e228) --- .../proxy/server/LookupProxyHandler.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java index 30e11a185a038..d9d208085fd2a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java @@ -215,20 +215,16 @@ public void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata part private void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata partitionMetadata, long clientRequestId) { TopicName topicName = TopicName.get(partitionMetadata.getTopic()); - URI brokerURI; - try { - String availableBrokerServiceURL = getBrokerServiceUrl(clientRequestId); - if (availableBrokerServiceURL == null) { - log.warn("No available broker for {} to lookup partition metadata", topicName); - return; - } - brokerURI = new URI(availableBrokerServiceURL); - } catch (URISyntaxException e) { - proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.MetadataError, - e.getMessage(), clientRequestId)); + + String serviceUrl = getBrokerServiceUrl(clientRequestId); + if (serviceUrl == null) { + log.warn("No available broker for {} to lookup partition metadata", topicName); + return; + } + InetSocketAddress addr = getAddr(serviceUrl, clientRequestId); + if (addr == null) { return; } - InetSocketAddress addr = new InetSocketAddress(brokerURI.getHost(), brokerURI.getPort()); if (log.isDebugEnabled()) { log.debug("Getting connections to '{}' for Looking up topic '{}' with clientReq Id '{}'", addr, From 67a75da81b3e3692e015076acff1e1b20a39adfe Mon Sep 17 00:00:00 2001 From: mattison chao Date: Wed, 25 May 2022 16:24:58 +0800 Subject: [PATCH 572/823] Fix cherry-picking issue with #15653 --- .../java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 7b235c17602a2..a2e1dad5b1012 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -94,7 +94,6 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.TenantOperation; -import org.apache.pulsar.common.policies.data.TopicHashPositions; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.ValidateResult; import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; From d3796e24c7a9d4a0174e0e987ffa4cbba4c50ccc Mon Sep 17 00:00:00 2001 From: mattison chao Date: Wed, 25 May 2022 17:06:21 +0800 Subject: [PATCH 573/823] Fix cherry-picking issue with #15137 --- .../broker/transaction/pendingack/impl/MLPendingAckStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 5fc210ca74165..e6d16fb7eae90 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -76,7 +76,7 @@ public class MLPendingAckStore implements PendingAckStore { private final AtomicLong currentIndexLag = new AtomicLong(0); private volatile long maxIndexLag; - protected PositionImpl maxAckPosition = PositionImpl.EARLIEST; + protected PositionImpl maxAckPosition = PositionImpl.earliest; private final LogIndexLagBackoff logIndexBackoff; /** From 7856268fb2aa05c015be5e68e57c46d8966d347f Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 11 May 2022 15:38:15 +0800 Subject: [PATCH 574/823] [cleanup][broker] Override close method to avoid caching exception (#15529) (cherry picked from commit 526979a87e3cea5c4c90667ce86aed52927fd631) --- .../org/apache/pulsar/broker/service/AbstractTopic.java | 4 +--- .../apache/pulsar/broker/service/PrecisPublishLimiter.java | 2 +- .../apache/pulsar/broker/service/PublishRateLimiter.java | 5 +++++ .../pulsar/broker/service/PublishRateLimiterDisable.java | 2 +- .../pulsar/broker/service/PublishRateLimiterImpl.java | 2 +- .../broker/service/nonpersistent/NonPersistentTopic.java | 6 +----- .../pulsar/broker/service/persistent/PersistentTopic.java | 6 +----- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index ba4756647a55f..fbdbd611d50c6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -834,9 +834,7 @@ private void updatePublishDispatcher(Optional optPolicies) { } // attach the resource-group level rate limiters, if set - String rgName = policies != null && policies.resource_group_name != null - ? policies.resource_group_name - : null; + String rgName = policies.resource_group_name; if (rgName != null) { final ResourceGroup resourceGroup = brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java index e61597e2d139f..67cc46d95fada 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java @@ -133,7 +133,7 @@ public boolean tryAcquire(int numbers, long bytes) { } @Override - public void close() throws Exception { + public void close() { rateLimitFunction.apply(); replaceLimiters(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java index 397887978b2b5..931f35cfa1bd4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java @@ -71,4 +71,9 @@ public interface PublishRateLimiter extends AutoCloseable { * @param bytes */ boolean tryAcquire(int numbers, long bytes); + + /** + * Close the limiter. + */ + void close(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java index 81c4b82317f83..72c8132128e19 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java @@ -63,7 +63,7 @@ public boolean tryAcquire(int numbers, long bytes) { } @Override - public void close() throws Exception { + public void close() { // No-op } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java index 0e1200edc31cb..f1646684b82cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java @@ -110,7 +110,7 @@ public boolean tryAcquire(int numbers, long bytes) { } @Override - public void close() throws Exception { + public void close() { // no-op } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index c6ea96179d6f8..1dbe95fb15a30 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -462,11 +462,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); producers.values().forEach(producer -> futures.add(producer.disconnect())); if (topicPublishRateLimiter != null) { - try { - topicPublishRateLimiter.close(); - } catch (Exception e) { - log.warn("Error closing topicPublishRateLimiter for topic {}", topic, e); - } + topicPublishRateLimiter.close(); } subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); if (this.resourceGroupPublishLimiter != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 5505c940af65d..6bcfa36320c39 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1270,11 +1270,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); producers.values().forEach(producer -> futures.add(producer.disconnect())); if (topicPublishRateLimiter != null) { - try { - topicPublishRateLimiter.close(); - } catch (Exception e) { - log.warn("Error closing topicPublishRateLimiter for topic {}", topic, e); - } + topicPublishRateLimiter.close(); } subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); if (this.resourceGroupPublishLimiter != null) { From 25f5cec3923055df0cf3ac81b6c3d10937f612a6 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 26 May 2022 00:39:23 +0800 Subject: [PATCH 575/823] [branch-2.9][fix][broker] Fix jdk API compatibility issues. (#15779) * [branch-2.9][fix][broker] Fix jdk API compatibility issues. * Fix compile issue --- .../broker/loadbalance/impl/ModularLoadManagerWrapper.java | 6 +++++- .../transaction/TopicTransactionBufferRecoverTest.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java index f2d61d2dcd1e6..ac55d679219f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,8 +70,10 @@ public Optional getLeastLoaded(final ServiceUnitId serviceUnit) { return leastLoadedBroker.map(s -> { String webServiceUrl = getBrokerWebServiceUrl(s); String brokerZnodeName = getBrokerZnodeName(s, webServiceUrl); + Map map = new HashMap<>(); + map.put(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName); return new SimpleResourceUnit(webServiceUrl, - new PulsarResourceDescription(), Map.of(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName)); + new PulsarResourceDescription(), Collections.unmodifiableMap(map)); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index dddda0f962db6..fb2968acd885a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -495,7 +495,7 @@ public void testTransactionBufferRecoverThrowException() throws Exception { doThrow(new RuntimeException("test")).when(reader).hasMoreEvents(); // check reader close topic checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceOriginal, - transactionBufferSnapshotService, originalTopic, field, producer); + transactionBufferSnapshotService, originalTopic, field); doReturn(true).when(reader).hasMoreEvents(); // mock reader can't read snapshot fail throw PulsarClientException From 033a3b6e39bde79bf7eea146181805ca541f6a75 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sat, 21 May 2022 17:15:56 +0800 Subject: [PATCH 576/823] [optimize][txn] Optimize transaction lowWaterMark to clean useless data faster (#15592) (cherry picked from commit fcf5e148b055d617db37eef3c40d4004e74190a5) --- .../buffer/impl/TopicTransactionBuffer.java | 72 +++++---- .../pendingack/impl/PendingAckHandleImpl.java | 49 ++++-- .../buffer/TransactionLowWaterMarkTest.java | 142 +++++++++++++++++- .../pendingack/PendingAckPersistentTest.java | 24 +-- 4 files changed, 223 insertions(+), 64 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 2ddac0862942f..7bb19d8008695 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import lombok.SneakyThrows; @@ -96,8 +97,13 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final CompletableFuture transactionBufferFuture = new CompletableFuture<>(); + /** + * The map is used to store the lowWaterMarks which key is TC ID and value is lowWaterMark of the TC. + */ private final ConcurrentHashMap lowWaterMarks = new ConcurrentHashMap<>(); + private final Semaphore handleLowWaterMark = new Semaphore(1); + public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; @@ -285,13 +291,6 @@ public CompletableFuture openTransactionBufferReader(Tx @Override public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { - lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { - if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { - return lowWaterMark; - } else { - return oldLowWaterMark; - } - }); if (log.isDebugEnabled()) { log.debug("Transaction {} commit on topic {}.", txnID.toString(), topic.getName()); } @@ -332,13 +331,6 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { @Override public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { - lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { - if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { - return lowWaterMark; - } else { - return oldLowWaterMark; - } - }); if (log.isDebugEnabled()) { log.debug("Transaction {} abort on topic {}.", txnID.toString(), topic.getName()); } @@ -358,12 +350,12 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { aborts.put(txnID, (PositionImpl) position); updateMaxReadPosition(txnID); - handleLowWaterMark(txnID, lowWaterMark); changeMaxReadPositionAndAddAbortTimes.getAndIncrement(); clearAbortedTransactions(); takeSnapshotByChangeTimes(); } completableFuture.complete(null); + handleLowWaterMark(txnID, lowWaterMark); } @Override @@ -384,30 +376,36 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { - if (!ongoingTxns.isEmpty()) { - TxnID firstTxn = ongoingTxns.firstKey(); - if (firstTxn.getMostSigBits() == txnID.getMostSigBits() && lowWaterMark >= firstTxn.getLeastSigBits()) { - ByteBuf abortMarker = Markers.newTxnAbortMarker(-1L, - firstTxn.getMostSigBits(), firstTxn.getLeastSigBits()); - try { - topic.getManagedLedger().asyncAddEntry(abortMarker, new AsyncCallbacks.AddEntryCallback() { - @Override - public void addComplete(Position position, ByteBuf entryData, Object ctx) { - synchronized (TopicTransactionBuffer.this) { - aborts.put(firstTxn, (PositionImpl) position); - updateMaxReadPosition(firstTxn); - } - } - - @Override - public void addFailed(ManagedLedgerException exception, Object ctx) { - log.error("Failed to abort low water mark for txn {}", txnID, exception); - } - }, null); - } finally { - abortMarker.release(); + lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { + if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { + return lowWaterMark; + } else { + return oldLowWaterMark; + } + }); + if (handleLowWaterMark.tryAcquire()) { + if (!ongoingTxns.isEmpty()) { + TxnID firstTxn = ongoingTxns.firstKey(); + long tCId = firstTxn.getMostSigBits(); + Long lowWaterMarkOfFirstTxnId = lowWaterMarks.get(tCId); + if (lowWaterMarkOfFirstTxnId != null && firstTxn.getLeastSigBits() <= lowWaterMarkOfFirstTxnId) { + abortTxn(firstTxn, lowWaterMarkOfFirstTxnId) + .thenRun(() -> { + log.warn("Successes to abort low water mark for txn [{}], topic [{}]," + + " lowWaterMark [{}]", firstTxn, topic.getName(), lowWaterMarkOfFirstTxnId); + handleLowWaterMark.release(); + }) + .exceptionally(ex -> { + log.warn("Failed to abort low water mark for txn {}, topic [{}], " + + "lowWaterMark [{}], ", firstTxn, topic.getName(), lowWaterMarkOfFirstTxnId, + ex); + handleLowWaterMark.release(); + return null; + }); + return; } } + handleLowWaterMark.release(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index ab5a8ff14542a..4fdc298f303f0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -28,10 +28,12 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -113,6 +115,13 @@ public class PendingAckHandleImpl extends PendingAckHandleState implements Pendi private final BlockingQueue acceptQueue = new LinkedBlockingDeque<>(); + /** + * The map is used to store the lowWaterMarks which key is TC ID and value is lowWaterMark of the TC. + */ + private final ConcurrentHashMap lowWaterMarks = new ConcurrentHashMap<>(); + + private final Semaphore handleLowWaterMark = new Semaphore(1); + @Getter private final ExecutorService internalPinnedExecutor; @@ -593,20 +602,34 @@ public CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long low } private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { - if (individualAckOfTransaction != null && !individualAckOfTransaction.isEmpty()) { - TxnID firstTxn = individualAckOfTransaction.firstKey(); - - if (firstTxn.getMostSigBits() == txnID.getMostSigBits() - && firstTxn.getLeastSigBits() <= lowWaterMark) { - abortTxn(firstTxn, null, lowWaterMark).thenRun(() -> { - log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " - + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); - }).exceptionally(e -> { - log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " - + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); - return null; - }); + lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { + if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { + return lowWaterMark; + } else { + return oldLowWaterMark; + } + }); + + if (handleLowWaterMark.tryAcquire()) { + if (individualAckOfTransaction != null && !individualAckOfTransaction.isEmpty()) { + TxnID firstTxn = individualAckOfTransaction.firstKey(); + long tCId = firstTxn.getMostSigBits(); + Long lowWaterMarkOfFirstTxnId = lowWaterMarks.get(tCId); + if (lowWaterMarkOfFirstTxnId != null && firstTxn.getLeastSigBits() <= lowWaterMarkOfFirstTxnId) { + abortTxn(firstTxn, null, lowWaterMarkOfFirstTxnId).thenRun(() -> { + log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, firstTxn, lowWaterMarkOfFirstTxnId); + handleLowWaterMark.release(); + }).exceptionally(ex -> { + log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, firstTxn, lowWaterMarkOfFirstTxnId); + handleLowWaterMark.release(); + return null; + }); + return; + } } + handleLowWaterMark.release(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java index ba0659892b452..1f012d7a6b10e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java @@ -31,13 +31,14 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; - import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -52,7 +53,6 @@ import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; - import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; @@ -71,7 +71,7 @@ @Test(groups = "broker") public class TransactionLowWaterMarkTest extends TransactionTestBase { - private static final String TOPIC = NAMESPACE1 + "/test-topic"; + private static final String TOPIC = "persistent://" + NAMESPACE1 + "/test-topic"; @BeforeMethod(alwaysRun = true) protected void setup() throws Exception { @@ -216,7 +216,7 @@ public void testPendingAckLowWaterMark() throws Exception { ConcurrentOpenHashMap>> topics = (ConcurrentOpenHashMap>>) field .get(getPulsarServiceList().get(i).getBrokerService()); - CompletableFuture> completableFuture = topics.get("persistent://" + TOPIC); + CompletableFuture> completableFuture = topics.get(TOPIC); if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { @@ -327,4 +327,138 @@ public void testTBLowWaterMarkEndToEnd() throws Exception { // no-op } } + + @Test + public void testLowWaterMarkForDifferentTC() throws Exception { + String subName = "sub"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(TOPIC) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(TOPIC) + .subscriptionName(subName) + .subscribe(); + + Transaction txn1 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + Transaction txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + while (txn2.getTxnID().getMostSigBits() == txn1.getTxnID().getMostSigBits()) { + txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + } + Transaction txn3 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + while (txn3.getTxnID().getMostSigBits() != txn2.getTxnID().getMostSigBits()) { + txn3 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + } + + Transaction txn4 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + while (txn4.getTxnID().getMostSigBits() != txn1.getTxnID().getMostSigBits()) { + txn4 = pulsarClient.newTransaction() + .withTransactionTimeout(500, TimeUnit.SECONDS) + .build().get(); + } + + for (int i = 0; i < 10; i++) { + producer.newMessage().send(); + } + + producer.newMessage(txn1).send(); + producer.newMessage(txn2).send(); + producer.newMessage(txn3).send(); + producer.newMessage(txn4).send(); + + Message message1 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message1.getMessageId(), txn1); + Message message2 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message2.getMessageId(), txn2); + Message message3 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message3.getMessageId(), txn3); + Message message4 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message4.getMessageId(), txn4); + + txn1.commit().get(); + txn2.commit().get(); + + Field field = TransactionImpl.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(txn1, TransactionImpl.State.OPEN); + field.set(txn2, TransactionImpl.State.OPEN); + + producer.newMessage(txn1).send(); + producer.newMessage(txn2).send(); + + Message message5 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message5.getMessageId(), txn1); + Message message6 = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledgeAsync(message6.getMessageId(), txn2); + + txn3.commit().get(); + TxnID txnID1 = txn1.getTxnID(); + TxnID txnID2 = txn2.getTxnID(); + Awaitility.await().untilAsserted(() -> { + assertTrue(checkTxnIsOngoingInTP(txnID1, subName)); + assertTrue(checkTxnIsOngoingInTP(txnID2, subName)); + assertTrue(checkTxnIsOngoingInTB(txnID1)); + assertTrue(checkTxnIsOngoingInTB(txnID2)); + }); + + txn4.commit().get(); + + Awaitility.await().untilAsserted(() -> { + assertFalse(checkTxnIsOngoingInTP(txnID1, subName)); + assertFalse(checkTxnIsOngoingInTP(txnID2, subName)); + assertFalse(checkTxnIsOngoingInTB(txnID1)); + assertFalse(checkTxnIsOngoingInTB(txnID2)); + }); + } + + private boolean checkTxnIsOngoingInTP(TxnID txnID, String subName) throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService() + .getTopic(TopicName.get(TOPIC).toString(), false) + .get().get(); + + PersistentSubscription persistentSubscription = persistentTopic.getSubscription(subName); + + Field field1 = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field1.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field1.get(persistentSubscription); + + Field field2 = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); + field2.setAccessible(true); + LinkedMap> individualAckOfTransaction = + (LinkedMap>) field2.get(pendingAckHandle); + return individualAckOfTransaction.containsKey(txnID); + } + + private boolean checkTxnIsOngoingInTB(TxnID txnID) throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService() + .getTopic(TopicName.get(TOPIC).toString(), false) + .get().get(); + + TopicTransactionBuffer topicTransactionBuffer = + (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); + Field field3 = TopicTransactionBuffer.class.getDeclaredField("ongoingTxns"); + field3.setAccessible(true); + LinkedMap ongoingTxns = + (LinkedMap) field3.get(topicTransactionBuffer); + return ongoingTxns.containsKey(txnID); + + } + + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index a64707f1ae889..6683be138da66 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; @@ -45,7 +44,6 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; @@ -55,8 +53,6 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.testng.Assert; @@ -545,24 +541,32 @@ public void testPendingAckLowWaterMarkRemoveFirstTxn() throws Exception { .get(); PersistentSubscription persistentSubscription = persistentTopic.getSubscription(subName); + Field field1 = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field1.setAccessible(true); + PendingAckHandleImpl oldPendingAckHandle = (PendingAckHandleImpl) field1.get(persistentSubscription); + Field field2 = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); + field2.setAccessible(true); + LinkedMap> oldIndividualAckOfTransaction = + (LinkedMap>) field2.get(oldPendingAckHandle); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(oldIndividualAckOfTransaction.size(), 0)); + PendingAckHandleImpl pendingAckHandle = new PendingAckHandleImpl(persistentSubscription); Method method = PendingAckHandleImpl.class.getDeclaredMethod("initPendingAckStore"); method.setAccessible(true); method.invoke(pendingAckHandle); - Field field1 = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); - field1.setAccessible(true); - CompletableFuture completableFuture = - (CompletableFuture) field1.get(pendingAckHandle); + Field field3 = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field3.setAccessible(true); Awaitility.await().until(() -> { + CompletableFuture completableFuture = + (CompletableFuture) field3.get(pendingAckHandle); completableFuture.get(); return true; }); - Field field2 = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); - field2.setAccessible(true); + LinkedMap> individualAckOfTransaction = (LinkedMap>) field2.get(pendingAckHandle); From fb426867c6534fc1cd2aae7c070b2b201bcc97ce Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Thu, 26 May 2022 08:24:14 +0800 Subject: [PATCH 577/823] [fix][sql] Fix the decimal type error convert in json schema (#15687) In the current sql implementation, If using `JSON` schema and querying for decimal type, there will be the following two errors: 1. The data type is displayed as varchar. 2. Loss of precision because scientific notation is used to display. ``` presto> select bigdecimal, typeof(bigdecimal) as devimal_type from pulsar."public/default".test_avro2; bigdecimal | devimal_type -----------------------+-------------- 1.2345678912345678E36 | varchar 1.2345678912345678E36 | varchar (2 rows) ``` The original data is: `1234567891234567891234567891234567.89` - When getting jsonNode, use `BIG_DECIMAL` instead of float and double. - `PulsarJsonFieldDecoder` increases the processing of Decimal types (cherry picked from commit 0c6e2ca24fd368d11a769233bb2041f6cc4a8374) --- .../schema/generic/GenericJsonReader.java | 27 +++++++++---------- .../decoder/json/PulsarJsonFieldDecoder.java | 22 +++++++++++++++ .../json/PulsarJsonRowDecoderFactory.java | 11 +++++--- .../presto/decoder/avro/TestAvroDecoder.java | 13 +++++++-- .../presto/decoder/json/TestJsonDecoder.java | 20 ++++++++++++++ 5 files changed, 72 insertions(+), 21 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonReader.java index f0b2c86508b3b..1a95e9be152da 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonReader.java @@ -18,35 +18,31 @@ */ package org.apache.pulsar.client.impl.schema.generic; +import static java.nio.charset.StandardCharsets.UTF_8; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaReader; - import org.apache.pulsar.common.schema.SchemaInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -import static java.nio.charset.StandardCharsets.UTF_8; - public class GenericJsonReader implements SchemaReader { - private final ObjectMapper objectMapper; + private final ObjectReader objectReader; private final byte[] schemaVersion; private final List fields; private SchemaInfo schemaInfo; public GenericJsonReader(List fields, SchemaInfo schemaInfo){ - this.fields = fields; - this.schemaVersion = null; - this.objectMapper = new ObjectMapper(); - this.schemaInfo = schemaInfo; + this(null, fields, schemaInfo); } public GenericJsonReader(List fields){ @@ -58,16 +54,17 @@ public GenericJsonReader(byte[] schemaVersion, List fields){ } public GenericJsonReader(byte[] schemaVersion, List fields, SchemaInfo schemaInfo){ - this.objectMapper = new ObjectMapper(); this.fields = fields; this.schemaVersion = schemaVersion; this.schemaInfo = schemaInfo; + ObjectMapper objectMapper = new ObjectMapper(); + this.objectReader = objectMapper.reader().with(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); } @Override public GenericJsonRecord read(byte[] bytes, int offset, int length) { try { - JsonNode jn = objectMapper.readTree(new String(bytes, offset, length, UTF_8)); + JsonNode jn = objectReader.readTree(new String(bytes, offset, length, UTF_8)); return new GenericJsonRecord(schemaVersion, fields, jn, schemaInfo); } catch (IOException ioe) { throw new SchemaSerializationException(ioe); @@ -77,7 +74,7 @@ public GenericJsonRecord read(byte[] bytes, int offset, int length) { @Override public GenericRecord read(InputStream inputStream) { try { - JsonNode jn = objectMapper.readTree(inputStream); + JsonNode jn = objectReader.readTree(inputStream); return new GenericJsonRecord(schemaVersion, fields, jn, schemaInfo); } catch (IOException ioe) { throw new SchemaSerializationException(ioe); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 960d8f42bdf7a..a14cd6b3ba2f2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -58,6 +58,8 @@ import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.Decimals; import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.MapType; @@ -70,6 +72,7 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.math.BigInteger; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -120,6 +123,9 @@ private static Pair getNumRangeByType(Type type) { } private boolean isSupportedType(Type type) { + if (type instanceof DecimalType) { + return true; + } if (isVarcharType(type)) { return true; } @@ -228,6 +234,13 @@ public static long getLong(JsonNode value, Type type, String columnName, long mi return floatToIntBits((Float) parseFloat(value.asText())); } + // If it is decimalType, need to eliminate the decimal point, + // and give it to presto to set the decimal point + if (type instanceof DecimalType) { + String decimalLong = value.asText().replace(".", ""); + return Long.valueOf(decimalLong); + } + long longValue; if (value.isIntegralNumber() && !value.isBigInteger()) { longValue = value.longValue(); @@ -267,6 +280,15 @@ public static double getDouble(JsonNode value, Type type, String columnName) { private static Slice getSlice(JsonNode value, Type type, String columnName) { String textValue = value.isValueNode() ? value.asText() : value.toString(); + + // If it is decimalType, need to eliminate the decimal point, + // and give it to presto to set the decimal point + if (type instanceof DecimalType) { + textValue = textValue.replace(".", ""); + BigInteger bigInteger = new BigInteger(textValue); + return Decimals.encodeUnscaledValue(bigInteger); + } + Slice slice = utf8Slice(textValue); if (isVarcharType(type)) { slice = truncateToLength(slice, type); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 843c2c7f8362f..b94a963cffd1a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -34,6 +34,7 @@ import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.RealType; @@ -128,11 +129,13 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: - // In the current implementation, since JsonSchema is generated by Avro, - // there may exist LogicalTypes.Decimal. - // Mapping decimalType with varcharType in JsonSchema. + // When the precision <= 0, throw Exception. + // When the precision > 0 and <= 18, use ShortDecimalType. and mapping Long + // When the precision > 18 and <= 36, use LongDecimalType. and mapping Slice + // When the precision > 36, throw Exception. if (logicalType instanceof LogicalTypes.Decimal) { - return createUnboundedVarcharType(); + LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; + return DecimalType.createDecimalType(decimal.getPrecision(), decimal.getScale()); } return VarbinaryType.VARBINARY; case INT: diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java index 7b270c7995be0..2478300dcaa16 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java @@ -89,8 +89,6 @@ public void testPrimitiveType() { message.longField = 222L; message.timestampField = System.currentTimeMillis(); message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_1; - message.decimalField = BigDecimal.valueOf(2233, 2); - message.longDecimalField = new BigDecimal("1234567891234567891234567891.23"); LocalTime now = LocalTime.now(ZoneId.systemDefault()); message.timeField = now.toSecondOfDay() * 1000; @@ -130,6 +128,17 @@ public void testPrimitiveType() { PulsarColumnHandle enumFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "enumField", VARCHAR, false, false, "enumField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); checkValue(decodedRow, enumFieldColumnHandle, message.enumField.toString()); + } + + @Test + public void testDecimal() { + DecoderTestMessage message = new DecoderTestMessage(); + message.decimalField = BigDecimal.valueOf(2233, 2); + message.longDecimalField = new BigDecimal("1234567891234567891234567891.23"); + + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(schema.encode(message)); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); PulsarColumnHandle decimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "decimalField", DecimalType.createDecimalType(4, 2), false, false, "decimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java index 2a22b58a03fb2..0b8a8f84eda0e 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java @@ -24,6 +24,7 @@ import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.PrestoException; import io.prestosql.spi.type.*; +import java.math.BigDecimal; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; @@ -119,6 +120,25 @@ public void testPrimitiveType() { } + @Test + public void testDecimal() { + DecoderTestMessage message = new DecoderTestMessage(); + message.decimalField = BigDecimal.valueOf(2233, 2); + message.longDecimalField = new BigDecimal("1234567891234567891234567891.23"); + + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(schema.encode(message)); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + PulsarColumnHandle decimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "decimalField", DecimalType.createDecimalType(4, 2), false, false, "decimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, decimalFieldColumnHandle, message.decimalField); + + PulsarColumnHandle longDecimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "longDecimalField", DecimalType.createDecimalType(30, 2), false, false, "longDecimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, longDecimalFieldColumnHandle, message.longDecimalField); + } + @Test public void testArray() { DecoderTestMessage message = new DecoderTestMessage(); From 90c4653aea848953696599c72553952b3ef0b53f Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 25 May 2022 05:54:55 -0700 Subject: [PATCH 578/823] [fix][broker] Fix creating producer failure when set backlog quota. (#15663) When trying to reproduce the problem of #15609 using the master's code, it was found that the master also had this bug. The root cause is: When there is only one ledger in the ManagedLedger, after the current ledger is closed, it has the timestamp and exceeds the time set by the backlog-qutoa, resulting in the failure to create the producer. The added test could reproduce this. So when there is only one ledger, we should not exclude it. If revert this patch, the added test will fail. (cherry picked from commit 3a8045851f7e9ea62da104dab2b7fe2b47a95ca9) --- .../service/persistent/PersistentTopic.java | 47 +++++++++----- .../systopic/PartitionedSystemTopicTest.java | 64 +++++++++++++++++++ 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 6bcfa36320c39..db824b0706bd2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2626,22 +2626,9 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { return false; } } else { - Long ledgerId = ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition().getLedgerId(); + PositionImpl slowestPosition = ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition(); try { - org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo - ledgerInfo = ledger.getLedgerInfo(ledgerId).get(); - if (ledgerInfo != null && ledgerInfo.hasTimestamp() && ledgerInfo.getTimestamp() > 0 - && ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp() - > backlogQuotaLimitInSecond * 1000) { - if (log.isDebugEnabled()) { - log.debug("Time based backlog quota exceeded, quota {}, age of ledger " - + "slowest cursor currently on {}", backlogQuotaLimitInSecond * 1000, - ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp()); - } - return true; - } else { - return false; - } + return slowestReaderTimeBasedBacklogQuotaCheck(slowestPosition); } catch (Exception e) { log.error("[{}][{}] Error reading entry for precise time based backlog check", topicName, e); return false; @@ -2649,6 +2636,36 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { } } + private boolean slowestReaderTimeBasedBacklogQuotaCheck(PositionImpl slowestPosition) + throws ExecutionException, InterruptedException { + int backlogQuotaLimitInSecond = getBacklogQuota(BacklogQuota.BacklogQuotaType.message_age).getLimitTime(); + Long ledgerId = slowestPosition.getLedgerId(); + if (((ManagedLedgerImpl) ledger).getLedgersInfo().lastKey().equals(ledgerId)) { + return false; + } + int result; + org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo + ledgerInfo = ledger.getLedgerInfo(ledgerId).get(); + if (ledgerInfo != null && ledgerInfo.hasTimestamp() && ledgerInfo.getTimestamp() > 0 + && ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp() + > backlogQuotaLimitInSecond * 1000 && (result = slowestPosition.compareTo( + new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1))) <= 0) { + if (result < 0) { + if (log.isDebugEnabled()) { + log.debug("Time based backlog quota exceeded, quota {}, age of ledger " + + "slowest cursor currently on {}", backlogQuotaLimitInSecond * 1000, + ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp()); + } + return true; + } else { + return slowestReaderTimeBasedBacklogQuotaCheck( + ((ManagedLedgerImpl) ledger).getNextValidPosition(slowestPosition)); + } + } else { + return false; + } + } + @Override public boolean isReplicated() { return !replicators.isEmpty(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index d506f7127678f..cf45d614f0fb4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -27,14 +27,18 @@ import org.apache.pulsar.broker.admin.impl.BrokersBase; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.events.EventsTopicNames; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.common.naming.TopicName; @@ -43,6 +47,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.mockito.Mockito; +import org.powermock.reflect.Whitebox; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -50,6 +55,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -65,6 +71,8 @@ protected void setup() throws Exception { conf.setAllowAutoTopicCreation(false); conf.setAllowAutoTopicCreationType("partitioned"); conf.setDefaultNumPartitions(PARTITIONS); + conf.setManagedLedgerMaxEntriesPerLedger(1); + conf.setBrokerDeleteInactiveTopicsEnabled(false); conf.setSystemTopicEnabled(true); conf.setTopicLevelPoliciesEnabled(true); @@ -170,4 +178,60 @@ public void testHealthCheckTopicNotOffload() throws Exception { }); } + @Test + private void testSetBacklogCausedCreatingProducerFailure() throws Exception { + final String ns = "prop/ns-test"; + final String topic = ns + "/topic-1"; + + admin.namespaces().createNamespace(ns, 2); + admin.topics().createPartitionedTopic(String.format("persistent://%s", topic), 1); + BacklogQuota quota = BacklogQuota.builder() + .limitTime(2) + .limitSize(-1) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(); + admin.namespaces().setBacklogQuota(ns, quota, BacklogQuota.BacklogQuotaType.message_age); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + String partition0 = TopicName.get(String.format("persistent://%s", topic)).getPartition(0).toString(); + Optional topicReference = pulsar.getBrokerService().getTopicReference(partition0); + Assert.assertTrue(topicReference.isPresent()); + PersistentTopic persistentTopic = (PersistentTopic) topicReference.get(); + ManagedLedgerConfig config = persistentTopic.getManagedLedger().getConfig(); + config.setMinimumRolloverTime(1, TimeUnit.SECONDS); + config.setMaximumRolloverTime(1, TimeUnit.SECONDS); + persistentTopic.getManagedLedger().setConfig(config); + Whitebox.invokeMethod(persistentTopic.getManagedLedger(), "updateLastLedgerCreatedTimeAndScheduleRolloverTask"); + String msg1 = "msg-1"; + producer.send(msg1); + Thread.sleep(3 * 1000); + + Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub-1") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + Message receive = consumer2.receive(); + consumer2.acknowledge(receive); + + Thread.sleep(3 * 1000); + + try { + Producer producerN = PulsarClient.builder() + .maxBackoffInterval(3, TimeUnit.SECONDS) + .operationTimeout(5, TimeUnit.SECONDS) + .serviceUrl(lookupUrl.toString()).connectionTimeout(2, TimeUnit.SECONDS).build() + .newProducer(Schema.STRING).topic(topic).sendTimeout(3, TimeUnit.SECONDS).create(); + Assert.assertTrue(producerN.isConnected()); + producerN.close(); + } catch (Exception ex) { + Assert.fail("failed to create producer"); + } + } } From 19bd78a25bd9b3bd07438c3a9869014d9d900fa3 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 26 May 2022 12:04:05 +0800 Subject: [PATCH 579/823] [branch-2.9] fix:remove the loadbalance/bundle-data node (#15730) --- .../broker/resources/NamespaceResources.java | 37 ++++++++++++++++++- .../broker/admin/impl/NamespacesBase.java | 4 +- .../pulsar/broker/admin/impl/TenantsBase.java | 2 + .../pulsar/broker/admin/AdminApiTest2.java | 5 +++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index 190d25144daeb..8f677f069d578 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -54,7 +54,7 @@ public class NamespaceResources extends BaseResources { private static final String POLICIES_READONLY_FLAG_PATH = "/admin/flags/policies-readonly"; private static final String NAMESPACE_BASE_PATH = "/namespace"; - + private static final String BUNDLE_DATA_BASE_PATH = "/loadbalance/bundle-data"; public NamespaceResources(MetadataStore localStore, MetadataStore configurationStore, int operationTimeoutSec) { super(configurationStore, Policies.class, operationTimeoutSec); this.configurationStore = configurationStore; @@ -281,4 +281,39 @@ public CompletableFuture clearPartitionedTopicMetadataAsync(NamespaceName return completableFuture; } } + + + // clear resource of `/loadbalance/bundle-data/{tenant}/{namespace}/` for zk-node + public CompletableFuture deleteBundleDataAsync(NamespaceName ns) { + final String namespaceBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, ns.toString()); + CompletableFuture future = new CompletableFuture(); + deleteRecursiveAsync(this, namespaceBundlePath).whenComplete((ignore, ex) -> { + if (ex instanceof MetadataStoreException.NotFoundException) { + future.complete(null); + } else if (ex != null) { + future.completeExceptionally(ex); + } else { + future.complete(null); + } + }); + + return future; + } + + // clear resource of `/loadbalance/bundle-data/{tenant}/` for zk-node + public CompletableFuture deleteBundleDataTenantAsync(String tenant) { + final String tenantBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, tenant); + CompletableFuture future = new CompletableFuture(); + deleteRecursiveAsync(this, tenantBundlePath).whenComplete((ignore, ex) -> { + if (ex instanceof MetadataStoreException.NotFoundException) { + future.complete(null); + } else if (ex != null) { + future.completeExceptionally(ex); + } else { + future.complete(null); + } + }); + + return future; + } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index a2e1dad5b1012..edbe1bfe385b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -324,7 +324,9 @@ protected CompletableFuture internalClearZkSources() { // z-node can be deleted now .thenCompose(ignore -> namespaceResources().deletePoliciesAsync(namespaceName)) // clear z-node of local policies - .thenCompose(ignore -> getLocalPolicies().deleteLocalPoliciesAsync(namespaceName)); + .thenCompose(ignore -> getLocalPolicies().deleteLocalPoliciesAsync(namespaceName)) + // clear /loadbalance/bundle-data + .thenCompose(ignore -> namespaceResources().deleteBundleDataAsync(namespaceName)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java index 209a532970191..3ec725e6f8041 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java @@ -265,6 +265,8 @@ protected void internalDeleteTenant(AsyncResponse asyncResponse, String tenant) .clearTenantPersistence(tenant)) .thenCompose(ignore -> pulsar().getPulsarResources().getNamespaceResources() .deleteTenantAsync(tenant)) + .thenCompose(ignore -> pulsar().getPulsarResources().getNamespaceResources() + .deleteBundleDataTenantAsync(tenant)) .whenComplete((ignore, ex) -> { if (ex != null) { log.error("[{}] Failed to delete tenant {}", clientAppId(), tenant, ex); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java index 0bf34a16fe020..baafbc1ba4253 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java @@ -1311,7 +1311,9 @@ public void testDeleteTenant() throws Exception { assertFalse(admin.tenants().getTenants().contains(tenant)); final String managedLedgersPath = "/managed-ledgers/" + tenant; + final String bundleDataPath = "/loadbalance/bundle-data/" + tenant; assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); + assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); } @Test @@ -1355,6 +1357,9 @@ public void testDeleteNamespace() throws Exception { final String managedLedgersPath = "/managed-ledgers/" + namespace; assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); + + final String bundleDataPath = "/loadbalance/bundle-data/" + namespace; + assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); } @Test From 5cd5ded1ae18227b6a5e630f1a302f5810322db4 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 26 May 2022 12:56:25 +0800 Subject: [PATCH 580/823] [branch-2.9][broker][test] Fix flaky test `testDeleteNamespaceBeforeCommit` (#15798) --- .../pulsar/broker/transaction/TransactionProduceTest.java | 6 +++--- .../pulsar/broker/transaction/TransactionTestBase.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index 350dfa8c3d061..316c2f9f9df05 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -91,7 +91,7 @@ public void produceAndCommitTest() throws Exception { @Test public void testDeleteNamespaceBeforeCommit() throws Exception { - final String topic = NAMESPACE1 + "/testDeleteTopicBeforeCommit"; + final String topic = "persistent://" + NAMESPACE3 + "/testDeleteTopicBeforeCommit"; PulsarClient pulsarClient = this.pulsarClient; Transaction tnx = pulsarClient.newTransaction() .withTransactionTimeout(60, TimeUnit.SECONDS) @@ -113,8 +113,8 @@ public void testDeleteNamespaceBeforeCommit() throws Exception { outProducer.newMessage(tnx).value(content.getBytes(UTF_8)).send(); try { - admin.namespaces().deleteNamespace(NAMESPACE1, true); - } catch (Exception ignore) {} + admin.namespaces().deleteNamespace(NAMESPACE3, true); + } catch (Exception ignore) { } tnx.commit().get(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 2829aa9f9131d..b1a82802c1f46 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -86,6 +86,7 @@ public abstract class TransactionTestBase extends TestRetrySupport { public static final String TENANT = "tnx"; protected static final String NAMESPACE1 = TENANT + "/ns1"; + protected static final String NAMESPACE3 = TENANT + "/ns3"; protected ServiceConfiguration conf = new ServiceConfiguration(); public void internalSetup() throws Exception { @@ -130,6 +131,7 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int admin.tenants().createTenant(TENANT, new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); admin.namespaces().createNamespace(NAMESPACE1); + admin.namespaces().createNamespace(NAMESPACE3); if (numPartitions == 0) { admin.topics().createNonPartitionedTopic(topic); } else { From d592b76fa506d099fb26d243936a55a2aa27c728 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 26 May 2022 21:31:02 +0800 Subject: [PATCH 581/823] [branch-2.9] Fix cherry-pick issue with #15051 (#15800) --- .../java/org/apache/pulsar/broker/service/ServerCnxTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index d2dac88efe852..be77a50eea52d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -1813,7 +1813,7 @@ public void testTopicIsNotReady() throws Exception { channel.finish(); } - @Test + @Test(enabled = false) public void testNeverDelayConsumerFutureWhenNotFail() throws Exception{ // Mock ServerCnx.field: consumers ConcurrentLongHashMap.Builder mapBuilder = Mockito.mock(ConcurrentLongHashMap.Builder.class); From 48d60865d252d32b7aad70d4662f04d765f45f0c Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Fri, 27 May 2022 17:32:11 +0800 Subject: [PATCH 582/823] [fix][branch-2.9] Fix wrong unit of NIC speed on linux (#15770) --- .../broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java | 8 ++++---- .../broker/loadbalance/LoadReportNetworkLimitTest.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 9bd5f4e65247b..95cefd35d6073 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -248,7 +248,7 @@ private Path getNicSpeedPath(String nic) { private double getTotalNicLimitKbps(List nics) { // Use the override value as configured. Return the total max speed across all available NICs, converted // from Gbps into Kbps - return overrideBrokerNicSpeedGbps.map(aDouble -> aDouble * nics.size() * 1024 * 1024) + return overrideBrokerNicSpeedGbps.map(aDouble -> aDouble * nics.size() * 1000 * 1000) .orElseGet(() -> nics.stream().mapToDouble(nicPath -> { // Nic speed is in Mbits/s, return kbits/s try { @@ -258,7 +258,7 @@ private double getTotalNicLimitKbps(List nics) { + " config [loadBalancerOverrideBrokerNicSpeedGbps] to override it.", nicPath), e); return 0d; } - }).sum() * 1024); + }).sum() * 1000); } private Path getNicTxPath(String nic) { @@ -277,7 +277,7 @@ private double getTotalNicUsageRxKb(List nics) { log.error("Failed to read rx_bytes for NIC " + s, e); return 0d; } - }).sum() * 8 / 1024; + }).sum() * 8d / 1000; } private double getTotalNicUsageTxKb(List nics) { @@ -288,7 +288,7 @@ private double getTotalNicUsageTxKb(List nics) { log.error("Failed to read tx_bytes for NIC " + s, e); return 0d; } - }).sum() * 8 / 1024; + }).sum() * 8d / 1000; } private static long readLongFromFile(String path) throws IOException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java index 460210d33d6bf..2d9dc1abb0efc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java @@ -56,8 +56,8 @@ public void checkLoadReportNicSpeed() throws Exception { LoadManagerReport report = admin.brokerStats().getLoadReport(); if (SystemUtils.IS_OS_LINUX) { - assertEquals(report.getBandwidthIn().limit, nicCount * 5.4 * 1024 * 1024); - assertEquals(report.getBandwidthOut().limit, nicCount * 5.4 * 1024 * 1024); + assertEquals(report.getBandwidthIn().limit, nicCount * 5.4 * 1000 * 1000); + assertEquals(report.getBandwidthOut().limit, nicCount * 5.4 * 1000 * 1000); } else { // On non-Linux system we don't report the network usage assertEquals(report.getBandwidthIn().limit, -1.0); From 15fdf15ff6f05e514b88143446da9439a7bab17b Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sun, 29 May 2022 10:09:30 +0800 Subject: [PATCH 583/823] [branch-2.9] Add publishRateLimitedTimes to topic metrics. (#15739) --- .../pulsar/broker/service/AbstractTopic.java | 9 ++ .../pulsar/broker/service/ServerCnx.java | 13 +++ .../apache/pulsar/broker/service/Topic.java | 5 + .../service/persistent/PersistentTopic.java | 3 +- .../prometheus/NamespaceStatsAggregator.java | 1 + .../broker/stats/prometheus/TopicStats.java | 5 + .../broker/stats/PrometheusMetricsTest.java | 105 +++++++++++++++++- .../policies/data/stats/TopicStatsImpl.java | 5 + 8 files changed, 143 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index fbdbd611d50c6..69c51c77d44a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -116,6 +116,10 @@ public abstract class AbstractTopic implements Topic { private LongAdder bytesInCounter = new LongAdder(); private LongAdder msgInCounter = new LongAdder(); + private static final AtomicLongFieldUpdater RATE_LIMITED_UPDATER = + AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "publishRateLimitedTimes"); + protected volatile long publishRateLimitedTimes = 0; + protected volatile Optional topicEpoch = Optional.empty(); private volatile boolean hasExclusiveProducer; // pointer to the exclusive producer @@ -634,6 +638,11 @@ protected void checkTopicFenced() throws BrokerServiceException { } } + @Override + public long increasePublishLimitedTimes() { + return RATE_LIMITED_UPDATER.incrementAndGet(this); + } + protected void internalAddProducer(Producer producer) throws BrokerServiceException { if (isProducersExceeded()) { log.warn("[{}] Attempting to add producer to topic which reached max producers limit", topic); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 3c420fe43b365..58035b84c3885 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2512,6 +2512,7 @@ public void startSendOperation(Producer producer, int msgSize, int numMessages) // When the quota of pending send requests is reached, stop reading from socket to cause backpressure on // client connection, possibly shared between multiple producers ctx.channel().config().setAutoRead(false); + recordRateLimitMetrics(producers); autoReadDisabledRateLimiting = isPublishRateExceeded; throttledConnections.inc(); } @@ -2533,6 +2534,17 @@ public void startSendOperation(Producer producer, int msgSize, int numMessages) } } + private void recordRateLimitMetrics(ConcurrentLongHashMap> producers) { + producers.forEach((key, producerFuture) -> { + if (producerFuture != null && producerFuture.isDone()) { + Producer p = producerFuture.getNow(null); + if (p != null && p.getTopic() != null) { + p.getTopic().increasePublishLimitedTimes(); + } + } + }); + } + @Override public void completedSendOperation(boolean isNonPersistentTopic, int msgSize) { if (pendingBytesPerThread.get().addAndGet(-msgSize) < resumeThresholdPendingBytesPerThread @@ -2577,6 +2589,7 @@ public void enableCnxAutoRead() { public void disableCnxAutoRead() { if (ctx != null && ctx.channel().config().isAutoRead()) { ctx.channel().config().setAutoRead(false); + recordRateLimitMetrics(producers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 9dd945f78da33..14e8e3fe00418 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -129,6 +129,11 @@ default boolean isMarkerMessage() { */ void recordAddLatency(long latency, TimeUnit unit); + /** + * increase the publishing limited times. + */ + long increasePublishLimitedTimes(); + CompletableFuture subscribe(TransportCnx cnx, String subscriptionName, long consumerId, SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageId startMessageId, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index db824b0706bd2..22531e8ed09a4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1682,7 +1682,7 @@ public ManagedLedger getManagedLedger() { public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace, boolean hydratePublishers) { - + this.publishRateLimitedTimes = 0; TopicStatsHelper topicStatsHelper = threadLocalTopicStats.get(); topicStatsHelper.reset(); @@ -1931,6 +1931,7 @@ public TopicStatsImpl getStats(boolean getPreciseBacklog, boolean subscriptionBa stats.waitingPublishers = getWaitingProducersCount(); stats.bytesOutCounter = bytesOutFromRemovedSubscriptions.longValue(); stats.msgOutCounter = msgOutFromRemovedSubscriptions.longValue(); + stats.publishRateLimitedTimes = publishRateLimitedTimes; subscriptions.forEach((name, subscription) -> { SubscriptionStatsImpl subStats = subscription.getStats(getPreciseBacklog, subscriptionBacklogSize); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 9df9ad4c7c624..2025c59c1a1f7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -148,6 +148,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include stats.msgOutCounter = tStatus.msgOutCounter; stats.bytesOutCounter = tStatus.bytesOutCounter; stats.averageMsgSize = tStatus.averageMsgSize; + stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes; stats.producersCount = 0; topic.getProducers().values().forEach(producer -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index bfa427e5e3a7c..599c3d82f6fc5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -44,6 +44,8 @@ class TopicStats { public long msgBacklog; + long publishRateLimitedTimes; + long backlogQuotaLimit; long backlogQuotaLimitTime; @@ -82,6 +84,7 @@ public void reset() { managedLedgerStats.reset(); msgBacklog = 0; + publishRateLimitedTimes = 0L; backlogQuotaLimit = 0; backlogQuotaLimitTime = -1; @@ -133,6 +136,8 @@ static void printTopicStats(SimpleTextOutputStream stream, String cluster, Strin splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, + splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, "pulsar_storage_offloaded_size", stats.managedLedgerStats .offloadedStorageUsed, splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index b4094e488f363..1304d5c7ae2f9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -55,12 +55,12 @@ import javax.crypto.SecretKey; import javax.naming.AuthenticationException; import lombok.Cleanup; -import org.apache.bookkeeper.client.BookKeeper; import org.apache.commons.io.IOUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentMessageExpiryMonitor; @@ -74,7 +74,6 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.compaction.Compactor; -import org.apache.pulsar.compaction.TwoPhaseCompactor; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -95,6 +94,108 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + resetConfig(); + } + + @Test + public void testPublishRateLimitedTimes() throws Exception { + cleanup(); + checkPublishRateLimitedTimes(true); + cleanup(); + checkPublishRateLimitedTimes(false); + } + + private void checkPublishRateLimitedTimes(boolean preciseRateLimit) throws Exception { + if (preciseRateLimit) { + conf.setBrokerPublisherThrottlingTickTimeMillis(10000000); + conf.setMaxPublishRatePerTopicInMessages(1); + conf.setMaxPublishRatePerTopicInBytes(1); + conf.setBrokerPublisherThrottlingMaxMessageRate(100000); + conf.setBrokerPublisherThrottlingMaxByteRate(10000000); + } else { + conf.setBrokerPublisherThrottlingTickTimeMillis(1); + conf.setBrokerPublisherThrottlingMaxMessageRate(1); + conf.setBrokerPublisherThrottlingMaxByteRate(1); + } + conf.setStatsUpdateFrequencyInSecs(100000000); + conf.setPreciseTopicPublishRateLimiterEnable(preciseRateLimit); + setup(); + String ns1 = "prop/ns-abc1" + UUID.randomUUID(); + admin.namespaces().createNamespace(ns1, 1); + String topicName = "persistent://" + ns1 + "/metrics" + UUID.randomUUID(); + String topicName2 = "persistent://" + ns1 + "/metrics" + UUID.randomUUID(); + String topicName3 = "persistent://" + ns1 + "/metrics" + UUID.randomUUID(); + // Use another connection + @Cleanup + PulsarClient client2 = newPulsarClient(lookupUrl.toString(), 0); + + Producer producer = pulsarClient.newProducer().producerName("my-pub").enableBatching(false) + .topic(topicName).create(); + Producer producer2 = pulsarClient.newProducer().producerName("my-pub-2").enableBatching(false) + .topic(topicName2).create(); + Producer producer3 = client2.newProducer().producerName("my-pub-2").enableBatching(false) + .topic(topicName3).create(); + producer.sendAsync(new byte[11]); + + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopic(topicName, false).get().get(); + Field field = AbstractTopic.class.getDeclaredField("publishRateLimitedTimes"); + field.setAccessible(true); + Awaitility.await().untilAsserted(() -> { + long value = (long) field.get(persistentTopic); + assertEquals(value, 1); + }); + @Cleanup + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + String metricsStr = statsOut.toString(); + Multimap metrics = parseMetrics(metricsStr); + assertTrue(metrics.containsKey("pulsar_publish_rate_limit_times")); + metrics.get("pulsar_publish_rate_limit_times").forEach(item -> { + if (ns1.equals(item.tags.get("namespace"))) { + if (item.tags.get("topic").equals(topicName)) { + assertEquals(item.value, 1); + return; + } else if (item.tags.get("topic").equals(topicName2)) { + assertEquals(item.value, 1); + return; + } else if (item.tags.get("topic").equals(topicName3)) { + //When using precise rate limiting, we only trigger the rate limiting of the topic, + // so if the topic is not using the same connection, the rate limiting times will be 0 + //When using asynchronous rate limiting, we will trigger the broker-level rate limiting, + // and all connections will be limited at this time. + if (preciseRateLimit) { + assertEquals(item.value, 0); + } else { + assertEquals(item.value, 1); + } + return; + } + fail("should not fail"); + } + }); + // Stats updater will reset the stats + pulsar.getBrokerService().updateRates(); + Awaitility.await().untilAsserted(() -> { + long value = (long) field.get(persistentTopic); + assertEquals(value, 0); + }); + + @Cleanup + ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut2); + String metricsStr2 = statsOut2.toString(); + Multimap metrics2 = parseMetrics(metricsStr2); + assertTrue(metrics2.containsKey("pulsar_publish_rate_limit_times")); + metrics2.get("pulsar_publish_rate_limit_times").forEach(item -> { + if (ns1.equals(item.tags.get("namespace"))) { + assertEquals(item.value, 0); + } + }); + + producer.close(); + producer2.close(); + producer3.close(); } @Test diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java index f3b3944b5fdc3..de016b0cf8f8a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java @@ -76,6 +76,9 @@ public class TopicStatsImpl implements TopicStats { /** Get estimated total unconsumed or backlog size in bytes. */ public long backlogSize; + /** The number of times the publishing rate limit was triggered. */ + public long publishRateLimitedTimes; + /** Space used to store the offloaded messages for the topic/. */ public long offloadedStorageSize; @@ -160,6 +163,7 @@ public void reset() { this.lastOffloadLedgerId = 0; this.lastOffloadFailureTimeStamp = 0; this.lastOffloadSuccessTimeStamp = 0; + this.publishRateLimitedTimes = 0L; this.compaction.reset(); } @@ -182,6 +186,7 @@ public TopicStatsImpl add(TopicStats ts) { this.averageMsgSize = newAverageMsgSize; this.storageSize += stats.storageSize; this.backlogSize += stats.backlogSize; + this.publishRateLimitedTimes += stats.publishRateLimitedTimes; this.offloadedStorageSize += stats.offloadedStorageSize; this.nonContiguousDeletedMessagesRanges += stats.nonContiguousDeletedMessagesRanges; this.nonContiguousDeletedMessagesRangesSerializedSize += stats.nonContiguousDeletedMessagesRangesSerializedSize; From 817e617dd1cdf0ced0ca98d811c39ea782e212c8 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Tue, 31 May 2022 11:14:48 +0200 Subject: [PATCH 584/823] [fix][python]Fix generated Python protobuf code not compatible with latest protobuf package (#15846) * [fix][python]Fix generated Python protobuf code not compatible with latest protobuf package Protobuf latest 4.21 (v21) version broke compatibility with files generated with protoc < 3.19 This fix downgrades protobuf python package to 3.20.1. See https://developers.google.com/protocol-buffers/docs/news/2022-05-06 * Limit protobuf version to 3.20.* in setup.py Co-authored-by: Lari Hotari (cherry picked from commit 7800fbd5d8cdbeb307f98a8d9408bb5309f41c0d) (cherry picked from commit 7fb23cf735c5994c427e290d74d87e1e033cf80c) --- pulsar-client-cpp/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-cpp/python/setup.py b/pulsar-client-cpp/python/setup.py index d012dfc13df59..3f172fe3d212a 100644 --- a/pulsar-client-cpp/python/setup.py +++ b/pulsar-client-cpp/python/setup.py @@ -82,7 +82,7 @@ def build_extension(self, ext): # functions dependencies extras_require["functions"] = sorted( { - "protobuf>=3.6.1", + "protobuf>=3.6.1,<=3.20.*", "grpcio<1.28,>=1.8.2", "apache-bookkeeper-client>=4.9.2", "prometheus_client", From 0cdf66ab7fc1a3681edf5776fe9bf817274bad96 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 1 Jun 2022 00:00:01 -0500 Subject: [PATCH 585/823] Switch to rely on Netty for Hostname Verification (#15824) * Switch to relying on Netty for Hostname Verification - Add "subjectAltName = DNS:localhost, IP:127.0.0.1" to unit test certs Co-authored-by: Lari Hotari (cherry picked from commit aa7700dbf45303fab8c874bd9e5fcf95745d2777) --- .../authentication/tls/broker-cert.pem | 74 ++++++------ .../resources/authentication/tls/cacert.pem | 110 +++++++++--------- .../authentication/tls/client-cert.pem | 74 ++++++------ build/regenerate_certs_for_tests.sh | 25 ++-- ...enticationTlsHostnameVerificationTest.java | 34 ++---- .../internal/http/AsyncHttpConnector.java | 9 +- .../apache/pulsar/client/impl/ClientCnx.java | 48 -------- .../apache/pulsar/client/impl/HttpClient.java | 1 + .../client/impl/PulsarChannelInitializer.java | 7 ++ .../util/NettyClientSslContextRefresher.java | 3 +- .../pulsar/common/util/SecurityUtility.java | 10 ++ .../proxy/server/AdminProxyHandler.java | 7 +- .../proxy/server/DirectProxyHandler.java | 101 +++++++++++----- .../pulsar/proxy/server/ProxyConnection.java | 9 +- .../server/ServiceChannelInitializer.java | 66 +---------- .../server/ProxyWithAuthorizationTest.java | 70 ++++++----- .../broker-cacert.pem | 110 +++++++++--------- .../broker-cert.pem | 74 ++++++------ .../client-cacert.pem | 110 +++++++++--------- .../client-cert.pem | 74 ++++++------ .../proxy-cacert.pem | 110 +++++++++--------- .../ProxyWithAuthorizationTest/proxy-cert.pem | 74 ++++++------ .../resources/authentication/tls/cacert.pem | 110 +++++++++--------- .../authentication/tls/client-cert.pem | 74 ++++++------ .../authentication/tls/server-cert.pem | 74 ++++++------ 25 files changed, 714 insertions(+), 744 deletions(-) diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem index 7f9effa6e92d3..e9be840d3a083 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:76 + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:05 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: a0:1a:81:9d:d2:e1:66:dd:c4:cc:fc:63:04:ac:ec: a7:35 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 3a:38:c8:85:48:ed:84:c9:f4:bc:ef:b4:4b:a1:46:9c:97:9b: - 5f:7e:1a:ff:9b:dc:93:0e:7e:ab:de:09:21:30:1f:7f:2a:f7: - 94:d1:b3:07:3d:b1:71:4f:72:90:1f:41:3d:fe:34:14:ac:5a: - 39:02:f1:a4:8a:d1:d3:c0:48:da:6f:37:dc:b5:1d:60:29:e6: - c5:b0:ce:b4:52:8d:f6:6b:59:0b:e4:c8:f1:1a:40:3a:4f:bd: - e2:dd:32:2f:21:3c:33:d7:61:5f:86:cd:94:31:31:f1:ff:c6: - 08:9e:67:bc:8f:9d:bf:38:a8:8c:ff:3f:1f:fb:24:ab:bb:7c: - fb:1b:c3:1b:62:b4:dd:21:d3:7b:19:92:16:b7:7d:f6:95:ee: - 14:a0:83:de:c5:05:d8:af:44:1d:f7:eb:32:e2:03:ac:c9:12: - df:11:b6:af:f8:b9:24:ae:55:3e:25:ae:2a:b2:d3:b6:6a:e9: - f9:28:e6:e0:46:98:66:2c:0d:a3:fe:c7:82:48:13:80:f2:b2: - d1:5c:7d:bb:11:1c:60:62:1b:f7:1a:11:e1:ee:29:70:f1:95: - c1:67:c4:f1:e2:d5:f4:24:49:0d:6e:2f:65:7b:48:cd:40:f9: - c9:26:a3:c7:41:20:d1:6e:2c:38:8e:1b:bc:93:fa:22:39:3d: - 2a:f6:ba:77 + 88:1d:a7:42:a1:1c:87:45:4a:e6:5e:aa:9c:7b:71:2e:5c:9e: + 11:85:0f:a3:c5:b4:ea:73:9e:b7:61:9d:4a:e9:cd:1a:c5:2e: + 03:be:a3:2b:b6:12:6a:15:03:04:3f:fb:4a:09:0d:84:0e:dd: + c0:63:2b:0f:13:fb:1f:98:64:49:48:e7:96:d5:41:c4:ca:94: + bf:ab:c5:ea:80:2c:ee:1f:ab:12:54:74:f1:f1:56:ea:03:c0: + 1c:0d:8d:b9:6e:b0:d0:5f:21:c1:d3:e3:45:df:cf:64:69:13: + 6c:54:79:06:7d:53:46:77:3c:21:cc:c4:6a:5f:f9:9a:07:0f: + a5:95:20:f0:0e:93:07:48:96:a9:2c:28:50:21:d7:f8:13:4f: + b8:ca:aa:1f:a6:41:7c:71:1f:ad:11:3f:3d:1e:e9:81:3c:86: + c1:af:2d:39:a0:13:9f:99:ec:9a:47:44:df:28:02:a7:1d:6a: + 8d:c0:1e:24:e8:19:fc:1d:dc:67:29:04:be:0a:d6:c5:81:59: + 27:2c:f5:e5:df:ba:0b:c6:50:e5:b3:bd:73:12:3e:2c:ef:a6: + 8a:ed:eb:86:9a:45:45:52:a3:44:78:12:60:17:e2:3a:32:92: + 03:6e:89:89:16:c5:e0:bc:be:a7:cb:93:4b:d8:56:33:a0:a0: + 53:b2:0d:a5 -----BEGIN CERTIFICATE----- -MIIC7zCCAdcCFAwmFd+PcR1qMdDar2TvgN6smkZ2MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFcxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYw -FAYDVQQLEw1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv -1f0fZ+ORA5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICW -yR8sxFXro3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHa -kroqKO4wKD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CF -gNlo5sZTA3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPX -zDhsg915JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAEw -DQYJKoZIhvcNAQELBQADggEBADo4yIVI7YTJ9LzvtEuhRpyXm19+Gv+b3JMOfqve -CSEwH38q95TRswc9sXFPcpAfQT3+NBSsWjkC8aSK0dPASNpvN9y1HWAp5sWwzrRS -jfZrWQvkyPEaQDpPveLdMi8hPDPXYV+GzZQxMfH/xgieZ7yPnb84qIz/Px/7JKu7 -fPsbwxtitN0h03sZkha3ffaV7hSgg97FBdivRB336zLiA6zJEt8Rtq/4uSSuVT4l -riqy07Zq6fko5uBGmGYsDaP+x4JIE4DystFcfbsRHGBiG/caEeHuKXDxlcFnxPHi -1fQkSQ1uL2V7SM1A+ckmo8dBINFuLDiOG7yT+iI5PSr2unc= +MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgUwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj +aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCWxvY2FsaG9zdDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+/ty2YrZ322qMT1GIPmL4c +ookium/V/R9n45EDmICBDu3Y9nB/LDZoPVPqWDqm1YlmS70eV3ETbUsR5UCldoQk +kkBYgJbJHyzEVeujeXNwXDeaie0vumvjgnxpSgJUi4FePL9MisvqLF6D57cQCF+C +WKOJ0dqSuioo7jAoP1uuEHGWx+ESxbAarURvRDoRSpo8D40GgHs07z9s9F7FRFQe +yN3HgIWA2WjmxlMDd+H+GGEHdwVM7Vm8XUE4au9dobJgmNRIKJUCig79z3sb0hHM +EAxQc9fMOGyD3XkmqpDIm4SGvFnpYmn0mBvEgHh+oBqBndLhZt3EzPxjBKzspzUC +AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQCIHadCoRyHRUrmXqqce3EuXJ4RhQ+jxbTqc563YZ1K6c0axS4DvqMr +thJqFQMEP/tKCQ2EDt3AYysPE/sfmGRJSOeW1UHEypS/q8XqgCzuH6sSVHTx8Vbq +A8AcDY25brDQXyHB0+NF389kaRNsVHkGfVNGdzwhzMRqX/maBw+llSDwDpMHSJap +LChQIdf4E0+4yqofpkF8cR+tET89HumBPIbBry05oBOfmeyaR0TfKAKnHWqNwB4k +6Bn8HdxnKQS+CtbFgVknLPXl37oLxlDls71zEj4s76aK7euGmkVFUqNEeBJgF+I6 +MpIDbomJFsXgvL6ny5NL2FYzoKBTsg2l -----END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem index 90fbb9b8898fb..21bbaba213f69 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem @@ -2,76 +2,76 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - 10:50:a0:5c:8e:cf:88:33:b6:b5:d2:1e:38:bf:78:56:2a:f1:09:22 + 70:4c:6b:e0:aa:cc:01:77:f2:1f:04:8c:d4:72:03:a5:32:5f:c7:be Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: CN = CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: - 00:c4:92:ca:40:ce:8d:71:dd:e9:2b:e3:3b:b7:17: - 1d:25:bf:12:66:c0:cb:32:18:32:3e:24:ea:e1:26: - 1a:97:e8:85:4b:19:8e:c0:0a:da:a6:57:ec:31:a6: - a8:68:d9:8e:5c:a2:00:54:30:11:47:a6:0e:84:0d: - 6d:e3:48:a8:a6:e3:42:63:97:ef:91:c0:3a:bc:db: - 77:77:3b:d0:45:fc:c5:a8:3a:74:dc:82:4e:83:ed: - f9:9d:a0:30:11:0c:d9:20:7b:a6:04:60:a1:9c:41: - 33:c6:04:d2:a7:e8:b1:46:e6:35:5e:fd:ca:2e:42: - 2f:f4:0c:f7:6e:8d:60:f5:cf:82:7a:e3:eb:ed:d0: - a1:51:a9:78:8d:14:2d:ca:ea:cc:fa:ae:a9:f9:6c: - df:5c:cb:83:4a:42:22:5c:48:3e:a6:63:70:43:63: - ff:3f:d8:1f:88:e1:91:7b:49:b9:67:10:8a:60:51: - 24:68:db:68:24:5f:10:a5:a2:b3:95:83:7e:3c:88: - 9c:1c:52:6a:2c:03:52:aa:90:90:85:21:78:a7:20: - b0:e2:dc:79:b4:b7:57:f0:be:df:3b:fc:21:23:ee: - ff:63:5d:0b:0d:3d:ab:61:54:8c:2d:96:44:7b:42: - 10:60:3b:1d:a8:ab:33:01:e7:96:74:08:a6:f9:9d: - ba:cf + 00:dc:9c:01:30:5f:c5:42:48:10:78:30:5d:66:20: + 0e:74:61:f6:82:74:9f:6f:b2:ed:00:9e:6c:21:b6: + 83:21:6b:54:34:e8:a9:dc:81:83:7a:0e:9f:cc:3d: + eb:97:ee:cf:ca:0e:5f:96:81:dc:e7:75:88:91:2f: + d5:65:74:c2:d8:67:58:d8:41:6a:5f:a9:79:dc:29: + 36:4a:b8:39:20:d2:f8:a8:59:9f:e3:be:f9:61:80: + 1b:ce:63:bb:12:56:06:b9:77:4e:6a:40:65:9b:bf: + 5b:f8:27:88:f5:ff:40:ee:47:bc:2d:8e:c3:a6:62: + 0d:18:76:d1:f5:af:1a:6b:25:4e:d4:55:15:f0:e3: + 97:1b:68:eb:75:b8:80:ea:64:ef:7e:e2:f0:5c:da: + 6d:d6:16:7b:0f:5e:ae:72:47:5a:df:0b:8a:e0:74: + c1:b7:82:0d:97:41:d7:84:16:51:40:37:15:a1:eb: + 70:0c:f1:5a:26:39:11:1e:97:b9:36:32:ce:16:b9: + 42:ad:31:5b:1e:89:f5:3e:07:0e:d6:fc:9a:46:8e: + 87:89:90:5c:f3:00:e4:9b:ce:7b:93:fe:9a:d8:65: + ec:49:5c:e8:eb:41:3d:53:bc:ce:e8:6d:44:ec:76: + 3f:e6:9b:13:e4:f8:d0:1c:00:e6:4f:73:e1:b0:27: + 6f:99 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - C6:91:71:A0:C9:1F:A9:5A:87:7B:E5:10:FB:9A:2A:12:90:44:7D:A0 + 8B:30:D2:81:7C:BE:AB:4D:76:37:19:2B:69:5E:DB:F7:81:95:73:F5 X509v3 Authority Key Identifier: - keyid:C6:91:71:A0:C9:1F:A9:5A:87:7B:E5:10:FB:9A:2A:12:90:44:7D:A0 + keyid:8B:30:D2:81:7C:BE:AB:4D:76:37:19:2B:69:5E:DB:F7:81:95:73:F5 X509v3 Basic Constraints: critical CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 5d:c2:68:9e:66:fb:67:39:fc:5e:2f:ba:4c:f0:20:3f:f9:4a: - e2:b9:05:56:d6:5e:da:01:c7:8b:1a:70:e6:67:61:84:71:67: - a8:11:bc:7c:4d:58:d0:52:44:71:19:47:87:60:cb:16:12:25: - b2:b0:95:13:ff:52:00:36:78:2d:d3:ce:4e:c6:7d:1b:e5:8e: - 37:23:8a:ef:c2:44:88:e2:bc:47:c4:ef:23:f5:8b:6d:fc:39: - 3c:cb:7e:70:7c:60:51:33:5a:38:3a:fd:cc:8f:2c:08:d5:07: - 06:f9:89:77:96:8e:60:21:e5:05:98:37:d6:c4:b7:a3:43:9e: - 87:13:9d:12:c4:8f:6a:ad:a9:67:c4:3a:7e:14:77:c3:75:72: - 95:e6:25:a2:14:e7:77:4d:8f:dd:45:ae:f0:f6:f3:fe:2b:cf: - ea:0e:f8:61:66:45:db:9f:6b:e4:5e:b8:d4:04:41:68:e9:7c: - a4:7e:c8:1c:4d:ec:49:49:57:a4:46:95:e8:0f:55:ea:08:2e: - b9:7a:62:e2:be:05:00:d5:81:5f:60:60:58:4e:19:bc:24:ee: - 0e:17:63:da:fd:40:44:c2:5f:7d:e9:26:b4:80:4d:db:88:4f: - 31:a4:16:93:fd:a8:70:94:50:f1:23:92:20:fb:26:c3:9a:71: - b1:9c:c9:db + 02:4c:80:4f:a4:b5:f4:70:be:82:cf:3a:ed:40:f9:97:17:22: + 07:5d:e0:9b:4e:54:f8:4b:64:99:f5:07:7f:87:5b:9c:60:ec: + 9f:69:e6:00:97:5a:cd:14:59:31:45:be:b7:bd:c4:ce:57:82: + 1a:4a:62:ce:8e:c8:59:d5:62:43:8b:94:c0:ab:c2:cc:3a:a0: + 69:d3:65:15:82:35:de:85:64:e6:7b:d9:3a:22:12:77:f7:71: + 82:86:d7:6c:e5:69:d5:3a:f2:a7:25:f7:dc:f3:6f:cb:eb:85: + 48:44:63:e2:6d:3c:82:eb:3a:c0:e1:bd:9d:3a:12:11:66:1f: + 05:8f:49:65:31:d6:cf:26:06:46:ba:73:c7:ad:61:fc:14:5f: + 68:d1:ee:02:5f:4b:98:b6:5b:0c:98:4e:61:7b:cb:35:ee:44: + a1:ce:e1:00:a2:56:f0:0d:72:3b:58:66:e8:9a:dc:62:d5:95: + 3e:5a:48:21:a8:7c:f8:1f:5a:13:db:53:33:11:3e:e6:14:39: + cd:2b:3f:77:5b:ee:f7:0c:59:69:2f:46:9a:34:56:89:05:8e: + 40:94:94:3f:95:f6:fa:f9:1a:e8:1a:80:7b:1d:f7:0c:a1:be: + e2:38:98:fd:0f:e7:68:4d:7d:fe:ae:5f:e3:32:c6:5d:37:77: + 7a:28:ce:cc -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUEFCgXI7PiDO2tdIeOL94VirxCSIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIxMDQyMzE3MDg1MVoXDTMxMDQyMTE3 -MDg1MVowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAxJLKQM6Ncd3pK+M7txcdJb8SZsDLMhgyPiTq4SYal+iFSxmOwAra -plfsMaaoaNmOXKIAVDARR6YOhA1t40iopuNCY5fvkcA6vNt3dzvQRfzFqDp03IJO -g+35naAwEQzZIHumBGChnEEzxgTSp+ixRuY1Xv3KLkIv9Az3bo1g9c+CeuPr7dCh -Ual4jRQtyurM+q6p+WzfXMuDSkIiXEg+pmNwQ2P/P9gfiOGRe0m5ZxCKYFEkaNto -JF8QpaKzlYN+PIicHFJqLANSqpCQhSF4pyCw4tx5tLdX8L7fO/whI+7/Y10LDT2r -YVSMLZZEe0IQYDsdqKszAeeWdAim+Z26zwIDAQABo1MwUTAdBgNVHQ4EFgQUxpFx -oMkfqVqHe+UQ+5oqEpBEfaAwHwYDVR0jBBgwFoAUxpFxoMkfqVqHe+UQ+5oqEpBE -faAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAXcJonmb7Zzn8 -Xi+6TPAgP/lK4rkFVtZe2gHHixpw5mdhhHFnqBG8fE1Y0FJEcRlHh2DLFhIlsrCV -E/9SADZ4LdPOTsZ9G+WONyOK78JEiOK8R8TvI/WLbfw5PMt+cHxgUTNaODr9zI8s -CNUHBvmJd5aOYCHlBZg31sS3o0OehxOdEsSPaq2pZ8Q6fhR3w3VyleYlohTnd02P -3UWu8Pbz/ivP6g74YWZF259r5F641ARBaOl8pH7IHE3sSUlXpEaV6A9V6gguuXpi -4r4FANWBX2BgWE4ZvCTuDhdj2v1ARMJffekmtIBN24hPMaQWk/2ocJRQ8SOSIPsm -w5pxsZzJ2w== +MIIDAzCCAeugAwIBAgIUcExr4KrMAXfyHwSM1HIDpTJfx74wDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA3JwBMF/FQkgQeDBdZiAOdGH2gnSfb7LtAJ5sIbaDIWtUNOip3IGD +eg6fzD3rl+7Pyg5floHc53WIkS/VZXTC2GdY2EFqX6l53Ck2Srg5INL4qFmf4775 +YYAbzmO7ElYGuXdOakBlm79b+CeI9f9A7ke8LY7DpmINGHbR9a8aayVO1FUV8OOX +G2jrdbiA6mTvfuLwXNpt1hZ7D16uckda3wuK4HTBt4INl0HXhBZRQDcVoetwDPFa +JjkRHpe5NjLOFrlCrTFbHon1PgcO1vyaRo6HiZBc8wDkm857k/6a2GXsSVzo60E9 +U7zO6G1E7HY/5psT5PjQHADmT3PhsCdvmQIDAQABo1MwUTAdBgNVHQ4EFgQUizDS +gXy+q012NxkraV7b94GVc/UwHwYDVR0jBBgwFoAUizDSgXy+q012NxkraV7b94GV +c/UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAkyAT6S19HC+ +gs867UD5lxciB13gm05U+EtkmfUHf4dbnGDsn2nmAJdazRRZMUW+t73EzleCGkpi +zo7IWdViQ4uUwKvCzDqgadNlFYI13oVk5nvZOiISd/dxgobXbOVp1TrypyX33PNv +y+uFSERj4m08gus6wOG9nToSEWYfBY9JZTHWzyYGRrpzx61h/BRfaNHuAl9LmLZb +DJhOYXvLNe5Eoc7hAKJW8A1yO1hm6JrcYtWVPlpIIah8+B9aE9tTMxE+5hQ5zSs/ +d1vu9wxZaS9GmjRWiQWOQJSUP5X2+vka6BqAex33DKG+4jiY/Q/naE19/q5f4zLG +XTd3eijOzA== -----END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem index e79bac70987d9..e5d9e6e74b233 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:77 + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:06 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = superUser Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: b6:98:ef:dd:03:82:58:a3:32:dc:90:a1:b6:a6:1e: e1:0b Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 6f:c2:2f:41:a4:a0:45:10:33:61:20:27:d2:74:40:f9:80:3b: - 06:88:91:c3:b8:4d:1a:c4:fd:39:9e:3a:c8:41:de:31:4e:ef: - 8b:06:ce:17:e2:8e:b5:ee:43:92:0a:44:3d:55:e9:85:81:49: - c9:19:44:15:f1:bd:ec:1e:cb:34:44:b1:01:c0:96:49:30:a4: - 5a:64:44:6e:59:d9:b1:17:bf:01:13:b7:45:53:8c:8d:a7:79: - fc:19:b4:a9:b5:9b:6f:16:8e:b3:de:5e:2a:db:01:f2:3e:b0: - 8f:23:4f:8f:49:ee:d5:b7:98:54:6e:b5:be:8b:fc:05:87:e3: - 8b:2e:70:28:2c:75:75:c3:76:a4:0d:5e:71:67:30:ec:69:cc: - 2b:43:69:3b:e8:78:89:51:98:07:cb:21:e9:7a:76:a9:b3:e8: - e6:19:e7:32:ae:3a:b8:24:c4:20:d8:c2:dc:91:99:d1:9b:8f: - 77:3c:e7:a8:53:ee:91:fe:ed:2b:86:18:0a:55:44:46:78:a1: - 78:41:a5:e9:fe:8b:db:bb:10:2e:72:52:b7:54:81:84:8b:f7: - 29:f3:86:29:7f:f8:e2:d8:51:d8:b2:3c:c2:78:7c:a4:11:9c: - 0a:42:64:1b:13:cc:91:1a:08:d9:ed:f1:23:5f:fd:b3:89:bb: - 7a:cc:96:8d + 90:62:ba:7b:6f:45:95:7a:71:2f:e7:88:0c:64:b8:6c:05:86: + 7f:47:08:ce:d6:e2:5a:32:13:0c:82:ad:a7:af:f0:a2:f7:86: + 79:87:1a:89:78:95:b1:9f:be:c5:8b:39:fd:12:94:b6:e1:69: + ff:fa:1e:c3:82:d8:6c:03:80:45:ac:1c:06:70:bb:77:c3:41: + 5f:b6:9d:fe:36:6f:ae:23:6c:bf:43:79:8e:74:85:8e:96:89: + a9:c4:6d:d9:fa:05:ba:a8:11:7c:82:45:94:3d:9f:b6:7c:2f: + 4e:6d:37:c3:fb:79:7e:0c:d2:15:fa:0e:ea:2d:c9:24:f3:34: + 13:6f:db:d7:55:e1:0c:2f:7e:fe:4c:3b:fa:7e:03:26:0f:6a: + 95:d2:22:ce:27:71:6a:97:ac:36:0a:20:ec:19:a0:78:23:0c: + 54:f3:b1:dd:33:36:7c:b7:61:23:70:8f:7f:c8:5f:e8:9e:b5: + 02:31:4d:b3:40:b0:7b:b2:ee:14:a7:69:22:8b:38:85:5d:04: + 6e:d5:44:41:31:a7:4b:71:86:fb:81:cd:3d:db:96:23:0b:bc: + e1:67:46:0e:87:86:91:4e:1a:35:37:af:a4:ac:9a:de:e3:4f: + 82:47:f1:c4:16:58:11:8f:76:d2:4d:df:a1:c6:a2:8f:33:6d: + 72:15:28:76 -----BEGIN CERTIFICATE----- -MIIC7zCCAdcCFAwmFd+PcR1qMdDar2TvgN6smkZ3MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFcxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYw -FAYDVQQLEw1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1o -sXBGO95E+RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgF -hMrmDCESWDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5Xi -Nr3brotaaGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36 -ukqN6MZaoVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpX -ABuREzH9QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAG/CL0GkoEUQM2EgJ9J0QPmAOwaIkcO4TRrE/Tme -OshB3jFO74sGzhfijrXuQ5IKRD1V6YWBSckZRBXxveweyzREsQHAlkkwpFpkRG5Z -2bEXvwETt0VTjI2nefwZtKm1m28WjrPeXirbAfI+sI8jT49J7tW3mFRutb6L/AWH -44sucCgsdXXDdqQNXnFnMOxpzCtDaTvoeIlRmAfLIel6dqmz6OYZ5zKuOrgkxCDY -wtyRmdGbj3c856hT7pH+7SuGGApVREZ4oXhBpen+i9u7EC5yUrdUgYSL9ynzhil/ -+OLYUdiyPMJ4fKQRnApCZBsTzJEaCNnt8SNf/bOJu3rMlo0= +MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgYwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj +aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCXN1cGVyVXNlcjCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1DfZhA+bBbvK7bwAutJpCW +4GI47WixcEY73kT5FFGGEOvKkOeI6PmRheDdtbQUuXjjhtVUbWjsFJK0+CJbBT3t +MSVlCAWEyuYMIRJYMscaYKNP0kqeKBl8RYQAjInc3orlT4iRzKTxgUVMfcL/4sGJ +xhJzleI2vduui1poapBR3iuIX6pn9KjjY9y+GYLMnX/mjfuCviIBPVYTO1sEtOjF +GOYuDfq6So3oxlqhUZpKYtev3bT84tXNrplsXGFWC9cMGndc9TpqVLWeM6ypdSia +dq/QelcAG5ETMf1CiCFHBRABL1m7xzrZ4VhMG2xxtpjv3QOCWKMy3JChtqYe4QsC +AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQCQYrp7b0WVenEv54gMZLhsBYZ/RwjO1uJaMhMMgq2nr/Ci94Z5hxqJ +eJWxn77Fizn9EpS24Wn/+h7DgthsA4BFrBwGcLt3w0Fftp3+Nm+uI2y/Q3mOdIWO +lompxG3Z+gW6qBF8gkWUPZ+2fC9ObTfD+3l+DNIV+g7qLckk8zQTb9vXVeEML37+ +TDv6fgMmD2qV0iLOJ3Fql6w2CiDsGaB4IwxU87HdMzZ8t2EjcI9/yF/onrUCMU2z +QLB7su4Up2kiiziFXQRu1URBMadLcYb7gc0925YjC7zhZ0YOh4aRTho1N6+krJre +40+CR/HEFlgRj3bSTd+hxqKPM21yFSh2 -----END CERTIFICATE----- diff --git a/build/regenerate_certs_for_tests.sh b/build/regenerate_certs_for_tests.sh index 7e4cf8474e234..fb0274cc19316 100755 --- a/build/regenerate_certs_for_tests.sh +++ b/build/regenerate_certs_for_tests.sh @@ -34,7 +34,7 @@ function reissue_certificate() { keyfile=$1 certfile=$2 openssl x509 -x509toreq -in $certfile -signkey $keyfile -out ${certfile}.csr - openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -out $certfile -days 3650 -CAcreateserial + openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -out $certfile -days 3650 -CAcreateserial -extfile <(printf "subjectAltName = DNS:localhost, IP:127.0.0.1") } generate_ca @@ -44,6 +44,16 @@ reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/server-key.pem \ $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem +# use same CA key and cert for ProxyWithAuthorizationTest/client-cacert.pem +cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem +reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-key.pem \ + $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem + +# use same CA key and cert for ProxyWithAuthorizationTest/proxy-cacert.pem +cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem +reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem \ + $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem + generate_ca cp ca-cert.pem $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem \ @@ -56,18 +66,5 @@ cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/Prox reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-key.pem \ $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem -generate_ca -cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem -reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-key.pem \ - $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem - -generate_ca -cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem -reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem \ - $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem - - - - cd $ROOT_DIR rm -rf /tmp/keygendir$$ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java index bb8a02143e5e7..157b35a8aa9ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java @@ -18,8 +18,7 @@ */ package org.apache.pulsar.client.api; -import static org.mockito.Mockito.spy; - +import com.google.common.collect.Sets; import java.lang.reflect.Method; import java.util.HashMap; import java.util.HashSet; @@ -27,16 +26,12 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; - import org.apache.pulsar.broker.authentication.AuthenticationProviderBasic; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.auth.AuthenticationTls; -import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.tls.PublicSuffixMatcher; import org.apache.pulsar.common.tls.TlsHostnameVerifier; import org.apache.pulsar.common.policies.data.ClusterDataImpl; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -44,8 +39,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.google.common.collect.Sets; - @Test(groups = "broker-api") public class AuthenticationTlsHostnameVerificationTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(AuthenticationTlsHostnameVerificationTest.class); @@ -65,8 +58,13 @@ public class AuthenticationTlsHostnameVerificationTest extends ProducerConsumerB private final String BASIC_CONF_FILE_PATH = "./src/test/resources/authentication/basic/.htpasswd"; private boolean hostnameVerificationEnabled = true; + private String clientTrustCertFilePath = TLS_TRUST_CERT_FILE_PATH; protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + super.stopBroker(); + if (methodName.equals("testAnonymousSyncProducerAndConsumer")) { conf.setAnonymousUserRole("anonymousUser"); } @@ -74,7 +72,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); - conf.setTlsAllowInsecureConnection(true); + conf.setTlsAllowInsecureConnection(false); Set superUserRoles = new HashSet<>(); superUserRoles.add("localhost"); @@ -96,7 +94,7 @@ protected void setup() throws Exception { conf.setClusterName("test"); conf.setNumExecutorThreadPoolSize(5); - super.init(); + startBroker(); setupClient(); } @@ -109,22 +107,11 @@ protected void setupClient() throws Exception { Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); - admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_MIM_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true) - .authentication(authTls).build()); replacePulsarClient(PulsarClient.builder() .serviceUrl(pulsar.getBrokerServiceUrlTls()) .statsInterval(0, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_MIM_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true) + .tlsTrustCertsFilePath(clientTrustCertFilePath) .authentication(authTls).enableTls(true).enableTlsHostnameVerification(hostnameVerificationEnabled)); - - admin.clusters().createCluster("test", ClusterData.builder() - .serviceUrl(brokerUrl.toString()) - .build()); - - admin.tenants().createTenant("my-property", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); - admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test")); } @AfterMethod(alwaysRun = true) @@ -157,10 +144,11 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname log.info("-- Starting {} test --", methodName); this.hostnameVerificationEnabled = hostnameVerificationEnabled; + clientTrustCertFilePath = TLS_MIM_TRUST_CERT_FILE_PATH; // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setTlsTrustCertsFilePath(TLS_MIM_TRUST_CERT_FILE_PATH); + conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH); conf.setBrokerClientAuthenticationParameters( diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java index 95ea0717b97e8..2b08bfc0048de 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java @@ -129,7 +129,7 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, params != null ? params.getKeyStoreType() : null, params != null ? params.getKeyStorePath() : null, params != null ? params.getKeyStorePassword() : null, - conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), + conf.isTlsAllowInsecureConnection(), conf.getTlsTrustStoreType(), conf.getTlsTrustStorePath(), conf.getTlsTrustStorePassword(), @@ -148,12 +148,12 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, sslCtx = authData.getTlsTrustStoreStream() == null ? SecurityUtility.createAutoRefreshSslContextForClient( sslProvider, - conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), + conf.isTlsAllowInsecureConnection(), conf.getTlsTrustCertsFilePath(), authData.getTlsCerificateFilePath(), authData.getTlsPrivateKeyFilePath(), null, autoCertRefreshTimeSeconds, delayer) : SecurityUtility.createNettySslContextForClient( sslProvider, - conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), + conf.isTlsAllowInsecureConnection(), authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), authData.getTlsPrivateKey(), conf.getTlsCiphers(), @@ -161,7 +161,7 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, } else { sslCtx = SecurityUtility.createNettySslContextForClient( sslProvider, - conf.isTlsAllowInsecureConnection() || !conf.isTlsHostnameVerificationEnable(), + conf.isTlsAllowInsecureConnection(), conf.getTlsTrustCertsFilePath(), conf.getTlsCiphers(), conf.getTlsProtocols()); @@ -169,6 +169,7 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, confBuilder.setSslContext(sslCtx); } } + confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); } httpClient = new DefaultAsyncHttpClient(confBuilder.build()); this.readTimeout = Duration.ofMillis(readTimeoutMs); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 3b71f6a62228d..20325ade4f6b1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -26,12 +26,10 @@ import com.google.common.collect.Queues; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.EventLoopGroup; import io.netty.channel.unix.Errors.NativeIoException; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; -import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.Promise; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -48,7 +46,6 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import javax.net.ssl.SSLSession; import lombok.Getter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Pair; @@ -156,9 +153,6 @@ public class ClientCnx extends PulsarHandler { protected String proxyToTargetBrokerAddress = null; // Remote hostName with which client is connected protected String remoteHostName = null; - private boolean isTlsHostnameVerificationEnable; - - private static final TlsHostnameVerifier HOSTNAME_VERIFIER = new TlsHostnameVerifier(); private ScheduledFuture timeoutTask; private SocketAddress localAddress; @@ -224,7 +218,6 @@ public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, in this.maxNumberOfRejectedRequestPerConnection = conf.getMaxNumberOfRejectedRequestPerConnection(); this.operationTimeoutMs = conf.getOperationTimeoutMs(); this.state = State.None; - this.isTlsHostnameVerificationEnable = conf.isTlsHostnameVerificationEnable(); this.protocolVersion = protocolVersion; } @@ -325,14 +318,6 @@ public static boolean isKnownException(Throwable t) { @Override protected void handleConnected(CommandConnected connected) { - - if (isTlsHostnameVerificationEnable && remoteHostName != null && !verifyTlsHostName(remoteHostName, ctx)) { - // close the connection if host-verification failed with the broker - log.warn("[{}] Failed to verify hostname of {}", ctx.channel(), remoteHostName); - ctx.close(); - return; - } - checkArgument(state == State.SentConnectFrame || state == State.Connecting); if (connected.hasMaxMessageSize()) { if (log.isDebugEnabled()) { @@ -1084,39 +1069,6 @@ private void incrementRejectsAndMaybeClose() { } } - /** - * verifies host name provided in x509 Certificate in tls session - * - * it matches hostname with below scenarios - * - * 
    -     *  1. Supports IPV4 and IPV6 host matching
-     *  2. Supports wild card matching for DNS-name
-     *  eg:
-     *     HostName                     CN           Result
-     * 1.  localhost                    localhost    PASS
-     * 2.  localhost                    local*       PASS
-     * 3.  pulsar1-broker.com           pulsar*.com  PASS
-     *
    - * - * @param ctx - * @return true if hostname is verified else return false - */ - private boolean verifyTlsHostName(String hostname, ChannelHandlerContext ctx) { - ChannelHandler sslHandler = ctx.channel().pipeline().get("tls"); - - SSLSession sslSession = null; - if (sslHandler != null) { - sslSession = ((SslHandler) sslHandler).engine().getSession(); - if (log.isDebugEnabled()) { - log.debug("Verifying HostName for {}, Cipher {}, Protocols {}", hostname, sslSession.getCipherSuite(), - sslSession.getProtocol()); - } - return HOSTNAME_VERIFIER.verify(hostname, sslSession); - } - return false; - } - void registerConsumer(final long consumerId, final ConsumerImpl consumer) { consumers.put(consumerId, consumer); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index 285a7202c7206..323f3bcad5e70 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -137,6 +137,7 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, } confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); + confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); } catch (GeneralSecurityException e) { throw new PulsarClientException.InvalidConfigurationException(e); } catch (Exception e) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index 497793d792d8e..bac1cd9ba419f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -51,6 +51,7 @@ public class PulsarChannelInitializer extends ChannelInitializer private final Supplier clientCnxSupplier; @Getter private final boolean tlsEnabled; + private final boolean tlsHostnameVerificationEnabled; private final boolean tlsEnabledWithKeyStore; private final InetSocketAddress socks5ProxyAddress; private final String socks5ProxyUsername; @@ -66,6 +67,7 @@ public PulsarChannelInitializer(ClientConfigurationData conf, Supplier initTls(Channel ch, InetSocketAddress sniHost) { ? new SslHandler(nettySSLContextAutoRefreshBuilder.get() .createSSLEngine(sniHost.getHostString(), sniHost.getPort())) : sslContextSupplier.get().newHandler(ch.alloc(), sniHost.getHostString(), sniHost.getPort()); + + if (tlsHostnameVerificationEnabled) { + SecurityUtility.configureSSLHandler(handler); + } + ch.pipeline().addFirst(TLS_HANDLER, handler); initTlsFuture.complete(ch); } catch (Throwable t) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java index e1fef9aaa9b10..9e050b7058d8b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java @@ -49,8 +49,7 @@ public NettyClientSslContextRefresher(SslProvider sslProvider, boolean allowInse AuthenticationDataProvider authData, Set ciphers, Set protocols, - long delayInSeconds) - throws IOException, GeneralSecurityException { + long delayInSeconds) { super(delayInSeconds); this.tlsAllowInsecureConnection = allowInsecure; this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index 6b715bbf6dc9d..5abad5924c4a8 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -21,6 +21,7 @@ import io.netty.handler.ssl.ClientAuth; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslProvider; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import java.io.BufferedReader; @@ -57,7 +58,9 @@ import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; +import javax.net.ssl.SSLParameters; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import lombok.extern.slf4j.Slf4j; @@ -549,6 +552,13 @@ private static void setupClientAuthentication(SslContextBuilder builder, } } + public static void configureSSLHandler(SslHandler handler) { + SSLEngine sslEngine = handler.engine(); + SSLParameters sslParameters = sslEngine.getSSLParameters(); + sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); + sslEngine.setSSLParameters(sslParameters); + } + public static Provider resolveProvider(String providerName) throws NoSuchAlgorithmException { Provider provider = null; if (!StringUtils.isEmpty(providerName)) { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index d4eb47742682d..bd1bad27b2b6a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -280,10 +280,11 @@ protected HttpClient newHttpClient() { ); } - - SslContextFactory contextFactory = new SslContextFactory.Client(true); + SslContextFactory contextFactory = new SslContextFactory.Client(); contextFactory.setSslContext(sslCtx); - + if (!config.isTlsHostnameVerificationEnabled()) { + contextFactory.setEndpointIdentificationAlgorithm(null); + } return new JettyHttpClient(contextFactory); } catch (Exception e) { LOG.error("new jetty http client exception ", e); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 24802f60a3db0..8ffcdb0acd5c1 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -21,13 +21,13 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static org.apache.commons.lang3.StringUtils.isEmpty; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; @@ -37,18 +37,19 @@ import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.handler.codec.haproxy.HAProxyProtocolVersion; import io.netty.handler.codec.haproxy.HAProxyProxiedProtocol; +import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import javax.net.ssl.SSLSession; import lombok.Getter; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.AuthData; @@ -57,7 +58,10 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarDecoder; import org.apache.pulsar.common.stats.Rate; -import org.apache.pulsar.common.tls.TlsHostnameVerifier; +import org.apache.pulsar.common.util.NettyClientSslContextRefresher; +import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,6 +83,11 @@ public class DirectProxyHandler { private AuthenticationDataProvider authenticationDataProvider; private final ProxyService service; private final Runnable onHandshakeCompleteAction; + private final boolean tlsHostnameVerificationEnabled; + private final boolean tlsEnabledWithKeyStore; + private final boolean tlsEnabledWithBroker; + private final SslContextAutoRefreshBuilder clientSslCtxRefresher; + private final NettySSLContextAutoRefreshBuilder clientSSLContextAutoRefreshBuilder; public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) { this.service = service; @@ -89,11 +98,59 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) this.originalPrincipal = proxyConnection.clientAuthRole; this.clientAuthData = proxyConnection.clientAuthData; this.clientAuthMethod = proxyConnection.clientAuthMethod; + this.tlsEnabledWithBroker = service.getConfiguration().isTlsEnabledWithBroker(); + this.tlsHostnameVerificationEnabled = service.getConfiguration().isTlsHostnameVerificationEnabled(); + this.tlsEnabledWithKeyStore = service.getConfiguration().isTlsEnabledWithKeyStore(); this.onHandshakeCompleteAction = proxyConnection::cancelKeepAliveTask; + ProxyConfiguration config = service.getConfiguration(); + + if (tlsEnabledWithBroker) { + AuthenticationDataProvider authData = null; + + if (!isEmpty(config.getBrokerClientAuthenticationPlugin())) { + try { + authData = AuthenticationFactory.create(config.getBrokerClientAuthenticationPlugin(), + config.getBrokerClientAuthenticationParameters()).getAuthData(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + + if (tlsEnabledWithKeyStore) { + clientSSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( + config.getBrokerClientSslProvider(), + config.isTlsAllowInsecureConnection(), + config.getBrokerClientTlsTrustStoreType(), + config.getBrokerClientTlsTrustStore(), + config.getBrokerClientTlsTrustStorePassword(), + config.getBrokerClientTlsCiphers(), + config.getBrokerClientTlsProtocols(), + config.getTlsCertRefreshCheckDurationSec(), + authData); + clientSslCtxRefresher = null; + } else { + SslProvider sslProvider = null; + if (config.getBrokerClientSslProvider() != null) { + sslProvider = SslProvider.valueOf(config.getBrokerClientSslProvider()); + } + clientSslCtxRefresher = new NettyClientSslContextRefresher( + sslProvider, + config.isTlsAllowInsecureConnection(), + config.getBrokerClientTrustCertsFilePath(), + authData, + config.getBrokerClientTlsCiphers(), + config.getBrokerClientTlsProtocols(), + config.getTlsCertRefreshCheckDurationSec() + ); + clientSSLContextAutoRefreshBuilder = null; + } + } else { + clientSSLContextAutoRefreshBuilder = null; + clientSslCtxRefresher = null; + } } - public void connect(String brokerHostAndPort, InetSocketAddress targetBrokerAddress, - int protocolVersion, Supplier sslHandlerSupplier) { + public void connect(String brokerHostAndPort, InetSocketAddress targetBrokerAddress, int protocolVersion) { ProxyConfiguration config = service.getConfiguration(); // Start the connection attempt. @@ -121,8 +178,16 @@ public void connect(String brokerHostAndPort, InetSocketAddress targetBrokerAddr b.handler(new ChannelInitializer() { @Override protected void initChannel(SocketChannel ch) { - if (sslHandlerSupplier != null) { - ch.pipeline().addLast(TLS_HANDLER, sslHandlerSupplier.get()); + if (tlsEnabledWithBroker) { + String host = targetBrokerAddress.getHostString(); + int port = targetBrokerAddress.getPort(); + SslHandler handler = tlsEnabledWithKeyStore + ? new SslHandler(clientSSLContextAutoRefreshBuilder.get().createSSLEngine(host, port)) + : clientSslCtxRefresher.get().newHandler(ch.alloc(), host, port); + if (tlsHostnameVerificationEnabled) { + SecurityUtility.configureSSLHandler(handler); + } + ch.pipeline().addLast(TLS_HANDLER, handler); } int brokerProxyReadTimeoutMs = service.getConfiguration().getBrokerProxyReadTimeoutMs(); if (brokerProxyReadTimeoutMs > 0) { @@ -338,15 +403,6 @@ protected void handleConnected(CommandConnected connected) { log.debug("[{}] [{}] Received Connected from broker", inboundChannel, outboundChannel); } - if (config.isTlsHostnameVerificationEnabled() && remoteHostName != null - && !verifyTlsHostName(remoteHostName, ctx)) { - // close the connection if host-verification failed with the - // broker - log.warn("[{}] Failed to verify hostname of {}", ctx.channel(), remoteHostName); - ctx.close(); - return; - } - state = BackendState.HandshakeCompleted; onHandshakeCompleteAction.run(); @@ -409,17 +465,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { log.warn("[{}] [{}] Caught exception: {}", inboundChannel, outboundChannel, cause.getMessage(), cause); ctx.close(); } - - private boolean verifyTlsHostName(String hostname, ChannelHandlerContext ctx) { - ChannelHandler sslHandler = ctx.channel().pipeline().get("tls"); - - SSLSession sslSession; - if (sslHandler != null) { - sslSession = ((SslHandler) sslHandler).engine().getSession(); - return (new TlsHostnameVerifier()).verify(hostname, sslSession); - } - return false; - } } private static final Logger log = LoggerFactory.getLogger(DirectProxyHandler.class); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index eeabced97b065..d9f0f5db38fd4 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -82,7 +82,6 @@ public class ProxyConnection extends PulsarHandler { private final DnsAddressResolverGroup dnsAddressResolverGroup; AuthenticationDataSource authenticationData; private State state; - private final Supplier sslHandlerSupplier; private LookupProxyHandler lookupProxyHandler = null; @Getter @@ -131,13 +130,11 @@ ConnectionPool getConnectionPool() { return connectionPool; } - public ProxyConnection(ProxyService proxyService, Supplier sslHandlerSupplier, - DnsAddressResolverGroup dnsAddressResolverGroup) { + public ProxyConnection(ProxyService proxyService, DnsAddressResolverGroup dnsAddressResolverGroup) { super(30, TimeUnit.SECONDS); this.service = proxyService; this.dnsAddressResolverGroup = dnsAddressResolverGroup; this.state = State.Init; - this.sslHandlerSupplier = sslHandlerSupplier; this.brokerProxyValidator = service.getBrokerProxyValidator(); } @@ -360,8 +357,7 @@ private void handleBrokerConnected(DirectProxyHandler directProxyHandler, Comman private void connectToBroker(InetSocketAddress brokerAddress) { checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop"); DirectProxyHandler directProxyHandler = new DirectProxyHandler(service, this); - directProxyHandler.connect(proxyToBrokerUrl, brokerAddress, - protocolVersionToAdvertise, sslHandlerSupplier); + directProxyHandler.connect(proxyToBrokerUrl, brokerAddress, protocolVersionToAdvertise); } public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) { @@ -531,6 +527,7 @@ ClientConfigurationData createClientConfiguration() { clientConf.setAuthentication(this.getClientAuthentication()); if (proxyConfig.isTlsEnabledWithBroker()) { clientConf.setUseTls(true); + clientConf.setTlsHostnameVerificationEnable(proxyConfig.isTlsHostnameVerificationEnabled()); if (proxyConfig.isBrokerClientTlsEnabledWithKeyStore()) { clientConf.setUseKeyStoreTls(true); clientConf.setTlsTrustStoreType(proxyConfig.getBrokerClientTlsTrustStoreType()); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 2ce2a93819f10..db2574f0df161 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -18,18 +18,13 @@ */ package org.apache.pulsar.proxy.server; -import static org.apache.commons.lang3.StringUtils.isEmpty; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder; -import org.apache.pulsar.common.util.NettyClientSslContextRefresher; import org.apache.pulsar.common.util.NettyServerSslContextBuilder; import io.netty.channel.ChannelInitializer; @@ -52,9 +47,7 @@ public class ServiceChannelInitializer extends ChannelInitializer private final int brokerProxyReadTimeoutMs; private SslContextAutoRefreshBuilder serverSslCtxRefresher; - private SslContextAutoRefreshBuilder clientSslCtxRefresher; private NettySSLContextAutoRefreshBuilder serverSSLContextAutoRefreshBuilder; - private NettySSLContextAutoRefreshBuilder clientSSLContextAutoRefreshBuilder; public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration serviceConfig, boolean enableTls) throws Exception { @@ -95,44 +88,6 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s } else { this.serverSslCtxRefresher = null; } - - if (serviceConfig.isTlsEnabledWithBroker()) { - AuthenticationDataProvider authData = null; - - if (!isEmpty(serviceConfig.getBrokerClientAuthenticationPlugin())) { - authData = AuthenticationFactory.create(serviceConfig.getBrokerClientAuthenticationPlugin(), - serviceConfig.getBrokerClientAuthenticationParameters()).getAuthData(); - } - - if (tlsEnabledWithKeyStore) { - clientSSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - serviceConfig.getBrokerClientSslProvider(), - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getBrokerClientTlsTrustStoreType(), - serviceConfig.getBrokerClientTlsTrustStore(), - serviceConfig.getBrokerClientTlsTrustStorePassword(), - serviceConfig.getBrokerClientTlsCiphers(), - serviceConfig.getBrokerClientTlsProtocols(), - serviceConfig.getTlsCertRefreshCheckDurationSec(), - authData); - } else { - SslProvider sslProvider = null; - if (serviceConfig.getBrokerClientSslProvider() != null) { - sslProvider = SslProvider.valueOf(serviceConfig.getBrokerClientSslProvider()); - } - clientSslCtxRefresher = new NettyClientSslContextRefresher( - sslProvider, - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getBrokerClientTrustCertsFilePath(), - authData, - serviceConfig.getBrokerClientTlsCiphers(), - serviceConfig.getBrokerClientTlsProtocols(), - serviceConfig.getTlsCertRefreshCheckDurationSec() - ); - } - } else { - this.clientSslCtxRefresher = null; - } } @Override @@ -156,25 +111,6 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder( Commands.DEFAULT_MAX_MESSAGE_SIZE + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, 4)); - Supplier sslHandlerSupplier = null; - if (clientSslCtxRefresher != null) { - sslHandlerSupplier = new Supplier() { - @Override - public SslHandler get() { - return clientSslCtxRefresher.get().newHandler(ch.alloc()); - } - }; - } else if (clientSSLContextAutoRefreshBuilder != null) { - sslHandlerSupplier = new Supplier() { - @Override - public SslHandler get() { - return new SslHandler(clientSSLContextAutoRefreshBuilder.get().createSSLEngine()); - } - }; - } - - ch.pipeline().addLast("handler", - new ProxyConnection(proxyService, sslHandlerSupplier, proxyService.getDnsAddressResolverGroup())); - + ch.pipeline().addLast("handler", new ProxyConnection(proxyService, proxyService.getDnsAddressResolverGroup())); } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index d813777f7eb2d..dd06f33b79a86 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -19,15 +19,13 @@ package org.apache.pulsar.proxy.server; import static org.mockito.Mockito.spy; - import com.google.common.collect.Sets; - +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; - import lombok.Cleanup; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.authentication.AuthenticationService; @@ -145,20 +143,24 @@ public Object[][] protocolsCiphersProviderCodecProvider() { }; } - @BeforeMethod @Override - protected void setup() throws Exception { - + protected void doInitConf() throws Exception { + super.doInitConf(); // enable tls and auth&auth at broker conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); + conf.setTopicLevelPoliciesEnabled(false); + conf.setProxyRoles(Collections.singleton("Proxy")); + conf.setAdvertisedAddress(null); conf.setBrokerServicePortTls(Optional.of(0)); + conf.setBrokerServicePort(Optional.empty()); conf.setWebServicePortTls(Optional.of(0)); + conf.setWebServicePort(Optional.empty()); conf.setTlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_BROKER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_BROKER_KEY_FILE_PATH); - conf.setTlsAllowInsecureConnection(true); + conf.setTlsAllowInsecureConnection(false); Set superUserRoles = new HashSet<>(); superUserRoles.add("superUser"); @@ -168,20 +170,24 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_BROKER_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_BROKER_KEY_FILE_PATH); conf.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); - Set providers = new HashSet<>(); - providers.add(AuthenticationProviderTls.class.getName()); - conf.setAuthenticationProviders(providers); + conf.setAuthenticationProviders(Collections.singleton(AuthenticationProviderTls.class.getName())); conf.setClusterName("proxy-authorization"); conf.setNumExecutorThreadPoolSize(5); + } + @BeforeMethod + @Override + protected void setup() throws Exception { super.init(); // start proxy service proxyConfig.setAuthenticationEnabled(true); proxyConfig.setAuthorizationEnabled(false); + proxyConfig.setForwardAuthorizationCredentials(true); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); + proxyConfig.setAdvertisedAddress(null); proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); @@ -198,7 +204,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_PROXY_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_PROXY_KEY_FILE_PATH); - proxyConfig.setAuthenticationProviders(providers); + proxyConfig.setAuthenticationProviders(Collections.singleton(AuthenticationProviderTls.class.getName())); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( @@ -240,11 +246,11 @@ public void testProxyAuthorization() throws Exception { @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), PulsarClient.builder()); - String namespaceName = "my-property/proxy-authorization/my-ns"; + String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - admin.tenants().createTenant("my-property", + admin.tenants().createTenant("my-tenant", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); admin.namespaces().createNamespace(namespaceName); @@ -254,11 +260,11 @@ public void testProxyAuthorization() throws Exception { Sets.newHashSet(AuthAction.consume, AuthAction.produce)); Consumer consumer = proxyClient.newConsumer() - .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1") + .topic("persistent://my-tenant/my-ns/my-topic1") .subscriptionName("my-subscriber-name").subscribe(); Producer producer = proxyClient.newProducer(Schema.BYTES) - .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1").create(); + .topic("persistent://my-tenant/my-ns/my-topic1").create(); final int msgs = 10; for (int i = 0; i < msgs; i++) { String message = "my-message-" + i; @@ -294,11 +300,11 @@ public void testTlsHostVerificationProxyToClient(boolean hostnameVerificationEna PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), PulsarClient.builder().enableTlsHostnameVerification(hostnameVerificationEnabled)); - String namespaceName = "my-property/proxy-authorization/my-ns"; + String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrl(brokerUrlTls.toString()).build()); - admin.tenants().createTenant("my-property", + admin.tenants().createTenant("my-tenant", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); admin.namespaces().createNamespace(namespaceName); @@ -308,7 +314,7 @@ public void testTlsHostVerificationProxyToClient(boolean hostnameVerificationEna Sets.newHashSet(AuthAction.consume, AuthAction.produce)); try { - proxyClient.newConsumer().topic("persistent://my-property/proxy-authorization/my-ns/my-topic1") + proxyClient.newConsumer().topic("persistent://my-tenant/my-ns/my-topic1") .subscriptionName("my-subscriber-name").subscribe(); if (hostnameVerificationEnabled) { Assert.fail("Connection should be failed due to hostnameVerification enabled"); @@ -344,13 +350,13 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), - PulsarClient.builder().operationTimeout(1, TimeUnit.SECONDS)); + PulsarClient.builder().operationTimeout(15, TimeUnit.SECONDS)); - String namespaceName = "my-property/proxy-authorization/my-ns"; + String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - admin.tenants().createTenant("my-property", + admin.tenants().createTenant("my-tenant", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); admin.namespaces().createNamespace(namespaceName); @@ -360,7 +366,7 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna Sets.newHashSet(AuthAction.consume, AuthAction.produce)); try { - proxyClient.newConsumer().topic("persistent://my-property/proxy-authorization/my-ns/my-topic1") + proxyClient.newConsumer().topic("persistent://my-tenant/my-ns/my-topic1") .subscriptionName("my-subscriber-name").subscribe(); if (hostnameVerificationEnabled) { Assert.fail("Connection should be failed due to hostnameVerification enabled"); @@ -382,12 +388,12 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtocols, boolean expectFailure) throws Exception { log.info("-- Starting {} test --", methodName); - String namespaceName = "my-property/proxy-authorization/my-ns"; + String namespaceName = "my-tenant/my-ns"; createAdminClient(); - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrl(brokerUrlTls.toString()).build()); - admin.tenants().createTenant("my-property", + admin.tenants().createTenant("my-tenant", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); admin.namespaces().createNamespace(namespaceName); @@ -399,8 +405,10 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc ProxyConfiguration proxyConfig = new ProxyConfiguration(); proxyConfig.setAuthenticationEnabled(true); proxyConfig.setAuthorizationEnabled(false); + proxyConfig.setForwardAuthorizationCredentials(true); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); + proxyConfig.setAdvertisedAddress(null); proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); @@ -447,7 +455,7 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc @Cleanup PulsarClient proxyClient = createPulsarClient("pulsar://localhost:" + proxyService.getListenPortTls().get(), PulsarClient.builder()); Consumer consumer = proxyClient.newConsumer() - .topic("persistent://my-property/proxy-authorization/my-ns/my-topic1") + .topic("persistent://my-tenant/my-ns/my-topic1") .subscriptionName("my-subscriber-name").subscribe(); if (expectFailure) { @@ -469,7 +477,7 @@ private void createAdminClient() throws Exception { authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true) + .tlsTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH) .authentication(AuthenticationTls.class.getName(), authParams).build()); } @@ -483,7 +491,7 @@ private PulsarClient createPulsarClient(String proxyServiceUrl, ClientBuilder cl authTls.configure(authParams); return clientBuilder.serviceUrl(proxyServiceUrl).statsInterval(0, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true) + .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH) .authentication(authTls).enableTls(true) .operationTimeout(1000, TimeUnit.MILLISECONDS).build(); } diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem index df21a4968bfb2..7d2d58d8d7a06 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem @@ -2,76 +2,76 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - 37:55:7a:ae:71:6b:5f:f0:0d:f7:11:df:b5:f9:ce:e1:65:a4:0c:a4 + 40:cd:a5:a5:35:76:ee:02:57:8b:30:8f:2a:12:34:03:45:c5:96:8c Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: CN = CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: - 00:ce:29:c8:45:af:07:8e:79:1e:55:66:7b:93:af: - 09:2c:72:fd:d5:33:38:30:a9:b5:50:92:90:33:b0: - 55:b0:c4:6b:37:4a:ba:5b:76:4d:52:0b:9f:58:b2: - c5:95:8c:47:6d:2b:07:0a:f5:74:43:ec:7d:36:bf: - 3e:8c:d6:13:31:ce:fc:d1:77:b0:ac:3c:ae:69:4b: - bd:5d:93:bd:84:57:51:a7:ef:03:2e:ae:3e:93:73: - 8b:1e:39:90:8b:32:e2:0a:dd:b8:20:83:98:76:91: - 75:d6:d5:db:43:7b:f4:c9:4e:23:52:e3:11:55:05: - 48:b8:82:47:ea:32:0b:56:1b:07:11:f3:06:c7:4a: - d5:6b:87:c2:2e:e2:9a:8c:9d:54:ca:5e:96:08:02: - 5d:17:42:4d:73:86:08:ab:6e:2e:f3:a8:c3:a3:c1: - bd:88:63:5e:69:7e:fa:af:31:8d:3a:49:ed:e8:cf: - 80:15:ca:d4:2b:fe:84:3d:aa:27:7e:98:36:48:4f: - 3b:27:90:1d:c1:fe:4e:13:b0:5e:a5:32:6e:16:38: - 2e:b7:d1:f3:6b:18:a5:3e:b6:d7:07:42:21:c7:d9: - 8e:d6:8c:a5:bf:25:9e:5c:fc:c7:12:18:59:23:b9: - 3d:39:45:3d:1c:81:e2:f2:29:91:05:20:46:b2:52: - 06:51 + 00:d8:d5:00:e0:6b:4f:4e:8a:67:08:e9:e3:3f:23: + ef:15:1d:82:10:85:f3:3b:77:9c:96:c1:aa:eb:90: + 41:0b:5b:ae:77:d9:a3:f1:cf:2a:32:40:78:33:6a: + 81:b9:c2:cd:91:36:98:df:41:84:c0:62:8a:a1:03: + 89:8d:2b:b8:91:49:a9:e8:a2:90:ad:b9:cd:23:84: + bc:60:1f:6f:b5:81:9f:9c:cf:d5:26:a8:a5:b6:4d: + 59:5f:5c:7f:da:e8:1d:3d:04:f3:b8:ef:f8:d5:73: + c6:fd:6a:b1:91:ae:16:b7:45:21:9a:1a:1a:76:74: + 01:40:ee:fc:3c:67:be:6a:7f:f4:a3:82:37:ee:43: + 41:f5:67:d5:d5:64:9c:d8:53:75:34:4d:23:80:b5: + 59:13:c2:27:47:8e:20:32:6f:f6:b3:70:bf:5e:15: + 08:7e:d1:bf:aa:4d:06:6b:0d:17:21:eb:95:47:52: + fa:d7:97:ef:1a:5d:63:26:17:36:01:20:ac:57:50: + 34:f0:57:49:38:3d:9c:68:6a:87:91:38:b6:76:9d: + bc:e9:4e:c2:58:54:8d:8a:32:05:9e:ba:cb:f0:d0: + ec:91:67:1d:77:bf:d5:02:77:d4:22:78:94:f4:9a: + 49:fa:ef:b2:9b:30:1a:8a:f0:a7:9a:2b:e5:e9:c7: + 36:c5 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - EF:DA:58:74:AA:21:F9:9E:19:7E:44:2B:84:32:93:F4:0F:79:18:3B + DD:AC:A0:40:6E:E9:2B:49:F2:35:DB:B4:E9:98:AD:58:7B:37:6B:55 X509v3 Authority Key Identifier: - keyid:EF:DA:58:74:AA:21:F9:9E:19:7E:44:2B:84:32:93:F4:0F:79:18:3B + keyid:DD:AC:A0:40:6E:E9:2B:49:F2:35:DB:B4:E9:98:AD:58:7B:37:6B:55 X509v3 Basic Constraints: critical CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 2e:f5:b6:f7:fc:50:89:16:1e:ea:8c:ec:57:54:f6:ca:d3:19: - 65:fe:da:c5:73:53:f6:d0:1e:26:96:f2:d3:03:55:8d:6e:c4: - cd:8c:2d:7a:ea:fa:38:6c:ed:fa:d5:23:b8:52:c1:e3:52:04: - 3d:46:8c:2d:b6:b2:47:68:41:92:f6:47:24:50:78:47:5e:2a: - 9b:df:85:a8:92:0d:49:17:eb:51:e8:b2:69:3c:4a:f3:9f:5f: - ea:fd:b2:08:3c:30:1a:93:be:d3:c3:b3:c7:60:7c:ea:f4:15: - 43:bd:3f:b1:d0:69:3c:84:5b:05:01:55:d7:d5:87:fb:58:53: - 03:d8:91:5f:e8:e0:37:88:82:ea:dc:1c:2d:a0:8d:82:68:65: - 6e:ea:0d:2a:e1:aa:cc:b3:d1:ce:a8:2b:2d:ed:e4:ba:0f:7f: - 51:48:d2:4b:2f:7c:eb:02:01:4f:2c:b6:06:c1:9a:97:2c:b7: - 6c:b7:06:86:d1:8b:cc:d6:d4:c3:ff:b5:65:c5:92:eb:9c:68: - 6d:99:d8:4a:6d:7a:ac:fe:dc:f3:12:f8:bb:2b:0a:b9:d8:1e: - 87:b6:e9:8b:51:32:f3:7b:0b:1a:29:57:4c:7d:5a:b6:9c:83: - 23:e5:35:2b:98:83:aa:7c:ef:24:3a:74:a8:86:22:32:06:fb: - 03:b7:01:9d + 07:0c:90:05:fa:2c:c9:4e:05:ec:6b:7d:99:9c:52:2a:20:34: + 46:ac:8d:24:81:f9:a7:f3:1d:03:32:45:82:9a:61:af:1f:63: + 25:6b:97:ca:93:78:e5:d7:87:81:b6:29:22:d4:0d:8d:ed:0e: + bd:85:80:6c:38:e9:86:3c:bd:ee:ff:26:78:0a:f0:a7:54:0b: + af:27:9e:8b:83:b7:10:e9:44:0d:4a:7e:a8:e2:aa:1c:06:f8: + 18:f1:c4:c9:e4:bb:17:41:59:94:b4:dc:78:53:fb:1b:43:57: + 82:59:de:6c:03:52:9a:28:cb:e4:9e:ea:c5:00:93:e0:27:b4: + 4b:e6:b3:c5:88:2d:14:33:10:ff:b0:23:4e:5d:ea:17:97:7d: + f4:e2:c8:fe:c3:4a:77:83:64:ef:c9:b6:3e:77:64:32:07:91: + bd:e1:58:9a:e1:38:ab:eb:d2:e3:cb:05:7c:c7:f3:2b:47:bf: + 36:64:7e:32:5a:62:44:07:c8:8e:9d:55:1a:99:c4:14:5a:66: + ed:5f:8b:ab:dd:eb:36:28:cd:77:47:84:00:ae:a7:34:0e:0d: + 77:df:67:72:08:94:75:52:1b:4a:71:4d:31:5d:aa:1b:aa:b6: + e0:d6:86:52:7c:26:ae:1f:96:ab:06:32:cb:7a:f3:bb:76:3e: + 08:53:9f:64 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUN1V6rnFrX/AN9xHftfnO4WWkDKQwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIxMDQyMzE3MDg1MVoXDTMxMDQyMTE3 -MDg1MVowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAzinIRa8HjnkeVWZ7k68JLHL91TM4MKm1UJKQM7BVsMRrN0q6W3ZN -UgufWLLFlYxHbSsHCvV0Q+x9Nr8+jNYTMc780XewrDyuaUu9XZO9hFdRp+8DLq4+ -k3OLHjmQizLiCt24IIOYdpF11tXbQ3v0yU4jUuMRVQVIuIJH6jILVhsHEfMGx0rV -a4fCLuKajJ1Uyl6WCAJdF0JNc4YIq24u86jDo8G9iGNeaX76rzGNOknt6M+AFcrU -K/6EPaonfpg2SE87J5Adwf5OE7BepTJuFjgut9HzaxilPrbXB0Ihx9mO1oylvyWe -XPzHEhhZI7k9OUU9HIHi8imRBSBGslIGUQIDAQABo1MwUTAdBgNVHQ4EFgQU79pY -dKoh+Z4ZfkQrhDKT9A95GDswHwYDVR0jBBgwFoAU79pYdKoh+Z4ZfkQrhDKT9A95 -GDswDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEALvW29/xQiRYe -6ozsV1T2ytMZZf7axXNT9tAeJpby0wNVjW7EzYwteur6OGzt+tUjuFLB41IEPUaM -LbayR2hBkvZHJFB4R14qm9+FqJINSRfrUeiyaTxK859f6v2yCDwwGpO+08Ozx2B8 -6vQVQ70/sdBpPIRbBQFV19WH+1hTA9iRX+jgN4iC6twcLaCNgmhlbuoNKuGqzLPR -zqgrLe3kug9/UUjSSy986wIBTyy2BsGalyy3bLcGhtGLzNbUw/+1ZcWS65xobZnY -Sm16rP7c8xL4uysKudgeh7bpi1Ey83sLGilXTH1atpyDI+U1K5iDqnzvJDp0qIYi -Mgb7A7cBnQ== +MIIDAzCCAeugAwIBAgIUQM2lpTV27gJXizCPKhI0A0XFlowwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA2NUA4GtPTopnCOnjPyPvFR2CEIXzO3eclsGq65BBC1uud9mj8c8q +MkB4M2qBucLNkTaY30GEwGKKoQOJjSu4kUmp6KKQrbnNI4S8YB9vtYGfnM/VJqil +tk1ZX1x/2ugdPQTzuO/41XPG/Wqxka4Wt0UhmhoadnQBQO78PGe+an/0o4I37kNB +9WfV1WSc2FN1NE0jgLVZE8InR44gMm/2s3C/XhUIftG/qk0Gaw0XIeuVR1L615fv +Gl1jJhc2ASCsV1A08FdJOD2caGqHkTi2dp286U7CWFSNijIFnrrL8NDskWcdd7/V +AnfUIniU9JpJ+u+ymzAaivCnmivl6cc2xQIDAQABo1MwUTAdBgNVHQ4EFgQU3ayg +QG7pK0nyNdu06ZitWHs3a1UwHwYDVR0jBBgwFoAU3aygQG7pK0nyNdu06ZitWHs3 +a1UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABwyQBfosyU4F +7Gt9mZxSKiA0RqyNJIH5p/MdAzJFgpphrx9jJWuXypN45deHgbYpItQNje0OvYWA +bDjphjy97v8meArwp1QLryeei4O3EOlEDUp+qOKqHAb4GPHEyeS7F0FZlLTceFP7 +G0NXglnebANSmijL5J7qxQCT4Ce0S+azxYgtFDMQ/7AjTl3qF5d99OLI/sNKd4Nk +78m2PndkMgeRveFYmuE4q+vS48sFfMfzK0e/NmR+MlpiRAfIjp1VGpnEFFpm7V+L +q93rNijNd0eEAK6nNA4Nd99ncgiUdVIbSnFNMV2qG6q24NaGUnwmrh+WqwYyy3rz +u3Y+CFOfZA== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem index edd9a025176fe..31743d0684670 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:78 + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:07 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache Pulsar, OU = Broker, CN = Broker Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: 07:f0:b0:06:4f:2c:4c:75:c2:37:ff:35:0d:b1:42: 06:0b Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 46:84:81:7e:4a:91:2a:c0:d7:0c:5a:a2:fb:6e:a2:e1:66:15: - b9:b3:50:1c:93:8c:68:ba:90:42:07:2c:d1:d9:22:53:c4:e7: - 74:a9:ac:0c:25:cb:ae:c9:a1:c9:35:49:5d:10:c6:ee:08:2a: - 23:f3:a4:87:24:92:c4:4e:35:b8:23:8e:be:ad:8c:5b:25:df: - 25:d4:49:8c:d6:11:bf:79:43:a2:88:7f:70:87:8c:fb:51:9a: - 4c:73:8d:10:e7:5b:fa:fb:76:f9:88:7a:6a:d0:bf:0f:65:1e: - 26:22:87:57:31:9a:c9:4c:62:cf:ef:00:2b:4e:2f:ee:d4:d8: - 0d:2f:7f:2e:14:21:d5:c3:25:ce:29:a3:f0:ee:c6:3d:d2:dc: - 7b:80:34:57:50:97:e7:79:d9:ca:39:10:73:2d:46:f4:98:de: - ec:be:98:1a:17:12:c3:9e:1f:0d:25:c8:4e:17:a1:4a:8d:6a: - 21:11:42:56:1a:16:79:12:e2:db:39:e1:5d:c4:2e:03:31:54: - d9:97:53:21:bc:f0:60:e1:ba:ff:f6:a5:4b:c1:39:4f:e1:87: - b7:63:9a:63:fa:a2:83:1c:b5:8e:fd:48:be:d5:50:40:0b:69: - 34:81:1e:d1:ca:c5:34:ff:bc:c3:ec:22:a5:3e:ca:31:fe:43: - 39:00:79:72 + 8d:1d:69:d2:44:1f:af:68:30:80:c1:91:b2:2f:9a:7e:ca:ff: + 38:46:8e:28:59:02:2d:e7:74:c4:3c:b3:ac:b3:22:53:e9:54: + 3a:e2:4d:4d:65:63:47:dd:38:86:ec:d1:7d:4f:fe:5d:c6:c8: + c8:10:b8:33:5a:4d:9e:83:e3:92:97:c5:f1:d8:e3:97:6d:01: + 50:03:de:25:d8:e4:de:62:70:b8:c4:55:5b:9f:8c:61:b8:d7: + f0:8f:6c:2d:80:cc:b8:7b:8b:b4:54:9a:d6:e1:f9:7f:52:99: + 7b:ef:23:88:61:e5:7c:85:5c:57:98:cc:a6:98:4b:71:84:5c: + ab:5e:82:48:5a:da:5f:d6:84:b5:52:43:df:3c:0f:95:06:29: + 00:94:f8:98:94:6d:1c:c8:76:21:7a:2f:61:34:ab:bd:27:59: + d1:41:99:91:69:68:f7:b6:65:21:e8:9a:b1:9b:ac:72:12:17: + 54:0b:56:08:bd:9d:6b:0e:35:4a:f8:97:b6:83:00:55:96:0c: + 66:13:06:c9:27:5f:cc:d0:81:4b:3e:6e:d2:85:cd:79:7a:8c: + a0:1e:d8:9b:e4:da:e9:ba:51:f1:29:0f:69:00:df:24:a0:55: + 5e:cd:d0:84:c9:4a:a8:b4:12:33:29:6f:8a:8c:d7:a1:b4:8b: + 4a:7d:a2:30 -----BEGIN CERTIFICATE----- -MIIC7DCCAdQCFAwmFd+PcR1qMdDar2TvgN6smkZ4MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFQxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1 -bHNhcjEPMA0GA1UECxMGQnJva2VyMQ8wDQYDVQQDEwZCcm9rZXIwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKd9wqEyUkyyliBhJfqJLJU9Y/B8qqCl9y -ks236kVHcfBjT1gaPfrOpnOQwKn3JfB2de2yAxe+2IpW809qTH4DZZXlReuNR+hg -Xp44dFBUZaDs2FxlYDQbloN9cdRdf+NiWWfo8NYkfcBuNwNUTD0MMzmbM+FSRMVD -2uruLPMcFi5GTHyfXU1u/owjnvd+nznBcQZS9CaaItTPxSU5qdLkJMbYSkii7nYl -yzzwv80Qd/+BEUMhzDvMEHoHhPzMAqJF3pEta9HtFxrQRvSufbOJ+DF3leVGsakx -1tjjRwCygYHbihzZ8c3jTTX2OJEN6gfwsAZPLEx1wjf/NQ2xQgYLAgMBAAEwDQYJ -KoZIhvcNAQELBQADggEBAEaEgX5KkSrA1wxaovtuouFmFbmzUByTjGi6kEIHLNHZ -IlPE53SprAwly67Jock1SV0Qxu4IKiPzpIckksRONbgjjr6tjFsl3yXUSYzWEb95 -Q6KIf3CHjPtRmkxzjRDnW/r7dvmIemrQvw9lHiYih1cxmslMYs/vACtOL+7U2A0v -fy4UIdXDJc4po/Duxj3S3HuANFdQl+d52co5EHMtRvSY3uy+mBoXEsOeHw0lyE4X -oUqNaiERQlYaFnkS4ts54V3ELgMxVNmXUyG88GDhuv/2pUvBOU/hh7djmmP6ooMc -tY79SL7VUEALaTSBHtHKxTT/vMPsIqU+yjH+QzkAeXI= +MIIDETCCAfmgAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgcwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowVDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj +aGUgUHVsc2FyMQ8wDQYDVQQLEwZCcm9rZXIxDzANBgNVBAMTBkJyb2tlcjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMp33CoTJSTLKWIGEl+okslT1j8H +yqoKX3KSzbfqRUdx8GNPWBo9+s6mc5DAqfcl8HZ17bIDF77YilbzT2pMfgNlleVF +641H6GBenjh0UFRloOzYXGVgNBuWg31x1F1/42JZZ+jw1iR9wG43A1RMPQwzOZsz +4VJExUPa6u4s8xwWLkZMfJ9dTW7+jCOe936fOcFxBlL0Jpoi1M/FJTmp0uQkxthK +SKLudiXLPPC/zRB3/4ERQyHMO8wQegeE/MwCokXekS1r0e0XGtBG9K59s4n4MXeV +5UaxqTHW2ONHALKBgduKHNnxzeNNNfY4kQ3qB/CwBk8sTHXCN/81DbFCBgsCAwEA +AaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUA +A4IBAQCNHWnSRB+vaDCAwZGyL5p+yv84Ro4oWQIt53TEPLOssyJT6VQ64k1NZWNH +3TiG7NF9T/5dxsjIELgzWk2eg+OSl8Xx2OOXbQFQA94l2OTeYnC4xFVbn4xhuNfw +j2wtgMy4e4u0VJrW4fl/Upl77yOIYeV8hVxXmMymmEtxhFyrXoJIWtpf1oS1UkPf +PA+VBikAlPiYlG0cyHYhei9hNKu9J1nRQZmRaWj3tmUh6Jqxm6xyEhdUC1YIvZ1r +DjVK+Je2gwBVlgxmEwbJJ1/M0IFLPm7Shc15eoygHtib5NrpulHxKQ9pAN8koFVe +zdCEyUqotBIzKW+KjNehtItKfaIw -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem index dc75fe9506eaf..127f56dd777a5 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem @@ -2,76 +2,76 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - 33:a3:2e:28:58:0b:7a:7b:3c:71:4e:51:1d:1d:16:f5:72:3d:99:01 + 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: CN = CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: - 00:d9:06:95:38:4a:ed:0d:ef:57:12:26:5e:2f:ea: - 3c:05:78:1e:36:90:6c:d6:8d:dc:18:e7:e0:24:d7: - 72:ae:d3:af:6a:ff:32:1f:ee:d8:93:9e:f4:53:88: - 0f:5d:d6:56:41:03:b9:1e:d7:d4:0d:d5:ae:27:20: - d8:8f:e3:7d:65:79:d3:00:c9:cc:f4:ef:f5:c9:f6: - 83:a4:45:b4:6d:11:ac:fc:55:f2:94:6b:75:74:d9: - f7:23:b2:5a:ba:a3:21:b4:6e:5a:2d:fc:84:32:ef: - 78:f5:d7:22:7c:e8:a8:15:aa:1d:9f:53:63:fd:77: - f4:d7:20:cc:21:34:1c:7a:22:a9:6a:de:90:06:ae: - 10:ff:96:21:61:9e:6d:21:f5:66:37:ef:a0:5a:a8: - 51:5f:22:24:9f:a9:a9:b3:21:10:f4:7a:d9:ee:c3: - 20:73:c3:48:0a:c7:98:7c:5f:04:7a:e1:eb:8c:d6: - f0:18:d7:e9:0c:11:cd:a1:81:f4:d4:67:c0:72:0f: - e3:90:86:92:97:bd:bc:44:df:b1:b3:6d:85:4f:6b: - fa:bf:9e:6a:1d:9c:77:23:3b:6f:89:38:fb:45:ff: - f5:76:b3:19:f7:7c:59:2b:07:ff:6a:4a:f5:93:4a: - 62:ef:18:3b:ea:54:8f:2d:c2:34:c8:a3:6f:ee:f8: - f2:a3 + 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: + 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: + 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: + d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: + d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: + 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: + 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: + b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: + 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: + fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: + ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: + 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: + 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: + ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: + 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: + 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: + 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: + 48:9d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - 86:1F:20:03:1D:EA:65:52:AA:D7:38:B7:A7:B1:DC:0A:02:F9:F2:02 + 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C X509v3 Authority Key Identifier: - keyid:86:1F:20:03:1D:EA:65:52:AA:D7:38:B7:A7:B1:DC:0A:02:F9:F2:02 + keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C X509v3 Basic Constraints: critical CA:TRUE Signature Algorithm: sha256WithRSAEncryption - c3:8a:4d:5b:3a:01:28:08:cc:cd:8b:cc:37:0d:0b:0c:45:dd: - c0:44:ee:36:9c:1d:7d:1f:b9:5a:a7:fd:9a:19:34:0f:8c:09: - 9d:24:f1:7b:a2:22:ef:7f:f3:4f:31:e2:b8:a5:f2:ec:d5:32: - 02:f3:10:c4:82:c4:a0:33:b0:50:53:b7:2e:3d:78:30:8e:b3: - c1:f8:51:4d:30:5b:40:65:6f:ad:b8:99:be:d8:cc:3b:43:00: - 2b:16:5c:9c:bd:83:24:a0:48:0d:cd:2e:29:74:a8:e6:bc:df: - f0:7c:2c:1f:03:72:f4:47:4d:88:e6:8f:53:77:25:23:57:0a: - 84:fb:38:e7:b0:84:57:2b:4d:5a:f0:94:34:8a:48:ca:dc:f7: - 08:b5:d5:1e:64:b4:03:c9:f3:3d:dd:f5:27:ac:f8:2b:d5:80: - ab:b5:b1:37:8e:ae:2f:03:c2:19:4d:37:d6:e2:76:24:a2:98: - ed:c8:c5:d0:65:29:4d:ce:0a:bf:d0:a3:3f:f6:03:47:fa:75: - 8c:06:22:fe:8a:13:9a:9c:17:f5:35:71:7d:66:b9:cd:ca:ac: - 1e:c3:09:c6:76:b0:6c:2b:45:fd:5b:a9:02:7b:e8:fa:65:32: - e3:8e:7d:25:6e:06:db:bc:fd:5b:ad:78:d3:e0:09:df:3d:9c: - 3b:56:c5:69 + 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: + 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: + 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: + 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: + 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: + ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: + 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: + a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: + a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: + 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: + 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: + 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: + fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: + 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: + ae:fe:19:34 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUM6MuKFgLens8cU5RHR0W9XI9mQEwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIxMDQyMzE3MDg1MVoXDTMxMDQyMTE3 -MDg1MVowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA2QaVOErtDe9XEiZeL+o8BXgeNpBs1o3cGOfgJNdyrtOvav8yH+7Y -k570U4gPXdZWQQO5HtfUDdWuJyDYj+N9ZXnTAMnM9O/1yfaDpEW0bRGs/FXylGt1 -dNn3I7JauqMhtG5aLfyEMu949dcifOioFaodn1Nj/Xf01yDMITQceiKpat6QBq4Q -/5YhYZ5tIfVmN++gWqhRXyIkn6mpsyEQ9HrZ7sMgc8NICseYfF8EeuHrjNbwGNfp -DBHNoYH01GfAcg/jkIaSl728RN+xs22FT2v6v55qHZx3IztviTj7Rf/1drMZ93xZ -Kwf/akr1k0pi7xg76lSPLcI0yKNv7vjyowIDAQABo1MwUTAdBgNVHQ4EFgQUhh8g -Ax3qZVKq1zi3p7HcCgL58gIwHwYDVR0jBBgwFoAUhh8gAx3qZVKq1zi3p7HcCgL5 -8gIwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAw4pNWzoBKAjM -zYvMNw0LDEXdwETuNpwdfR+5Wqf9mhk0D4wJnSTxe6Ii73/zTzHiuKXy7NUyAvMQ -xILEoDOwUFO3Lj14MI6zwfhRTTBbQGVvrbiZvtjMO0MAKxZcnL2DJKBIDc0uKXSo -5rzf8HwsHwNy9EdNiOaPU3clI1cKhPs457CEVytNWvCUNIpIytz3CLXVHmS0A8nz -Pd31J6z4K9WAq7WxN46uLwPCGU031uJ2JKKY7cjF0GUpTc4Kv9CjP/YDR/p1jAYi -/ooTmpwX9TVxfWa5zcqsHsMJxnawbCtF/VupAnvo+mUy4459JW4G27z9W6140+AJ -3z2cO1bFaQ== +MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs +/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 +8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 +AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS +/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO +bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh +Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud +9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU +yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D +VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w +Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX +9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn +eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk +rEibrv4ZNA== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem index 0ac579026ef26..1a21d9d41387f 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:79 + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:03 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache Pulsar, OU = Client, CN = Client Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: 8e:18:48:4c:5f:19:e9:b0:7b:22:d3:bc:42:32:45: 9a:d1 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - a4:bb:d2:e4:ba:17:1f:07:13:26:ac:e1:71:df:1e:d4:d7:a7: - 31:dd:df:ce:e6:bb:11:fb:cf:a5:66:d2:fb:0e:26:90:fd:94: - 0d:d2:d6:91:f3:65:75:ae:16:b6:92:2e:0a:41:b5:fc:ba:33: - 57:85:92:e8:a3:30:97:d9:26:dc:e0:37:da:c5:bd:5f:e9:dd: - db:81:cb:38:96:99:6e:d2:a5:6d:92:a8:6d:be:03:6f:a9:48: - 4a:a1:4b:91:f9:c3:11:85:79:1e:4e:77:98:ff:43:dd:e0:f9: - 8e:95:fe:f3:e2:eb:48:72:cf:04:fe:3d:78:b3:a8:ee:56:c8: - 12:c8:0a:3d:70:f4:86:42:d2:b9:54:4d:07:8c:45:ad:af:b9: - 43:c8:f9:ee:fc:5d:96:a2:b6:d5:d9:48:57:4e:b5:7d:c7:8c: - 35:21:99:13:9a:60:42:1f:39:4a:3a:1b:3b:e5:ab:1d:91:59: - 8a:e1:82:9e:70:79:f9:9a:6e:bb:a9:99:30:4d:93:c8:bf:95: - 91:a1:03:a3:ac:d8:cd:80:db:89:82:a7:e6:74:8d:53:b3:a6: - 7a:b9:ca:93:14:a2:01:08:bd:9f:4e:2d:0d:50:b3:aa:e8:a6: - a8:43:b5:d6:a4:1c:2f:62:7a:1f:1b:92:6b:2d:fa:12:c3:1a: - ed:8b:11:fe + 8b:88:90:00:1a:15:fa:11:f2:f0:35:6f:0f:f2:76:74:fc:8d: + bc:03:ee:a5:c5:21:17:c9:01:6b:58:93:fa:3e:7b:e0:0d:6d: + db:1f:2a:48:fa:15:34:66:b7:cb:be:82:c6:28:91:99:42:5a: + 36:b6:0b:2f:bb:85:14:88:a9:ea:dd:0a:7a:be:c4:e7:b2:2d: + 82:a9:37:bc:d9:5c:aa:03:2e:54:68:b1:b7:e8:d6:45:a5:8f: + 48:45:2c:9c:7a:55:0a:4a:07:1b:30:8a:49:6d:f4:62:b1:9e: + 92:0e:d9:34:44:6c:6d:e7:a3:18:bb:85:58:6d:da:20:83:d5: + ca:65:63:1e:3b:e6:df:7b:97:40:4f:b1:59:63:a9:b5:80:6f: + 97:51:53:a1:d3:29:1f:1a:26:05:17:59:3e:16:4f:5f:38:36: + 76:30:c6:bf:1e:3e:ed:39:83:91:31:58:01:13:59:5c:c5:e9: + d6:61:e0:f3:5f:c7:47:8a:5f:af:23:98:89:7b:b4:e6:f6:51: + 98:a0:26:31:c8:67:91:6d:d5:68:75:3d:4d:48:44:5f:3b:9c: + df:a7:87:a0:11:02:d2:13:5f:c1:4c:3f:3e:09:59:2e:fc:cb: + c2:c5:f0:f8:91:df:c3:dd:ad:c8:fc:44:23:9b:78:0d:3b:f2: + 82:f6:02:82 -----BEGIN CERTIFICATE----- -MIIC7DCCAdQCFAwmFd+PcR1qMdDar2TvgN6smkZ5MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFQxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1 -bHNhcjEPMA0GA1UECxMGQ2xpZW50MQ8wDQYDVQQDEwZDbGllbnQwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeHhC9ZBPBbHpJhgE7q6sd7LKTQWxsIfLm -FRtRzq1n/Rg+f3pkomJfLgtZtO3ZFw63vFBmQbfjxHHJc3M92G00gPLjuZiPK1QU -lbNRG9aRhc23NKJQtvGGbgcw+q5VoF35fByRUGJ9uxSGkgqsKT4oG5nKMGPcqV8F -+Dg+MBACn8yU10fgGvQcaJY9El5YIUEs7JatnghWg3qSX0vmvQEWcCivqicdxP6y -Cb+ltEfZWEv+QYEOokZXwTl8jeSxpyXmtN3zniTJ58CMGrSr3bkzvxHLvrsi9/yt -xEBB1+83CBqVRR/bFF8L+Ej/QSTLXI4YSExfGemweyLTvEIyRZrRAgMBAAEwDQYJ -KoZIhvcNAQELBQADggEBAKS70uS6Fx8HEyas4XHfHtTXpzHd387muxH7z6Vm0vsO -JpD9lA3S1pHzZXWuFraSLgpBtfy6M1eFkuijMJfZJtzgN9rFvV/p3duByziWmW7S -pW2SqG2+A2+pSEqhS5H5wxGFeR5Od5j/Q93g+Y6V/vPi60hyzwT+PXizqO5WyBLI -Cj1w9IZC0rlUTQeMRa2vuUPI+e78XZaittXZSFdOtX3HjDUhmROaYEIfOUo6Gzvl -qx2RWYrhgp5wefmabrupmTBNk8i/lZGhA6Os2M2A24mCp+Z0jVOzpnq5ypMUogEI -vZ9OLQ1Qs6ropqhDtdakHC9ieh8bkmst+hLDGu2LEf4= +MIIDETCCAfmgAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgMwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowVDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj +aGUgUHVsc2FyMQ8wDQYDVQQLEwZDbGllbnQxDzANBgNVBAMTBkNsaWVudDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN4eEL1kE8FsekmGATurqx3sspNB +bGwh8uYVG1HOrWf9GD5/emSiYl8uC1m07dkXDre8UGZBt+PEcclzcz3YbTSA8uO5 +mI8rVBSVs1Eb1pGFzbc0olC28YZuBzD6rlWgXfl8HJFQYn27FIaSCqwpPigbmcow +Y9ypXwX4OD4wEAKfzJTXR+Aa9Bxolj0SXlghQSzslq2eCFaDepJfS+a9ARZwKK+q +Jx3E/rIJv6W0R9lYS/5BgQ6iRlfBOXyN5LGnJea03fOeJMnnwIwatKvduTO/Ecu+ +uyL3/K3EQEHX7zcIGpVFH9sUXwv4SP9BJMtcjhhITF8Z6bB7ItO8QjJFmtECAwEA +AaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUA +A4IBAQCLiJAAGhX6EfLwNW8P8nZ0/I28A+6lxSEXyQFrWJP6PnvgDW3bHypI+hU0 +ZrfLvoLGKJGZQlo2tgsvu4UUiKnq3Qp6vsTnsi2CqTe82VyqAy5UaLG36NZFpY9I +RSycelUKSgcbMIpJbfRisZ6SDtk0RGxt56MYu4VYbdogg9XKZWMeO+bfe5dAT7FZ +Y6m1gG+XUVOh0ykfGiYFF1k+Fk9fODZ2MMa/Hj7tOYORMVgBE1lcxenWYeDzX8dH +il+vI5iJe7Tm9lGYoCYxyGeRbdVodT1NSERfO5zfp4egEQLSE1/BTD8+CVku/MvC +xfD4kd/D3a3I/EQjm3gNO/KC9gKC -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem index cb22ab5057372..127f56dd777a5 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem @@ -2,76 +2,76 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - 2d:fc:78:73:ca:55:1e:32:12:3e:ef:08:24:cf:63:95:1e:ad:ea:ae + 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: CN = CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: - 00:c3:e0:f7:5d:bb:9a:76:ee:84:c6:2d:79:3f:a6: - 4b:3b:1f:32:31:d9:65:80:d3:02:13:23:2a:f1:2f: - e6:ac:bc:24:d1:cb:b9:5b:ed:cb:63:fe:31:e4:e6: - b8:f3:13:72:be:48:57:cb:d1:70:0f:67:16:6d:26: - bc:23:1c:64:30:ee:c8:0e:0e:68:d9:43:7e:42:74: - 7a:d4:59:a4:76:67:70:9f:85:aa:f3:9f:6c:e6:a1: - b5:06:3c:1d:46:38:45:05:df:88:cc:3a:ad:6c:72: - 96:69:55:d0:b2:a8:ed:fd:b8:07:6b:5c:6d:1c:0d: - 98:c2:88:3f:59:3c:d6:6c:ab:df:dd:3a:c0:5c:fe: - 86:74:38:bc:00:d4:f0:50:ea:f0:e6:74:23:48:6d: - 63:77:c7:f6:e2:94:f8:1b:0f:51:98:f6:fb:e0:20: - 58:c1:b6:a0:58:08:6f:ad:05:f7:71:90:b3:1a:5b: - 24:88:0b:ed:71:26:aa:84:c2:21:97:76:e7:d5:77: - 30:62:15:d4:30:5e:f9:aa:bc:7f:1f:50:5e:92:47: - f2:92:c0:85:cf:ce:33:07:24:e9:ee:b7:04:0d:b7: - 9f:82:ae:a0:b6:73:51:8f:fe:bd:2c:f3:b5:76:61: - 3c:da:c6:c0:bd:44:46:6f:43:9d:47:b6:0a:80:a5: - fe:3b + 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: + 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: + 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: + d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: + d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: + 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: + 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: + b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: + 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: + fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: + ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: + 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: + 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: + ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: + 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: + 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: + 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: + 48:9d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - 4E:9B:EB:E2:41:17:D1:24:AF:39:02:BC:42:D6:81:B7:62:6D:E3:57 + 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C X509v3 Authority Key Identifier: - keyid:4E:9B:EB:E2:41:17:D1:24:AF:39:02:BC:42:D6:81:B7:62:6D:E3:57 + keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C X509v3 Basic Constraints: critical CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 16:01:53:ab:85:57:5f:92:b9:24:85:c5:70:02:fa:fe:ae:ff: - e9:3e:36:24:6e:9e:34:dd:7c:56:f9:31:a1:d1:ae:63:af:3c: - 2c:e5:8e:47:34:df:b0:1c:33:48:3f:e7:32:fd:a8:38:99:a6: - ef:e1:7b:65:92:80:1e:68:e5:98:db:c5:50:4a:35:53:e5:86: - 89:56:85:0c:6e:da:64:28:68:33:dc:29:3f:41:8b:cf:9c:ec: - fc:74:15:19:ff:da:0a:ef:d0:51:67:97:ad:2f:e4:8a:94:52: - 96:18:bd:77:b3:2b:79:9a:f8:de:af:0f:a2:65:c4:f2:88:3a: - 57:79:18:e1:d8:7c:e0:52:da:35:8c:dd:d9:75:0d:72:e9:e8: - d0:a7:a6:0b:49:88:6d:ed:86:45:25:72:15:4e:2a:0b:6f:9c: - 2f:48:75:28:b0:aa:cd:15:7f:ae:b3:b7:ec:75:d9:63:c8:46: - 8f:84:49:1c:e2:db:95:7b:3d:bb:fd:98:45:53:56:3c:3c:de: - 60:16:f9:14:b8:7e:27:37:be:f0:69:b5:a0:18:bc:83:1e:c1: - 3a:11:9b:a3:1d:1f:a6:9c:7e:c9:aa:7c:53:44:9e:1d:cb:ca: - c8:22:7f:cc:ad:e6:fa:51:54:4d:b5:a1:e6:e3:04:4e:49:1e: - 67:9c:93:30 + 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: + 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: + 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: + 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: + 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: + ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: + 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: + a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: + a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: + 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: + 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: + 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: + fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: + 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: + ae:fe:19:34 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIULfx4c8pVHjISPu8IJM9jlR6t6q4wDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIxMDQyMzE3MDg1MVoXDTMxMDQyMTE3 -MDg1MVowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAw+D3Xbuadu6Exi15P6ZLOx8yMdllgNMCEyMq8S/mrLwk0cu5W+3L -Y/4x5Oa48xNyvkhXy9FwD2cWbSa8IxxkMO7IDg5o2UN+QnR61Fmkdmdwn4Wq859s -5qG1BjwdRjhFBd+IzDqtbHKWaVXQsqjt/bgHa1xtHA2Ywog/WTzWbKvf3TrAXP6G -dDi8ANTwUOrw5nQjSG1jd8f24pT4Gw9RmPb74CBYwbagWAhvrQX3cZCzGlskiAvt -cSaqhMIhl3bn1XcwYhXUMF75qrx/H1BekkfyksCFz84zByTp7rcEDbefgq6gtnNR -j/69LPO1dmE82sbAvURGb0OdR7YKgKX+OwIDAQABo1MwUTAdBgNVHQ4EFgQUTpvr -4kEX0SSvOQK8QtaBt2Jt41cwHwYDVR0jBBgwFoAUTpvr4kEX0SSvOQK8QtaBt2Jt -41cwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAFgFTq4VXX5K5 -JIXFcAL6/q7/6T42JG6eNN18VvkxodGuY688LOWORzTfsBwzSD/nMv2oOJmm7+F7 -ZZKAHmjlmNvFUEo1U+WGiVaFDG7aZChoM9wpP0GLz5zs/HQVGf/aCu/QUWeXrS/k -ipRSlhi9d7MreZr43q8PomXE8og6V3kY4dh84FLaNYzd2XUNcuno0KemC0mIbe2G -RSVyFU4qC2+cL0h1KLCqzRV/rrO37HXZY8hGj4RJHOLblXs9u/2YRVNWPDzeYBb5 -FLh+Jze+8Gm1oBi8gx7BOhGbox0fppx+yap8U0SeHcvKyCJ/zK3m+lFUTbWh5uME -TkkeZ5yTMA== +MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs +/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 +8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 +AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS +/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO +bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh +Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud +9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU +yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D +VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w +Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX +9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn +eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk +rEibrv4ZNA== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem index a4c03e3c2eaa4..e2c1e5a230c26 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:7a + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:04 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache Pulsar, OU = Proxy, CN = Proxy Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: 29:e1:23:c4:ed:a0:1c:f6:2a:ed:dc:c0:df:97:a9: f3:8d Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 7b:27:a8:2a:54:35:76:e5:f8:a7:60:8d:e7:35:12:69:38:f3: - 32:af:25:0f:69:1a:b1:af:79:e5:7c:94:5c:8f:aa:76:95:54: - 35:b4:bb:64:20:1a:91:1e:b3:e4:d1:06:72:24:c3:35:bd:9c: - f6:54:61:d9:39:22:99:42:08:d4:97:aa:7d:82:46:fc:77:58: - df:93:29:03:6c:ba:1c:13:d1:42:49:32:f1:38:09:d3:3e:43: - 89:1b:61:c4:40:f3:ac:4c:c1:36:2f:28:bd:57:a0:de:35:82: - c9:da:93:5f:09:d6:e8:5b:cd:15:45:b3:28:22:7d:48:00:c4: - 55:0f:f6:de:d9:c2:0a:39:5e:69:a4:50:9b:3f:e1:06:44:8a: - 13:af:0b:56:8d:70:c4:9f:d1:a2:b4:25:09:8b:19:47:e8:d2: - 98:49:2a:a0:8b:fe:8c:cb:23:d8:f8:e6:28:c6:d9:0b:10:7c: - d3:ce:48:07:8d:c7:56:bb:c9:e8:d7:a8:a1:24:93:bf:5f:d2: - a9:f1:35:b7:40:ad:08:bf:89:63:e5:49:40:13:e7:1e:6a:77: - 7f:9a:5b:07:0c:eb:80:77:b0:ac:fa:8a:9d:b8:83:53:a1:1e: - 0e:14:2b:c9:50:96:81:c2:c0:0b:d1:c6:b6:2e:ea:98:3e:7b: - ee:5f:09:f7 + 8d:b6:2c:5f:87:13:06:a8:66:ce:11:2a:2c:20:1e:c7:ee:50: + 75:a7:d1:7c:ad:c6:ec:d1:18:d0:fa:aa:00:fa:08:f9:0f:cc: + df:59:9a:6b:1c:18:07:15:84:d0:9a:24:8d:dd:46:79:9c:dc: + 9e:3e:97:10:24:b2:9d:d4:f6:c5:79:58:87:7c:a6:af:cf:69: + 23:fb:43:7a:0f:4d:26:e0:e9:66:c5:ad:fa:88:e2:c5:6e:6a: + ce:70:0c:8f:73:01:d6:fd:a9:1f:31:49:41:17:45:22:cc:a6: + 71:e4:f4:0f:0f:2e:3e:49:0b:5f:04:94:36:49:fa:72:42:c9: + 25:75:84:9a:dc:16:cb:69:44:44:e5:3a:ff:26:f6:44:42:4c: + 6c:e2:56:d6:3e:bc:f2:8b:83:de:e2:91:70:65:b9:d0:dd:a3: + d1:de:53:27:77:13:2d:86:27:c3:40:2f:c1:a5:50:1c:5a:44: + 51:b4:29:11:c3:30:9d:1a:96:25:7a:d6:05:70:ad:06:0d:f2: + 9b:b1:b6:82:39:06:c7:7c:b2:49:04:19:e4:7e:87:b8:d8:42: + 1d:ab:ed:d0:b0:7f:79:6b:89:75:2f:6a:26:67:3d:33:57:5f: + 5a:49:52:98:3b:2a:e5:43:d7:f9:97:ca:75:cd:6f:e9:e4:66: + b6:d6:c2:c7 -----BEGIN CERTIFICATE----- -MIIC6jCCAdICFAwmFd+PcR1qMdDar2TvgN6smkZ6MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFIxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1 -bHNhcjEOMAwGA1UECxMFUHJveHkxDjAMBgNVBAMTBVByb3h5MIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw1zFrRfc9NTE6hxgWiRGE9nPwM2DLi+CcOXg -jTO9lbXPxvBU1Y29hw1ibB0/UmZ0/wYzHDzV7S5j2ZbG8ZiCx5RKvGTymzpU7IGZ -vBSCQ4cMa9oDjKoLQdf+J8T5iIE0sf8q4G3QR93BEaVUqVMyzY/2dViOBeTZsaxp -/rZUw602BKJ39VO2dIPVagHglrWir1CPtdedp8K9+DGGCV98CrLbNOGAJRdffW+L -3I7V+c/P9faPav4+lgDJVrDQ40beuaaKXpuOf+oZzKJbdSI8HTZI5PIaAZVhwfB6 -J52DlnTMqQRCCFM0mC6344P58qMp4SPE7aAc9irt3MDfl6nzjQIDAQABMA0GCSqG -SIb3DQEBCwUAA4IBAQB7J6gqVDV25finYI3nNRJpOPMyryUPaRqxr3nlfJRcj6p2 -lVQ1tLtkIBqRHrPk0QZyJMM1vZz2VGHZOSKZQgjUl6p9gkb8d1jfkykDbLocE9FC -STLxOAnTPkOJG2HEQPOsTME2Lyi9V6DeNYLJ2pNfCdboW80VRbMoIn1IAMRVD/be -2cIKOV5ppFCbP+EGRIoTrwtWjXDEn9GitCUJixlH6NKYSSqgi/6MyyPY+OYoxtkL -EHzTzkgHjcdWu8no16ihJJO/X9Kp8TW3QK0Iv4lj5UlAE+ceand/mlsHDOuAd7Cs -+oqduINToR4OFCvJUJaBwsAL0ca2LuqYPnvuXwn3 +MIIDDzCCAfegAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgQwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj +aGUgUHVsc2FyMQ4wDAYDVQQLEwVQcm94eTEOMAwGA1UEAxMFUHJveHkwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDXMWtF9z01MTqHGBaJEYT2c/AzYMu +L4Jw5eCNM72Vtc/G8FTVjb2HDWJsHT9SZnT/BjMcPNXtLmPZlsbxmILHlEq8ZPKb +OlTsgZm8FIJDhwxr2gOMqgtB1/4nxPmIgTSx/yrgbdBH3cERpVSpUzLNj/Z1WI4F +5NmxrGn+tlTDrTYEonf1U7Z0g9VqAeCWtaKvUI+1152nwr34MYYJX3wKsts04YAl +F199b4vcjtX5z8/19o9q/j6WAMlWsNDjRt65popem45/6hnMolt1IjwdNkjk8hoB +lWHB8HonnYOWdMypBEIIUzSYLrfjg/nyoynhI8TtoBz2Ku3cwN+XqfONAgMBAAGj +HjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOC +AQEAjbYsX4cTBqhmzhEqLCAex+5QdafRfK3G7NEY0PqqAPoI+Q/M31maaxwYBxWE +0Jokjd1GeZzcnj6XECSyndT2xXlYh3ymr89pI/tDeg9NJuDpZsWt+ojixW5qznAM +j3MB1v2pHzFJQRdFIsymceT0Dw8uPkkLXwSUNkn6ckLJJXWEmtwWy2lEROU6/yb2 +REJMbOJW1j688ouD3uKRcGW50N2j0d5TJ3cTLYYnw0AvwaVQHFpEUbQpEcMwnRqW +JXrWBXCtBg3ym7G2gjkGx3yySQQZ5H6HuNhCHavt0LB/eWuJdS9qJmc9M1dfWklS +mDsq5UPX+ZfKdc1v6eRmttbCxw== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem index b607fb9d131bf..127f56dd777a5 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem @@ -2,76 +2,76 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - 7f:c3:12:28:23:73:86:8e:bb:d6:e6:21:43:e3:72:e8:01:17:3e:d1 + 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: CN = CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: - 00:b3:6a:94:67:7c:33:90:4e:db:b9:94:b0:a6:1a: - 69:77:bb:33:31:fe:3c:8b:6d:8a:f1:cf:07:d9:87: - 86:ad:45:cf:4c:e3:e7:35:d5:4b:a3:76:27:9b:30: - b1:82:3f:57:29:c9:f0:be:25:49:25:16:64:58:cc: - b0:f1:01:2e:19:69:52:c8:38:64:61:16:b4:a7:ba: - 76:2b:54:e6:a5:80:6c:b6:6c:8a:3c:c1:06:c2:e1: - c1:f3:18:6b:87:08:4b:bb:54:f4:b3:72:1d:f2:ce: - 47:18:5f:82:d3:88:c9:39:7b:71:fc:71:1a:aa:7e: - 55:6c:35:7f:83:c1:60:e7:7d:b1:80:d0:17:7a:ed: - e7:0d:87:8b:59:e3:18:47:e9:cf:de:0d:0e:c6:3e: - 5c:eb:6e:f4:43:95:31:01:2d:e8:f2:ba:8a:bf:ed: - 82:0c:7c:14:14:13:0e:fb:ae:f0:3a:7c:29:ee:55: - 29:ca:46:7a:be:05:9f:fa:75:65:4c:f5:fb:cf:fe: - 92:8d:78:e2:e1:41:55:32:2c:36:a2:ac:96:43:aa: - e2:60:5a:ff:a6:e2:3f:5b:fc:d4:d3:af:cf:78:45: - b5:e7:6e:7d:b6:fa:c4:05:84:a6:49:a7:ac:16:8e: - b2:17:ac:75:76:f0:29:df:c8:da:a2:01:05:25:08: - 4d:8f + 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: + 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: + 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: + d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: + d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: + 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: + 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: + b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: + 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: + fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: + ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: + 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: + 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: + ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: + 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: + 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: + 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: + 48:9d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - 09:93:47:8E:5F:F3:BD:19:A2:77:FD:09:BA:13:A9:B6:C6:75:4E:B0 + 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C X509v3 Authority Key Identifier: - keyid:09:93:47:8E:5F:F3:BD:19:A2:77:FD:09:BA:13:A9:B6:C6:75:4E:B0 + keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C X509v3 Basic Constraints: critical CA:TRUE Signature Algorithm: sha256WithRSAEncryption - a1:52:44:1e:c0:a1:73:48:98:dd:91:b9:a7:e1:da:c5:48:65: - d2:6d:38:77:b5:fa:f6:f7:c5:e4:b7:51:28:ea:f1:6c:9e:82: - 80:6d:6f:56:9c:3b:31:b8:71:0e:ad:17:f9:8e:c6:7e:87:a9: - 5f:30:1c:0e:17:c8:c7:c2:3c:96:3d:7d:01:a9:ce:d0:cd:c3: - 55:6b:ce:64:35:53:93:c6:8c:4c:3d:0d:38:01:17:7b:e2:d8: - b3:a5:78:46:77:fc:7e:da:16:f8:96:d0:72:35:89:c3:15:8c: - 38:37:8b:7f:ff:01:f9:84:b2:e9:8d:11:64:82:36:e7:ef:86: - a6:de:11:d9:78:b4:07:6c:18:89:aa:d6:6d:a2:d8:24:98:40: - 85:5d:ba:5c:36:75:ad:e8:25:03:2d:94:69:d1:ce:d9:8f:9b: - fd:79:5d:4b:30:7a:de:18:08:5a:54:e9:7b:7d:e2:cb:20:65: - 99:4c:5a:31:de:c8:2c:01:b1:c8:d1:30:1d:33:bd:ef:9b:43: - 4d:ac:7d:20:1f:c3:10:53:2e:1a:99:d5:6c:62:0e:15:b3:bd: - 3c:88:58:88:0c:4f:06:21:b7:a4:8c:eb:9f:63:2e:5e:1d:c8: - 91:39:9a:2b:e3:bf:e4:0a:bd:6e:4d:71:15:4d:e1:af:01:15: - 99:38:25:12 + 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: + 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: + 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: + 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: + 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: + ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: + 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: + a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: + a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: + 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: + 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: + 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: + fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: + 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: + ae:fe:19:34 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUf8MSKCNzho671uYhQ+Ny6AEXPtEwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIxMDQyMzE3MDg1MVoXDTMxMDQyMTE3 -MDg1MVowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs2qUZ3wzkE7buZSwphppd7szMf48i22K8c8H2YeGrUXPTOPnNdVL -o3YnmzCxgj9XKcnwviVJJRZkWMyw8QEuGWlSyDhkYRa0p7p2K1TmpYBstmyKPMEG -wuHB8xhrhwhLu1T0s3Id8s5HGF+C04jJOXtx/HEaqn5VbDV/g8Fg532xgNAXeu3n -DYeLWeMYR+nP3g0Oxj5c6270Q5UxAS3o8rqKv+2CDHwUFBMO+67wOnwp7lUpykZ6 -vgWf+nVlTPX7z/6SjXji4UFVMiw2oqyWQ6riYFr/puI/W/zU06/PeEW15259tvrE -BYSmSaesFo6yF6x1dvAp38jaogEFJQhNjwIDAQABo1MwUTAdBgNVHQ4EFgQUCZNH -jl/zvRmid/0JuhOptsZ1TrAwHwYDVR0jBBgwFoAUCZNHjl/zvRmid/0JuhOptsZ1 -TrAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAoVJEHsChc0iY -3ZG5p+HaxUhl0m04d7X69vfF5LdRKOrxbJ6CgG1vVpw7MbhxDq0X+Y7GfoepXzAc -DhfIx8I8lj19AanO0M3DVWvOZDVTk8aMTD0NOAEXe+LYs6V4Rnf8ftoW+JbQcjWJ -wxWMODeLf/8B+YSy6Y0RZII25++Gpt4R2Xi0B2wYiarWbaLYJJhAhV26XDZ1regl -Ay2UadHO2Y+b/XldSzB63hgIWlTpe33iyyBlmUxaMd7ILAGxyNEwHTO975tDTax9 -IB/DEFMuGpnVbGIOFbO9PIhYiAxPBiG3pIzrn2MuXh3IkTmaK+O/5Aq9bk1xFU3h -rwEVmTglEg== +MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs +/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 +8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 +AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS +/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO +bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh +Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud +9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU +yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D +VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w +Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX +9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn +eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk +rEibrv4ZNA== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem index 0fc458dbe5363..192d686246f1a 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:74 + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:01 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = superUser Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: b6:98:ef:dd:03:82:58:a3:32:dc:90:a1:b6:a6:1e: e1:0b Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 33:40:2a:38:48:99:a0:fe:68:4d:07:3b:08:ae:af:a1:7c:ea: - 70:ab:a7:c8:32:b4:ff:9f:5a:51:3b:2b:a2:aa:21:75:44:7d: - be:e7:fb:08:b9:81:e5:4c:cf:01:86:f9:06:63:4f:ce:7a:1d: - cb:1e:9e:8f:d5:0a:54:53:69:91:05:10:2c:b0:4f:d4:3a:b5: - 25:0e:25:4c:eb:67:64:d7:85:29:77:63:30:da:2a:77:3f:59: - c2:8c:e9:02:57:49:93:3a:51:91:1a:b2:59:4d:d5:69:c9:9d: - cc:e2:4f:b2:6c:5b:ba:45:68:c7:f5:18:f4:1d:b8:0c:eb:fd: - 0a:cf:10:5d:dc:3e:26:49:03:33:37:40:f7:96:88:82:99:5c: - 38:8d:cc:3b:de:b5:b9:ee:f9:ac:ae:ce:03:9a:1e:a7:f8:02: - 73:2e:af:e7:b0:22:cb:3d:a3:ca:85:16:e9:e6:e2:d6:bf:1c: - 1a:4c:ea:14:49:52:84:67:38:97:c7:b3:30:72:cc:c6:08:e5: - 40:0a:87:da:19:98:26:4f:0b:54:43:a2:a0:ea:51:b2:23:88: - d2:b4:0e:82:4f:02:92:a4:fb:27:e2:06:15:76:e7:27:f2:a2: - e4:23:7b:24:ca:e6:80:93:2b:cd:54:ca:1b:9b:fd:d9:59:d1: - 96:31:25:7b + 96:c2:23:2d:46:d0:3d:23:0e:ab:3d:b6:1e:31:96:00:eb:ae: + 17:ac:6e:c0:d4:1a:8d:0f:36:63:27:02:49:4e:24:cf:d3:80: + 88:3a:4f:d0:f1:e5:1c:df:2d:8a:ab:ae:8d:48:77:a0:d0:dc: + d5:80:1c:a1:3d:0d:49:64:bf:cb:39:84:c9:f3:5d:e0:2d:ba: + a0:f2:ac:03:85:44:a1:97:6b:0b:de:ed:a7:49:19:46:b2:18: + 49:21:62:43:52:36:6f:47:6c:21:6b:5e:41:85:28:71:6c:22: + 27:35:76:82:ed:ac:ad:d7:fa:9d:4c:7d:6f:44:7e:06:dd:8a: + 11:32:0c:d9:d0:f6:63:2a:40:ae:0d:5a:df:9e:d7:91:8a:db: + 2d:95:f3:19:f0:8f:1e:34:e3:b2:31:67:38:74:fd:3f:e6:49: + 5e:53:eb:88:ae:b1:45:71:0e:67:97:3c:99:4e:c7:ea:1e:02: + 67:b4:54:ef:4f:10:55:4a:70:c0:eb:41:e4:50:d4:48:5e:70: + c5:0f:79:f2:06:3d:35:ea:ce:5d:13:8e:14:65:fc:98:21:16: + 2d:5d:6d:f8:e0:6b:c7:c6:e4:8a:ca:c9:38:1f:93:27:86:28: + ef:96:e7:ad:6c:4a:9e:10:78:48:00:f4:4a:43:dc:87:1d:e3: + d3:39:53:68 -----BEGIN CERTIFICATE----- -MIIC7zCCAdcCFAwmFd+PcR1qMdDar2TvgN6smkZ0MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFcxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYw -FAYDVQQLEw1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1o -sXBGO95E+RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgF -hMrmDCESWDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5Xi -Nr3brotaaGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36 -ukqN6MZaoVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpX -ABuREzH9QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBADNAKjhImaD+aE0HOwiur6F86nCrp8gytP+fWlE7 -K6KqIXVEfb7n+wi5geVMzwGG+QZjT856Hcseno/VClRTaZEFECywT9Q6tSUOJUzr -Z2TXhSl3YzDaKnc/WcKM6QJXSZM6UZEasllN1WnJncziT7JsW7pFaMf1GPQduAzr -/QrPEF3cPiZJAzM3QPeWiIKZXDiNzDvetbnu+ayuzgOaHqf4AnMur+ewIss9o8qF -Funm4ta/HBpM6hRJUoRnOJfHszByzMYI5UAKh9oZmCZPC1RDoqDqUbIjiNK0DoJP -ApKk+yfiBhV25yfyouQjeyTK5oCTK81Uyhub/dlZ0ZYxJXs= +MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgEwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj +aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCXN1cGVyVXNlcjCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1DfZhA+bBbvK7bwAutJpCW +4GI47WixcEY73kT5FFGGEOvKkOeI6PmRheDdtbQUuXjjhtVUbWjsFJK0+CJbBT3t +MSVlCAWEyuYMIRJYMscaYKNP0kqeKBl8RYQAjInc3orlT4iRzKTxgUVMfcL/4sGJ +xhJzleI2vduui1poapBR3iuIX6pn9KjjY9y+GYLMnX/mjfuCviIBPVYTO1sEtOjF +GOYuDfq6So3oxlqhUZpKYtev3bT84tXNrplsXGFWC9cMGndc9TpqVLWeM6ypdSia +dq/QelcAG5ETMf1CiCFHBRABL1m7xzrZ4VhMG2xxtpjv3QOCWKMy3JChtqYe4QsC +AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQCWwiMtRtA9Iw6rPbYeMZYA664XrG7A1BqNDzZjJwJJTiTP04CIOk/Q +8eUc3y2Kq66NSHeg0NzVgByhPQ1JZL/LOYTJ813gLbqg8qwDhUShl2sL3u2nSRlG +shhJIWJDUjZvR2wha15BhShxbCInNXaC7ayt1/qdTH1vRH4G3YoRMgzZ0PZjKkCu +DVrfnteRitstlfMZ8I8eNOOyMWc4dP0/5kleU+uIrrFFcQ5nlzyZTsfqHgJntFTv +TxBVSnDA60HkUNRIXnDFD3nyBj016s5dE44UZfyYIRYtXW344GvHxuSKysk4H5Mn +hijvluetbEqeEHhIAPRKQ9yHHePTOVNo -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem index 0f8bc17b9ed76..c09434c85d20a 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem @@ -1,13 +1,13 @@ Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: - 0c:26:15:df:8f:71:1d:6a:31:d0:da:af:64:ef:80:de:ac:9a:46:75 + 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:02 Signature Algorithm: sha256WithRSAEncryption Issuer: CN = CARoot Validity - Not Before: Apr 23 17:08:51 2021 GMT - Not After : Apr 21 17:08:51 2031 GMT + Not Before: May 30 13:38:24 2022 GMT + Not After : May 27 13:38:24 2032 GMT Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -32,37 +32,41 @@ Certificate: a0:1a:81:9d:d2:e1:66:dd:c4:cc:fc:63:04:ac:ec: a7:35 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 81:a7:27:69:49:e6:1b:c0:f2:a6:10:c2:ef:c7:64:27:69:53: - 3c:bd:8e:7c:b7:b8:bd:2a:02:d4:ab:4b:f3:7b:25:e8:1e:d8: - 3d:88:00:04:6c:a0:da:67:57:65:5d:a2:b6:1d:9a:8c:c7:bd: - 27:53:78:6a:61:3f:61:c1:23:d5:34:65:f1:49:ec:20:5d:f1: - 01:90:99:e8:e6:99:17:ae:c3:ed:e5:da:c4:f1:8c:89:e8:38: - c1:01:e0:84:27:bf:01:f5:ee:62:87:55:6c:63:fc:45:12:d3: - 2f:f7:e2:b9:f0:33:d0:84:1e:6b:23:7b:3e:ae:25:f6:ff:11: - 12:f4:12:63:b6:88:5d:01:aa:ce:c9:e4:d8:78:a2:2d:4c:9a: - 50:4d:57:80:6a:4b:2d:19:4c:61:21:6a:7a:06:2b:cf:82:ae: - f3:61:b0:ef:62:ae:3b:2d:2d:0d:c8:da:75:49:72:5a:1c:8b: - 15:c2:bb:07:5b:37:81:f6:42:e4:84:29:4c:cb:fc:4d:e1:86: - 9b:86:af:1f:03:08:58:b0:15:4c:72:fd:e6:62:e2:b2:37:ca: - eb:a4:67:ec:12:8f:95:57:d7:e7:cf:fe:b5:f9:4a:55:66:c4: - 2f:af:e9:65:a9:54:a8:9d:1a:1e:9a:9e:ec:60:bf:b5:ef:2b: - b6:d5:02:e9 + 88:89:d7:52:b3:61:49:73:7d:ee:aa:6f:47:11:cd:52:f1:ef: + 9a:63:5f:43:a9:4f:66:c8:36:dd:44:24:ba:4f:c3:6c:94:90: + 85:5e:29:fb:65:cf:03:3b:37:16:5e:88:07:70:97:54:93:f0: + f3:09:d7:65:60:09:00:fd:7f:dd:6a:ab:25:3a:30:c4:89:34: + 43:82:f6:f5:f4:2d:39:3d:21:90:c4:00:27:c5:6a:23:41:20: + c6:42:35:56:91:17:fa:31:90:09:6a:4c:e4:a7:53:ae:61:b6: + d3:5b:82:71:08:d0:0b:af:34:0f:9b:bd:bc:8c:1c:31:43:43: + 97:82:9a:ac:2a:53:ca:11:ce:6f:64:ac:86:c1:f0:62:14:aa: + c3:dd:15:5b:1c:02:6f:bb:40:87:17:b7:e5:9d:93:9a:51:c9: + 1e:7a:8c:d1:22:75:44:f1:9d:90:4b:3e:1f:6c:ab:6f:e3:be: + cd:c7:15:9d:04:84:4a:1b:a7:ac:64:5d:d7:3e:23:98:b9:49: + dd:85:dd:80:4c:46:08:9b:f5:df:eb:19:c8:57:70:ac:43:f9: + d6:9c:1b:1b:2a:94:cf:c1:35:56:a2:f4:b1:00:5d:9e:1e:36: + 54:72:ab:aa:ef:49:b2:f0:dc:cf:5b:22:51:bf:e4:c9:57:dc: + d0:48:0d:f2 -----BEGIN CERTIFICATE----- -MIIC7zCCAdcCFAwmFd+PcR1qMdDar2TvgN6smkZ1MA0GCSqGSIb3DQEBCwUAMBEx -DzANBgNVBAMMBkNBUm9vdDAeFw0yMTA0MjMxNzA4NTFaFw0zMTA0MjExNzA4NTFa -MFcxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYw -FAYDVQQLEw1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv -1f0fZ+ORA5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICW -yR8sxFXro3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHa -kroqKO4wKD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CF -gNlo5sZTA3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPX -zDhsg915JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAIGnJ2lJ5hvA8qYQwu/HZCdpUzy9jny3uL0qAtSr -S/N7Jege2D2IAARsoNpnV2VdorYdmozHvSdTeGphP2HBI9U0ZfFJ7CBd8QGQmejm -mReuw+3l2sTxjInoOMEB4IQnvwH17mKHVWxj/EUS0y/34rnwM9CEHmsjez6uJfb/ -ERL0EmO2iF0Bqs7J5Nh4oi1MmlBNV4BqSy0ZTGEhanoGK8+CrvNhsO9irjstLQ3I -2nVJclocixXCuwdbN4H2QuSEKUzL/E3hhpuGrx8DCFiwFUxy/eZi4rI3yuukZ+wS -j5VX1+fP/rX5SlVmxC+v6WWpVKidGh6anuxgv7XvK7bVAuk= +MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgIwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz +MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj +aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCWxvY2FsaG9zdDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+/ty2YrZ322qMT1GIPmL4c +ookium/V/R9n45EDmICBDu3Y9nB/LDZoPVPqWDqm1YlmS70eV3ETbUsR5UCldoQk +kkBYgJbJHyzEVeujeXNwXDeaie0vumvjgnxpSgJUi4FePL9MisvqLF6D57cQCF+C +WKOJ0dqSuioo7jAoP1uuEHGWx+ESxbAarURvRDoRSpo8D40GgHs07z9s9F7FRFQe +yN3HgIWA2WjmxlMDd+H+GGEHdwVM7Vm8XUE4au9dobJgmNRIKJUCig79z3sb0hHM +EAxQc9fMOGyD3XkmqpDIm4SGvFnpYmn0mBvEgHh+oBqBndLhZt3EzPxjBKzspzUC +AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQCIiddSs2FJc33uqm9HEc1S8e+aY19DqU9myDbdRCS6T8NslJCFXin7 +Zc8DOzcWXogHcJdUk/DzCddlYAkA/X/daqslOjDEiTRDgvb19C05PSGQxAAnxWoj +QSDGQjVWkRf6MZAJakzkp1OuYbbTW4JxCNALrzQPm728jBwxQ0OXgpqsKlPKEc5v +ZKyGwfBiFKrD3RVbHAJvu0CHF7flnZOaUckeeozRInVE8Z2QSz4fbKtv477NxxWd +BIRKG6esZF3XPiOYuUndhd2ATEYIm/Xf6xnIV3CsQ/nWnBsbKpTPwTVWovSxAF2e +HjZUcquq70my8NzPWyJRv+TJV9zQSA3y -----END CERTIFICATE----- From 50d95028a21e9284299b60704873a8daaa42c410 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 1 Jun 2022 02:14:49 -0500 Subject: [PATCH 586/823] Configure DLog Bookie, Pulsar, and Admin clients via pass through config (#15818) (cherry picked from commit aa673498f88d0ed4f9d5788a5036355834ea5119) --- conf/broker.conf | 14 ++- conf/functions_worker.yml | 20 +++ conf/proxy.conf | 4 + conf/websocket.conf | 4 + .../broker/BookKeeperClientFactoryImpl.java | 16 +-- .../apache/pulsar/broker/PulsarService.java | 29 +++-- .../broker/namespace/NamespaceService.java | 6 + .../pulsar/broker/service/BrokerService.java | 21 +++- .../pulsar/compaction/CompactorTool.java | 6 + ...ternalClientConfigurationOverrideTest.java | 115 ++++++++++++++++++ ...PulsarClientConfigurationOverrideTest.java | 56 +++++++++ .../proxy/ProxyConfigurationTest.java | 6 + .../client/admin/PulsarAdminBuilder.java | 23 ++++ .../internal/PulsarAdminBuilderImpl.java | 9 +- .../client/internal/PropertiesUtils.java | 64 ++++++++++ .../src/test/resources/test_worker_config.yml | 3 + .../functions/worker/PulsarWorkerService.java | 12 +- .../pulsar/functions/worker/WorkerUtils.java | 45 ++++++- .../functions/worker/WorkerUtilsTest.java | 19 +++ .../bookkeeper/BookKeeperPackagesStorage.java | 8 ++ ...ookKeeperPackagesStorageConfiguration.java | 4 + .../core/PackagesStorageConfiguration.java | 6 + .../DefaultPackagesStorageConfiguration.java | 5 + .../pulsar/proxy/server/ProxyConnection.java | 15 ++- .../pulsar/websocket/WebSocketService.java | 7 +- site2/docs/reference-configuration.md | 21 ++++ 26 files changed, 504 insertions(+), 34 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarClientConfigurationOverrideTest.java create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PropertiesUtils.java diff --git a/conf/broker.conf b/conf/broker.conf index 3d11ce2538ad7..db8b618fff8f7 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -656,6 +656,9 @@ brokerClientTlsCiphers= # used by the internal client to authenticate with Pulsar brokers brokerClientTlsProtocols= +# You can add extra configuration options for the Pulsar Client and the Pulsar Admin Client +# by prefixing them with "brokerClient_". These configurations are applied after hard coded configuration +# and before the above brokerClient configurations named above. ### --- Authentication --- ### @@ -897,8 +900,11 @@ managedLedgerDefaultAckQuorum=2 # in case of lack of enough bookies #bookkeeper_opportunisticStriping=false -# you can add other configuration options for the BookKeeper client -# by prefixing them with bookkeeper_ +# You can add other configuration options for the BookKeeper client +# by prefixing them with "bookkeeper_". These configurations are applied +# to all bookkeeper clients started by the broker (including the managed ledger bookkeeper clients as well as +# the BookkeeperPackagesStorage bookkeeper client), except the distributed log bookkeeper client. +# The dlog bookkeeper client is configured in the functions worker configuration file. # How frequently to flush the cursor positions that were accumulated due to rate limiting. (seconds). # Default is 60 seconds @@ -1349,4 +1355,8 @@ packagesReplicas=1 # The bookkeeper ledger root path packagesManagementLedgerRootPath=/ledgers +# When using BookKeeperPackagesStorageProvider, you can configure the +# bookkeeper client by prefixing configurations with "bookkeeper_". +# This config applies to managed ledger bookkeeper clients, as well. + ### --- Packages management service configuration variables (end) --- ### diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 0b228d26a58e2..a0449cbb2365b 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -362,3 +362,23 @@ validateConnectorConfig: false # Whether to initialize distributed log metadata by runtime. # If it is set to true, you must ensure that it has been initialized by "bin/pulsar initialize-cluster-metadata" command. initializedDlogMetadata: false +########################### +# Arbitrary Configuration +########################### +# When a configuration parameter is not explicitly named in the WorkerConfig class, it is only accessible from the +# properties map. This map can be configured by supplying values to the properties map in this config file. + +# Configure the DLog bookkeeper client by prefixing configurations with "bookkeeper_". Because these are arbitrary, they +# must be added to the properties map to get correctly applied. This configuration applies to the Dlog bookkeeper client +# in both the standalone function workers and function workers initialized in the broker. + +# You can add extra configuration options for the Pulsar Client and the Pulsar Admin Client +# by prefixing them with "brokerClient_". These configurations are applied after hard coded configuration +# and before the above brokerClient configurations named above. + +## For example, when using the token authentication provider (AuthenticationProviderToken), you must configure several +## custom configurations. Here is a sample for configuring one of the necessary configs: +#properties: +# tokenPublicKey: "file:///path/to/my/key" +# tokenPublicAlg: "RSA256" + diff --git a/conf/proxy.conf b/conf/proxy.conf index 77129ccc71dc0..2454b9bf20c00 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -163,6 +163,10 @@ tlsEnabledWithBroker=false # Tls cert refresh duration in seconds (set 0 to check on every new connection) tlsCertRefreshCheckDurationSec=300 +# You can add extra configuration options for the Pulsar Client +# by prefixing them with "brokerClient_". These configurations are applied after hard coded configuration +# and before the above brokerClient configurations named above. + ##### --- Rate Limiting --- ##### # Max concurrent inbound connections. The proxy will reject requests beyond that. diff --git a/conf/websocket.conf b/conf/websocket.conf index 535fade4ea65a..4fe6f7e37b6d8 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -92,6 +92,10 @@ brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= brokerClientTrustCertsFilePath= +# You can add extra configuration options for the Pulsar Client +# by prefixing them with "brokerClient_". These configurations are applied after hard coded configuration +# and before the above brokerClient configurations named above. + # When this parameter is not empty, unauthenticated users perform as anonymousUserRole anonymousUserRole= diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java index 9a09deb166b0f..ce91ecf907cb2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.extern.slf4j.Slf4j; @@ -42,6 +41,7 @@ import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.zookeeper.ZkBookieRackAffinityMapping; @@ -147,15 +147,11 @@ ClientConfiguration createBkClientConfiguration(ServiceConfiguration conf) { conf.getBookkeeperClientGetBookieInfoIntervalSeconds(), TimeUnit.SECONDS); bkConf.setGetBookieInfoRetryIntervalSeconds( conf.getBookkeeperClientGetBookieInfoRetryIntervalSeconds(), TimeUnit.SECONDS); - Properties allProps = conf.getProperties(); - allProps.forEach((key, value) -> { - String sKey = key.toString(); - if (sKey.startsWith("bookkeeper_") && value != null) { - String bkExtraConfigKey = sKey.substring(11); - log.info("Extra BookKeeper client configuration {}, setting {}={}", sKey, bkExtraConfigKey, value); - bkConf.setProperty(bkExtraConfigKey, value); - } - }); + PropertiesUtils.filterAndMapProperties(conf.getProperties(), "bookkeeper_") + .forEach((key, value) -> { + log.info("Applying BookKeeper client configuration setting {}={}", key, value); + bkConf.setProperty(key, value); + }); return bkConf; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index a777ced4f956f..85f21e0bf8f64 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -127,6 +127,8 @@ import org.apache.pulsar.client.api.transaction.TransactionBufferClient; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; @@ -1328,7 +1330,16 @@ protected synchronized OrderedScheduler getOffloaderScheduler(OffloadPoliciesImp public synchronized PulsarClient getClient() throws PulsarServerException { if (this.client == null) { try { - ClientConfigurationData conf = new ClientConfigurationData(); + ClientConfigurationData initialConf = new ClientConfigurationData(); + initialConf.setStatsIntervalSeconds(0); + + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + Map overrides = PropertiesUtils + .filterAndMapProperties(this.getConfiguration().getProperties(), "brokerClient_"); + ClientConfigurationData conf = + ConfigurationDataUtils.loadData(overrides, initialConf, ClientConfigurationData.class); conf.setServiceUrl(this.getConfiguration().isTlsEnabled() ? this.brokerServiceUrlTls : this.brokerServiceUrl); conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); @@ -1356,8 +1367,6 @@ public synchronized PulsarClient getClient() throws PulsarServerException { this.getConfiguration().getBrokerClientAuthenticationPlugin(), this.getConfiguration().getBrokerClientAuthenticationParameters())); } - - conf.setStatsIntervalSeconds(0); this.client = new PulsarClientImpl(conf, ioEventLoopGroup); } catch (Exception e) { throw new PulsarServerException(e); @@ -1377,10 +1386,16 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { + ", webServiceAddressTls: " + webServiceAddressTls + ", webServiceAddress: " + webServiceAddress); } - PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(adminApiUrl) // - .authentication(// - conf.getBrokerClientAuthenticationPlugin(), // - conf.getBrokerClientAuthenticationParameters()); + PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(adminApiUrl); + + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + builder.loadConf(PropertiesUtils.filterAndMapProperties(config.getProperties(), "brokerClient_")); + + builder.authentication( + conf.getBrokerClientAuthenticationPlugin(), + conf.getBrokerClientAuthenticationParameters()); if (conf.isBrokerClientTlsEnabled()) { builder.tlsCiphers(config.getBrokerClientTlsCiphers()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 58c39ac914338..4d910cb901dfc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -67,6 +67,7 @@ import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.lookup.data.LookupData; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -1272,6 +1273,11 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { .enableTcpNoDelay(false) .statsInterval(0, TimeUnit.SECONDS); + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + clientBuilder.loadConf(PropertiesUtils.filterAndMapProperties(config.getProperties(), "brokerClient_")); + if (pulsar.getConfiguration().isAuthenticationEnabled()) { clientBuilder.authentication(pulsar.getConfiguration().getBrokerClientAuthenticationPlugin(), pulsar.getConfiguration().getBrokerClientAuthenticationParameters()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 5dc5edf9f0849..a932e2fa505c6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -127,6 +127,7 @@ import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.configuration.BindAddress; import org.apache.pulsar.common.configuration.FieldContext; @@ -1135,6 +1136,12 @@ public PulsarClient getReplicationClient(String cluster, Optional c .enableTcpNoDelay(false) .connectionsPerBroker(pulsar.getConfiguration().getReplicationConnectionsPerBroker()) .statsInterval(0, TimeUnit.SECONDS); + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + clientBuilder.loadConf(PropertiesUtils.filterAndMapProperties(pulsar.getConfiguration().getProperties(), + "brokerClient_")); + if (data.getAuthenticationPlugin() != null && data.getAuthenticationParameters() != null) { clientBuilder.authentication(data.getAuthenticationPlugin(), data.getAuthenticationParameters()); } else if (pulsar.getConfiguration().isAuthenticationEnabled()) { @@ -1210,10 +1217,16 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c boolean isTlsUrl = conf.isBrokerClientTlsEnabled() && isNotBlank(data.getServiceUrlTls()); String adminApiUrl = isTlsUrl ? data.getServiceUrlTls() : data.getServiceUrl(); - PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(adminApiUrl) - .authentication( - conf.getBrokerClientAuthenticationPlugin(), - conf.getBrokerClientAuthenticationParameters()); + PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(adminApiUrl); + + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + builder.loadConf(PropertiesUtils.filterAndMapProperties(conf.getProperties(), "brokerClient_")); + + builder.authentication( + conf.getBrokerClientAuthenticationPlugin(), + conf.getBrokerClientAuthenticationParameters()); if (isTlsUrl) { builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index ac028ef871b79..35ca089e5df9b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -37,6 +37,7 @@ import org.apache.pulsar.broker.ServiceConfigurationUtils; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.netty.EventLoopUtil; @@ -105,6 +106,11 @@ public static void main(String[] args) throws Exception { ClientBuilder clientBuilder = PulsarClient.builder(); + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + clientBuilder.loadConf(PropertiesUtils.filterAndMapProperties(brokerConfig.getProperties(), "brokerClient_")); + if (isNotBlank(brokerConfig.getBrokerClientAuthenticationPlugin())) { clientBuilder.authentication(brokerConfig.getBrokerClientAuthenticationPlugin(), brokerConfig.getBrokerClientAuthenticationParameters()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java new file mode 100644 index 0000000000000..775636c9489fb --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; +import java.util.Properties; + +public class BrokerInternalClientConfigurationOverrideTest extends BrokerTestBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testPulsarServiceAdminClientConfiguration() throws PulsarServerException { + Properties config = pulsar.getConfiguration().getProperties(); + config.setProperty("brokerClient_operationTimeoutMs", "60000"); + config.setProperty("brokerClient_statsIntervalSeconds", "10"); + ClientConfigurationData clientConf = ((PulsarAdminImpl) pulsar.getAdminClient()).getClientConfigData(); + Assert.assertEquals(clientConf.getOperationTimeoutMs(), 60000); + Assert.assertEquals(clientConf.getStatsIntervalSeconds(), 10); + } + + @Test + public void testPulsarServicePulsarClientConfiguration() throws PulsarServerException { + Properties config = pulsar.getConfiguration().getProperties(); + config.setProperty("brokerClient_operationTimeoutMs", "60000"); + config.setProperty("brokerClient_statsIntervalSeconds", "10"); + pulsar.getConfiguration().setBrokerClientAuthenticationParameters("sensitive"); + ClientConfigurationData clientConf = ((PulsarClientImpl) pulsar.getClient()).getConfiguration(); + Assert.assertEquals(clientConf.getOperationTimeoutMs(), 60000); + // Config should override internal default, which is 0. + Assert.assertEquals(clientConf.getStatsIntervalSeconds(), 10); + Assert.assertEquals(clientConf.getAuthParams(), "sensitive"); + } + + @Test + public void testBrokerServicePulsarClientConfiguration() { + // This data only needs to have the service url for this test. + ClusterData data = ClusterData.builder().serviceUrl("http://localhost:8080").build(); + + // Set the configs and set some configs that won't apply + Properties config = pulsar.getConfiguration().getProperties(); + config.setProperty("brokerClient_operationTimeoutMs", "60000"); + config.setProperty("brokerClient_statsIntervalSeconds", "10"); + config.setProperty("memoryLimitBytes", "10"); + config.setProperty("brokerClient_memoryLimitBytes", "100000"); + + PulsarClientImpl client = (PulsarClientImpl) pulsar.getBrokerService() + .getReplicationClient("an_arbitrary_name", Optional.of(data)); + ClientConfigurationData clientConf = client.getConfiguration(); + Assert.assertEquals(clientConf.getOperationTimeoutMs(), 60000); + // Config should override internal default, which is 0. + Assert.assertEquals(clientConf.getStatsIntervalSeconds(), 10); + // This config defaults to 0 (for good reason), but it could be overridden by configuration. + Assert.assertEquals(clientConf.getMemoryLimitBytes(), 100000); + } + + @Test + public void testNamespaceServicePulsarClientConfiguration() { + // This data only needs to have the service url for this test. + ClusterDataImpl data = (ClusterDataImpl) ClusterData.builder().serviceUrl("http://localhost:8080").build(); + + // Set the configs and set some configs that won't apply + Properties config = pulsar.getConfiguration().getProperties(); + config.setProperty("brokerClient_operationTimeoutMs", "60000"); + config.setProperty("brokerClient_statsIntervalSeconds", "10"); + config.setProperty("memoryLimitBytes", "10"); + config.setProperty("brokerClient_memoryLimitBytes", "100000"); + + PulsarClientImpl client = pulsar.getNamespaceService().getNamespaceClient(data); + ClientConfigurationData clientConf = client.getConfiguration(); + Assert.assertEquals(clientConf.getOperationTimeoutMs(), 60000); + // Config should override internal default, which is 0. + Assert.assertEquals(clientConf.getStatsIntervalSeconds(), 10); + // This config defaults to 0 (for good reason), but it could be overridden by configuration. + Assert.assertEquals(clientConf.getMemoryLimitBytes(), 100000); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarClientConfigurationOverrideTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarClientConfigurationOverrideTest.java new file mode 100644 index 0000000000000..4f885ecc46b07 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarClientConfigurationOverrideTest.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.internal.PropertiesUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.Map; + +public class PulsarClientConfigurationOverrideTest { + @Test + public void testFilterAndMapProperties() { + // Create a default config + ServiceConfiguration conf = new ServiceConfiguration(); + conf.getProperties().setProperty("keepAliveIntervalSeconds", "15"); + conf.getProperties().setProperty("brokerClient_keepAliveIntervalSeconds", "25"); + + // Apply the filtering and mapping logic + Map result = PropertiesUtils.filterAndMapProperties(conf.getProperties(), "brokerClient_"); + + // Ensure the results match expectations + Assert.assertEquals(result.size(), 1, "The filtered map should have one entry."); + Assert.assertNull(result.get("brokerClient_keepAliveIntervalSeconds"), + "The mapped prop should not be in the result."); + Assert.assertEquals(result.get("keepAliveIntervalSeconds"), "25", "The original value is overridden."); + + // Create sample ClientBuilder + ClientBuilder builder = PulsarClient.builder(); + Assert.assertEquals( + ((ClientBuilderImpl) builder).getClientConfigurationData().getKeepAliveIntervalSeconds(), 30); + // Note: this test would fail if any @Secret fields were set before the loadConf and the accessed afterwards. + builder.loadConf(result); + Assert.assertEquals( + ((ClientBuilderImpl) builder).getClientConfigurationData().getKeepAliveIntervalSeconds(), 25); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java index ec4937bdd2149..184f86340fa9a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java @@ -66,6 +66,9 @@ public Object[][] setProxyConfig() { public void configTest(int numIoThreads, int connectionsPerBroker) throws Exception { config.setWebSocketNumIoThreads(numIoThreads); config.setWebSocketConnectionsPerBroker(connectionsPerBroker); + config.getProperties().setProperty("brokerClient_serviceUrl", "https://broker.com:8080"); + config.setServiceUrl("http://localhost:8080"); + config.getProperties().setProperty("brokerClient_lookupTimeoutMs", "100"); WebSocketService service = spyWithClassAndConstructorArgs(WebSocketService.class, config); doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service).createMetadataStore(anyString(), anyInt()); service.start(); @@ -73,6 +76,9 @@ public void configTest(int numIoThreads, int connectionsPerBroker) throws Except PulsarClientImpl client = (PulsarClientImpl) service.getPulsarClient(); assertEquals(client.getConfiguration().getNumIoThreads(), numIoThreads); assertEquals(client.getConfiguration().getConnectionsPerBroker(), connectionsPerBroker); + assertEquals(client.getConfiguration().getServiceUrl(), "http://localhost:8080", + "brokerClient_ configs take precedence"); + assertEquals(client.getConfiguration().getLookupTimeoutMs(), 100); service.close(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java index 9f8b4be140908..c685c1f77936d 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java @@ -36,6 +36,29 @@ public interface PulsarAdminBuilder { */ PulsarAdmin build() throws PulsarClientException; + /** + * Load the configuration from provided config map. + * + *

    Example: + * + * 

    +     * {@code
+     * Map config = new HashMap<>();
+     * config.put("serviceHttpUrl", "http://localhost:6650");
+     *
+     * PulsarAdminBuilder builder = ...;
+     * builder = builder.loadConf(config);
+     *
+     * PulsarAdmin client = builder.build();
+     * }
+     *
    + * + * @param config + * configuration to load + * @return the client builder instance + */ + PulsarAdminBuilder loadConf(Map config); + /** * Create a copy of the current client builder. *

    diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 70463b7fb4e9a..d86b9e73457ca 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -28,10 +28,11 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; public class PulsarAdminBuilderImpl implements PulsarAdminBuilder { - protected final ClientConfigurationData conf; + protected ClientConfigurationData conf; private int connectTimeout = PulsarAdminImpl.DEFAULT_CONNECT_TIMEOUT_SECONDS; private int readTimeout = PulsarAdminImpl.DEFAULT_READ_TIMEOUT_SECONDS; private int requestTimeout = PulsarAdminImpl.DEFAULT_REQUEST_TIMEOUT_SECONDS; @@ -62,6 +63,12 @@ public PulsarAdminBuilder clone() { return new PulsarAdminBuilderImpl(conf.clone()); } + @Override + public PulsarAdminBuilder loadConf(Map config) { + conf = ConfigurationDataUtils.loadData(config, conf, ClientConfigurationData.class); + return this; + } + @Override public PulsarAdminBuilder serviceHttpUrl(String serviceHttpUrl) { conf.setServiceUrl(serviceHttpUrl); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PropertiesUtils.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PropertiesUtils.java new file mode 100644 index 0000000000000..4a418b1d5158e --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PropertiesUtils.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.internal; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Internal utility methods for filtering and mapping {@link Properties} objects. + */ +public class PropertiesUtils { + + /** + * Filters the {@link Properties} object so that only properties with the configured prefix are retained, + * and then removes that prefix and puts the key value pairs into the result map. + * @param props - the properties object to filter + * @param prefix - the prefix to filter against and then remove for keys in the resulting map + * @return a map of properties + */ + public static Map filterAndMapProperties(Properties props, String prefix) { + return filterAndMapProperties(props, prefix, ""); + } + + /** + * Filters the {@link Properties} object so that only properties with the configured prefix are retained, + * and then replaces the srcPrefix with the targetPrefix when putting the key value pairs in the resulting map. + * @param props - the properties object to filter + * @param srcPrefix - the prefix to filter against and then remove for keys in the resulting map + * @param targetPrefix - the prefix to add to keys in the result map + * @return a map of properties + */ + public static Map filterAndMapProperties(Properties props, String srcPrefix, String targetPrefix) { + Map result = new HashMap<>(); + int prefixLength = srcPrefix.length(); + props.forEach((keyObject, value) -> { + if (!(keyObject instanceof String)) { + return; + } + String key = (String) keyObject; + if (key.startsWith(srcPrefix) && value != null) { + String truncatedKey = key.substring(prefixLength); + result.put(targetPrefix + truncatedKey, value); + } + }); + return result; + } +} diff --git a/pulsar-functions/src/test/resources/test_worker_config.yml b/pulsar-functions/src/test/resources/test_worker_config.yml index 4614ca3cfd1c2..f0ecf2bd71bc6 100644 --- a/pulsar-functions/src/test/resources/test_worker_config.yml +++ b/pulsar-functions/src/test/resources/test_worker_config.yml @@ -23,4 +23,7 @@ pulsarServiceUrl: pulsar://localhost:6650 functionMetadataTopicName: test-function-metadata-topic numFunctionPackageReplicas: 3 maxPendingAsyncRequests: 200 +properties: + # Fake Bookkeeper Client config to be applied to the DLog Bookkeeper Client + bookkeeper_testKey: "fakeValue" diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index ef92e85853b7b..9ac0374675620 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -133,7 +133,8 @@ public PulsarAdmin newPulsarAdmin(String pulsarServiceUrl, WorkerConfig workerCo workerConfig.getBrokerClientAuthenticationParameters(), workerConfig.getBrokerClientTrustCertsFilePath(), workerConfig.isTlsAllowInsecureConnection(), - workerConfig.isTlsEnableHostnameVerification()); + workerConfig.isTlsEnableHostnameVerification(), + workerConfig); } else { return WorkerUtils.getPulsarAdminClient( pulsarServiceUrl, @@ -141,7 +142,8 @@ public PulsarAdmin newPulsarAdmin(String pulsarServiceUrl, WorkerConfig workerCo null, null, workerConfig.isTlsAllowInsecureConnection(), - workerConfig.isTlsEnableHostnameVerification()); + workerConfig.isTlsEnableHostnameVerification(), + workerConfig); } } @@ -156,7 +158,8 @@ public PulsarClient newPulsarClient(String pulsarServiceUrl, WorkerConfig worker workerConfig.isUseTls(), workerConfig.getBrokerClientTrustCertsFilePath(), workerConfig.isTlsAllowInsecureConnection(), - workerConfig.isTlsEnableHostnameVerification()); + workerConfig.isTlsEnableHostnameVerification(), + workerConfig); } else { return WorkerUtils.getPulsarClient( pulsarServiceUrl, @@ -165,7 +168,8 @@ public PulsarClient newPulsarClient(String pulsarServiceUrl, WorkerConfig worker null, null, workerConfig.isTlsAllowInsecureConnection(), - workerConfig.isTlsEnableHostnameVerification()); + workerConfig.isTlsEnableHostnameVerification(), + workerConfig); } } }; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java index 4f3ee0428666c..741a89bc397bd 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java @@ -40,6 +40,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl; @@ -158,6 +159,13 @@ public static DistributedLogConfiguration getDlogConf(WorkerConfig workerConfig) workerConfig.getBookkeeperClientAuthenticationParameters()); } } + // Map arbitrary bookkeeper client configuration into DLog Config. Note that this only configures the + // bookie client. + PropertiesUtils.filterAndMapProperties(workerConfig.getProperties(), "bookkeeper_", "bkc.") + .forEach((key, value) -> { + log.info("Applying DLog BookKeeper client configuration setting {}={}", key, value); + conf.setProperty(key, value); + }); return conf; } @@ -194,12 +202,20 @@ public static URI initializeDlogNamespace(InternalConfigurationData internalConf } public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl) { - return getPulsarAdminClient(pulsarWebServiceUrl, null, null, null, null, null); + return getPulsarAdminClient(pulsarWebServiceUrl, null, null, null, null, null, null); } public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl, String authPlugin, String authParams, String tlsTrustCertsFilePath, Boolean allowTlsInsecureConnection, Boolean enableTlsHostnameVerificationEnable) { + return getPulsarAdminClient(pulsarWebServiceUrl, authPlugin, authParams, tlsTrustCertsFilePath, + allowTlsInsecureConnection, enableTlsHostnameVerificationEnable, null); + } + + public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl, String authPlugin, String authParams, + String tlsTrustCertsFilePath, Boolean allowTlsInsecureConnection, + Boolean enableTlsHostnameVerificationEnable, + WorkerConfig workerConfig) { log.info("Create Pulsar Admin to service url {}: " + "authPlugin = {}, authParams = {}, " + "tlsTrustCerts = {}, allowTlsInsecureConnector = {}, enableTlsHostnameVerification = {}", @@ -207,6 +223,13 @@ public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl, Strin tlsTrustCertsFilePath, allowTlsInsecureConnection, enableTlsHostnameVerificationEnable); try { PulsarAdminBuilder adminBuilder = PulsarAdmin.builder().serviceHttpUrl(pulsarWebServiceUrl); + if (workerConfig != null) { + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + adminBuilder.loadConf( + PropertiesUtils.filterAndMapProperties(workerConfig.getProperties(), "brokerClient_")); + } if (isNotBlank(authPlugin) && isNotBlank(authParams)) { adminBuilder.authentication(authPlugin, authParams); } @@ -219,6 +242,7 @@ public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl, Strin if (enableTlsHostnameVerificationEnable != null) { adminBuilder.enableTlsHostnameVerification(enableTlsHostnameVerificationEnable); } + return adminBuilder.build(); } catch (PulsarClientException e) { log.error("Error creating pulsar admin client", e); @@ -228,17 +252,33 @@ public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl, Strin public static PulsarClient getPulsarClient(String pulsarServiceUrl) { return getPulsarClient(pulsarServiceUrl, null, null, null, - null, null, null); + null, null, null, null); } public static PulsarClient getPulsarClient(String pulsarServiceUrl, String authPlugin, String authParams, Boolean useTls, String tlsTrustCertsFilePath, Boolean allowTlsInsecureConnection, Boolean enableTlsHostnameVerificationEnable) { + return getPulsarClient(pulsarServiceUrl, authPlugin, authParams, useTls, tlsTrustCertsFilePath, + allowTlsInsecureConnection, enableTlsHostnameVerificationEnable, null); + } + + public static PulsarClient getPulsarClient(String pulsarServiceUrl, String authPlugin, String authParams, + Boolean useTls, String tlsTrustCertsFilePath, + Boolean allowTlsInsecureConnection, + Boolean enableTlsHostnameVerificationEnable, + WorkerConfig workerConfig) { try { ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(pulsarServiceUrl); + if (workerConfig != null) { + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + clientBuilder.loadConf( + PropertiesUtils.filterAndMapProperties(workerConfig.getProperties(), "brokerClient_")); + } if (isNotBlank(authPlugin) && isNotBlank(authParams)) { clientBuilder.authentication(authPlugin, authParams); @@ -255,7 +295,6 @@ && isNotBlank(authParams)) { if (enableTlsHostnameVerificationEnable != null) { clientBuilder.enableTlsHostnameVerification(enableTlsHostnameVerificationEnable); } - return clientBuilder.build(); } catch (PulsarClientException e) { log.error("Error creating pulsar client", e); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java index d899db1323748..b2e0f0f354cbc 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java @@ -40,8 +40,13 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.fail; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import org.apache.distributedlog.DistributedLogConfiguration; public class WorkerUtilsTest { @@ -99,4 +104,18 @@ public Boolean get() { } } + + @Test + public void testDLogConfiguration() throws URISyntaxException, IOException { + // The config yml is seeded with a fake bookie config. + URL yamlUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); + WorkerConfig config = WorkerConfig.load(yamlUrl.toURI().getPath()); + + // Map the config. + DistributedLogConfiguration dlogConf = WorkerUtils.getDlogConf(config); + + // Verify the outcome. + assertEquals(dlogConf.getString("bkc.testKey"), "fakeValue", + "The bookkeeper client config mapping should apply."); + } } \ No newline at end of file diff --git a/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorage.java b/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorage.java index f0db59351f5fc..e3147c0e8bc34 100644 --- a/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorage.java +++ b/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorage.java @@ -36,6 +36,7 @@ import org.apache.distributedlog.impl.metadata.BKDLConfig; import org.apache.distributedlog.metadata.DLMetadata; import org.apache.distributedlog.namespace.NamespaceDriver; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.packages.management.core.PackagesStorage; import org.apache.pulsar.packages.management.core.PackagesStorageConfiguration; import org.apache.zookeeper.KeeperException; @@ -72,6 +73,13 @@ public void initialize() { configuration.getBookkeeperClientAuthenticationParameters()); } } + // Map arbitrary bookkeeper client configuration into DLog Config. Note that this only configures the + // bookie client. + PropertiesUtils.filterAndMapProperties(configuration.getProperties(), "bookkeeper_", "bkc.") + .forEach((key, value) -> { + log.info("Applying DLog BookKeeper client configuration setting {}={}", key, value); + conf.setProperty(key, value); + }); try { this.namespace = NamespaceBuilder.newBuilder() .conf(conf).clientId(NS_CLIENT_ID).uri(initializeDlogNamespace()).build(); diff --git a/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorageConfiguration.java b/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorageConfiguration.java index 226b80abeaa30..ce6acecdd5100 100644 --- a/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorageConfiguration.java +++ b/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/BookKeeperPackagesStorageConfiguration.java @@ -58,6 +58,10 @@ String getBookkeeperClientAuthenticationParameters() { return getProperty("bookkeeperClientAuthenticationParameters"); } + @Override + public Properties getProperties() { + return configuration.getProperties(); + } @Override public String getProperty(String key) { diff --git a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/PackagesStorageConfiguration.java b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/PackagesStorageConfiguration.java index b4044a6338c20..5c346a0d05c42 100644 --- a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/PackagesStorageConfiguration.java +++ b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/PackagesStorageConfiguration.java @@ -50,4 +50,10 @@ public interface PackagesStorageConfiguration { * a group of the property */ void setProperty(Properties properties); + + /** + * Get all properties for the configuration. + * @return all properties for the configuration + */ + Properties getProperties(); } diff --git a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/DefaultPackagesStorageConfiguration.java b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/DefaultPackagesStorageConfiguration.java index cb35048a360b5..d3c5d7494b370 100644 --- a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/DefaultPackagesStorageConfiguration.java +++ b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/DefaultPackagesStorageConfiguration.java @@ -39,4 +39,9 @@ public void setProperty(String key, String value) { public void setProperty(Properties properties) { this.properties = properties; } + + @Override + public Properties getProperties() { + return this.properties; + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index d9f0f5db38fd4..0f41208fde292 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -28,6 +28,7 @@ import java.net.SocketAddress; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadLocalRandom; @@ -47,6 +48,8 @@ import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarChannelInitializer; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarHandler; @@ -521,9 +524,17 @@ protected void handleLookup(CommandLookupTopic lookup) { } ClientConfigurationData createClientConfiguration() { - ClientConfigurationData clientConf = new ClientConfigurationData(); - clientConf.setServiceUrl(service.getServiceUrl()); + ClientConfigurationData initialConf = new ClientConfigurationData(); + initialConf.setServiceUrl(service.getServiceUrl()); ProxyConfiguration proxyConfig = service.getConfiguration(); + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + Map overrides = PropertiesUtils + .filterAndMapProperties(proxyConfig.getProperties(), "brokerClient_"); + ClientConfigurationData clientConf = ConfigurationDataUtils + .loadData(overrides, initialConf, ClientConfigurationData.class); + clientConf.setAuthentication(this.getClientAuthentication()); if (proxyConfig.isTlsEnabledWithBroker()) { clientConf.setUseTls(true); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java index b2873d778ab2b..7a0f19bad1086 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java @@ -37,6 +37,7 @@ import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; @@ -181,6 +182,11 @@ private PulsarClient createClientInstance(ClusterData clusterData) throws IOExce .ioThreads(config.getWebSocketNumIoThreads()) // .connectionsPerBroker(config.getWebSocketConnectionsPerBroker()); + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + clientBuilder.loadConf(PropertiesUtils.filterAndMapProperties(config.getProperties(), "brokerClient_")); + if (isNotBlank(config.getBrokerClientAuthenticationPlugin()) && isNotBlank(config.getBrokerClientAuthenticationParameters())) { clientBuilder.authentication(config.getBrokerClientAuthenticationPlugin(), @@ -198,7 +204,6 @@ && isNotBlank(config.getBrokerClientAuthenticationParameters())) { } else { clientBuilder.serviceUrl(clusterData.getServiceUrl()); } - return clientBuilder.build(); } diff --git a/site2/docs/reference-configuration.md b/site2/docs/reference-configuration.md index f8d1e87f890b2..3226007209fd6 100644 --- a/site2/docs/reference-configuration.md +++ b/site2/docs/reference-configuration.md @@ -352,6 +352,15 @@ brokerServiceCompactionThresholdInBytes|If the estimated backlog size is greater | managedLedgerInfoCompressionType | Compression type of managed ledger information.

    Available options are `NONE`, `LZ4`, `ZLIB`, `ZSTD`, and `SNAPPY`).

    If this value is `NONE` or invalid, the `managedLedgerInfo` is not compressed.

    **Note** that after enabling this configuration, if you want to degrade a broker, you need to change the value to `NONE` and make sure all ledger metadata is saved without compression. | None | | additionalServlets | Additional servlet name.

    If you have multiple additional servlets, separate them by commas.

    For example, additionalServlet_1, additionalServlet_2 | N/A | | additionalServletDirectory | Location of broker additional servlet NAR directory | ./brokerAdditionalServlet | +#### Configuration Override For Clients Internal to Broker + +It's possible to configure some clients by using the appropriate prefix. + +|Prefix|Description| +|brokerClient_| Configure **all** the broker's Pulsar Clients and Pulsar Admin Clients. These configurations are applied after hard coded configuration and before the above brokerClient configurations named above.| +|bookkeeper_| Configure the broker's bookkeeper clients used by managed ledgers and the BookkeeperPackagesStorage bookkeeper client. Takes precedence over most other configuration values.| + +Note: when running the function worker within the broker, these prefixed configurations do not apply to any of those clients. You must instead configure those clients using the `functions_worker.yml`. ## Client @@ -677,6 +686,12 @@ You can set the log level and configuration in the [log4j2.yaml](https://github |tlsCertificateFilePath||| |tlsKeyFilePath ||| |tlsTrustCertsFilePath||| +#### Configuration Override For Clients Internal to WebSocket + +It's possible to configure some clients by using the appropriate prefix. + +|Prefix|Description| +|brokerClient_| Configure **all** the broker's Pulsar Clients. These configurations are applied after hard coded configuration and before the above brokerClient configurations named above.| ## Pulsar proxy @@ -734,6 +749,12 @@ The [Pulsar proxy](concepts-architecture-overview.md#pulsar-proxy) can be config |haProxyProtocolEnabled | Enable or disable the [HAProxy](http://www.haproxy.org/) protocol. |false| | numIOThreads | Number of threads used for Netty IO. | 2 * Runtime.getRuntime().availableProcessors() | | numAcceptorThreads | Number of threads used for Netty Acceptor. | 1 | +#### Configuration Override For Clients Internal to Proxy + +It's possible to configure some clients by using the appropriate prefix. + +|Prefix|Description| +|brokerClient_| Configure **all** the proxy's Pulsar Clients. These configurations are applied after hard coded configuration and before the above brokerClient configurations named above.| ## ZooKeeper From b274e449aace4c1fdd8f59e3f075636e9f4b6ade Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 1 Jun 2022 10:57:20 +0300 Subject: [PATCH 587/823] Bump version to 2.9.3-SNAPSHOT --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- deployment/terraform-ansible/deploy-pulsar.yaml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/java-version-trim-agent/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 128 files changed, 128 insertions(+), 128 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index ddca4e6494ada..9b36a306b979c 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index fb84e6fe63d32..616aa02c01a43 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 4fa3ead0eadd4..389cfb7f2e90c 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 979803c96f58c..49bc0d85da445 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 5e2454863e555..003bc118cfcc7 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.2 + 2.9.3-SNAPSHOT jar Pulsar Build Tools diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml index cdde04d8aab5c..a06b73b99852e 100644 --- a/deployment/terraform-ansible/deploy-pulsar.yaml +++ b/deployment/terraform-ansible/deploy-pulsar.yaml @@ -39,7 +39,7 @@ zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" service_url: "{{ pulsar_service_url }}" http_url: "{{ pulsar_web_url }}" - pulsar_version: "2.9.2" + pulsar_version: "2.9.3-SNAPSHOT" - name: Download Pulsar binary package unarchive: src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 155b7da8a080d..4ee3328cc4d8b 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index ff4055cafd909..1248ea1682609 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 72afff04ad6ab..f79f26d45fb26 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 5fa966d476187..ed5401ad39650 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index df9617f314ec9..adb423253ef31 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 94fabef4d82be..a3534d2c0c83d 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index a897b00b648cb..b9e4f8d158a49 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 36ebfa4161072..d75e18aab9e45 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 8067c21f6939d..2b45d8f6ae38b 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 85d189f87b5b5..f32bc87c46e27 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 63525923b0003..509d00380abc0 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 4e05ec6072ed9..06955be60d7bf 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index c6ff0570e1bde..fdd52671868fe 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 660097361abad..92f27eb385ca7 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 77757b42fa6d8..9e755b12161f1 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 289105f78f2f8..68a53c806ffef 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 60b175c3d4ecd..2d4605a7a0153 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 6078062dc1e69..ac7d7ec5354da 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 59cf04834338d..b17e509d89a9f 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 1f3346fd0da53..6c0bde533061e 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 60840c9f37f44..e5147c4837a8a 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index faebb94d05622..0a3dc027dfb86 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index f6db9d39ee693..74f20a15f4caf 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 2722b42911976..37d12891d049c 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index c9ef5d87e4106..e7808b61a03ea 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 40f5f8407b8d5..11f5fac560a32 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 020af46f5f855..8e7a0b4ecad6e 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 9f9528b58e80e..3b24ced062b6e 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 97002332870b7..891bd2f7f8b88 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 2397289149c1e..593d5842448e7 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 9252d603f32b4..27838ecddf998 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 089ac49c5d579..f5c6cd2e06392 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index fd715f87e0015..d42d74752e5f5 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index afbc14ec0eec1..759e6acccf810 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 6e42f63c66acc..1392e9badc7ba 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index d5d37d325562b..61935a3fa9af2 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index ad37c3588e0fe..f6cf87853f7d2 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 254ed1437d129..732dee3f597ea 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index b5c22b5cff2af..d05780478a168 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 36591e6f2f194..727304d30e78a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 871a07a9658dc..2506936143b58 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 6a11a7551fdc1..37e0267590edd 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 2037b5d3f6194..4278a6553ab11 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 30c8375a0c7b9..247a6393a81fd 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 676a983f20dc2..b16f364689af8 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 8bc57600a8759..7b76ab13e098a 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 8bc73937e2029..83395d20966b3 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 176d920631e2f..f06c5a609b1f3 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 6ce7e8c415c03..70778879c23df 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index b66f5bddd275b..3833961d94c57 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 4c5b57d67fbbd..889f0ab150cb2 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 311197c9f2756..bf92cea841c28 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index d4a711ed01186..71606c0548153 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 917981f55658c..f1c47fb5dfa2a 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 1a49c3eb32b6e..81dad1f245cee 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 5401ace104fc4..21423a08143bd 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index c6c8c58f95e27..53910d0cff22f 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 0a0f831bef961..229623deee7a0 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index c963ae626c7e4..a68c942d97180 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index cde53aeb6359b..e5badf75a677e 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 79b92b89999c2..f7005c635e9ff 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 5bcf07941e921..81942862019f0 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 38c8b53a6bd0b..e8f597224e582 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 0a173aa8650e8..fbdaa6345abb0 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 081620e25ec6d..3faf309b68328 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 4a7a2a335e4b7..0fb45c9a5c009 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index b259cbb1de580..a760ee9584693 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index cde101487ff3e..5025e47bf2dbc 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 2f3e178a9de09..beec6c6ab96e5 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index d6dabf60355ba..8a23962c7388d 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 3fa60cd1807d7..ef84dd16c2e00 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index b1fcbc010f996..5e48ab354e27b 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 2db6fb2ef7d99..6bc9c0a025cd3 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 483a855523e08..67b887c1e07ff 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 5f95ff1e50211..aad83eaeeaa06 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 127c6f72ef6fa..b53a8ed9282df 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index e1d65e87302bf..67327a8f4d46c 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index af7f025330897..8199c1fe3df18 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 4df689dfff5d2..c22f704565290 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index f3060d71166bc..a6222fda0a7e0 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 5d6166e13ffad..76e8aa5fbd05a 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 1678a9294dc89..d37dd1e8f9f08 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 2bf4d0baf3452..739f1acade64f 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index bdf1081106964..6a4d3df44430b 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 5974289444586..350bcfd05a345 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index e573833baba2d..8824f35891b16 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 7291f3e88f29e..bb10dec5cdc53 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 76fcf52c380fa..72a9162706f89 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 780e216c96484..f8f8a2a6491ee 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.2 + 2.9.3-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 809def86ed1dc..468a8debdb903 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index e688a104a16f2..eb93d9900c971 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 168397dacb44d..c205dcad6d505 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index f6f776ce015a3..ad838c8a96bab 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index e132a5de6ace0..0faafd53034dd 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-proxy diff --git a/pulsar-sql/java-version-trim-agent/pom.xml b/pulsar-sql/java-version-trim-agent/pom.xml index 2f1171408b601..cc2026bb53386 100644 --- a/pulsar-sql/java-version-trim-agent/pom.xml +++ b/pulsar-sql/java-version-trim-agent/pom.xml @@ -24,7 +24,7 @@ pulsar-sql org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 5aac5ffa4564a..bf4166cb75fb3 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 2785c979c7a5e..66fb2213af547 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2 + 2.9.3-SNAPSHOT pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index b1f9486286769..f49d541cdd570 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2 + 2.9.3-SNAPSHOT pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 129f0854bc81f..80896ed2c8a5a 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.2 + 2.9.3-SNAPSHOT pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index c31872944da0d..9e686cde439c5 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index ddc76bebb24e9..58ca5ad6fc17a 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index b4ddf32ae19de..b274931049df1 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index e0b566a0c3434..7aeb860a9eaf3 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 2fff29d1c0165..8a85e3f8d641d 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index e2c29fe00ae0b..4b21f3e8ed25c 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index e52e5b7350f2c..1bead7e397c59 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index b18cd2eb75c9b..c4db8a1ade2de 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.2 + 2.9.3-SNAPSHOT testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 80f166bb0c575..acb01809c8f13 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 9a61cb41d4f6e..e0ac02e7f4747 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 559bc5fe6f658..9cc9619d2a388 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 20e71e217ce44..06d6f18a678d2 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 46558ca492aa2..063a4c1a50075 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index c9e16e115f056..40518114309b2 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.2 + 2.9.3-SNAPSHOT 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index f2681751f3eda..e114b1c6bd945 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 1314c15dcb45c..16a0eb7d6bd5a 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT integration diff --git a/tests/pom.xml b/tests/pom.xml index 512c1529b195f..e34f6678186bb 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index a84660e326864..dbb9be2d0d670 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 9bdd8fea40493..664c56cfd5044 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 281cf4cd64c02..191c31527877e 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.2 + 2.9.3-SNAPSHOT pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index f839ad30810ee..3e3c1985f0b6d 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 716aed18b98f0..b776161662dc8 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.2 + 2.9.3-SNAPSHOT .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 5b8ae94a8e323..132083817932c 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.2 + 2.9.3-SNAPSHOT .. From 362c126a38536b378891439859c348824929700e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 19 Apr 2022 17:18:25 +0800 Subject: [PATCH 588/823] [PIP-146] ManagedCursorInfo compression (#14542) Fixes #14529 The cursor data is managed by ZooKeeper/etcd metadata store. When cursor data becomes more and more, the data size will increase and will take a lot of time to pull the data. Therefore, it is necessary to add compression for the cursor, which can reduce the size of data and reduce the time of pulling data. - Add a named `ManagedCursorInfoMetadata` message to `MLDataFormats.proto` for as compression metadata - Add the `managedCursorInfoCompressionType` to `org.apache.pulsar.broker.ServiceConfiguration` and `org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig` - This feature is the same as the implementation of ManagedLedgerInfo compression, so the code is optimized to avoid duplication (cherry picked from commit 43987336c28585021943af72d7b3a35ff8d95c19) --- .../mledger/ManagedLedgerFactoryConfig.java | 5 + .../impl/ManagedLedgerFactoryImpl.java | 3 +- .../mledger/impl/MetaStoreImpl.java | 183 +++++++++++------- .../src/main/proto/MLDataFormats.proto | 5 + .../impl/ManagedCursorInfoMetadataTest.java | 96 +++++++++ .../impl/ManagedLedgerInfoMetadataTest.java | 15 +- .../pulsar/broker/ServiceConfiguration.java | 5 + .../broker/ManagedLedgerClientFactory.java | 1 + 8 files changed, 236 insertions(+), 77 deletions(-) create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java index a00c161641083..25fcb377e3e11 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java @@ -86,4 +86,9 @@ public class ManagedLedgerFactoryConfig { * ManagedLedgerInfo compression type. If the compression type is null or invalid, don't compress data. */ private String managedLedgerInfoCompressionType = MLDataFormats.CompressionType.NONE.name(); + + /** + * ManagedCursorInfo compression type. If the compression type is null or invalid, don't compress data. + */ + private String managedCursorInfoCompressionType = MLDataFormats.CompressionType.NONE.name(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index f8806824d8f0c..e3d24f5522211 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -184,7 +184,8 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, this.bookkeeperFactory = bookKeeperGroupFactory; this.isBookkeeperManaged = isBookkeeperManaged; this.metadataStore = metadataStore; - this.store = new MetaStoreImpl(metadataStore, scheduledExecutor, config.getManagedLedgerInfoCompressionType()); + this.store = new MetaStoreImpl(metadataStore, scheduledExecutor, config.getManagedLedgerInfoCompressionType(), + config.getManagedCursorInfoCompressionType()); this.config = config; this.mbean = new ManagedLedgerFactoryMBeanImpl(this); this.entryCacheManager = new EntryCacheManager(this); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index 5ad62b228bce0..e1c72d2fc0dcb 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -41,6 +41,7 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; import org.apache.bookkeeper.util.SafeRunnable; +import org.apache.commons.lang.StringUtils; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; @@ -57,30 +58,39 @@ public class MetaStoreImpl implements MetaStore { private final MetadataStore store; private final OrderedExecutor executor; - private static final int MAGIC_MANAGED_LEDGER_INFO_METADATA = 0x4778; // 0100 0111 0111 1000 - private final CompressionType compressionType; + private static final int MAGIC_MANAGED_INFO_METADATA = 0x4778; // 0100 0111 0111 1000 + private final CompressionType ledgerInfoCompressionType; + private final CompressionType cursorInfoCompressionType; public MetaStoreImpl(MetadataStore store, OrderedExecutor executor) { this.store = store; this.executor = executor; - this.compressionType = CompressionType.NONE; + this.ledgerInfoCompressionType = CompressionType.NONE; + this.cursorInfoCompressionType = CompressionType.NONE; } - public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String compressionType) { + public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String ledgerInfoCompressionType, + String cursorInfoCompressionType) { this.store = store; this.executor = executor; - CompressionType finalCompressionType; - if (compressionType != null) { - try { - finalCompressionType = CompressionType.valueOf(compressionType); - } catch (Exception e) { - log.error("Failed to get compression type {} error msg: {}.", compressionType, e.getMessage()); - throw e; - } - } else { - finalCompressionType = CompressionType.NONE; + this.ledgerInfoCompressionType = parseCompressionType(ledgerInfoCompressionType); + this.cursorInfoCompressionType = parseCompressionType(cursorInfoCompressionType); + } + + private CompressionType parseCompressionType(String value) { + if (StringUtils.isEmpty(value)) { + return CompressionType.NONE; + } + + CompressionType compressionType; + try { + compressionType = CompressionType.valueOf(value); + } catch (Exception e) { + log.error("Failed to get compression type {} error msg: {}.", value, e.getMessage()); + throw e; } - this.compressionType = finalCompressionType; + + return compressionType; } @Override @@ -174,7 +184,7 @@ public void asyncGetCursorInfo(String ledgerName, String cursorName, .thenAcceptAsync(optRes -> { if (optRes.isPresent()) { try { - ManagedCursorInfo info = ManagedCursorInfo.parseFrom(optRes.get().getValue()); + ManagedCursorInfo info = parseManagedCursorInfo(optRes.get().getValue()); callback.operationComplete(info, optRes.get().getStat()); } catch (InvalidProtocolBufferException e) { callback.operationFailed(getException(e)); @@ -196,7 +206,7 @@ public void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedC info.getCursorsLedgerId(), info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); String path = PREFIX + ledgerName + "/" + cursorName; - byte[] content = info.toByteArray(); // Binary format + byte[] content = compressCursorInfo(info); long expectedVersion; @@ -306,32 +316,97 @@ private static MetaStoreException getException(Throwable t) { } } + public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) { + if (ledgerInfoCompressionType.equals(CompressionType.NONE)) { + return managedLedgerInfo.toByteArray(); + } + MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata + .newBuilder() + .setCompressionType(ledgerInfoCompressionType) + .setUncompressedSize(managedLedgerInfo.getSerializedSize()) + .build(); + return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(), + mlInfoMetadata.getSerializedSize(), ledgerInfoCompressionType); + } + + public byte[] compressCursorInfo(ManagedCursorInfo managedCursorInfo) { + if (cursorInfoCompressionType.equals(CompressionType.NONE)) { + return managedCursorInfo.toByteArray(); + } + MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata + .newBuilder() + .setCompressionType(cursorInfoCompressionType) + .setUncompressedSize(managedCursorInfo.getSerializedSize()) + .build(); + return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(), + metadata.getSerializedSize(), cursorInfoCompressionType); + } + + public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProtocolBufferException { + ByteBuf byteBuf = Unpooled.wrappedBuffer(data); + + byte[] metadataBytes = extractCompressMetadataBytes(byteBuf); + if (metadataBytes != null) { + try { + MLDataFormats.ManagedLedgerInfoMetadata metadata = + MLDataFormats.ManagedLedgerInfoMetadata.parseFrom(metadataBytes); + return ManagedLedgerInfo.parseFrom(getCompressionCodec(metadata.getCompressionType()) + .decode(byteBuf, metadata.getUncompressedSize()).nioBuffer()); + } catch (Exception e) { + log.error("Failed to parse managedLedgerInfo metadata, " + + "fall back to parse managedLedgerInfo directly.", e); + return ManagedLedgerInfo.parseFrom(data); + } finally { + byteBuf.release(); + } + } else { + return ManagedLedgerInfo.parseFrom(data); + } + } + + public ManagedCursorInfo parseManagedCursorInfo(byte[] data) throws InvalidProtocolBufferException { + ByteBuf byteBuf = Unpooled.wrappedBuffer(data); + + byte[] metadataBytes = extractCompressMetadataBytes(byteBuf); + if (metadataBytes != null) { + try { + MLDataFormats.ManagedCursorInfoMetadata metadata = + MLDataFormats.ManagedCursorInfoMetadata.parseFrom(metadataBytes); + return ManagedCursorInfo.parseFrom(getCompressionCodec(metadata.getCompressionType()) + .decode(byteBuf, metadata.getUncompressedSize()).nioBuffer()); + } catch (Exception e) { + log.error("Failed to parse ManagedCursorInfo metadata, " + + "fall back to parse ManagedCursorInfo directly", e); + return ManagedCursorInfo.parseFrom(data); + } finally { + byteBuf.release(); + } + } else { + return ManagedCursorInfo.parseFrom(data); + } + } + /** - * Compress ManagedLedgerInfo data. + * Compress Managed Info data such as LedgerInfo, CursorInfo. * * compression data structure * [MAGIC_NUMBER](2) + [METADATA_SIZE](4) + [METADATA_PAYLOAD] + [MANAGED_LEDGER_INFO_PAYLOAD] - */ - public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) { + */ + private byte[] compressManagedInfo(byte[] info, byte[] metadata, int metadataSerializedSize, + MLDataFormats.CompressionType compressionType) { if (compressionType == null || compressionType.equals(CompressionType.NONE)) { - return managedLedgerInfo.toByteArray(); + return info; } ByteBuf metadataByteBuf = null; ByteBuf encodeByteBuf = null; try { - MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata - .newBuilder() - .setCompressionType(compressionType) - .setUncompressedSize(managedLedgerInfo.getSerializedSize()) - .build(); - metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer( - mlInfoMetadata.getSerializedSize() + 6, mlInfoMetadata.getSerializedSize() + 6); - metadataByteBuf.writeShort(MAGIC_MANAGED_LEDGER_INFO_METADATA); - metadataByteBuf.writeInt(mlInfoMetadata.getSerializedSize()); - metadataByteBuf.writeBytes(mlInfoMetadata.toByteArray()); - + metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(metadataSerializedSize + 6, + metadataSerializedSize + 6); + metadataByteBuf.writeShort(MAGIC_MANAGED_INFO_METADATA); + metadataByteBuf.writeInt(metadataSerializedSize); + metadataByteBuf.writeBytes(metadata); encodeByteBuf = getCompressionCodec(compressionType) - .encode(Unpooled.wrappedBuffer(managedLedgerInfo.toByteArray())); + .encode(Unpooled.wrappedBuffer(info)); CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer(); compositeByteBuf.addComponent(true, metadataByteBuf); compositeByteBuf.addComponent(true, encodeByteBuf); @@ -348,42 +423,14 @@ public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) { } } - public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProtocolBufferException { - ByteBuf byteBuf = Unpooled.wrappedBuffer(data); - if (byteBuf.readableBytes() > 0 && byteBuf.readShort() == MAGIC_MANAGED_LEDGER_INFO_METADATA) { - ByteBuf decodeByteBuf = null; - try { - int metadataSize = byteBuf.readInt(); - byte[] metadataBytes = new byte[metadataSize]; - byteBuf.readBytes(metadataBytes); - MLDataFormats.ManagedLedgerInfoMetadata metadata = - MLDataFormats.ManagedLedgerInfoMetadata.parseFrom(metadataBytes); - - long unpressedSize = metadata.getUncompressedSize(); - decodeByteBuf = getCompressionCodec(metadata.getCompressionType()) - .decode(byteBuf, (int) unpressedSize); - byte[] decodeBytes; - // couldn't decode data by ZLIB compression byteBuf array() directly - if (decodeByteBuf.hasArray() && !CompressionType.ZLIB.equals(metadata.getCompressionType())) { - decodeBytes = decodeByteBuf.array(); - } else { - decodeBytes = new byte[decodeByteBuf.readableBytes() - decodeByteBuf.readerIndex()]; - decodeByteBuf.readBytes(decodeBytes); - } - return ManagedLedgerInfo.parseFrom(decodeBytes); - } catch (Exception e) { - log.error("Failed to parse managedLedgerInfo metadata, " - + "fall back to parse managedLedgerInfo directly.", e); - return ManagedLedgerInfo.parseFrom(data); - } finally { - if (decodeByteBuf != null) { - decodeByteBuf.release(); - } - byteBuf.release(); - } - } else { - return ManagedLedgerInfo.parseFrom(data); + private byte[] extractCompressMetadataBytes(ByteBuf data) { + if (data.readableBytes() > 0 && data.readShort() == MAGIC_MANAGED_INFO_METADATA) { + int metadataSize = data.readInt(); + byte[] metadataBytes = new byte[metadataSize]; + data.readBytes(metadataBytes); + return metadataBytes; } + return null; } private CompressionCodec getCompressionCodec(CompressionType compressionType) { diff --git a/managed-ledger/src/main/proto/MLDataFormats.proto b/managed-ledger/src/main/proto/MLDataFormats.proto index a3528b664e29f..4671816c1a199 100644 --- a/managed-ledger/src/main/proto/MLDataFormats.proto +++ b/managed-ledger/src/main/proto/MLDataFormats.proto @@ -137,3 +137,8 @@ message ManagedLedgerInfoMetadata { required CompressionType compressionType = 1; required int32 uncompressedSize = 2; } + +message ManagedCursorInfoMetadata { + required CompressionType compressionType = 1; + required int32 uncompressedSize = 2; +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java new file mode 100644 index 0000000000000..8b95876d0ae3c --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.junit.Assert.assertEquals; +import static org.testng.Assert.expectThrows; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.pulsar.common.api.proto.CompressionType; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * ManagedCursorInfo metadata test. + */ +@Slf4j +public class ManagedCursorInfoMetadataTest { + private final String INVALID_TYPE = "INVALID_TYPE"; + + @DataProvider(name = "compressionTypeProvider") + private Object[][] compressionTypeProvider() { + return new Object[][]{ + {null}, + {INVALID_TYPE}, + {CompressionType.NONE.name()}, + {CompressionType.LZ4.name()}, + {CompressionType.ZLIB.name()}, + {CompressionType.ZSTD.name()}, + {CompressionType.SNAPPY.name()} + }; + } + + @Test(dataProvider = "compressionTypeProvider") + public void testEncodeAndDecode(String compressionType) throws IOException { + long ledgerId = 10000; + MLDataFormats.ManagedCursorInfo.Builder builder = MLDataFormats.ManagedCursorInfo.newBuilder(); + + builder.setCursorsLedgerId(ledgerId); + builder.setMarkDeleteLedgerId(ledgerId); + + List batchedEntryDeletionIndexInfos = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + MLDataFormats.NestedPositionInfo nestedPositionInfo = MLDataFormats.NestedPositionInfo.newBuilder() + .setEntryId(i).setLedgerId(i).build(); + MLDataFormats.BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = MLDataFormats + .BatchedEntryDeletionIndexInfo.newBuilder().setPosition(nestedPositionInfo).build(); + batchedEntryDeletionIndexInfos.add(batchedEntryDeletionIndexInfo); + } + builder.addAllBatchedEntryDeletionIndexInfo(batchedEntryDeletionIndexInfos); + + MetaStoreImpl metaStore; + if (INVALID_TYPE.equals(compressionType)) { + IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> { + new MetaStoreImpl(null, null, null, compressionType); + }); + assertEquals("No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType." + + compressionType, compressionTypeEx.getMessage()); + return; + } else { + metaStore = new MetaStoreImpl(null, null, null, compressionType); + } + + MLDataFormats.ManagedCursorInfo managedCursorInfo = builder.build(); + byte[] compressionBytes = metaStore.compressCursorInfo(managedCursorInfo); + log.info("[{}] Uncompressed data size: {}, compressed data size: {}", + compressionType, managedCursorInfo.getSerializedSize(), compressionBytes.length); + if (compressionType == null || compressionType.equals(CompressionType.NONE.name())) { + Assert.assertEquals(compressionBytes.length, managedCursorInfo.getSerializedSize()); + } + + // parse compression data and unCompression data, check their results. + MLDataFormats.ManagedCursorInfo info1 = metaStore.parseManagedCursorInfo(compressionBytes); + MLDataFormats.ManagedCursorInfo info2 = metaStore.parseManagedCursorInfo(managedCursorInfo.toByteArray()); + Assert.assertEquals(info1, info2); + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java index 2f27489aeb9f6..91bc7f143a4ae 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java @@ -19,6 +19,12 @@ package org.apache.bookkeeper.mledger.impl; import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.offload.OffloadUtils; import org.apache.bookkeeper.mledger.proto.MLDataFormats; @@ -28,13 +34,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - /** * ManagedLedgerInfo metadata test. */ @@ -91,7 +90,7 @@ public void testEncodeAndDecode(String compressionType) throws IOException { MetaStoreImpl metaStore; try { - metaStore = new MetaStoreImpl(null, null, compressionType); + metaStore = new MetaStoreImpl(null, null, compressionType, null); if ("INVALID_TYPE".equals(compressionType)) { Assert.fail("The managedLedgerInfo compression type is invalid, should fail."); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index dc5640f2315fd..bbdbc31b3f16c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1706,6 +1706,11 @@ public class ServiceConfiguration implements PulsarConfiguration { + "If value is invalid or NONE, then save the ManagedLedgerInfo bytes data directly.") private String managedLedgerInfoCompressionType = "NONE"; + @FieldContext(category = CATEGORY_STORAGE_ML, + doc = "ManagedCursorInfo compression type, option values (NONE, LZ4, ZLIB, ZSTD, SNAPPY). \n" + + "If value is NONE, then save the ManagedCursorInfo bytes data directly.") + private String managedCursorInfoCompressionType = "NONE"; + /*** --- Load balancer --- ****/ @FieldContext( category = CATEGORY_LOAD_BALANCER, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java index b615628c08fa1..6a9aa8877e252 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java @@ -71,6 +71,7 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata managedLedgerFactoryConfig.setCursorPositionFlushSeconds(conf.getManagedLedgerCursorPositionFlushSeconds()); managedLedgerFactoryConfig.setManagedLedgerInfoCompressionType(conf.getManagedLedgerInfoCompressionType()); managedLedgerFactoryConfig.setStatsPeriodSeconds(conf.getManagedLedgerStatsPeriodSeconds()); + managedLedgerFactoryConfig.setManagedCursorInfoCompressionType(conf.getManagedCursorInfoCompressionType()); Configuration configuration = new ClientConfiguration(); if (conf.isBookkeeperClientExposeStatsToPrometheus()) { From 12ee3f56072123e99179015a50df60a7492d6a41 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Mon, 6 Jun 2022 16:31:14 +0800 Subject: [PATCH 589/823] [improve][tiered storage] Reduce cpu usage when offloading the ledger (#15063) * [imporve][tiered storage] Reduce cpu usage when offloading the ledger --- *Motivation* When offloading a ledger, the BlockAwareSegmentInputStreamImpl will wrap the ledger handler and make it can stream output. Then the JCloud will read the stream as the payload and upload to the storage. In the JCloud implementation, it read the stream with a buffer https://github.com/apache/jclouds/blob/36f351cd18925d2bb27bf7ad2c5d75e555da377a/core/src/main/java/org/jclouds/io/ByteStreams2.java#L68 In the current offload implementation, the read will call multiple times to construct the buffer and then return the data. After implement the read(byte[] b, int off, int len), the cpu usage reduced almost 10%. *Modifications* - Add read(byte[] b, int off, int len) implementation in the BlockAwareSegmentInputStreamImpl (cherry picked from commit 938ab7befc57a23e5a2bcb0f8bfe5c714c4d0018) --- .../BlockAwareSegmentInputStreamImpl.java | 117 ++++++- .../BlockAwareSegmentInputStreamTest.java | 328 ++++++++++++++++-- 2 files changed, 410 insertions(+), 35 deletions(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java index a4ffdea65098f..b69f9f5e78544 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState; import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; import java.io.IOException; @@ -27,6 +28,7 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.bookkeeper.client.api.LedgerEntries; import org.apache.bookkeeper.client.api.LedgerEntry; import org.apache.bookkeeper.client.api.ReadHandle; @@ -44,6 +46,9 @@ public class BlockAwareSegmentInputStreamImpl extends BlockAwareSegmentInputStre private static final Logger log = LoggerFactory.getLogger(BlockAwareSegmentInputStreamImpl.class); static final int[] BLOCK_END_PADDING = new int[]{ 0xFE, 0xDC, 0xDE, 0xAD }; + static final byte[] BLOCK_END_PADDING_BYTES = Ints.toByteArray(0xFEDCDEAD); + + private final ByteBuf paddingBuf = PulsarByteBufAllocator.DEFAULT.buffer(128, 128); private final ReadHandle ledger; private final long startEntryId; @@ -65,6 +70,9 @@ public class BlockAwareSegmentInputStreamImpl extends BlockAwareSegmentInputStre static final int ENTRY_HEADER_SIZE = 4 /* entry size */ + 8 /* entry id */; // Keep a list of all entries ByteBuf, each ByteBuf contains 2 buf: entry header and entry content. private List entriesByteBuf = null; + private int currentOffset = 0; + private final AtomicBoolean close = new AtomicBoolean(false); + public BlockAwareSegmentInputStreamImpl(ReadHandle ledger, long startEntryId, int blockSize) { this.ledger = ledger; @@ -76,6 +84,52 @@ public BlockAwareSegmentInputStreamImpl(ReadHandle ledger, long startEntryId, in this.entriesByteBuf = Lists.newLinkedList(); } + private ByteBuf readEntries(int len) throws IOException { + checkState(bytesReadOffset >= DataBlockHeaderImpl.getDataStartOffset()); + checkState(bytesReadOffset < blockSize); + + // once reach the end of entry buffer, read more, if there is more + if (bytesReadOffset < dataBlockFullOffset + && entriesByteBuf.isEmpty() + && startEntryId + blockEntryCount <= ledger.getLastAddConfirmed()) { + entriesByteBuf = readNextEntriesFromLedger(startEntryId + blockEntryCount, ENTRIES_PER_READ); + } + + if (!entriesByteBuf.isEmpty() + && bytesReadOffset + entriesByteBuf.get(0).readableBytes() <= blockSize) { + // always read from the first ByteBuf in the list, once read all of its content remove it. + ByteBuf entryByteBuf = entriesByteBuf.get(0); + int readableBytes = entryByteBuf.readableBytes(); + int read = Math.min(readableBytes, len); + ByteBuf buf = entryByteBuf.slice(currentOffset, read); + buf.retain(); + currentOffset += read; + entryByteBuf.readerIndex(currentOffset); + bytesReadOffset += read; + + if (entryByteBuf.readableBytes() == 0) { + entryByteBuf.release(); + entriesByteBuf.remove(0); + blockEntryCount++; + currentOffset = 0; + } + + return buf; + } else { + // no space for a new entry or there are no more entries + // set data block full, return end padding + if (dataBlockFullOffset == blockSize) { + dataBlockFullOffset = bytesReadOffset; + } + paddingBuf.clear(); + for (int i = 0; i < Math.min(len, paddingBuf.capacity()); i++) { + paddingBuf.writeByte(BLOCK_END_PADDING_BYTES[(bytesReadOffset++ - dataBlockFullOffset) + % BLOCK_END_PADDING_BYTES.length]); + } + return paddingBuf.retain(); + } + } + // read ledger entries. private int readEntries() throws IOException { checkState(bytesReadOffset >= DataBlockHeaderImpl.getDataStartOffset()); @@ -143,6 +197,46 @@ private List readNextEntriesFromLedger(long start, long maxNumberEntrie } } + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (b == null) { + throw new NullPointerException("The given bytes are null"); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException("off=" + off + ", len=" + len + ", b.length=" + b.length); + } else if (len == 0) { + return 0; + } + + int offset = off; + int readLen = len; + int readBytes = 0; + // reading header + if (dataBlockHeaderStream.available() > 0) { + int read = dataBlockHeaderStream.read(b, off, len); + offset += read; + readLen -= read; + readBytes += read; + bytesReadOffset += read; + } + if (readLen == 0) { + return readBytes; + } + + // reading ledger entries + if (bytesReadOffset < blockSize) { + readLen = Math.min(readLen, blockSize - bytesReadOffset); + ByteBuf readEntries = readEntries(readLen); + int read = readEntries.readableBytes(); + readEntries.readBytes(b, offset, read); + readEntries.release(); + readBytes += read; + return readBytes; + } + + // reached end + return -1; + } + @Override public int read() throws IOException { // reading header @@ -162,11 +256,20 @@ public int read() throws IOException { @Override public void close() throws IOException { - super.close(); - dataBlockHeaderStream.close(); - if (!entriesByteBuf.isEmpty()) { - entriesByteBuf.forEach(buf -> buf.release()); - entriesByteBuf.clear(); + // The close method will be triggered twice in the BlobStoreManagedLedgerOffloader#offload method. + // The stream resource used by the try-with block which will called the close + // And through debug, writeBlobStore.uploadMultipartPart in the offload method also will trigger + // the close method. + // So we add the close variable to avoid release paddingBuf twice. + if (!close.compareAndSet(false, true)) { + super.close(); + dataBlockHeaderStream.close(); + if (!entriesByteBuf.isEmpty()) { + entriesByteBuf.forEach(buf -> buf.release()); + entriesByteBuf.clear(); + } + paddingBuf.clear(); + paddingBuf.release(); } } @@ -185,6 +288,10 @@ public int getBlockSize() { return blockSize; } + public int getDataBlockFullOffset() { + return dataBlockFullOffset; + } + @Override public int getBlockEntryCount() { return blockEntryCount; diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java index 5cf6bd5650003..0cd4bbd70a9e6 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java @@ -19,6 +19,7 @@ package org.apache.bookkeeper.mledger.offload.jcloud.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.fail; import static org.testng.internal.junit.ArrayAsserts.assertArrayEquals; @@ -28,6 +29,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Iterator; @@ -44,6 +46,7 @@ import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.mledger.offload.jcloud.DataBlockHeader; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import org.testng.collections.Lists; @@ -206,8 +209,16 @@ public CompletableFuture closeAsync() { } } - @Test - public void testHaveEndPadding() throws Exception { + @DataProvider(name = "useBufferRead") + public static Object[][] useBufferRead() { + return new Object[][]{ + {Boolean.TRUE}, + {Boolean.FALSE} + }; + } + + @Test(dataProvider = "useBufferRead") + public void testHaveEndPadding(boolean useBufferRead) throws Exception { int ledgerId = 1; int entrySize = 8; int lac = 160; @@ -226,7 +237,12 @@ public void testHaveEndPadding() throws Exception { // verify read inputStream // 1. read header. 128 byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()]; - ByteStreams.readFully(inputStream, headerB); + if (useBufferRead) { + int ret = inputStream.read(headerB, 0, DataBlockHeaderImpl.getDataStartOffset()); + assertEquals(DataBlockHeaderImpl.getDataStartOffset(), ret); + } else { + ByteStreams.readFully(inputStream, headerB); + } DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB)); assertEquals(headerRead.getBlockLength(), blockSize); assertEquals(headerRead.getFirstEntryId(), 0); @@ -240,9 +256,18 @@ public void testHaveEndPadding() throws Exception { byte lengthBuf[] = new byte[4]; byte entryIdBuf[] = new byte[8]; byte content[] = new byte[entrySize]; - inputStream.read(lengthBuf); - inputStream.read(entryIdBuf); - inputStream.read(content); + if (useBufferRead) { + int read = inputStream.read(lengthBuf, 0, 4); + assertEquals(read, 4); + read = inputStream.read(entryIdBuf, 0, 8); + assertEquals(read, 8); + read = inputStream.read(content, 0, entrySize); + assertEquals(read, entrySize); + } else { + inputStream.read(lengthBuf); + inputStream.read(entryIdBuf); + inputStream.read(content); + } assertEquals(entrySize, Ints.fromByteArray(lengthBuf)); assertEquals(i, Longs.fromByteArray(entryIdBuf)); @@ -256,13 +281,36 @@ public void testHaveEndPadding() throws Exception { int left = blockSize - DataBlockHeaderImpl.getDataStartOffset() - expectedEntryCount * (entrySize + 4 + 8); assertEquals(left, 5); byte padding[] = new byte[left]; - inputStream.read(padding); + if (useBufferRead) { + int ret = 0; + int offset = 0; + while ((ret = inputStream.read(padding, offset, padding.length - offset)) > 0) { + offset += ret; + } + assertEquals(inputStream.read(padding, 0, padding.length), -1); + } else { + int len = left; + int offset = 0; + byte[] buf = new byte[4]; + while (len > 0) { + int ret = inputStream.read(buf); + for (int i = 0; i < ret; i++) { + padding[offset++] = buf[i]; + } + len -= ret; + } + } ByteBuf paddingBuf = Unpooled.wrappedBuffer(padding); IntStream.range(0, paddingBuf.capacity()/4).forEach(i -> assertEquals(Integer.toHexString(paddingBuf.readInt()), Integer.toHexString(0xFEDCDEAD))); // 4. reach end. + if (useBufferRead) { + byte[] b = new byte[4]; + int ret = inputStream.read(b, 0, 4); + assertEquals(ret, -1); + } assertEquals(inputStream.read(), -1); assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount); @@ -272,8 +320,8 @@ public void testHaveEndPadding() throws Exception { inputStream.close(); } - @Test - public void testNoEndPadding() throws Exception { + @Test(dataProvider = "useBufferRead") + public void testNoEndPadding(boolean useBufferRead) throws Exception { int ledgerId = 1; int entrySize = 8; int lac = 120; @@ -293,7 +341,12 @@ public void testNoEndPadding() throws Exception { // verify read inputStream // 1. read header. 128 byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()]; - ByteStreams.readFully(inputStream, headerB); + if (useBufferRead) { + int ret = inputStream.read(headerB, 0, DataBlockHeaderImpl.getDataStartOffset()); + assertEquals(DataBlockHeaderImpl.getDataStartOffset(), ret); + } else { + ByteStreams.readFully(inputStream, headerB); + } DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB)); assertEquals(headerRead.getBlockLength(), blockSize); assertEquals(headerRead.getFirstEntryId(), 0); @@ -307,9 +360,18 @@ public void testNoEndPadding() throws Exception { byte lengthBuf[] = new byte[4]; byte entryIdBuf[] = new byte[8]; byte content[] = new byte[entrySize]; - inputStream.read(lengthBuf); - inputStream.read(entryIdBuf); - inputStream.read(content); + if (useBufferRead) { + int read = inputStream.read(lengthBuf, 0, 4); + assertEquals(read, 4); + read = inputStream.read(entryIdBuf, 0, 8); + assertEquals(read, 8); + read = inputStream.read(content, 0, entrySize); + assertEquals(read, entrySize); + } else { + inputStream.read(lengthBuf); + inputStream.read(entryIdBuf); + inputStream.read(content); + } assertEquals(entrySize, Ints.fromByteArray(lengthBuf)); assertEquals(i, Longs.fromByteArray(entryIdBuf)); @@ -324,6 +386,11 @@ public void testNoEndPadding() throws Exception { assertEquals(left, 0); // 4. reach end. + if (useBufferRead) { + byte[] b = new byte[4]; + int ret = inputStream.read(b, 0, 4); + assertEquals(ret, -1); + } assertEquals(inputStream.read(), -1); assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount); @@ -333,8 +400,8 @@ public void testNoEndPadding() throws Exception { inputStream.close(); } - @Test - public void testReadTillLac() throws Exception { + @Test(dataProvider = "useBufferRead") + public void testReadTillLac(boolean useBufferRead) throws Exception { // simulate last data block read. int ledgerId = 1; int entrySize = 8; @@ -354,7 +421,12 @@ public void testReadTillLac() throws Exception { // verify read inputStream // 1. read header. 128 byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()]; - ByteStreams.readFully(inputStream, headerB); + if (useBufferRead) { + int ret = inputStream.read(headerB, 0, DataBlockHeaderImpl.getDataStartOffset()); + assertEquals(DataBlockHeaderImpl.getDataStartOffset(), ret); + } else { + ByteStreams.readFully(inputStream, headerB); + } DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB)); assertEquals(headerRead.getBlockLength(), blockSize); assertEquals(headerRead.getFirstEntryId(), 0); @@ -368,9 +440,18 @@ public void testReadTillLac() throws Exception { byte lengthBuf[] = new byte[4]; byte entryIdBuf[] = new byte[8]; byte content[] = new byte[entrySize]; - inputStream.read(lengthBuf); - inputStream.read(entryIdBuf); - inputStream.read(content); + if (useBufferRead) { + int read = inputStream.read(lengthBuf, 0, 4); + assertEquals(read, 4); + read = inputStream.read(entryIdBuf, 0, 8); + assertEquals(read, 8); + read = inputStream.read(content, 0, entrySize); + assertEquals(read, entrySize); + } else { + inputStream.read(lengthBuf); + inputStream.read(entryIdBuf); + inputStream.read(content); + } assertEquals(entrySize, Ints.fromByteArray(lengthBuf)); assertEquals(i, Longs.fromByteArray(entryIdBuf)); @@ -385,6 +466,11 @@ public void testReadTillLac() throws Exception { assertEquals(left, 0); // 4. reach end. + if (useBufferRead) { + byte[] b = new byte[4]; + int ret = inputStream.read(b, 0, 4); + assertEquals(ret, -1); + } assertEquals(inputStream.read(), -1); assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount); @@ -394,8 +480,8 @@ public void testReadTillLac() throws Exception { inputStream.close(); } - @Test - public void testNoEntryPutIn() throws Exception { + @Test(dataProvider = "useBufferRead") + public void testNoEntryPutIn(boolean useBufferRead) throws Exception { // simulate first entry size over the block size budget, it shouldn't be added. // 2 entries, each with bigger size than block size, so there should no entry added into block. int ledgerId = 1; @@ -416,7 +502,12 @@ public void testNoEntryPutIn() throws Exception { // verify read inputStream // 1. read header. 128 byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()]; - ByteStreams.readFully(inputStream, headerB); + if (useBufferRead) { + int ret = inputStream.read(headerB, 0, DataBlockHeaderImpl.getDataStartOffset()); + assertEquals(DataBlockHeaderImpl.getDataStartOffset(), ret); + } else { + ByteStreams.readFully(inputStream, headerB); + } DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB)); assertEquals(headerRead.getBlockLength(), blockSize); assertEquals(headerRead.getFirstEntryId(), 0); @@ -424,13 +515,36 @@ public void testNoEntryPutIn() throws Exception { // 2. since no entry put in, it should only get padding after header. byte padding[] = new byte[blockSize - DataBlockHeaderImpl.getDataStartOffset()]; - inputStream.read(padding); + if (useBufferRead) { + int ret = 0; + int offset = 0; + while ((ret = inputStream.read(padding, offset, padding.length - offset)) > 0) { + offset += ret; + } + assertEquals(inputStream.read(padding, 0, padding.length), -1); + } else { + int len = padding.length; + int offset = 0; + byte[] buf = new byte[4]; + while (len > 0) { + int ret = inputStream.read(buf); + for (int i = 0; i < ret; i++) { + padding[offset++] = buf[i]; + } + len -= ret; + } + } ByteBuf paddingBuf = Unpooled.wrappedBuffer(padding); IntStream.range(0, paddingBuf.capacity()/4).forEach(i -> assertEquals(Integer.toHexString(paddingBuf.readInt()), Integer.toHexString(0xFEDCDEAD))); // 3. reach end. + if (useBufferRead) { + byte[] b = new byte[4]; + int ret = inputStream.read(b, 0, 4); + assertEquals(ret, -1); + } assertEquals(inputStream.read(), -1); assertEquals(inputStream.getBlockEntryCount(), 0); @@ -440,8 +554,8 @@ public void testNoEntryPutIn() throws Exception { inputStream.close(); } - @Test - public void testPaddingOnLastBlock() throws Exception { + @Test(dataProvider = "useBufferRead") + public void testPaddingOnLastBlock(boolean useBufferRead) throws Exception { int ledgerId = 1; int entrySize = 1000; int lac = 0; @@ -460,7 +574,12 @@ public void testPaddingOnLastBlock() throws Exception { // verify read inputStream // 1. read header. 128 byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()]; - ByteStreams.readFully(inputStream, headerB); + if (useBufferRead) { + int ret = inputStream.read(headerB, 0, DataBlockHeaderImpl.getDataStartOffset()); + assertEquals(DataBlockHeaderImpl.getDataStartOffset(), ret); + } else { + ByteStreams.readFully(inputStream, headerB); + } DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB)); assertEquals(headerRead.getBlockLength(), blockSize); assertEquals(headerRead.getFirstEntryId(), 0); @@ -474,9 +593,18 @@ public void testPaddingOnLastBlock() throws Exception { byte lengthBuf[] = new byte[4]; byte entryIdBuf[] = new byte[8]; byte content[] = new byte[entrySize]; - inputStream.read(lengthBuf); - inputStream.read(entryIdBuf); - inputStream.read(content); + if (useBufferRead) { + int read = inputStream.read(lengthBuf, 0, 4); + assertEquals(read, 4); + read = inputStream.read(entryIdBuf, 0, 8); + assertEquals(read, 8); + read = inputStream.read(content, 0, entrySize); + assertEquals(read, entrySize); + } else { + inputStream.read(lengthBuf); + inputStream.read(entryIdBuf); + inputStream.read(content); + } assertEquals(entrySize, Ints.fromByteArray(lengthBuf)); assertEquals(i, Longs.fromByteArray(entryIdBuf)); @@ -490,13 +618,36 @@ public void testPaddingOnLastBlock() throws Exception { int consumedBytes = DataBlockHeaderImpl.getDataStartOffset() + expectedEntryCount * (entrySize + BlockAwareSegmentInputStreamImpl.ENTRY_HEADER_SIZE); byte padding[] = new byte[blockSize - consumedBytes]; - inputStream.read(padding); + if (useBufferRead) { + int ret = 0; + int offset = 0; + while ((ret = inputStream.read(padding, offset, padding.length - offset)) > 0) { + offset += ret; + } + assertEquals(inputStream.read(padding, 0, padding.length), -1); + } else { + int len = blockSize - consumedBytes; + int offset = 0; + byte[] buf = new byte[4]; + while (len > 0) { + int ret = inputStream.read(buf); + for (int i = 0; i < ret; i++) { + padding[offset++] = buf[i]; + } + len -= ret; + } + } ByteBuf paddingBuf = Unpooled.wrappedBuffer(padding); IntStream.range(0, paddingBuf.capacity()/4).forEach(i -> assertEquals(Integer.toHexString(paddingBuf.readInt()), Integer.toHexString(0xFEDCDEAD))); // 3. reach end. + if (useBufferRead) { + byte[] b = new byte[4]; + int ret = inputStream.read(b, 0, 4); + assertEquals(ret, -1); + } assertEquals(inputStream.read(), -1); assertEquals(inputStream.getBlockEntryCount(), 1); @@ -530,4 +681,121 @@ public void testOnlyNegativeOnEOF() throws Exception { } } + @Test + public void testOnlyNegativeOnEOFWithBufferedRead() throws IOException { + int ledgerId = 1; + int entrySize = 10000; + int lac = 0; + + Random r = new Random(0); + ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac, () -> (byte)r.nextInt()); + + int blockSize = DataBlockHeaderImpl.getDataStartOffset() + entrySize * 2; + BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0, blockSize); + + int bytesRead = 0; + int ret; + int offset = 0; + int resetOffsetCount = 0; + byte[] buf = new byte[1024]; + while ((ret = inputStream.read(buf, offset, buf.length - offset)) > 0) { + bytesRead += ret; + int currentOffset = offset; + offset = (offset + ret) % buf.length; + if (offset < currentOffset) { + resetOffsetCount++; + } + } + assertEquals(bytesRead, blockSize); + assertNotEquals(resetOffsetCount, 0); + } + + // This test is for testing the read(byte[] buf, int off, int len) method can work properly + // on the offset not 0. + @Test + public void testReadTillLacWithSmallBuffer() throws Exception { + // simulate last data block read. + int ledgerId = 1; + int entrySize = 8; + int lac = 89; + ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac); + + // set block size equals to (header + lac_entry) size. + int blockSize = DataBlockHeaderImpl.getDataStartOffset() + (1 + lac) * (entrySize + 4 + 8); + BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0, blockSize); + int expectedEntryCount = (blockSize - DataBlockHeaderImpl.getDataStartOffset()) / (entrySize + 4 + 8); + + // verify get methods + assertEquals(inputStream.getLedger(), readHandle); + assertEquals(inputStream.getStartEntryId(), 0); + assertEquals(inputStream.getBlockSize(), blockSize); + + // verify read inputStream + // 1. read header. 128 + byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()]; + // read twice to test the offset not 0 case + int ret = inputStream.read(headerB, 0, 66); + assertEquals(ret, 66); + ret = inputStream.read(headerB, 66, headerB.length - 66); + assertEquals(headerB.length - 66, ret); + DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB)); + assertEquals(headerRead.getBlockLength(), blockSize); + assertEquals(headerRead.getFirstEntryId(), 0); + + byte[] entryData = new byte[entrySize]; + Arrays.fill(entryData, (byte)0xB); // 0xB is MockLedgerEntry.blockPadding + + // 2. read Ledger entries. 96 * 20 + IntStream.range(0, expectedEntryCount).forEach(i -> { + try { + byte lengthBuf[] = new byte[4]; + byte entryIdBuf[] = new byte[8]; + byte content[] = new byte[entrySize]; + + int read = inputStream.read(lengthBuf, 0, 4); + assertEquals(read, 4); + read = inputStream.read(entryIdBuf, 0, 8); + assertEquals(read, 8); + + Random random = new Random(System.currentTimeMillis()); + int o = 0; + int totalRead = 0; + int maxReadTime = 10; + while (o != content.length) { + int r; + if (maxReadTime-- == 0) { + r = entrySize - o; + } else { + r = random.nextInt(entrySize - o); + } + read = inputStream.read(content, o, r); + totalRead += read; + o += r; + } + assertEquals(totalRead, entrySize); + + assertEquals(entrySize, Ints.fromByteArray(lengthBuf)); + assertEquals(i, Longs.fromByteArray(entryIdBuf)); + assertArrayEquals(entryData, content); + } catch (Exception e) { + fail("meet exception", e); + } + }); + + // 3. should have no padding + int left = blockSize - DataBlockHeaderImpl.getDataStartOffset() - expectedEntryCount * (entrySize + 4 + 8); + assertEquals(left, 0); + assertEquals(inputStream.getBlockSize(), inputStream.getDataBlockFullOffset()); + + // 4. reach end. + byte[] b = new byte[4]; + ret = inputStream.read(b, 0, 4); + assertEquals(ret, -1); + + assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount); + assertEquals(inputStream.getBlockEntryBytesCount(), entrySize * expectedEntryCount); + assertEquals(inputStream.getEndEntryId(), expectedEntryCount - 1); + + inputStream.close(); + } } From 935ce6e945e4ce08e43169801dd30e3de109dd82 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 8 Jun 2022 17:48:49 +0300 Subject: [PATCH 590/823] [ML] When skipping updating mark delete position, execute callback with executor to prevent deadlock (#15971) (cherry picked from commit bbc404bf6c8778e1e788b2b48af60de925256587) --- .../apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index ee9474a0019e9..412b778a3d6c6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1827,7 +1827,8 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { log.info("Skipping updating mark delete position to {}. The persisted mark delete position {} " + "is later.", mdEntry.newPosition, persistentMarkDeletePosition); } - mdEntry.triggerComplete(); + // run with executor to prevent deadlock + ledger.getExecutor().executeOrdered(ledger.getName(), safeRun(() -> mdEntry.triggerComplete())); return; } @@ -1845,7 +1846,8 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { log.info("Skipping updating mark delete position to {}. The mark delete position update " + "in progress {} is later.", mdEntry.newPosition, inProgressLatest); } - mdEntry.triggerComplete(); + // run with executor to prevent deadlock + ledger.getExecutor().executeOrdered(ledger.getName(), safeRun(() -> mdEntry.triggerComplete())); return; } From ec09b60a1988355cae47012b36252ea5d19c8782 Mon Sep 17 00:00:00 2001 From: Kay Johansen Date: Wed, 8 Jun 2022 21:52:42 -0600 Subject: [PATCH 591/823] [fix][pulsar] Bump pyyaml from 5.3.1 to 5.4.1 to solve CVE-2020-14343 (#15989) --- docker/pulsar/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index d30c6c3b681a7..ce7be90e223f1 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -46,7 +46,7 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get -y dist-upgrade \ && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ - python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ + python3 python3-dev python3-setuptools python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ && apt-get -y --purge autoremove \ @@ -58,6 +58,7 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +RUN pip3 install pyyaml==5.4.1 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security From 43234c63bab1a84d26448ddb663b4f6ba9352223 Mon Sep 17 00:00:00 2001 From: Jim Baugh Date: Thu, 9 Jun 2022 13:20:49 -0600 Subject: [PATCH 592/823] Allow PULSAR_MEM & PULSAR_GC to be Overridden in pulsar_tool_env.sh (#15868) The pulsar_tool_env.sh sets the PULSAR_MEM and PULSAR_GC environment variables without allowing them to be overridden. This change keps the default values but allows PULSAR_MEM & PULSAR_GC to be overridden which aligns with the pulsar_env.sh file. This allows adjustments to be made to the memory settings when more memory is needed. Co-authored-by: Jim Baugh ### Motivation The pulsar_tool_env.sh sets the PULSAR_MEM environment variable without allowing it to be overridden. When running an pulsar-admin function (e.g. running the kafka to pulsar connector) we can hit java memory issues without a way to change the memory settings. This PR resolves this issue. ### Modifications This change keeps the default value but allows PULSAR_MEM to be overridden which aligns with the pulsar_env.sh file. This allows adjustments to be made to the memory settings when more memory is needed. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: *If `yes` was chosen, please highlight the changes* - Dependencies (does it add or upgrade a dependency): (no) - The public API: (no) - The schema: (no) - The default values of configurations: (no) - The wire protocol: (no) - The rest endpoints: (no) - The admin cli options: (no) - Anything that affects deployment: (no) ### Documentation Check the box below or label this PR directly. Need to update docs? - [X] `doc-not-needed` (Please explain why) There is not currently documentation around the pulsar_tools_env.sh PULSAR_MEM setting. This change doesn't change the default behavior. (cherry picked from commit fa6288e87ec557e99a65131a48ee1f1e6a46781f) --- conf/pulsar_tools_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh index 0f14977954971..80f8d3f1d160f 100755 --- a/conf/pulsar_tools_env.sh +++ b/conf/pulsar_tools_env.sh @@ -42,10 +42,10 @@ # PULSAR_GLOBAL_ZK_CONF= # Extra options to be passed to the jvm -PULSAR_MEM="-Xmx128m -XX:MaxDirectMemorySize=128m" +PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} # Garbage collection options -PULSAR_GC=" -client " +PULSAR_GC=${PULSAR_GC:-" -client "} # Extra options to be passed to the jvm PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} ${PULSAR_GC_LOG} -Dio.netty.leakDetectionLevel=disabled" From 55e4a1d66a7cd7853d37c76a47fcb965bf3e937c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E5=BB=B6?= Date: Mon, 11 Apr 2022 12:34:36 +0800 Subject: [PATCH 593/823] [cleanup] [broker] Remove useless code to avoid confusion in OpReadEntry#checkReadCompletion. (#15104) (cherry picked from commit 93761284b9f6875da0403f5fedb6ccbfbbcd7315) --- .../org/apache/bookkeeper/mledger/impl/OpReadEntry.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java index b9c82914a76b9..d7eb0467f56c6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java @@ -137,12 +137,8 @@ void checkReadCompletion() { // op readPosition is smaller or equals maxPosition then can read again if (entries.size() < count && cursor.hasMoreEntries() && maxPosition.compareTo(readPosition) > 0) { - // We still have more entries to read from the next ledger, schedule a new async operation - if (nextReadPosition.getLedgerId() != readPosition.getLedgerId()) { - cursor.ledger.startReadOperationOnLedger(nextReadPosition, OpReadEntry.this); - } - // Schedule next read in a different thread + // We still have more entries to read from the next ledger, schedule a new async operation cursor.ledger.getExecutor().execute(safeRun(() -> { readPosition = cursor.ledger.startReadOperationOnLedger(nextReadPosition, OpReadEntry.this); cursor.ledger.asyncReadEntries(OpReadEntry.this); From 960b80f59e8b8700e6a5d30aca7ceb784d267ced Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 9 May 2022 09:11:19 +0800 Subject: [PATCH 594/823] [fix][broker] Fix MultiRolesTokenAuthorizationProvider `authorize` issue. (#15454) (cherry picked from commit 19f61d53b88bb195fabb367be722694902c79d22) --- .../MultiRolesTokenAuthorizationProvider.java | 37 +++------------ .../apache/pulsar/common/util/FutureUtil.java | 47 ++++++++++++++++++- .../pulsar/common/util/FutureUtilTest.java | 45 ++++++++++++++++++ 3 files changed, 98 insertions(+), 31 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index c508ccbd5b419..b8f46a5248352 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -23,10 +23,15 @@ import io.jsonwebtoken.JwtParser; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.RequiredTypeException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; -import org.apache.pulsar.broker.cache.ConfigurationCacheService; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -39,14 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.function.Function; - public class MultiRolesTokenAuthorizationProvider extends PulsarAuthorizationProvider { private static final Logger log = LoggerFactory.getLogger(MultiRolesTokenAuthorizationProvider.class); @@ -137,27 +134,7 @@ public CompletableFuture authorize(AuthenticationDataSource authenticat } List> futures = new ArrayList<>(roles.size()); roles.forEach(r -> futures.add(authorizeFunc.apply(r))); - return CompletableFuture.supplyAsync(() -> { - do { - try { - List> doneFutures = new ArrayList<>(); - FutureUtil.waitForAny(futures).get(); - for (CompletableFuture future : futures) { - if (!future.isDone()) continue; - doneFutures.add(future); - if (future.get()) { - futures.forEach(f -> { - if (!f.isDone()) f.cancel(false); - }); - return true; - } - } - futures.removeAll(doneFutures); - } catch (InterruptedException | ExecutionException ignored) { - } - } while (!futures.isEmpty()); - return false; - }); + return FutureUtil.waitForAny(futures, ret -> (boolean) ret).thenApply(v -> v.isPresent()); } /** diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index 2cdd9fce99508..dac204db98e81 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -19,6 +19,7 @@ package org.apache.pulsar.common.util; import java.time.Duration; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -28,7 +29,9 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * This class is aimed at simplifying work with {@code CompletableFuture}. @@ -51,10 +54,52 @@ public static CompletableFuture waitForAll(List waitForAny(List> futures) { + public static CompletableFuture waitForAny(Collection> futures) { return CompletableFuture.anyOf(futures.toArray(new CompletableFuture[0])); } + /** + * Return a future that represents the completion of any future that match the predicate in the provided Collection. + * + * @param futures futures to wait any + * @param tester if any future match the predicate + * @return a new CompletableFuture that is completed when any of the given CompletableFutures match the tester + */ + public static CompletableFuture> waitForAny(Collection> futures, + Predicate tester) { + return waitForAny(futures).thenCompose(v -> { + if (tester.test(v)) { + futures.forEach(f -> { + if (!f.isDone()) { + f.cancel(true); + } + }); + return CompletableFuture.completedFuture(Optional.of(v)); + } + Collection> doneFutures = futures.stream() + .filter(f -> f.isDone()) + .collect(Collectors.toList()); + futures.removeAll(doneFutures); + Optional value = doneFutures.stream() + .filter(f -> !f.isCompletedExceptionally()) + .map(CompletableFuture::join) + .filter(tester) + .findFirst(); + if (!value.isPresent()) { + if (futures.size() == 0) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return waitForAny(futures, tester); + } + futures.forEach(f -> { + if (!f.isDone()) { + f.cancel(true); + } + }); + return CompletableFuture.completedFuture(Optional.of(value.get())); + }); + } + /** * Return a future that represents the completion of the futures in the provided list. diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java index b9458bf8e1efd..0de407676567e 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java @@ -25,13 +25,18 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.time.Duration; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeoutException; import lombok.Cleanup; +import org.assertj.core.util.Lists; import org.testng.annotations.Test; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class FutureUtilTest { @@ -91,4 +96,44 @@ public void testCreatingFutureWithTimeoutHandling() { assertEquals(executionException.getCause(), e); } } + + @Test + public void testWaitForAny() { + CompletableFuture f1 = new CompletableFuture<>(); + CompletableFuture f2 = new CompletableFuture<>(); + CompletableFuture f3 = new CompletableFuture<>(); + CompletableFuture f4 = new CompletableFuture<>(); + f1.complete("1"); + f2.complete("2"); + f3.complete("3"); + f4.complete("4"); + CompletableFuture> ret = FutureUtil.waitForAny(Lists.newArrayList(f1, f2, f3, f4), p -> p.equals("3")); + assertEquals(ret.join().get(), "3"); + // test not matched predicate result + CompletableFuture f5 = new CompletableFuture<>(); + CompletableFuture f6 = new CompletableFuture<>(); + f5.complete("5"); + f6.complete("6"); + ret = FutureUtil.waitForAny(Lists.newArrayList(f5, f6), p -> p.equals("3")); + assertFalse(ret.join().isPresent()); + // test one complete, others are cancelled. + CompletableFuture f55 = new CompletableFuture<>(); + CompletableFuture f66 = new CompletableFuture<>(); + f55.complete("55"); + ret = FutureUtil.waitForAny(Lists.newArrayList(f55, f66), p -> p.equals("55")); + assertTrue(ret.join().isPresent()); + assertTrue(f66.isCancelled()); + // test with exception + CompletableFuture f7 = new CompletableFuture<>(); + CompletableFuture f8 = new CompletableFuture<>(); + f8.completeExceptionally(new RuntimeException("f7 exception")); + f8.completeExceptionally(new RuntimeException("f8 exception")); + ret = FutureUtil.waitForAny(Lists.newArrayList(f7, f8), p -> p.equals("3")); + try { + ret.join(); + fail("Should have failed"); + } catch (CompletionException ex) { + assertTrue(ex.getCause() instanceof RuntimeException); + } + } } \ No newline at end of file From b2de04362b302d8d4294e6e4073c9d046f9bb84d Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Wed, 25 May 2022 15:37:33 +0800 Subject: [PATCH 595/823] [feat] [tiered-storage] Add pure S3 provider for the offloader (#15710) * [improve] [tiered-storage] Add pure S3 provider for the offloader --- *Motivation* There have some cloud storages are compatible with S3 APIs, such as aliyun-oss. Some other storages also use the S3 APIs and want to offload the data into them, but we only support the AWS or the Aliyun. The PR https://github.com/apache/pulsar/pull/8985 provides the Aliyun offload provider, but it has a force limitation of the `S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS`. That is not a limitation on other storage service which compatible with S3 APIs. This PR provides a more general offload provider `S3` which uses pure JClouds S3 metadata and allows people to override the default JClouds properties through system properties. *Modifications* - Add the pure S3 offload provider (cherry picked from commit 047cb0e3117d55a79c0082c0dc3d2ab3c9bcd6f9) --- .../provider/JCloudBlobStoreProvider.java | 54 ++++++++++++++----- .../provider/TieredStorageConfiguration.java | 13 +++++ .../JCloudBlobStoreProviderTests.java | 31 ++++++++++- .../TieredStorageConfigurationTests.java | 17 ++++++ 4 files changed, 99 insertions(+), 16 deletions(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java index 44aa92ce92438..fc28c0291ce27 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java @@ -181,17 +181,34 @@ public void buildCredentials(TieredStorageConfiguration config) { ALIYUN_OSS("aliyun-oss", new AnonymousProviderMetadata(new S3ApiMetadata(), "")) { @Override public void validate(TieredStorageConfiguration config) throws IllegalArgumentException { - ALIYUN_OSS_VALIDATION.validate(config); + S3_VALIDATION.validate(config); } @Override public BlobStore getBlobStore(TieredStorageConfiguration config) { - return ALIYUN_OSS_BLOB_STORE_BUILDER.getBlobStore(config); + return S3_BLOB_STORE_BUILDER.getBlobStore(config); } @Override public void buildCredentials(TieredStorageConfiguration config) { - ALIYUN_OSS_CREDENTIAL_BUILDER.buildCredentials(config); + S3_CREDENTIAL_BUILDER.buildCredentials(config); + } + }, + + S3("S3", new AnonymousProviderMetadata(new S3ApiMetadata(), "")) { + @Override + public BlobStore getBlobStore(TieredStorageConfiguration config) { + return S3_BLOB_STORE_BUILDER.getBlobStore(config); + } + + @Override + public void buildCredentials(TieredStorageConfiguration config) { + S3_CREDENTIAL_BUILDER.buildCredentials(config); + } + + @Override + public void validate(TieredStorageConfiguration config) throws IllegalArgumentException { + S3_VALIDATION.validate(config); } }, @@ -374,12 +391,14 @@ public String getAWSSecretKey() { } }; - static final BlobStoreBuilder ALIYUN_OSS_BLOB_STORE_BUILDER = (TieredStorageConfiguration config) -> { + static final BlobStoreBuilder S3_BLOB_STORE_BUILDER = (TieredStorageConfiguration config) -> { ContextBuilder contextBuilder = ContextBuilder.newBuilder(config.getProviderMetadata()); contextBuilder.modules(Arrays.asList(new SLF4JLoggingModule())); Properties overrides = config.getOverrides(); - // For security reasons, OSS supports only virtual hosted style access. - overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true"); + if (ALIYUN_OSS.getDriver().equals(config.getDriver())) { + // For security reasons, OSS supports only virtual hosted style access. + overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true"); + } contextBuilder.overrides(overrides); contextBuilder.endpoint(config.getServiceEndpoint()); @@ -396,7 +415,7 @@ public String getAWSSecretKey() { } }; - static final ConfigValidation ALIYUN_OSS_VALIDATION = (TieredStorageConfiguration config) -> { + static final ConfigValidation S3_VALIDATION = (TieredStorageConfiguration config) -> { if (Strings.isNullOrEmpty(config.getServiceEndpoint())) { throw new IllegalArgumentException( "ServiceEndpoint must specified for " + config.getDriver() + " offload"); @@ -414,14 +433,21 @@ public String getAWSSecretKey() { } }; - static final CredentialBuilder ALIYUN_OSS_CREDENTIAL_BUILDER = (TieredStorageConfiguration config) -> { - String accountName = System.getenv("ALIYUN_OSS_ACCESS_KEY_ID"); - if (StringUtils.isEmpty(accountName)) { - throw new IllegalArgumentException("Couldn't get the aliyun oss access key id."); + static final CredentialBuilder S3_CREDENTIAL_BUILDER = (TieredStorageConfiguration config) -> { + String accountName = System.getenv().getOrDefault("ACCESS_KEY_ID", ""); + // For forward compatibility + if (StringUtils.isEmpty(accountName.trim())) { + accountName = System.getenv().getOrDefault("ALIYUN_OSS_ACCESS_KEY_ID", ""); + } + if (StringUtils.isEmpty(accountName.trim())) { + throw new IllegalArgumentException("Couldn't get the access key id."); + } + String accountKey = System.getenv().getOrDefault("ACCESS_KEY_ID", ""); + if (StringUtils.isEmpty(accountKey.trim())) { + accountKey = System.getenv().getOrDefault("ALIYUN_OSS_ACCESS_KEY_SECRET", ""); } - String accountKey = System.getenv("ALIYUN_OSS_ACCESS_KEY_SECRET"); - if (StringUtils.isEmpty(accountKey)) { - throw new IllegalArgumentException("Couldn't get the aliyun oss access key secret."); + if (StringUtils.isEmpty(accountKey.trim())) { + throw new IllegalArgumentException("Couldn't get the access key secret."); } Credentials credentials = new Credentials( accountName, accountKey); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java index c1054969a4280..18e3bbf0db8fe 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfiguration.java @@ -329,6 +329,19 @@ protected Properties getOverrides() { overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false"); } + // load more jclouds properties into the overrides + System.getProperties().entrySet().stream() + .filter(p -> p.getKey().toString().startsWith("jclouds")) + .forEach(jcloudsProp -> { + overrides.setProperty(jcloudsProp.getKey().toString(), jcloudsProp.getValue().toString()); + }); + + System.getenv().entrySet().stream() + .filter(p -> p.getKey().toString().startsWith("jclouds")) + .forEach(jcloudsProp -> { + overrides.setProperty(jcloudsProp.getKey().toString(), jcloudsProp.getValue().toString()); + }); + log.info("getOverrides: {}", overrides.toString()); return overrides; } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTests.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTests.java index 28e5829ba2a5e..4f0c60bc00708 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTests.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTests.java @@ -23,8 +23,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider; -import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; import org.testng.annotations.Test; public class JCloudBlobStoreProviderTests { @@ -105,4 +103,33 @@ public void transientValidationFailureTest() { config = new TieredStorageConfiguration(map); JCloudBlobStoreProvider.TRANSIENT.validate(config); } + + @Test() + public void s3ValidationTest() { + Map map = new HashMap<>(); + map.put("managedLedgerOffloadDriver", "S3"); + map.put("managedLedgerOffloadServiceEndpoint", "http://s3.service"); + map.put("managedLedgerOffloadBucket", "test-s3-bucket"); + TieredStorageConfiguration configuration = new TieredStorageConfiguration(map); + configuration.getProvider().validate(configuration); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "ServiceEndpoint must specified for S3 offload") + public void s3ValidationServiceEndpointMissed() { + Map map = new HashMap<>(); + map.put("managedLedgerOffloadDriver", "S3"); + TieredStorageConfiguration configuration = new TieredStorageConfiguration(map); + configuration.getProvider().validate(configuration); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "Bucket cannot be empty for S3 offload") + public void s3ValidationBucketMissed() { + Map map = new HashMap<>(); + map.put("managedLedgerOffloadDriver", "S3"); + map.put("managedLedgerOffloadServiceEndpoint", "http://s3.service"); + TieredStorageConfiguration configuration = new TieredStorageConfiguration(map); + configuration.getProvider().validate(configuration); + } } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java index f80f3ceaa1aa8..bf5e046bf70ab 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java @@ -22,6 +22,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; + import org.jclouds.domain.Credentials; import org.testng.annotations.Test; @@ -205,4 +207,19 @@ public final void gcsBackwardCompatiblePropertiesTest() { assertEquals(config.getMaxBlockSizeInBytes(), new Integer(12)); assertEquals(config.getReadBufferSizeInBytes(), new Integer(500)); } + + @Test + public void overridePropertiesTest() { + Map map = new HashMap<>(); + map.put("s3ManagedLedgerOffloadServiceEndpoint", "http://localhost"); + map.put("s3ManagedLedgerOffloadRegion", "my-region"); + System.setProperty("jclouds.SystemPropertyA", "A"); + System.setProperty("jclouds.region", "jclouds-region"); + TieredStorageConfiguration config = new TieredStorageConfiguration(map); + Properties properties = config.getOverrides(); + System.out.println(properties.toString()); + assertEquals(properties.get("jclouds.region"), "jclouds-region"); + assertEquals(config.getServiceEndpoint(), "http://localhost"); + assertEquals(properties.get("jclouds.SystemPropertyA"), "A"); + } } From c1e6606682053c2ad38d141faaab7314a910c692 Mon Sep 17 00:00:00 2001 From: Qiang Huang Date: Sat, 28 May 2022 10:58:35 +0800 Subject: [PATCH 596/823] fix bug in getNumberOfEntriesInStorage (#15627) (cherry picked from commit a43981109a9322d94082ae0d87d0de53b8f237e8) --- .../mledger/impl/ManagedCursorImpl.java | 2 +- .../mledger/impl/ManagedLedgerTest.java | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 412b778a3d6c6..604e3a4f3d89b 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -964,7 +964,7 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) { } public long getNumberOfEntriesInStorage() { - return ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition().getNext())); + return ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition())); } @Override diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index af5f6c807e3ff..8826f0d99fc88 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -2243,6 +2243,35 @@ public void testGetPositionAfterN() throws Exception { assertEquals(targetPosition.getEntryId(), 4); } + @Test + public void testGetNumberOfEntriesInStorage() throws Exception { + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setMaxEntriesPerLedger(5); + ManagedLedgerImpl managedLedger = + (ManagedLedgerImpl) factory.open("testGetNumberOfEntriesInStorage", managedLedgerConfig); + // open cursor to prevent ledger to be deleted when ledger rollover + ManagedCursorImpl managedCursor = (ManagedCursorImpl) managedLedger.openCursor("cursor"); + int numberOfEntries = 10; + for (int i = 0; i < numberOfEntries; i++) { + managedLedger.addEntry(("entry-" + i).getBytes(Encoding)); + } + + //trigger ledger rollover and wait for the new ledger created + Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); + stateUpdater.setAccessible(true); + stateUpdater.set(managedLedger, ManagedLedgerImpl.State.LedgerOpened); + managedLedger.rollCurrentLedgerIfFull(); + Awaitility.await().untilAsserted(() -> { + assertEquals(managedLedger.getLedgersInfo().size(), 2); + assertEquals(managedLedger.getState(), ManagedLedgerImpl.State.ClosedLedger); + }); + assertEquals(5, managedLedger.getLedgersInfoAsList().get(0).getEntries()); + assertEquals(5, managedLedger.getLedgersInfoAsList().get(1).getEntries()); + log.info("### ledgers {}", managedLedger.getLedgersInfo()); + long length = managedCursor.getNumberOfEntriesInStorage(); + assertEquals(length, numberOfEntries); + } + @Test public void testEstimatedBacklogSize() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testEstimatedBacklogSize"); From 89e005f939fa4400bc1be5a9058264d9e071622a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 1 Jun 2022 11:09:01 +0800 Subject: [PATCH 597/823] Fix NPE in MessageDeduplication. (#15820) (cherry picked from commit 01d7bfa681b23d1a236b1411b83e854c9ad9323f) --- .../broker/service/persistent/MessageDeduplication.java | 2 +- .../broker/service/persistent/MessageDuplicationTest.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 55d35201fe5be..5b29a4f15c733 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -490,7 +490,7 @@ public synchronized void purgeInactiveProducers() { hasInactive = true; } } - if (hasInactive) { + if (hasInactive && isEnabled()) { takeSnapshot(getManagedCursor().getMarkDeletedPosition()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index 76caccadcc96a..0e1b37b5160ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -202,6 +202,13 @@ public void testInactiveProducerRemove() throws Exception { messageDeduplication.purgeInactiveProducers(); assertEquals(inactiveProducers.size(), 3); + doReturn(false).when(messageDeduplication).isEnabled(); + inactiveProducers.put(producerName2, System.currentTimeMillis() - 80000); + inactiveProducers.put(producerName3, System.currentTimeMillis() - 80000); + messageDeduplication.purgeInactiveProducers(); + assertFalse(inactiveProducers.containsKey(producerName2)); + assertFalse(inactiveProducers.containsKey(producerName3)); + doReturn(true).when(messageDeduplication).isEnabled(); // Modify the inactive time of produce2 and produce3 // messageDeduplication.purgeInactiveProducers() will remove producer2 and producer3 inactiveProducers.put(producerName2, System.currentTimeMillis() - 70000); From d6565a5d3dac9910a8c361a4a08a3a7faf101331 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 1 Jun 2022 16:58:48 +0800 Subject: [PATCH 598/823] Fix avro conversion order of registration (#15863) ### Motivation Fixes #15858 The conversion that is registered first is a higher priority than the registered later, so `TimestampMillisConversion` should not be registered after `TimestampMicrosConversion`. ### Modifications Improve `avro` conversion order of registration. (cherry picked from commit 311fdb5dad09217c1706409feb3387d59285c51f) --- .../pulsar/client/impl/schema/AvroSchema.java | 3 ++- .../client/impl/schema/AvroSchemaTest.java | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index b34017e20aa1d..cff3ccdf8f689 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -117,8 +117,8 @@ public static void addLogicalTypeConversions(ReflectData reflectData, boolean js reflectData.addLogicalTypeConversion(new TimeConversions.DateConversion()); reflectData.addLogicalTypeConversion(new TimeConversions.TimeMillisConversion()); reflectData.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); - reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); if (jsr310ConversionEnabled) { + // The conversion that is registered first is higher priority than the registered later. reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMillisConversion()); } else { try { @@ -127,6 +127,7 @@ public static void addLogicalTypeConversions(ReflectData reflectData, boolean js } catch (ClassNotFoundException e) { // Skip if have not provide joda-time dependency. } + reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); } reflectData.addLogicalTypeConversion(new Conversions.UUIDConversion()); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 9e707af836796..d69f8bf66ba7a 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -42,6 +42,7 @@ import org.apache.avro.SchemaValidationException; import org.apache.avro.SchemaValidator; import org.apache.avro.SchemaValidatorBuilder; +import org.apache.avro.data.TimeConversions; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.BufferedBinaryEncoder; import org.apache.avro.reflect.AvroDefault; @@ -459,4 +460,24 @@ public void testAvroBigDecimal() { assertEquals(pojo2.value1, myBigDecimalPojo.value1); assertEquals(pojo2.value2, myBigDecimalPojo.value2); } + + + @Data + private static class TimestampStruct { + Instant value; + } + + @Test + public void testTimestampWithJsr310Conversion() { + AvroSchema schema = AvroSchema.of(TimestampStruct.class); + Assert.assertEquals( + schema.getAvroSchema().getFields().get(0).schema().getTypes().get(1).getLogicalType().getName(), + new TimeConversions.TimestampMicrosConversion().getLogicalTypeName()); + + AvroSchema schema2 = AvroSchema.of(SchemaDefinition.builder() + .withPojo(TimestampStruct.class).withJSR310ConversionEnabled(true).build()); + Assert.assertEquals( + schema2.getAvroSchema().getFields().get(0).schema().getTypes().get(1).getLogicalType().getName(), + new TimeConversions.TimestampMillisConversion().getLogicalTypeName()); + } } From 4a471d06ce173a99850866c095c54c88810e2ee4 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Sun, 5 Jun 2022 09:44:47 +0800 Subject: [PATCH 599/823] [Revert] [#15483] Remove sensitive msg from consumer/producer stats log (#15817) ### Motivation See #15483 The `@Secret` annotation works well, and introduced in #8910 ### Modifications - Revert the unneeded `@JsonIgnore` - remove `Assert.assertFalse(s.contains("Password"));` `Password` is printed in a key. The sensitive field's value is `****`. (cherry picked from commit 67361e8db632b0cd4c23198c5c569f3f2193fc70) --- .../pulsar/client/impl/conf/ClientConfigurationData.java | 7 ------- .../client/impl/conf/ClientConfigurationDataTest.java | 1 - 2 files changed, 8 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 093e3e198839b..3044b2a4c3bd4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -30,7 +30,6 @@ import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; -import lombok.Getter; import lombok.NoArgsConstructor; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.ProxyProtocol; @@ -61,7 +60,6 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "The implementation class of ServiceUrlProvider used to generate ServiceUrl." ) @JsonIgnore - @Getter(onMethod_ = @JsonIgnore) private transient ServiceUrlProvider serviceUrlProvider; @ApiModelProperty( @@ -257,8 +255,6 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "Password of TLS TrustStore." ) @Secret - @JsonIgnore - @Getter(onMethod_ = @JsonIgnore) private String tlsTrustStorePassword = null; @ApiModelProperty( @@ -318,10 +314,8 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "Password of SOCKS5 proxy." ) @Secret - @JsonIgnore private String socks5ProxyPassword; - @JsonIgnore public Authentication getAuthentication() { if (authentication == null) { this.authentication = AuthenticationDisabled.INSTANCE; @@ -377,7 +371,6 @@ public String getSocks5ProxyUsername() { return Objects.nonNull(socks5ProxyUsername) ? socks5ProxyUsername : System.getProperty("socks5Proxy.username"); } - @JsonIgnore public String getSocks5ProxyPassword() { return Objects.nonNull(socks5ProxyPassword) ? socks5ProxyPassword : System.getProperty("socks5Proxy.password"); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java index b5c30c9a7c6c4..c817ec996d480 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java @@ -48,7 +48,6 @@ public void testDoNotPrintSensitiveInfo() throws JsonProcessingException { clientConfigurationData.setSocks5ProxyPassword("yyyy"); clientConfigurationData.setAuthentication(new AuthenticationToken("zzzz")); String s = w.writeValueAsString(clientConfigurationData); - Assert.assertFalse(s.contains("Password")); Assert.assertFalse(s.contains("xxxx")); Assert.assertFalse(s.contains("yyyy")); Assert.assertFalse(s.contains("zzzz")); From dac1c6f39d17b4f64c5dcc8ccd342fc88ecbc2af Mon Sep 17 00:00:00 2001 From: Xiaoyu Hou Date: Mon, 6 Jun 2022 10:59:55 +0800 Subject: [PATCH 600/823] [fix][broker]Fast return if ack cumulative illegal (#15695) (cherry picked from commit 02dca31a78523a7d061ac1d6702ff6600a7f4ec4) --- .../pulsar/broker/service/Consumer.java | 2 + .../service/MessageCumulativeAckTest.java | 199 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index f7ed211fb2c7a..031574975d1c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -358,11 +358,13 @@ public CompletableFuture messageAcked(CommandAck ack) { if (ack.getAckType() == AckType.Cumulative) { if (ack.getMessageIdsCount() != 1) { log.warn("[{}] [{}] Received multi-message ack", subscription, consumerId); + return CompletableFuture.completedFuture(null); } if (Subscription.isIndividualAckMode(subType)) { log.warn("[{}] [{}] Received cumulative ack on shared subscription, ignoring", subscription, consumerId); + return CompletableFuture.completedFuture(null); } PositionImpl position = PositionImpl.earliest; if (ack.getMessageIdsCount() == 1) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java new file mode 100644 index 0000000000000..86754efc0c28f --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static java.util.Collections.emptyMap; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.common.api.proto.CommandAck.AckType.Cumulative; +import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive; +import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Failover; +import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Key_Shared; +import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Shared; +import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import java.net.InetSocketAddress; +import java.util.Optional; +import org.apache.bookkeeper.common.util.OrderedExecutor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.common.api.proto.CommandAck; +import org.apache.pulsar.common.api.proto.CommandSubscribe; +import org.apache.pulsar.common.api.proto.ProtocolVersion; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class MessageCumulativeAckTest { + private final int consumerId = 1; + private BrokerService brokerService; + private ServerCnx serverCnx; + private MetadataStore store; + protected PulsarService pulsar; + private OrderedExecutor executor; + private EventLoopGroup eventLoopGroup; + private PersistentSubscription sub; + + @BeforeMethod + public void setup() throws Exception { + executor = OrderedExecutor.newBuilder().numThreads(1).name("persistent-dispatcher-cumulative-ack-test").build(); + ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); + svcConfig.setBrokerShutdownTimeoutMs(0L); + svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + svcConfig.setClusterName("pulsar-cluster"); + pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); + doReturn(svcConfig).when(pulsar).getConfiguration(); + + ManagedLedgerFactory mlFactoryMock = mock(ManagedLedgerFactory.class); + doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory(); + doReturn(TransactionTestBase.createMockBookKeeper(executor)) + .when(pulsar).getBookKeeperClient(); + + store = MetadataStoreFactory.create("memory:local", MetadataStoreConfig.builder().build()); + doReturn(store).when(pulsar).getLocalMetadataStore(); + doReturn(store).when(pulsar).getConfigurationMetadataStore(); + + PulsarResources pulsarResources = new PulsarResources(store, store); + doReturn(pulsarResources).when(pulsar).getPulsarResources(); + + serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + doReturn(true).when(serverCnx).isActive(); + doReturn(true).when(serverCnx).isWritable(); + doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress(); + when(serverCnx.getRemoteEndpointProtocolVersion()).thenReturn(ProtocolVersion.v12.getValue()); + when(serverCnx.ctx()).thenReturn(mock(ChannelHandlerContext.class)); + doReturn(new PulsarCommandSenderImpl(null, serverCnx)) + .when(serverCnx).getCommandSender(); + + eventLoopGroup = new NioEventLoopGroup(); + brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); + doReturn(brokerService).when(pulsar).getBrokerService(); + + String topicName = TopicName.get("MessageCumulativeAckTest").toString(); + PersistentTopic persistentTopic = new PersistentTopic(topicName, mock(ManagedLedger.class), brokerService); + sub = spy(new PersistentSubscription(persistentTopic, "sub-1", + mock(ManagedCursorImpl.class), false)); + doNothing().when(sub).acknowledgeMessage(any(), any(), any()); + } + + @AfterMethod(alwaysRun = true) + public void shutdown() throws Exception { + if (brokerService != null) { + brokerService.close(); + brokerService = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + + executor.shutdown(); + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully().get(); + } + store.close(); + sub = null; + } + + @DataProvider(name = "individualAckModes") + public static Object[][] individualAckModes() { + return new Object[][]{ + {Shared}, + {Key_Shared}, + }; + } + + @DataProvider(name = "notIndividualAckModes") + public static Object[][] notIndividualAckModes() { + return new Object[][]{ + {Exclusive}, + {Failover}, + }; + } + + @Test(timeOut = 5000, dataProvider = "individualAckModes") + public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { + Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, + "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, + MessageId.latest, DEFAULT_CONSUMER_EPOCH); + + CommandAck commandAck = new CommandAck(); + commandAck.setAckType(Cumulative); + commandAck.setConsumerId(consumerId); + commandAck.addMessageId().setEntryId(0L).setLedgerId(1L); + + consumer.messageAcked(commandAck).get(); + verify(sub, never()).acknowledgeMessage(any(), any(), any()); + } + + @Test(timeOut = 5000, dataProvider = "notIndividualAckModes") + public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { + Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, + "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, + MessageId.latest, DEFAULT_CONSUMER_EPOCH); + + CommandAck commandAck = new CommandAck(); + commandAck.setAckType(Cumulative); + commandAck.setConsumerId(consumerId); + commandAck.addMessageId().setEntryId(0L).setLedgerId(1L); + + consumer.messageAcked(commandAck).get(); + verify(sub, times(1)).acknowledgeMessage(any(), any(), any()); + } + + @Test(timeOut = 5000) + public void testAckWithMoreThanNoneMessageIds() throws Exception { + Consumer consumer = new Consumer(sub, Failover, "topic-1", consumerId, 0, + "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, + MessageId.latest, DEFAULT_CONSUMER_EPOCH); + + CommandAck commandAck = new CommandAck(); + commandAck.setAckType(Cumulative); + commandAck.setConsumerId(consumerId); + commandAck.addMessageId().setEntryId(0L).setLedgerId(1L); + commandAck.addMessageId().setEntryId(0L).setLedgerId(2L); + + consumer.messageAcked(commandAck).get(); + verify(sub, never()).acknowledgeMessage(any(), any(), any()); + } +} From faac51937d2f9c95811e85446b3319f877dc9fd4 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 7 Jun 2022 12:45:00 +0800 Subject: [PATCH 601/823] [fix][txn]Fix transasction ack batch message (#15875) Fixes https://github.com/apache/pulsar/issues/15832 ### Motivation The transaction needs batch size to help determine whether the batch message is in the pending ack state. ### Modifications Returns the batch size of messageID directly. (cherry picked from commit f87b3708ae6f05a8d4d4d6cd0db1090724fbcf4b) --- .../pulsar/broker/service/Consumer.java | 7 +- .../pendingack/PendingAckPersistentTest.java | 71 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 031574975d1c4..f2a4677dbfc55 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -478,8 +478,11 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId()); ackedCount = batchSize; } - - positionsAcked.add(new MutablePair<>(position, (int) batchSize)); + if (msgId.hasBatchSize()) { + positionsAcked.add(new MutablePair<>(position, msgId.getBatchSize())); + } else { + positionsAcked.add(new MutablePair<>(position, (int) batchSize)); + } addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index 6683be138da66..bd22ff423a956 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -45,8 +45,10 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.transaction.TransactionImpl; @@ -574,4 +576,73 @@ public void testPendingAckLowWaterMarkRemoveFirstTxn() throws Exception { assertFalse(individualAckOfTransaction.containsKey(transaction2.getTxnID())); } + + @Test + public void testTransactionConflictExceptionWhenAckBatchMessage() throws Exception { + String topic = TopicName.get(TopicDomain.persistent.toString(), + NamespaceName.get(NAMESPACE1), "test").toString(); + + String subscriptionName = "my-subscription-batch"; + pulsarServiceList.get(0).getBrokerService() + .getManagedLedgerConfig(TopicName.get(topic)).get() + .setDeletionAtBatchIndexLevelEnabled(true); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(true) + .batchingMaxMessages(3) + // set batch max publish delay big enough to make sure entry has 3 messages + .batchingMaxPublishDelay(10, TimeUnit.SECONDS) + .topic(topic).create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(subscriptionName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Exclusive) + .isAckReceiptEnabled(true) + .topic(topic) + .subscribe(); + + List messageIds = new ArrayList<>(); + List> futureMessageIds = new ArrayList<>(); + + List messages = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + String message = "my-message-" + i; + messages.add(message); + CompletableFuture messageIdCompletableFuture = producer.sendAsync(message); + futureMessageIds.add(messageIdCompletableFuture); + } + + for (CompletableFuture futureMessageId : futureMessageIds) { + MessageId messageId = futureMessageId.get(); + messageIds.add(messageId); + } + + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.DAYS) + .build() + .get(); + + Message message1 = consumer.receive(); + Message message2 = consumer.receive(); + + BatchMessageIdImpl messageId = (BatchMessageIdImpl) message2.getMessageId(); + consumer.acknowledgeAsync(messageId, transaction).get(); + + Transaction transaction2 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.DAYS) + .build() + .get(); + transaction.commit().get(); + + try { + consumer.acknowledgeAsync(messageId, transaction2).get(); + fail(); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + } From 43ab20b735f3a7cd989369725206c79d82b4d4f0 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 7 Jun 2022 15:46:57 +0800 Subject: [PATCH 602/823] [fix][auth] Generate correct well-known OpenID configuration URL (#15928) (cherry picked from commit 304b03e7ff3eeff62c31f93738af488eb44abde0) --- pulsar-client-cpp/lib/auth/AuthOauth2.cc | 9 ++++++++- pulsar-client-cpp/lib/auth/AuthOauth2.h | 1 + pulsar-client-cpp/tests/AuthPluginTest.cc | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.cc b/pulsar-client-cpp/lib/auth/AuthOauth2.cc index c3dfe550a0c19..438239a46d69e 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.cc +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.cc @@ -143,6 +143,8 @@ ClientCredentialFlow::ClientCredentialFlow(ParamMap& params) audience_(params["audience"]), scope_(params["scope"]) {} +std::string ClientCredentialFlow::getTokenEndPoint() const { return tokenEndPoint_; } + static size_t curlWriteCallback(void* contents, size_t size, size_t nmemb, void* responseDataPtr) { ((std::string*)responseDataPtr)->append((char*)contents, size * nmemb); return size * nmemb; @@ -168,7 +170,12 @@ void ClientCredentialFlow::initialize() { curl_easy_setopt(handle, CURLOPT_CUSTOMREQUEST, "GET"); // set URL: well-know endpoint - curl_easy_setopt(handle, CURLOPT_URL, (issuerUrl_ + "/.well-known/openid-configuration").c_str()); + std::string wellKnownUrl = issuerUrl_; + if (wellKnownUrl.back() == '/') { + wellKnownUrl.pop_back(); + } + wellKnownUrl.append("/.well-known/openid-configuration"); + curl_easy_setopt(handle, CURLOPT_URL, wellKnownUrl.c_str()); // Write callback curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlWriteCallback); diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.h b/pulsar-client-cpp/lib/auth/AuthOauth2.h index a3658b353eefa..986919ddfcd9c 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.h +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.h @@ -57,6 +57,7 @@ class ClientCredentialFlow : public Oauth2Flow { void close(); ParamMap generateParamMap() const; + std::string getTokenEndPoint() const; private: std::string tokenEndPoint_; diff --git a/pulsar-client-cpp/tests/AuthPluginTest.cc b/pulsar-client-cpp/tests/AuthPluginTest.cc index be987e07c4860..01c19ebbea484 100644 --- a/pulsar-client-cpp/tests/AuthPluginTest.cc +++ b/pulsar-client-cpp/tests/AuthPluginTest.cc @@ -412,6 +412,26 @@ TEST(AuthPluginTest, testOauth2RequestBody) { ASSERT_EQ(flow2.generateParamMap(), expectedResult2); } +TEST(AuthPluginTest, testInitialize) { + std::string issuerUrl = "https://dev-kt-aa9ne.us.auth0.com"; + std::string expectedTokenEndPoint = issuerUrl + "/oauth/token"; + + ParamMap params; + params["issuer_url"] = issuerUrl; + params["client_id"] = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x"; + params["client_secret"] = "rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb"; + params["audience"] = "https://dev-kt-aa9ne.us.auth0.com/api/v2/"; + + ClientCredentialFlow flow1(params); + flow1.initialize(); + ASSERT_EQ(flow1.getTokenEndPoint(), expectedTokenEndPoint); + + params["issuer_url"] = issuerUrl + "/"; + ClientCredentialFlow flow2(params); + flow2.initialize(); + ASSERT_EQ(flow2.getTokenEndPoint(), expectedTokenEndPoint); +} + TEST(AuthPluginTest, testOauth2Failure) { ParamMap params; auto addKeyValue = [&](const std::string& key, const std::string& value) { From 53615e05cff4010ef0834f15f057b78c1eb6fc77 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 7 Jun 2022 21:52:05 +0800 Subject: [PATCH 603/823] [Fix][broker] Fix NPE when ledger id not found in `OpReadEntry` (#15837) (cherry picked from commit 7a3ad611f51511afca4bcaa1de299517a1907e8e) --- .../mledger/impl/ManagedLedgerImpl.java | 9 ++---- .../bookkeeper/mledger/impl/OpReadEntry.java | 4 +-- .../mledger/impl/ManagedLedgerTest.java | 32 +++++++++++++++++-- .../service/MessageCumulativeAckTest.java | 15 +++------ 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index d03d98a3f064c..c79b8a9f8f700 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2153,14 +2153,9 @@ void updateCursor(ManagedCursorImpl cursor, PositionImpl newPosition) { } } - PositionImpl startReadOperationOnLedger(PositionImpl position, OpReadEntry opReadEntry) { + PositionImpl startReadOperationOnLedger(PositionImpl position) { Long ledgerId = ledgers.ceilingKey(position.getLedgerId()); - if (null == ledgerId) { - opReadEntry.readEntriesFailed(new ManagedLedgerException.NoMoreEntriesToReadException("The ceilingKey(K key) method is used to return the " + - "least key greater than or equal to the given key, or null if there is no such key"), null); - } - - if (ledgerId != position.getLedgerId()) { + if (ledgerId != null && ledgerId != position.getLedgerId()) { // The ledger pointed by this position does not exist anymore. It was deleted because it was empty. We need // to skip on the next available ledger position = new PositionImpl(ledgerId, 0); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java index d7eb0467f56c6..27e99169e31cd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java @@ -48,7 +48,7 @@ class OpReadEntry implements ReadEntriesCallback { public static OpReadEntry create(ManagedCursorImpl cursor, PositionImpl readPositionRef, int count, ReadEntriesCallback callback, Object ctx, PositionImpl maxPosition) { OpReadEntry op = RECYCLER.get(); - op.readPosition = cursor.ledger.startReadOperationOnLedger(readPositionRef, op); + op.readPosition = cursor.ledger.startReadOperationOnLedger(readPositionRef); op.cursor = cursor; op.count = count; op.callback = callback; @@ -140,7 +140,7 @@ void checkReadCompletion() { // We still have more entries to read from the next ledger, schedule a new async operation cursor.ledger.getExecutor().execute(safeRun(() -> { - readPosition = cursor.ledger.startReadOperationOnLedger(nextReadPosition, OpReadEntry.this); + readPosition = cursor.ledger.startReadOperationOnLedger(nextReadPosition); cursor.ledger.asyncReadEntries(OpReadEntry.this); })); } else { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 8826f0d99fc88..317fb7e2b307e 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -52,6 +52,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -408,6 +409,33 @@ public void spanningMultipleLedgers() throws Exception { ledger.close(); } + @Test + public void testStartReadOperationOnLedgerWithEmptyLedgers() throws ManagedLedgerException, InterruptedException { + ManagedLedger ledger = factory.open("my_test_ledger_1"); + ManagedLedgerImpl ledgerImpl = (ManagedLedgerImpl) ledger; + NavigableMap ledgers = ledgerImpl.getLedgersInfo(); + LedgerInfo ledgerInfo = ledgers.firstEntry().getValue(); + ledgers.clear(); + ManagedCursor c1 = ledger.openCursor("c1"); + PositionImpl position = new PositionImpl(ledgerInfo.getLedgerId(), 0); + PositionImpl maxPosition = new PositionImpl(ledgerInfo.getLedgerId(), 99); + OpReadEntry opReadEntry = OpReadEntry.create((ManagedCursorImpl) c1, position, 20, + new ReadEntriesCallback() { + + @Override + public void readEntriesComplete(List entries, Object ctx) { + + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + + } + }, null, maxPosition); + Assert.assertEquals(opReadEntry.readPosition, position); + } + + @Test(timeOut = 20000) public void spanningMultipleLedgersWithSize() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1000000); @@ -2262,8 +2290,8 @@ public void testGetNumberOfEntriesInStorage() throws Exception { stateUpdater.set(managedLedger, ManagedLedgerImpl.State.LedgerOpened); managedLedger.rollCurrentLedgerIfFull(); Awaitility.await().untilAsserted(() -> { - assertEquals(managedLedger.getLedgersInfo().size(), 2); - assertEquals(managedLedger.getState(), ManagedLedgerImpl.State.ClosedLedger); + assertEquals(managedLedger.getLedgersInfo().size(), 3); + assertEquals(managedLedger.getState(), ManagedLedgerImpl.State.LedgerOpened); }); assertEquals(5, managedLedger.getLedgersInfoAsList().get(0).getEntries()); assertEquals(5, managedLedger.getLedgersInfoAsList().get(1).getEntries()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java index 86754efc0c28f..d45054fab7939 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java @@ -25,7 +25,6 @@ import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Failover; import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Key_Shared; import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Shared; -import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -39,7 +38,6 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import java.net.InetSocketAddress; -import java.util.Optional; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; @@ -79,7 +77,7 @@ public void setup() throws Exception { executor = OrderedExecutor.newBuilder().numThreads(1).name("persistent-dispatcher-cumulative-ack-test").build(); ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); - svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(1.0d); svcConfig.setClusterName("pulsar-cluster"); pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -89,7 +87,7 @@ public void setup() throws Exception { doReturn(TransactionTestBase.createMockBookKeeper(executor)) .when(pulsar).getBookKeeperClient(); - store = MetadataStoreFactory.create("memory:local", MetadataStoreConfig.builder().build()); + store = MetadataStoreFactory.create("memory://local", MetadataStoreConfig.builder().build()); doReturn(store).when(pulsar).getLocalMetadataStore(); doReturn(store).when(pulsar).getConfigurationMetadataStore(); @@ -154,8 +152,7 @@ public static Object[][] notIndividualAckModes() { @Test(timeOut = 5000, dataProvider = "individualAckModes") public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, - "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "Cons1", 50000, serverCnx, "myrole-1", emptyMap(), false, CommandSubscribe.InitialPosition.Latest, null, MessageId.latest); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); @@ -169,8 +166,7 @@ public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throw @Test(timeOut = 5000, dataProvider = "notIndividualAckModes") public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, - "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "Cons1", 50000, serverCnx, "myrole-1", emptyMap(), false, CommandSubscribe.InitialPosition.Latest, null, MessageId.latest); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); @@ -184,8 +180,7 @@ public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) th @Test(timeOut = 5000) public void testAckWithMoreThanNoneMessageIds() throws Exception { Consumer consumer = new Consumer(sub, Failover, "topic-1", consumerId, 0, - "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "Cons1", 50000, serverCnx, "myrole-1", emptyMap(), false, CommandSubscribe.InitialPosition.Latest, null, MessageId.latest); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); From 3411bf211761dbbdf9f78a5b768ca30a52363148 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 9 Jun 2022 11:55:18 +0800 Subject: [PATCH 604/823] [fix][txn] fix NPE of TransactionMetaStoreHandler (#15840) (cherry picked from commit f9b0912dc3b7768c604b3f1c039c4068bb0d5810) --- .../pulsar/client/impl/TransactionMetaStoreHandler.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 5b91a1cd84b75..82fc89ca0a1e3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -129,9 +129,6 @@ public void connectionOpened(ClientCnx cnx) { return; } - connectionHandler.setClientCnx(cnx); - cnx.registerTransactionMetaStoreHandler(transactionCoordinatorId, this); - // if broker protocol version < 19, don't send TcClientConnectRequest to broker. if (cnx.getRemoteEndpointProtocolVersion() > ProtocolVersion.v18.getValue()) { long requestId = client.newRequestId(); @@ -145,6 +142,8 @@ public void connectionOpened(ClientCnx cnx) { cnx.channel().close(); } + connectionHandler.setClientCnx(cnx); + cnx.registerTransactionMetaStoreHandler(transactionCoordinatorId, this); if (!this.connectFuture.isDone()) { this.connectFuture.complete(null); } @@ -168,6 +167,9 @@ public void connectionOpened(ClientCnx cnx) { } else { if (!changeToReadyState()) { cnx.channel().close(); + } else { + connectionHandler.setClientCnx(cnx); + cnx.registerTransactionMetaStoreHandler(transactionCoordinatorId, this); } this.connectFuture.complete(null); } From 88c5681b219d26526b3b5c4c69897ed1fee8d35d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 9 Jun 2022 09:08:44 +0800 Subject: [PATCH 605/823] Upgrade Netty Reactive Streams to 2.0.6 (#15990) ### Motivation https://nvd.nist.gov/vuln/detail/CVE-2019-20444#range-6908693 Upgrade from 2.0.4 to 2.0.6 (cherry picked from commit 3d7634a06ff48f070a0bfce3925ccd30f1981cf6) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 7 ++++++- pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 5808df85c688f..de9366b033692 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -332,7 +332,7 @@ The Apache Software License, Version 2.0 - com.google.guava-failureaccess-1.0.1.jar - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar - * Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.4.jar + * Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.6.jar * Swagger - io.swagger-swagger-annotations-1.6.2.jar - io.swagger-swagger-core-1.6.2.jar diff --git a/pom.xml b/pom.xml index 06955be60d7bf..88db1eca8c8dd 100644 --- a/pom.xml +++ b/pom.xml @@ -205,7 +205,7 @@ flexible messaging model and an intuitive client API. 5.3.15 4.5.13 5.3.3 - + 2.0.6 3.6.0 3.4.0 @@ -1244,6 +1244,11 @@ flexible messaging model and an intuitive client API. ${kotlin-stdlib.version} + + com.typesafe.netty + netty-reactive-streams + ${netty-reactive-streams.version} + diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 411a7645fe488..d137a9a809875 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -241,7 +241,7 @@ The Apache Software License, Version 2.0 - netty-handler-proxy-4.1.77.Final.jar - netty-common-4.1.77.Final.jar - netty-handler-4.1.77.Final.jar - - netty-reactive-streams-2.0.4.jar + - netty-reactive-streams-2.0.6.jar - netty-resolver-4.1.77.Final.jar - netty-resolver-dns-4.1.77.Final.jar - netty-tcnative-boringssl-static-2.0.52.Final.jar From 134337fcd38fb6e71e2ffe7ef94475fab34ae8dc Mon Sep 17 00:00:00 2001 From: lipenghui Date: Sat, 11 Jun 2022 17:03:17 +0800 Subject: [PATCH 606/823] [branch-2.9] Disable testReuseFork for broker_group_2 (#16010) --- build/run_unit_group.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 77082cf3568a4..2d99e84eff325 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -42,7 +42,7 @@ function broker_group_1() { } function broker_group_2() { - $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other' + $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other' -DtestReuseFork=false } function broker_client_api() { From d7cb6a108c237b664c1081044db3b4867eeb05c6 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 12 Jun 2022 19:28:46 +0800 Subject: [PATCH 607/823] [fix] [admin] [branch-2.9] fix reach max tenants error if the tenant already exists (#16007) --- .../pulsar/broker/admin/impl/TenantsBase.java | 37 +++++++++---------- .../apache/pulsar/broker/admin/AdminTest.java | 33 +++++++++++++++++ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java index 3ec725e6f8041..29b46990ca4ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java @@ -136,38 +136,37 @@ public void createTenant(@Suspended final AsyncResponse asyncResponse, return; } - tenantResources().listTenantsAsync().whenComplete((tenants, e) -> { - if (e != null) { - log.error("[{}] Failed to create tenant ", clientAppId, e.getCause()); - asyncResponse.resume(new RestException(e)); + tenantResources().tenantExistsAsync(tenant).thenAccept(exist -> { + if (exist) { + asyncResponse.resume(new RestException(Status.CONFLICT, "Tenant already exist")); return; } - - int maxTenants = pulsar().getConfiguration().getMaxTenants(); - // Due to the cost of distributed locks, no locks are added here. - // In a concurrent scenario, the threshold will be exceeded. - if (maxTenants > 0) { - if (tenants != null && tenants.size() >= maxTenants) { - asyncResponse.resume( - new RestException(Status.PRECONDITION_FAILED, "Exceed the maximum number of tenants")); + tenantResources().listTenantsAsync().whenComplete((tenants, e) -> { + if (e != null) { + log.error("[{}] Failed to create tenant ", clientAppId, e.getCause()); + asyncResponse.resume(new RestException(e)); return; } - } - tenantResources().tenantExistsAsync(tenant).thenAccept(exist -> { - if (exist) { - asyncResponse.resume(new RestException(Status.CONFLICT, "Tenant already exist")); - return; + int maxTenants = pulsar().getConfiguration().getMaxTenants(); + // Due to the cost of distributed locks, no locks are added here. + // In a concurrent scenario, the threshold will be exceeded. + if (maxTenants > 0) { + if (tenants != null && tenants.size() >= maxTenants) { + asyncResponse.resume( + new RestException(Status.PRECONDITION_FAILED, "Exceed the maximum number of tenants")); + return; + } } tenantResources().createTenantAsync(tenant, tenantInfo).thenAccept((r) -> { log.info("[{}] Created tenant {}", clientAppId(), tenant); asyncResponse.resume(Response.noContent().build()); }).exceptionally(ex -> { - log.error("[{}] Failed to create tenant {}", clientAppId, tenant, e); + log.error("[{}] Failed to create tenant {}", clientAppId, tenant, ex); asyncResponse.resume(new RestException(ex)); return null; }); }).exceptionally(ex -> { - log.error("[{}] Failed to create tenant {}", clientAppId(), tenant, e); + log.error("[{}] Failed to create tenant {}", clientAppId(), tenant, ex); asyncResponse.resume(new RestException(ex)); return null; }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index a2f04927de928..1e825d340c001 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -124,6 +124,12 @@ public AdminTest() { super(); } + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setMaxTenants(10); + } + @Override @BeforeMethod public void setup() throws Exception { @@ -627,6 +633,33 @@ public void properties() throws Throwable { assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); } + // Check max tenant count + int maxTenants = pulsar.getConfiguration().getMaxTenants(); + List tenants = pulsar.getPulsarResources().getTenantResources().listTenants(); + + for(int tenantSize = tenants.size();tenantSize < maxTenants; tenantSize++ ){ + final int tenantIndex = tenantSize; + Response obj = (Response)asynRequests(ctx -> + properties.createTenant(ctx, "test-tenant-" + tenantIndex, tenantInfo)); + Assert.assertTrue(obj.getStatus() < 400 && obj.getStatus() >= 200); + } + try { + response = asynRequests(ctx -> + properties.createTenant(ctx, "test-tenant-" + maxTenants, tenantInfo)); + fail("should have failed"); + } catch (RestException e) { + assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); + } + + // Check creating existing property when tenant reach max count. + try { + response = asynRequests(ctx -> + properties.createTenant(ctx, "test-tenant-" + (maxTenants-1), tenantInfo)); + fail("should have failed"); + } catch (RestException e) { + assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode()); + } + AsyncResponse response2 = mock(AsyncResponse.class); namespaces.deleteNamespace(response2, "my-tenant", "use", "my-namespace", false, false); ArgumentCaptor captor = ArgumentCaptor.forClass(Response.class); From 213b72f245b9af118ba3df643072100d64697ecf Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Sun, 12 Jun 2022 04:49:53 -0700 Subject: [PATCH 608/823] [cleanup][function] refine file io connector (#15250) (cherry picked from commit cbefe3ed9b907e0cf1bed2a16f26055dc23026b0) --- .../pulsar/io/file/FileListingThread.java | 6 +-- .../pulsar/io/file/FileSourceConfig.java | 4 +- .../pulsar/io/file/FileSourceConfigTests.java | 44 ++++++++++++++++--- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileListingThread.java b/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileListingThread.java index a13b923c25c47..4067141bf8bdc 100644 --- a/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileListingThread.java +++ b/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileListingThread.java @@ -106,10 +106,10 @@ public void run() { private Set performListing(final File directory, final FileFilter filter, final boolean recurseSubdirectories) { Path p = directory.toPath(); - if (!Files.isWritable(p) || !Files.isReadable(p)) { - throw new IllegalStateException("Directory '" + directory - + "' does not have sufficient permissions (i.e., not writable and readable)"); + if (!Files.isReadable(p)) { + throw new IllegalStateException("Cannot read directory: '" + directory); } + final Set queue = new HashSet<>(); if (!directory.exists()) { return queue; diff --git a/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSourceConfig.java b/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSourceConfig.java index 24835e0e3141b..a43afc44c7992 100644 --- a/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSourceConfig.java +++ b/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSourceConfig.java @@ -129,7 +129,7 @@ public void validate() { throw new IllegalArgumentException("Specified input directory does not exist"); } else if (!Files.isReadable(Paths.get(inputDirectory))) { throw new IllegalArgumentException("Specified input directory is not readable"); - } else if (Optional.ofNullable(keepFile).orElse(false) && !Files.isWritable(Paths.get(inputDirectory))) { + } else if (!Optional.ofNullable(keepFile).orElse(false) && !Files.isWritable(Paths.get(inputDirectory))) { throw new IllegalArgumentException("You have requested the consumed files to be deleted, but the " + "source directory is not writeable."); } @@ -166,4 +166,4 @@ public void validate() { throw new IllegalArgumentException("The property numWorkers must be greater than zero"); } } -} \ No newline at end of file +} diff --git a/pulsar-io/file/src/test/java/org/apache/pulsar/io/file/FileSourceConfigTests.java b/pulsar-io/file/src/test/java/org/apache/pulsar/io/file/FileSourceConfigTests.java index 64144e667adeb..4a4d8d2a86713 100644 --- a/pulsar-io/file/src/test/java/org/apache/pulsar/io/file/FileSourceConfigTests.java +++ b/pulsar-io/file/src/test/java/org/apache/pulsar/io/file/FileSourceConfigTests.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.io.file; +import static org.junit.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -29,6 +31,8 @@ public class FileSourceConfigTests { + private final static String INPUT_DIRECTORY = "/dev/null"; + @Test public final void loadFromYamlFileTest() throws IOException { File yamlFile = getFile("sinkConfig.yaml"); @@ -39,7 +43,7 @@ public final void loadFromYamlFileTest() throws IOException { @Test public final void loadFromMapTest() throws IOException { Map map = new HashMap (); - map.put("inputDirectory", "/tmp"); + map.put("inputDirectory", INPUT_DIRECTORY); map.put("keepFile", false); FileSourceConfig config = FileSourceConfig.load(map); @@ -49,7 +53,7 @@ public final void loadFromMapTest() throws IOException { @Test public final void validValidateTest() throws IOException { Map map = new HashMap (); - map.put("inputDirectory", "/tmp"); + map.put("inputDirectory", INPUT_DIRECTORY); FileSourceConfig config = FileSourceConfig.load(map); assertNotNull(config); @@ -70,7 +74,7 @@ public final void missingRequiredPropertiesTest() throws IOException { @Test(expectedExceptions = com.fasterxml.jackson.databind.exc.InvalidFormatException.class) public final void InvalidBooleanPropertyTest() throws IOException { Map map = new HashMap (); - map.put("inputDirectory", "/"); + map.put("inputDirectory", INPUT_DIRECTORY); map.put("recurse", "not a boolean"); FileSourceConfig config = FileSourceConfig.load(map); @@ -82,7 +86,7 @@ public final void InvalidBooleanPropertyTest() throws IOException { expectedExceptionsMessageRegExp = "The property pollingInterval must be greater than zero") public final void ZeroValueTest() throws IOException { Map map = new HashMap (); - map.put("inputDirectory", "/"); + map.put("inputDirectory", INPUT_DIRECTORY); map.put("pollingInterval", 0); FileSourceConfig config = FileSourceConfig.load(map); @@ -94,7 +98,7 @@ public final void ZeroValueTest() throws IOException { expectedExceptionsMessageRegExp = "The property minimumFileAge must be non-negative") public final void NegativeValueTest() throws IOException { Map map = new HashMap (); - map.put("inputDirectory", "/"); + map.put("inputDirectory", INPUT_DIRECTORY); map.put("minimumFileAge", "-50"); FileSourceConfig config = FileSourceConfig.load(map); @@ -106,14 +110,40 @@ public final void NegativeValueTest() throws IOException { expectedExceptionsMessageRegExp = "Invalid Regex pattern provided for fileFilter") public final void invalidFileFilterTest() throws IOException { Map map = new HashMap (); - map.put("inputDirectory", "/"); + map.put("inputDirectory", INPUT_DIRECTORY); map.put("fileFilter", "\\"); // Results in a single '\' being sent. FileSourceConfig config = FileSourceConfig.load(map); assertNotNull(config); config.validate(); } - + + @Test + public final void keepFileTest() throws IOException { + Map map = new HashMap (); + map.put("inputDirectory", "/"); // root directory that we cannot write to + map.put("keepFile", "true"); // even though no write permission on "/", we should still be able to read + + FileSourceConfig config = FileSourceConfig.load(map); + assertNotNull(config); + assertTrue(config.getKeepFile()); + config.validate(); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "You have requested the consumed files to be deleted, " + + "but the source directory is not writeable.") + public final void invalidKeepFileTest() throws IOException { + Map map = new HashMap (); + map.put("inputDirectory", "/"); // root directory that we cannot write to + map.put("keepFile", "false"); + + FileSourceConfig config = FileSourceConfig.load(map); + assertNotNull(config); + assertFalse(config.getKeepFile()); + config.validate(); + } + private File getFile(String name) { ClassLoader classLoader = getClass().getClassLoader(); return new File(classLoader.getResource(name).getFile()); From e1b207e698e1bc09fc08656f36dc2cd7fbda323a Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 12 Mar 2022 11:20:57 +0800 Subject: [PATCH 609/823] Fix wrong prompt exception when get non-persistent topic list without GET_BUDNLE permission (#14638) Fixes #14191 ### Motivation We have some big issues with the permission part. We only have the permission with [doc](https://pulsar.apache.org/docs/en/admin-api-permissions/) mentioned. But if user do it according to the doc, they will face the same issue that #14191 described. We don't have GET_BUNDLE in the grant interface but given the prompt message to the user. And currently, only the admin role could have the permission. This pr is not solving the permission issue but fixing the prompt message first, not giving 500 error to the user. Then I open an issue https://github.com/apache/pulsar/issues/14639 to discuss refactoring the permission part. ### Modification - Return 403 to the user when permission is denied. (cherry picked from commit ca6e82422ecbc5272d3dda2c90873c5a493764e1) --- .../broker/admin/v1/NonPersistentTopics.java | 25 ++++++------ .../broker/admin/v2/NonPersistentTopics.java | 31 +++++++-------- .../pulsar/broker/auth/AuthorizationTest.java | 39 +++++++++++++++++-- 3 files changed, 62 insertions(+), 33 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java index 2a5ab77909773..68a6c71994211 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java @@ -198,6 +198,7 @@ public void getList(@Suspended final AsyncResponse asyncResponse, @PathParam("pr Policies policies = null; NamespaceName nsName = null; try { + validateNamespaceName(property, cluster, namespace); validateNamespaceOperation(namespaceName, NamespaceOperation.GET_TOPICS); policies = getNamespacePolicies(property, cluster, namespace); nsName = NamespaceName.get(property, cluster, namespace); @@ -232,22 +233,19 @@ public void getList(@Suspended final AsyncResponse asyncResponse, @PathParam("pr } } - final List topics = Lists.newArrayList(); - FutureUtil.waitForAll(futures).handle((result, exception) -> { - for (int i = 0; i < futures.size(); i++) { - try { - if (futures.get(i).isDone() && futures.get(i).get() != null) { - topics.addAll(futures.get(i).get()); + FutureUtil.waitForAll(futures).whenComplete((result, ex) -> { + if (ex != null) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } else { + final List topics = Lists.newArrayList(); + for (int i = 0; i < futures.size(); i++) { + List topicList = futures.get(i).join(); + if (topicList != null) { + topics.addAll(topicList); } - } catch (InterruptedException | ExecutionException e) { - log.error("[{}] Failed to get list of topics under namespace {}/{}/{}", clientAppId(), property, - cluster, namespace, e); - asyncResponse.resume(new RestException(e instanceof ExecutionException ? e.getCause() : e)); - return null; } + asyncResponse.resume(topics); } - asyncResponse.resume(topics); - return null; }); } @@ -264,6 +262,7 @@ public List getListFromBundle(@PathParam("property") String property, @P @PathParam("bundle") String bundleRange) { log.info("[{}] list of topics on namespace bundle {}/{}/{}/{}", clientAppId(), property, cluster, namespace, bundleRange); + validateNamespaceName(property, cluster, namespace); validateNamespaceOperation(namespaceName, NamespaceOperation.GET_BUNDLE); Policies policies = getNamespacePolicies(property, cluster, namespace); if (!cluster.equals(Constants.GLOBAL_CLUSTER)) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index fa52cb4599714..38b1d725a703e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -403,26 +403,23 @@ public void getList( } } - final List topics = Lists.newArrayList(); - FutureUtil.waitForAll(futures).handle((result, exception) -> { - for (int i = 0; i < futures.size(); i++) { - try { - if (futures.get(i).isDone() && futures.get(i).get() != null) { - topics.addAll(futures.get(i).get()); + FutureUtil.waitForAll(futures).whenComplete((result, ex) -> { + if (ex != null) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } else { + final List topics = Lists.newArrayList(); + for (int i = 0; i < futures.size(); i++) { + List topicList = futures.get(i).join(); + if (topicList != null) { + topics.addAll(topicList); } - } catch (InterruptedException | ExecutionException e) { - log.error("[{}] Failed to get list of topics under namespace {}", clientAppId(), namespaceName, e); - asyncResponse.resume(new RestException(e instanceof ExecutionException ? e.getCause() : e)); - return null; } + final List nonPersistentTopics = + topics.stream() + .filter(name -> !TopicName.get(name).isPersistent()) + .collect(Collectors.toList()); + asyncResponse.resume(nonPersistentTopics); } - - final List nonPersistentTopics = - topics.stream() - .filter(name -> !TopicName.get(name).isPersistent()) - .collect(Collectors.toList()); - asyncResponse.resume(nonPersistentTopics); - return null; }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index b7a54d137fd70..4b18791fce08c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -18,14 +18,17 @@ */ package org.apache.pulsar.broker.auth; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import java.util.EnumSet; - import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; @@ -36,7 +39,6 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; - import com.google.common.collect.Sets; @Test(groups = "flaky") @@ -230,6 +232,37 @@ public void simple() throws Exception { admin.clusters().deleteCluster("c1"); } + @Test + public void testGetListWithoutGetBundleOp() throws Exception { + String tenant = "p1"; + String namespaceV1 = "p1/global/ns1"; + String namespaceV2 = "p1/ns2"; + admin.clusters().createCluster("c1", ClusterData.builder().build()); + admin.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); + admin.namespaces().createNamespace(namespaceV1, Sets.newHashSet("c1")); + admin.namespaces().grantPermissionOnNamespace(namespaceV1, "pass.pass2", EnumSet.of(AuthAction.produce)); + admin.namespaces().createNamespace(namespaceV2, Sets.newHashSet("c1")); + admin.namespaces().grantPermissionOnNamespace(namespaceV2, "pass.pass2", EnumSet.of(AuthAction.produce)); + PulsarAdmin admin2 = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null + ? brokerUrl.toString() + : brokerUrlTls.toString()) + .authentication(new MockAuthentication("pass.pass2")) + .build(); + when(pulsar.getAdminClient()).thenReturn(admin2); + try { + admin2.topics().getList(namespaceV1, TopicDomain.non_persistent); + } catch (Exception ex) { + assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + assertEquals(ex.getMessage(), "Unauthorized to validateNamespaceOperation for operation [GET_BUNDLE] on namespace [p1/global/ns1]"); + } + try { + admin2.topics().getList(namespaceV2, TopicDomain.non_persistent); + } catch (Exception ex) { + assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + assertEquals(ex.getMessage(), "Unauthorized to validateNamespaceOperation for operation [GET_BUNDLE] on namespace [p1/ns2]"); + } + } + private static void waitForChange() { try { Thread.sleep(100); From 6e7bd706ccaa9a4ef5afe8a55c4e208d7c585237 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 9 May 2022 22:05:07 +0800 Subject: [PATCH 610/823] Fix grant all permissions but can't list topic. (#15501) (cherry picked from commit 5155b1df876bd98d173e87753cca642b82b6595a) --- .../PulsarAuthorizationProvider.java | 2 +- .../pulsar/broker/auth/AuthorizationTest.java | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index 9aea1261cf216..097464bfb5f3d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -542,6 +542,7 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam namespaceName, role, authData, AuthAction.packages); case GET_TOPIC: case GET_TOPICS: + case GET_BUNDLE: return allowConsumeOrProduceOpsAsync(namespaceName, role, authData); case UNSUBSCRIBE: case CLEAR_BACKLOG: @@ -550,7 +551,6 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam case CREATE_TOPIC: case DELETE_TOPIC: case ADD_BUNDLE: - case GET_BUNDLE: case DELETE_BUNDLE: case GRANT_PERMISSION: case GET_PERMISSION: diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 4b18791fce08c..2596d243a9f87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.auth; import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -27,7 +26,6 @@ import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; @@ -233,7 +231,7 @@ public void simple() throws Exception { } @Test - public void testGetListWithoutGetBundleOp() throws Exception { + public void testGetListWithGetBundleOp() throws Exception { String tenant = "p1"; String namespaceV1 = "p1/global/ns1"; String namespaceV2 = "p1/ns2"; @@ -249,18 +247,8 @@ public void testGetListWithoutGetBundleOp() throws Exception { .authentication(new MockAuthentication("pass.pass2")) .build(); when(pulsar.getAdminClient()).thenReturn(admin2); - try { - admin2.topics().getList(namespaceV1, TopicDomain.non_persistent); - } catch (Exception ex) { - assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); - assertEquals(ex.getMessage(), "Unauthorized to validateNamespaceOperation for operation [GET_BUNDLE] on namespace [p1/global/ns1]"); - } - try { - admin2.topics().getList(namespaceV2, TopicDomain.non_persistent); - } catch (Exception ex) { - assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); - assertEquals(ex.getMessage(), "Unauthorized to validateNamespaceOperation for operation [GET_BUNDLE] on namespace [p1/ns2]"); - } + Assert.assertEquals(admin2.topics().getList(namespaceV1, TopicDomain.non_persistent).size(), 0); + Assert.assertEquals(admin2.topics().getList(namespaceV2, TopicDomain.non_persistent).size(), 0); } private static void waitForChange() { From a767d3734d677849c3d523c2b01e2895b23164fc Mon Sep 17 00:00:00 2001 From: visortelle Date: Mon, 13 Jun 2022 09:53:39 +0800 Subject: [PATCH 611/823] [fix][admin] Fix typo in validation message (#16021) (cherry picked from commit 8d8a19f786aadb2d18146f4bded3d550d2a2d040) --- .../main/java/org/apache/pulsar/broker/admin/AdminResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 3598e03b2ed29..fef7abf6d0711 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -806,7 +806,7 @@ protected void validatePersistencePolicies(PersistencePolicies persistence) { checkArgument( (persistence.getBookkeeperEnsemble() >= persistence.getBookkeeperWriteQuorum()) && (persistence.getBookkeeperWriteQuorum() >= persistence.getBookkeeperAckQuorum()), - String.format("Bookkeeper Ensemble (%s) >= WriteQuorum (%s) >= AckQuoru (%s)", + String.format("Bookkeeper Ensemble (%s) >= WriteQuorum (%s) >= AckQuorum (%s)", persistence.getBookkeeperEnsemble(), persistence.getBookkeeperWriteQuorum(), persistence.getBookkeeperAckQuorum())); From 2e78141e79571c9a795ee31a08d2d6aca393003c Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Sun, 12 Jun 2022 18:05:25 -0700 Subject: [PATCH 612/823] [Function] provide default error handler for function log appender (#15728) (cherry picked from commit f7635ec6d99bd5a13a31c7e9f17640746afec43c) --- .../functions/instance/LogAppender.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/LogAppender.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/LogAppender.java index 20354d4854a27..1b75eec7522dd 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/LogAppender.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/LogAppender.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.core.ErrorHandler; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.DefaultErrorHandler; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -35,6 +36,11 @@ * to a log topic. */ public class LogAppender implements Appender { + + private static final String LOG_LEVEL = "loglevel"; + private static final String INSTANCE = "instance"; + private static final String FQN = "fqn"; + private PulsarClient pulsarClient; private String logTopic; private String fqn; @@ -48,15 +54,16 @@ public LogAppender(PulsarClient pulsarClient, String logTopic, String fqn, Strin this.logTopic = logTopic; this.fqn = fqn; this.instance = instance; + this.errorHandler = new DefaultErrorHandler(this); } @Override public void append(LogEvent logEvent) { producer.newMessage() .value(logEvent.getMessage().getFormattedMessage().getBytes(StandardCharsets.UTF_8)) - .property("loglevel", logEvent.getLevel().name()) - .property("instance", instance) - .property("fqn", fqn) + .property(LOG_LEVEL, logEvent.getLevel().name()) + .property(INSTANCE, instance) + .property(FQN, fqn) .sendAsync(); } @@ -82,6 +89,12 @@ public ErrorHandler getHandler() { @Override public void setHandler(ErrorHandler errorHandler) { + if (errorHandler == null) { + throw new RuntimeException("The log error handler cannot be set to null"); + } + if (isStarted()) { + throw new RuntimeException("The log error handler cannot be changed once the appender is started"); + } this.errorHandler = errorHandler; } From 7349c23eeda45c170e17ef6f267e6c9d1084606b Mon Sep 17 00:00:00 2001 From: mattison chao Date: Mon, 13 Jun 2022 11:26:24 +0800 Subject: [PATCH 613/823] [fix][client] Remove consumer when close consumer command is received (#15761) (cherry picked from commit 5246c8e1cc44b96db6ba684e0ce64914cfd05a61) --- .../apache/pulsar/client/impl/ClientCnx.java | 7 ++++--- .../pulsar/client/impl/ClientCnxTest.java | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 20325ade4f6b1..52d10ad996e86 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -22,7 +22,7 @@ import static com.google.common.base.Preconditions.checkState; import static org.apache.pulsar.client.impl.TransactionMetaStoreHandler.getExceptionByServerError; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; - +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Queues; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -120,7 +120,8 @@ public class ClientCnx extends PulsarHandler { .expectedItems(16) .concurrencyLevel(1) .build(); - private final ConcurrentLongHashMap> consumers = + @VisibleForTesting + final ConcurrentLongHashMap> consumers = ConcurrentLongHashMap.>newBuilder() .expectedItems(16) .concurrencyLevel(1) @@ -737,7 +738,7 @@ protected void handleCloseProducer(CommandCloseProducer closeProducer) { protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) { log.info("[{}] Broker notification of Closed consumer: {}", remoteAddress, closeConsumer.getConsumerId()); final long consumerId = closeConsumer.getConsumerId(); - ConsumerImpl consumer = consumers.get(consumerId); + ConsumerImpl consumer = consumers.remove(consumerId); if (consumer != null) { consumer.connectionClosed(this); } else { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index 558c0bfa13f76..a3a00b1b70ec8 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -37,6 +37,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.BrokerMetadataException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.api.proto.CommandCloseConsumer; import org.apache.pulsar.common.api.proto.CommandError; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.protocol.Commands; @@ -152,4 +153,23 @@ public void testGetLastMessageIdWithError() throws Exception { eventLoop.shutdownGracefully(); } + + @Test + public void testHandleCloseConsumer() { + ThreadFactory threadFactory = new DefaultThreadFactory("testReceiveErrorAtSendConnectFrameState"); + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); + ClientConfigurationData conf = new ClientConfigurationData(); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + long consumerId = 1; + cnx.registerConsumer(consumerId, mock(ConsumerImpl.class)); + assertEquals(cnx.consumers.size(), 1); + + CommandCloseConsumer closeConsumer = new CommandCloseConsumer() + .setConsumerId(1); + cnx.handleCloseConsumer(closeConsumer); + assertEquals(cnx.consumers.size(), 0); + + eventLoop.shutdownGracefully(); + } } From f80225d5e6a453d588902d68b3483919ea7e9b85 Mon Sep 17 00:00:00 2001 From: visortelle Date: Wed, 15 Jun 2022 00:35:04 +0200 Subject: [PATCH 614/823] Fix wrong response type for swagger definitions (#16022) --- .../main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 37b735464c341..4fe4cf315607a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -288,7 +288,7 @@ public void setNamespaceReplicationClusters(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/messageTTL") - @ApiOperation(value = "Get the message TTL for the namespace") + @ApiOperation(value = "Get the message TTL for the namespace", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public Integer getNamespaceMessageTTL(@PathParam("tenant") String tenant, From 3211f91ae91239fb58bc1c709b66f41dcc9023ed Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 14 Jun 2022 18:09:50 -0500 Subject: [PATCH 615/823] Clean up C++ client curl configuration (#16064) --- pulsar-client-cpp/lib/auth/AuthOauth2.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.cc b/pulsar-client-cpp/lib/auth/AuthOauth2.cc index 438239a46d69e..31225b15a71c5 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.cc +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.cc @@ -186,8 +186,6 @@ void ClientCredentialFlow::initialize() { curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); char errorBuffer[CURL_ERROR_SIZE]; curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, errorBuffer); @@ -311,8 +309,6 @@ Oauth2TokenResultPtr ClientCredentialFlow::authenticate() { curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(handle, CURLOPT_POSTFIELDS, postData.c_str()); From 422588782146bc60721f532f8a4f7c2e1a91f1f2 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 15 Jun 2022 00:00:28 -0700 Subject: [PATCH 616/823] Avoid AuthenticationDataSource mutation for subscription name (#16065) The `authenticationData` field in `ServerCnx` is being mutated to add the `subscription` field that will be passed on to the authorization plugin. The problem is that `authenticationData` is scoped to the whole connection and it should be getting mutated for each consumer that is created on the connection. The current code leads to a race condition where the subscription name used in the authz plugin is already modified while we're looking at it. Instead, we should create a new object and enforce the final modifier. (cherry picked from commit e6b12c64b043903eb5ff2dc5186fe8030f157cfc) --- .../AuthenticationDataCommand.java | 33 +-------- .../AuthenticationDataSubscription.java | 74 +++++++++++++++++++ .../pulsar/broker/service/ServerCnx.java | 30 ++++---- 3 files changed, 88 insertions(+), 49 deletions(-) create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataCommand.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataCommand.java index efc332968afb2..31bc670a2de1e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataCommand.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataCommand.java @@ -30,22 +30,9 @@ public class AuthenticationDataCommand implements AuthenticationDataSource { protected final String authData; protected final SocketAddress remoteAddress; protected final SSLSession sslSession; - protected String subscription; public AuthenticationDataCommand(String authData) { - this(authData, null, null, null); - } - - public AuthenticationDataCommand(String authData, String subscription) { - this(authData, null, null, subscription); - } - - public AuthenticationDataCommand(String authData, SocketAddress remoteAddress, SSLSession sslSession, - String subscription) { - this.authData = authData; - this.remoteAddress = remoteAddress; - this.sslSession = sslSession; - this.subscription = subscription; + this(authData, null, null); } public AuthenticationDataCommand(String authData, SocketAddress remoteAddress, SSLSession sslSession) { @@ -100,22 +87,4 @@ public Certificate[] getTlsCertificates() { return null; } } - - /* - * Subscription - */ - @Override - public boolean hasSubscription() { - return this.subscription != null; - } - - @Override - public void setSubscription(String subscription) { - this.subscription = subscription; - } - - @Override - public String getSubscription() { - return subscription; - } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java new file mode 100644 index 0000000000000..f6723609908ac --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication; + +import java.net.SocketAddress; +import java.security.cert.Certificate; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AuthenticationDataSubscription implements AuthenticationDataSource { + private final AuthenticationDataSource authData; + private final String subscription; + + public AuthenticationDataSubscription(AuthenticationDataSource authData, String subscription) { + this.authData = authData; + this.subscription = subscription; + } + + @Override + public boolean hasDataFromCommand() { + return authData.hasDataFromCommand(); + } + + @Override + public String getCommandData() { + return authData.getCommandData(); + } + + @Override + public boolean hasDataFromPeer() { + return authData.hasDataFromPeer(); + } + + @Override + public SocketAddress getPeerAddress() { + return authData.getPeerAddress(); + } + + @Override + public boolean hasDataFromTls() { + return authData.hasDataFromTls(); + } + + @Override + public Certificate[] getTlsCertificates() { + return authData.getTlsCertificates(); + } + + @Override + public boolean hasSubscription() { + return this.subscription != null; + } + + @Override + public String getSubscription() { + return subscription; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 58035b84c3885..d4b53aeee5d78 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -66,8 +66,8 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.TransactionMetadataStoreService; -import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; import org.apache.pulsar.broker.intercept.BrokerInterceptor; @@ -115,7 +115,6 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; -import org.apache.pulsar.common.api.proto.CommandTcClientConnectRequest; import org.apache.pulsar.common.api.proto.CommandUnsubscribe; import org.apache.pulsar.common.api.proto.FeatureFlags; import org.apache.pulsar.common.api.proto.KeySharedMeta; @@ -375,19 +374,20 @@ private boolean invalidOriginalPrincipal(String originalPrincipal) { // // Incoming commands handling // //// - private CompletableFuture isTopicOperationAllowed(TopicName topicName, TopicOperation operation) { + private CompletableFuture isTopicOperationAllowed(TopicName topicName, TopicOperation operation, + AuthenticationDataSource authData) { if (!service.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } CompletableFuture isProxyAuthorizedFuture; if (originalPrincipal != null) { isProxyAuthorizedFuture = service.getAuthorizationService().allowTopicOperationAsync( - topicName, operation, originalPrincipal, getAuthenticationData()); + topicName, operation, originalPrincipal, authData); } else { isProxyAuthorizedFuture = CompletableFuture.completedFuture(true); } CompletableFuture isAuthorizedFuture = service.getAuthorizationService().allowTopicOperationAsync( - topicName, operation, authRole, authenticationData); + topicName, operation, authRole, authData); return isProxyAuthorizedFuture.thenCombine(isAuthorizedFuture, (isProxyAuthorized, isAuthorized) -> { if (!isProxyAuthorized) { log.warn("OriginalRole {} is not authorized to perform operation {} on topic {}", @@ -404,15 +404,9 @@ private CompletableFuture isTopicOperationAllowed(TopicName topicName, private CompletableFuture isTopicOperationAllowed(TopicName topicName, String subscriptionName, TopicOperation operation) { if (service.isAuthorizationEnabled()) { - if (authenticationData == null) { - authenticationData = new AuthenticationDataCommand("", subscriptionName); - } else { - authenticationData.setSubscription(subscriptionName); - } - if (originalAuthData != null) { - originalAuthData.setSubscription(subscriptionName); - } - return isTopicOperationAllowed(topicName, operation); + AuthenticationDataSource authData = + new AuthenticationDataSubscription(getAuthenticationData(), subscriptionName); + return isTopicOperationAllowed(topicName, operation, authData); } else { return CompletableFuture.completedFuture(true); } @@ -447,7 +441,8 @@ protected void handleLookup(CommandLookupTopic lookup) { lookupSemaphore.release(); return; } - isTopicOperationAllowed(topicName, TopicOperation.LOOKUP).thenApply(isAuthorized -> { + isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, getAuthenticationData()).thenApply( + isAuthorized -> { if (isAuthorized) { lookupTopicAsync(getBrokerService().pulsar(), topicName, authoritative, getPrincipal(), getAuthenticationData(), @@ -510,7 +505,8 @@ protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata pa lookupSemaphore.release(); return; } - isTopicOperationAllowed(topicName, TopicOperation.LOOKUP).thenApply(isAuthorized -> { + isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, getAuthenticationData()).thenApply( + isAuthorized -> { if (isAuthorized) { unsafeGetPartitionedTopicMetadataAsync(getBrokerService().pulsar(), topicName) .handle((metadata, ex) -> { @@ -1163,7 +1159,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { } CompletableFuture isAuthorizedFuture = isTopicOperationAllowed( - topicName, TopicOperation.PRODUCE + topicName, TopicOperation.PRODUCE, getAuthenticationData() ); isAuthorizedFuture.thenApply(isAuthorized -> { if (isAuthorized) { From e3b8e014eec7f27d08a79e5c78325e568751850b Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 16 Jun 2022 21:12:12 +0800 Subject: [PATCH 617/823] [branch-2.9] Fix compile issue by cherry-pick (#16086) --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d4b53aeee5d78..8b56b3a4b3dbd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -115,6 +115,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.api.proto.CommandTcClientConnectRequest; import org.apache.pulsar.common.api.proto.CommandUnsubscribe; import org.apache.pulsar.common.api.proto.FeatureFlags; import org.apache.pulsar.common.api.proto.KeySharedMeta; From 3e8445226f5e6b4b1bcfc0d96e237b0a09703b40 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 9 Jun 2022 11:09:20 -0700 Subject: [PATCH 618/823] Removing log4j-1.2-api from dependencies (#15991) --- buildtools/pom.xml | 4 ---- distribution/server/pom.xml | 5 ----- distribution/server/src/assemble/LICENSE.bin.txt | 1 - managed-ledger/pom.xml | 6 ------ 4 files changed, 16 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 003bc118cfcc7..1195df6d5c8df 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -81,10 +81,6 @@ org.apache.logging.log4j log4j-slf4j-impl - - org.apache.logging.log4j - log4j-1.2-api - org.slf4j jcl-over-slf4j diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index ed5401ad39650..f295d34a8ea26 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -100,11 +100,6 @@ ${project.version} - - org.apache.logging.log4j - log4j-1.2-api - - org.apache.logging.log4j log4j-api diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index de9366b033692..3418ff92248da 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -394,7 +394,6 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-core-2.17.1.jar - org.apache.logging.log4j-log4j-slf4j-impl-2.17.1.jar - org.apache.logging.log4j-log4j-web-2.17.1.jar - - org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - org.apache.bookkeeper-bookkeeper-common-4.14.5.jar diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 509d00380abc0..0461ae604cc09 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -101,12 +101,6 @@ test - - org.apache.logging.log4j - log4j-1.2-api - test - - org.slf4j slf4j-api From 1fa9c2e17977cb451c0379581c35c70920a393f3 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Fri, 17 Jun 2022 13:27:36 +0800 Subject: [PATCH 619/823] [branch-2.9][fix][security] Add timeout of sync methods and avoid call sync method for AuthoriationService (#16083) --- .../authorization/AuthorizationService.java | 24 ++++- .../admin/impl/PersistentTopicsBase.java | 89 +++++++++------- .../pulsar/broker/lookup/TopicLookupBase.java | 53 ++++++---- .../pulsar/broker/web/PulsarWebResource.java | 100 ++++++++++++++++-- 4 files changed, 190 insertions(+), 76 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 6943e95999bba..ae36a9bba6ece 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -43,6 +43,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import static java.util.concurrent.TimeUnit.SECONDS; @@ -396,11 +397,15 @@ public boolean allowTenantOperation(String tenantName, AuthenticationDataSource authData) { try { return allowTenantOperationAsync( - tenantName, operation, originalRole, role, authData).get(); + tenantName, operation, originalRole, role, authData).get( + conf.getZooKeeperOperationTimeoutSeconds(), SECONDS); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new RestException(e); } catch (ExecutionException e) { throw new RestException(e.getCause()); + } catch (TimeoutException e) { + throw new RestException(e); } } @@ -521,11 +526,15 @@ public boolean allowNamespacePolicyOperation(NamespaceName namespaceName, AuthenticationDataSource authData) { try { return allowNamespacePolicyOperationAsync( - namespaceName, policy, operation, originalRole, role, authData).get(); + namespaceName, policy, operation, originalRole, role, authData).get( + conf.getZooKeeperOperationTimeoutSeconds(), SECONDS); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new RestException(e); } catch (ExecutionException e) { throw new RestException(e.getCause()); + } catch (TimeoutException e) { + throw new RestException(e); } } @@ -585,11 +594,15 @@ public Boolean allowTopicPolicyOperation(TopicName topicName, AuthenticationDataSource authData) { try { return allowTopicPolicyOperationAsync( - topicName, policy, operation, originalRole, role, authData).get(); + topicName, policy, operation, originalRole, role, authData).get( + conf.getZooKeeperOperationTimeoutSeconds(), SECONDS); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new RestException(e); } catch (ExecutionException e) { throw new RestException(e.getCause()); + } catch (TimeoutException e) { + throw new RestException(e); } } @@ -667,9 +680,10 @@ public Boolean allowTopicOperation(TopicName topicName, TopicOperation operation, String originalRole, String role, - AuthenticationDataSource authData) { + AuthenticationDataSource authData) throws Exception { try { - return allowTopicOperationAsync(topicName, operation, originalRole, role, authData).get(); + return allowTopicOperationAsync(topicName, operation, originalRole, role, authData).get( + conf.getZooKeeperOperationTimeoutSeconds(), SECONDS); } catch (InterruptedException e) { throw new RestException(e); } catch (ExecutionException e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index b28a51014ad4e..4ff236af5ac27 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -233,7 +233,7 @@ protected void validateAdminAndClientPermission() { validateAdminAccessForTenant(topicName.getTenant()); } catch (Exception ve) { try { - checkAuthorization(pulsar(), topicName, clientAppId(), clientAuthData()); + checkAuthorizationAsync(pulsar(), topicName, clientAppId(), clientAuthData()); } catch (RestException re) { throw re; } catch (Exception e) { @@ -3559,46 +3559,55 @@ public static CompletableFuture getPartitionedTopicMet PulsarService pulsar, String clientAppId, String originalPrincipal, AuthenticationDataSource authenticationData, TopicName topicName) { CompletableFuture metadataFuture = new CompletableFuture<>(); - try { - // (1) authorize client - try { - checkAuthorization(pulsar, topicName, clientAppId, authenticationData); - } catch (RestException e) { - try { - validateAdminAccessForTenant(pulsar, - clientAppId, originalPrincipal, topicName.getTenant(), authenticationData); - } catch (RestException authException) { - log.warn("Failed to authorize {} on cluster {}", clientAppId, topicName.toString()); - throw new PulsarClientException(String.format("Authorization failed %s on topic %s with error %s", - clientAppId, topicName.toString(), authException.getMessage())); - } - } catch (Exception ex) { - // throw without wrapping to PulsarClientException that considers: unknown error marked as internal - // server error - log.warn("Failed to authorize {} on cluster {} with unexpected exception {}", clientAppId, - topicName.toString(), ex.getMessage(), ex); - throw ex; - } + CompletableFuture authorizationFuture = new CompletableFuture<>(); + checkAuthorizationAsync(pulsar, topicName, clientAppId, authenticationData) + .thenRun(() -> authorizationFuture.complete(null)) + .exceptionally(e -> { + Throwable throwable = FutureUtil.unwrapCompletionException(e); + if (throwable instanceof RestException) { + validateAdminAccessForTenantAsync(pulsar, + clientAppId, originalPrincipal, topicName.getTenant(), authenticationData) + .thenRun(() -> { + authorizationFuture.complete(null); + }).exceptionally(ex -> { + Throwable throwable2 = FutureUtil.unwrapCompletionException(ex); + if (throwable2 instanceof RestException) { + log.warn("Failed to authorize {} on topic {}", clientAppId, topicName); + authorizationFuture.completeExceptionally(new PulsarClientException( + String.format("Authorization failed %s on topic %s with error %s", + clientAppId, topicName, throwable2.getMessage()))); + } else { + authorizationFuture.completeExceptionally(throwable2); + } + return null; + }); + } else { + // throw without wrapping to PulsarClientException that considers: unknown error marked as + // internal server error + log.warn("Failed to authorize {} on topic {}", clientAppId, topicName, throwable); + authorizationFuture.completeExceptionally(throwable); + } + return null; + }); - // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can - // serve/redirect request else fail partitioned-metadata-request so, client fails while creating - // producer/consumer - checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject()) - .thenCompose(res -> pulsar.getBrokerService() - .fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) - .thenAccept(metadata -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Total number of partitions for topic {} is {}", clientAppId, topicName, - metadata.partitions); - } - metadataFuture.complete(metadata); - }).exceptionally(ex -> { - metadataFuture.completeExceptionally(ex.getCause()); - return null; - }); - } catch (Exception ex) { - metadataFuture.completeExceptionally(ex); - } + // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can + // serve/redirect request else fail partitioned-metadata-request so, client fails while creating + // producer/consumer + authorizationFuture.thenCompose(__ -> + checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject())) + .thenCompose(res -> + pulsar.getBrokerService().fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) + .thenAccept(metadata -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Total number of partitions for topic {} is {}", clientAppId, topicName, + metadata.partitions); + } + metadataFuture.complete(metadata); + }) + .exceptionally(e -> { + metadataFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); + return null; + }); return metadataFuture; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index dab1b293e9008..967059c07184b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -48,6 +48,7 @@ import org.apache.pulsar.common.policies.data.NamespaceOperation; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -219,23 +220,14 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe cluster); } validationFuture.complete(newLookupResponse(differentClusterData.getBrokerServiceUrl(), - differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId, false)); + differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, + requestId, false)); } else { // (2) authorize client - try { - checkAuthorization(pulsarService, topicName, clientAppId, authenticationData); - } catch (RestException authException) { - log.warn("Failed to authorized {} on cluster {}", clientAppId, topicName.toString()); - validationFuture.complete(newLookupErrorResponse(ServerError.AuthorizationError, - authException.getMessage(), requestId)); - return; - } catch (Exception e) { - log.warn("Unknown error while authorizing {} on cluster {}", clientAppId, topicName.toString()); - validationFuture.completeExceptionally(e); - return; - } - // (3) validate global namespace - checkLocalOrGetPeerReplicationCluster(pulsarService, topicName.getNamespaceObject()) + checkAuthorizationAsync(pulsarService, topicName, clientAppId, authenticationData).thenRun(() -> { + // (3) validate global namespace + checkLocalOrGetPeerReplicationCluster(pulsarService, + topicName.getNamespaceObject()) .thenAccept(peerClusterData -> { if (peerClusterData == null) { // (4) all validation passed: initiate lookup @@ -247,21 +239,36 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl()) && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) { validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, - "Redirected cluster's brokerService url is not configured", requestId)); + "Redirected cluster's brokerService url is not configured", + requestId)); return; } validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(), - peerClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId, + peerClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, + requestId, false)); - }).exceptionally(ex -> { - validationFuture.complete( - newLookupErrorResponse(ServerError.MetadataError, ex.getMessage(), requestId)); - return null; - }); + validationFuture.complete( + newLookupErrorResponse(ServerError.MetadataError, + FutureUtil.unwrapCompletionException(ex).getMessage(), requestId)); + return null; + }); + }) + .exceptionally(e -> { + Throwable throwable = FutureUtil.unwrapCompletionException(e); + if (throwable instanceof RestException) { + log.warn("Failed to authorized {} on cluster {}", clientAppId, topicName); + validationFuture.complete(newLookupErrorResponse(ServerError.AuthorizationError, + throwable.getMessage(), requestId)); + } else { + log.warn("Unknown error while authorizing {} on cluster {}", clientAppId, topicName); + validationFuture.completeExceptionally(throwable); + } + return null; + }); } }).exceptionally(ex -> { - validationFuture.completeExceptionally(ex); + validationFuture.completeExceptionally(FutureUtil.unwrapCompletionException(ex)); return null; }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 7b5f455ac970b..20652d14a539c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -245,6 +245,86 @@ protected void validateAdminAccessForTenant(String tenant) { } } + protected static CompletableFuture validateAdminAccessForTenantAsync( + PulsarService pulsar, String clientAppId, + String originalPrincipal, String tenant, + AuthenticationDataSource authenticationData) { + if (log.isDebugEnabled()) { + log.debug("check admin access on tenant: {} - Authenticated: {} -- role: {}", tenant, + (isClientAuthenticated(clientAppId)), clientAppId); + } + return pulsar.getPulsarResources().getTenantResources().getTenantAsync(tenant) + .thenCompose(tenantInfoOptional -> { + if (!tenantInfoOptional.isPresent()) { + throw new RestException(Status.NOT_FOUND, "Tenant does not exist"); + } + TenantInfo tenantInfo = tenantInfoOptional.get(); + if (pulsar.getConfiguration().isAuthenticationEnabled() && pulsar.getConfiguration() + .isAuthorizationEnabled()) { + if (!isClientAuthenticated(clientAppId)) { + throw new RestException(Status.FORBIDDEN, "Need to authenticate to perform the request"); + } + validateOriginalPrincipal(pulsar.getConfiguration().getProxyRoles(), clientAppId, + originalPrincipal); + if (pulsar.getConfiguration().getProxyRoles().contains(clientAppId)) { + AuthorizationService authorizationService = + pulsar.getBrokerService().getAuthorizationService(); + return authorizationService.isTenantAdmin(tenant, clientAppId, tenantInfo, + authenticationData) + .thenCompose(isTenantAdmin -> { + String debugMsg = "Successfully authorized {} (proxied by {}) on tenant {}"; + if (!isTenantAdmin) { + return authorizationService.isSuperUser(clientAppId, authenticationData) + .thenCombine(authorizationService.isSuperUser(originalPrincipal, + authenticationData), + (proxyAuthorized, originalPrincipalAuthorized) -> { + if (!proxyAuthorized || !originalPrincipalAuthorized) { + throw new RestException(Status.UNAUTHORIZED, + String.format( + "Proxy not authorized to access " + + "resource (proxy:%s,original:%s)" + , clientAppId, originalPrincipal)); + } else { + if (log.isDebugEnabled()) { + log.debug(debugMsg, originalPrincipal, + clientAppId, tenant); + } + return null; + } + }); + } else { + if (log.isDebugEnabled()) { + log.debug(debugMsg, originalPrincipal, clientAppId, tenant); + } + return CompletableFuture.completedFuture(null); + } + }); + } else { + return pulsar.getBrokerService() + .getAuthorizationService() + .isSuperUser(clientAppId, authenticationData) + .thenCompose(isSuperUser -> { + if (!isSuperUser) { + return pulsar.getBrokerService().getAuthorizationService() + .isTenantAdmin(tenant, clientAppId, tenantInfo, authenticationData); + } else { + return CompletableFuture.completedFuture(true); + } + }).thenAccept(authorized -> { + if (!authorized) { + throw new RestException(Status.UNAUTHORIZED, + "Don't have permission to administrate resources on this tenant"); + } else { + log.debug("Successfully authorized {} on tenant {}", clientAppId, tenant); + } + }); + } + } else { + return CompletableFuture.completedFuture(null); + } + }); + } + protected static void validateAdminAccessForTenant(PulsarService pulsar, String clientAppId, String originalPrincipal, String tenant, AuthenticationDataSource authenticationData) @@ -795,18 +875,22 @@ private static ClusterDataImpl getOwnerFromPeerClusterList(PulsarService pulsar, return null; } - protected static void checkAuthorization(PulsarService pulsarService, TopicName topicName, String role, - AuthenticationDataSource authenticationData) throws Exception { + protected static CompletableFuture checkAuthorizationAsync(PulsarService pulsarService, + TopicName topicName, String role, + AuthenticationDataSource authenticationData) { if (!pulsarService.getConfiguration().isAuthorizationEnabled()) { // No enforcing of authorization policies - return; + return CompletableFuture.completedFuture(null); } // get zk policy manager - if (!pulsarService.getBrokerService().getAuthorizationService().allowTopicOperation(topicName, - TopicOperation.LOOKUP, null, role, authenticationData)) { - log.warn("[{}] Role {} is not allowed to lookup topic", topicName, role); - throw new RestException(Status.UNAUTHORIZED, "Don't have permission to connect to this namespace"); - } + return pulsarService.getBrokerService().getAuthorizationService().allowTopicOperationAsync(topicName, + TopicOperation.LOOKUP, null, role, authenticationData).thenAccept(allow -> { + if (!allow) { + log.warn("[{}] Role {} is not allowed to lookup topic", topicName, role); + throw new RestException(Status.UNAUTHORIZED, + "Don't have permission to connect to this namespace"); + } + }); } // Used for unit tests access From dd9a5f1f91651b634600f66c53dcc6ad855fb669 Mon Sep 17 00:00:00 2001 From: mattison chao Date: Fri, 17 Jun 2022 13:41:07 +0800 Subject: [PATCH 620/823] Release 2.9.3 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- deployment/terraform-ansible/deploy-pulsar.yaml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- docker/grafana/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- kafka-connect-avro-converter-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker-shaded/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/java-version-trim-agent/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- pulsar-zookeeper-utils/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 128 files changed, 128 insertions(+), 128 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 9b36a306b979c..620061ab67a6a 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 616aa02c01a43..8fa49a3ea2023 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 389cfb7f2e90c..4350308bf7d29 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 49bc0d85da445..474d20035e6e6 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 1195df6d5c8df..e35e7bab1da4d 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.3-SNAPSHOT + 2.9.3 jar Pulsar Build Tools diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml index a06b73b99852e..db180ddfeb94c 100644 --- a/deployment/terraform-ansible/deploy-pulsar.yaml +++ b/deployment/terraform-ansible/deploy-pulsar.yaml @@ -39,7 +39,7 @@ zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" service_url: "{{ pulsar_service_url }}" http_url: "{{ pulsar_web_url }}" - pulsar_version: "2.9.3-SNAPSHOT" + pulsar_version: "2.9.3" - name: Download Pulsar binary package unarchive: src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 4ee3328cc4d8b..a1a63301483ea 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 1248ea1682609..07bc62ffb7603 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index f79f26d45fb26..4959dd90662de 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index f295d34a8ea26..eb8c3577724b3 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index adb423253ef31..4437c3fd48406 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index a3534d2c0c83d..80db6b2797ab6 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index b9e4f8d158a49..be92352bd46bc 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index d75e18aab9e45..8a64a759375c0 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 2b45d8f6ae38b..c690773089f8e 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index f32bc87c46e27..7c426485b04e7 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 0461ae604cc09..0ea22a9fa36ad 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pom.xml b/pom.xml index 88db1eca8c8dd..da348ec8f75e0 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index fdd52671868fe..f42eb59f5a91f 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 92f27eb385ca7..c82d2219a47b2 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 9e755b12161f1..b675405677541 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-broker-common diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 68a53c806ffef..2f759e0371eff 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 2d4605a7a0153..b579d3f987989 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index ac7d7ec5354da..70e75d446dae0 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index b17e509d89a9f..6ee0ac15be55d 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 6c0bde533061e..61b59f0bea143 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index e5147c4837a8a..2dcf6d9cc6578 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 0a3dc027dfb86..f2b96114531d4 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 74f20a15f4caf..697110084f18e 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 37d12891d049c..68ee34ce87b8e 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index e7808b61a03ea..dc541415c2c91 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 11f5fac560a32..054a3d2fdbe6a 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 8e7a0b4ecad6e..dafbea5b5bc6c 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 3b24ced062b6e..c433ef114be24 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 891bd2f7f8b88..b9c7ac32f475e 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 593d5842448e7..6cafa3a8d7e33 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 27838ecddf998..c20458d5129d8 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index f5c6cd2e06392..233b4f9e5b403 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index d42d74752e5f5..01fe5091bdbed 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 759e6acccf810..57c5e04858ea9 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 1392e9badc7ba..d95667f81ce37 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 61935a3fa9af2..705e04e15d03f 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index f6cf87853f7d2..0d96e2c9bb9ea 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 732dee3f597ea..c6f1957f88cff 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index d05780478a168..798c69f7a934d 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 727304d30e78a..966283570dc4a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 2506936143b58..4ea6780cf21cc 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 37e0267590edd..023c7569d1d01 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 4278a6553ab11..cf87a3fec237e 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 247a6393a81fd..062347816d9fd 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index b16f364689af8..27713485015fa 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 7b76ab13e098a..385f7fb6ed1ae 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 83395d20966b3..c8fa82a7bca44 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index f06c5a609b1f3..a8a7eb3e72da2 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 70778879c23df..640af6d592ee5 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 3833961d94c57..2c78f3d6c8024 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 889f0ab150cb2..07a5d9b2eac17 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index bf92cea841c28..76b6182f75d1d 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 71606c0548153..52e601a9cae3c 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index f1c47fb5dfa2a..269111778289b 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 81dad1f245cee..b13ce73c364da 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 21423a08143bd..e084d43a8a16a 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 53910d0cff22f..5dada914d85bb 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 229623deee7a0..afd23135eece4 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index a68c942d97180..f2af3781b3ea9 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index e5badf75a677e..56caeb765a832 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index f7005c635e9ff..b50799ac9d4e1 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 81942862019f0..4986fb547a805 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index e8f597224e582..a70335b402955 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index fbdaa6345abb0..67d4450ff2729 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 3faf309b68328..544119ae259e0 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 0fb45c9a5c009..25e2af6268cef 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index a760ee9584693..d4606db742e63 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 5025e47bf2dbc..60d02110748fc 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index beec6c6ab96e5..2505b0b6a56b4 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 8a23962c7388d..ba53000940b1d 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index ef84dd16c2e00..6ffdce617bf23 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 5e48ab354e27b..73460a1b6b742 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 6bc9c0a025cd3..6fa7af0f36d3e 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 67b887c1e07ff..ec9a9ac964e02 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index aad83eaeeaa06..59a15fca7d39a 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index b53a8ed9282df..c8c6918b94547 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 67327a8f4d46c..ec215407eb340 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 8199c1fe3df18..f68e9bcea2e45 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index c22f704565290..f43933dfc838c 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index a6222fda0a7e0..0fb41f92e7c8b 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 76e8aa5fbd05a..c2d4048c6ede2 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index d37dd1e8f9f08..41e3225a8b383 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 739f1acade64f..93032fedd87d8 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 6a4d3df44430b..b97c03d9d7307 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 350bcfd05a345..b9cca005d32ce 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 8824f35891b16..5b1a51ce8e648 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index bb10dec5cdc53..c258706560840 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 72a9162706f89..386f7b720ca93 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index f8f8a2a6491ee..0a6bef8ff46f6 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.3-SNAPSHOT + 2.9.3 pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 468a8debdb903..2072464a51f63 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index eb93d9900c971..a33ad930113db 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index c205dcad6d505..199f21d7a468a 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index ad838c8a96bab..4f0714cfd5815 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 0faafd53034dd..983bc627cbc52 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-proxy diff --git a/pulsar-sql/java-version-trim-agent/pom.xml b/pulsar-sql/java-version-trim-agent/pom.xml index cc2026bb53386..f564229b33013 100644 --- a/pulsar-sql/java-version-trim-agent/pom.xml +++ b/pulsar-sql/java-version-trim-agent/pom.xml @@ -24,7 +24,7 @@ pulsar-sql org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index bf4166cb75fb3..480cec5679ee3 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 66fb2213af547..87e112599825a 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.3-SNAPSHOT + 2.9.3 pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index f49d541cdd570..a3ef46f0e8741 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.3-SNAPSHOT + 2.9.3 pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 80896ed2c8a5a..dc5abfacde221 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.3-SNAPSHOT + 2.9.3 pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 9e686cde439c5..a2aea122c563c 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 58ca5ad6fc17a..7b60179835259 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.3-SNAPSHOT + 2.9.3 pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index b274931049df1..c785b8c6df9c2 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.3-SNAPSHOT + 2.9.3 pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 7aeb860a9eaf3..2760271368785 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 8a85e3f8d641d..64a46af2daa8d 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index 4b21f3e8ed25c..319d513a906c7 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 1bead7e397c59..1d84189f513f3 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index c4db8a1ade2de..7bc28b2bd7127 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.3-SNAPSHOT + 2.9.3 testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index acb01809c8f13..83a7442aa5a3b 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index e0ac02e7f4747..a1836f0ad4fed 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 9cc9619d2a388..555c5b118c8ee 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 06d6f18a678d2..185102bb6f128 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 063a4c1a50075..79b19f3ac9330 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 40518114309b2..d5f17ff3b59cd 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.3-SNAPSHOT + 2.9.3 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index e114b1c6bd945..b68ddad98f8fa 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 16a0eb7d6bd5a..97c72032aa1cf 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 integration diff --git a/tests/pom.xml b/tests/pom.xml index e34f6678186bb..a45e2c80ceed9 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index dbb9be2d0d670..bc1e44eb72fab 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 664c56cfd5044..1bdf4cee29e5e 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 191c31527877e..85fccd3178f5d 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.3-SNAPSHOT + 2.9.3 pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 3e3c1985f0b6d..e5f1c880cca83 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index b776161662dc8..9299ddda4f6ac 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.3-SNAPSHOT + 2.9.3 .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 132083817932c..ea9dd883fa004 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.3-SNAPSHOT + 2.9.3 .. From ce08aa3d77a3b6548c27a7c03791e75a503e0c10 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 19 Jun 2022 05:27:30 -0700 Subject: [PATCH 621/823] Use OrderedExecutor instead of OrderedScheduler for consumer dispatch (#16115) --- .../java/org/apache/pulsar/broker/service/BrokerService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index a932e2fa505c6..b81200f737905 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -76,7 +76,6 @@ import lombok.Getter; import lombok.Setter; import org.apache.bookkeeper.common.util.OrderedExecutor; -import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback; import org.apache.bookkeeper.mledger.LedgerOffloader; @@ -301,7 +300,7 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws ConcurrentOpenHashMap.newBuilder().build(); - this.topicOrderedExecutor = OrderedScheduler.newSchedulerBuilder() + this.topicOrderedExecutor = OrderedExecutor.newBuilder() .numThreads(pulsar.getConfiguration().getNumWorkerThreadsForNonPersistentTopic()) .name("broker-topic-workers").build(); final DefaultThreadFactory acceptorThreadFactory = new DefaultThreadFactory("pulsar-acceptor"); From 803173149df1f65d083fe66cc306750f843f4a45 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 21 Jun 2022 03:23:51 +0200 Subject: [PATCH 622/823] Transactions: fix race in TransactionMetaStoreHandler (#16147) ### Motivation Fixes #15375 the NPE happens because internalPinnedExecutor has not been assigned yet. ### Modifications Refactor the constructor, in a way that "this" is not accessed from another thread while the constructor is not done yet. (cherry picked from commit ec276da53231bc01eb44e9cdc106b5174915691f) --- .../pulsar/client/impl/TransactionMetaStoreHandler.java | 7 +++++-- .../impl/transaction/TransactionCoordinatorClientImpl.java | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 82fc89ca0a1e3..73df89dd6ae6a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -103,9 +103,12 @@ public TransactionMetaStoreHandler(long transactionCoordinatorId, PulsarClientIm .create(), this); this.connectFuture = connectFuture; - this.connectionHandler.grabCnx(); + this.internalPinnedExecutor = pulsarClient.getInternalExecutorService(); this.timer = pulsarClient.timer(); - internalPinnedExecutor = pulsarClient.getInternalExecutorService(); + } + + public void start() { + this.connectionHandler.grabCnx(); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java index e8baec784a7b1..c320d13e166f0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java @@ -94,6 +94,7 @@ public CompletableFuture startAsync() { i, pulsarClient, getTCAssignTopicName(i), connectFuture); handlers[i] = handler; handlerMap.put(i, handler); + handler.start(); } } else { handlers = new TransactionMetaStoreHandler[1]; @@ -103,6 +104,7 @@ public CompletableFuture startAsync() { getTCAssignTopicName(-1), connectFuture); handlers[0] = handler; handlerMap.put(0, handler); + handler.start(); } STATE_UPDATER.set(TransactionCoordinatorClientImpl.this, State.READY); From 741fd6c748c4c799ff51bd125ed5bc34fff6f1c7 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Wed, 15 Jun 2022 21:38:45 +0800 Subject: [PATCH 623/823] [fix][txn] Ack the same batch message different batchIndex with transaction (#16032) (cherry picked from commit 0a846ad5d61df7d4842cbcb78d7dfaa86156fa73) --- .../pendingack/impl/PendingAckHandleImpl.java | 14 +++++ .../broker/transaction/TransactionTest.java | 53 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 4fdc298f303f0..5b808f1dedbca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -22,6 +22,7 @@ import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.compareToWithAckSet; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.isAckSetOverlap; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -767,6 +768,19 @@ private void handleIndividualAck(TxnID txnID, List positionPair = positions.get(i); + positionPair.left = PositionImpl.get(positionPair.getLeft().getLedgerId(), + positionPair.getLeft().getEntryId(), + Arrays.copyOf(positionPair.left.getAckSet(), positionPair.left.getAckSet().length)); this.individualAckPositions.put(position, positions.get(i)); } else { MutablePair positionPair = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 0b59eda45234c..10aa86aed2cb3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -996,4 +996,57 @@ public void testConsistencyOfTransactionStatsAtEndTxn() throws Exception { transaction.commit().get(); } + + @Test + public void testPendingAckBatchMessageCommit() throws Exception { + String topic = NAMESPACE1 + "/testPendingAckBatchMessageCommit"; + + // enable batch index ack + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); + + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(true) + // ensure that batch message is sent + .batchingMaxPublishDelay(3, TimeUnit.SECONDS) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer() + .subscriptionType(SubscriptionType.Shared) + .topic(topic) + .subscriptionName("sub") + .subscribe(); + + // send batch message, the size is 5 + for (int i = 0; i < 5; i++) { + producer.sendAsync(("test" + i).getBytes()); + } + + producer.flush(); + + Transaction txn1 = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.MINUTES).build().get(); + // ack the first message with transaction + consumer.acknowledgeAsync(consumer.receive().getMessageId(), txn1).get(); + Transaction txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.MINUTES).build().get(); + // ack the second message with transaction + MessageId messageId = consumer.receive().getMessageId(); + consumer.acknowledgeAsync(messageId, txn2).get(); + + // commit the txn1 + txn1.commit().get(); + // abort the txn2 + txn2.abort().get(); + + Transaction txn3 = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.MINUTES).build().get(); + // repeat ack the second message, can ack successful + consumer.acknowledgeAsync(messageId, txn3).get(); + } } From ac68841124a0115dec1d8538ae7d66c2fd4da789 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 1 Jul 2022 18:13:43 -0700 Subject: [PATCH 624/823] Ensure ack-timeout task gets re-scheduled when there are exception in the final stage (#16337) --- .../client/impl/UnAckedMessageTracker.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java index ff07156972f62..020c925d0b061 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java @@ -122,6 +122,10 @@ public UnAckedMessageTracker(PulsarClientImpl client, ConsumerBase consumerBa timeout = client.timer().newTimeout(new TimerTask() { @Override public void run(Timeout t) throws Exception { + if (t.isCancelled()) { + return; + } + Set messageIds = TL_MESSAGE_IDS_SET.get(); messageIds.clear(); @@ -140,12 +144,16 @@ public void run(Timeout t) throws Exception { headPartition.clear(); timePartitions.addLast(headPartition); } finally { - writeLock.unlock(); - if (messageIds.size() > 0) { - consumerBase.onAckTimeoutSend(messageIds); - consumerBase.redeliverUnacknowledgedMessages(messageIds); + try { + timeout = client.timer().newTimeout(this, tickDurationInMs, TimeUnit.MILLISECONDS); + } finally { + writeLock.unlock(); + + if (!messageIds.isEmpty()) { + consumerBase.onAckTimeoutSend(messageIds); + consumerBase.redeliverUnacknowledgedMessages(messageIds); + } } - timeout = client.timer().newTimeout(this, tickDurationInMs, TimeUnit.MILLISECONDS); } } }, this.tickDurationInMs, TimeUnit.MILLISECONDS); From 641406e6cfd92a2142d64f1ba5a8d34cfba64fc9 Mon Sep 17 00:00:00 2001 From: LinChen <1572139390@qq.com> Date: Sat, 2 Jul 2022 06:44:08 +0800 Subject: [PATCH 625/823] fix select broker is none (#16316) * fix select broker is none * check style (cherry picked from commit cf4fe17d47b2b45126d1795a49641dc53ba37d19) --- .../broker/loadbalance/impl/ModularLoadManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 68833199bf47c..67a1ecc6f6c83 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -825,7 +825,11 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), brokerTopicLoadingPredicate); - broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + Optional brokerTmp = + placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (brokerTmp.isPresent()) { + broker = brokerTmp; + } } // Add new bundle to preallocated. From 6f12f5ed2ec5ca3c7da74cad02d9d01e1f7d1e66 Mon Sep 17 00:00:00 2001 From: mattison chao Date: Sat, 2 Jul 2022 10:30:13 +0800 Subject: [PATCH 626/823] [improve][broker] Optimise msgOutCounter and bytesOutCounter (#16214) (#16286) AbstractTopic#getBytesOutCounter and AbstractTopic#getMsgOutCounter both generate full stats only to pick the single required counter for the getters. These can be optimised to perform only the necessary work to calculate the counters. Provided a scoped implementation of the above methods for the single counter required for each getter. (cherry picked from commit 07c46fe9ef42a72de18be3267e70a51bbc883678) --- .../broker/service/AbstractSubscription.java | 42 ++++++++++ .../pulsar/broker/service/AbstractTopic.java | 17 ++++- .../pulsar/broker/service/Consumer.java | 8 ++ .../NonPersistentSubscription.java | 6 +- .../nonpersistent/NonPersistentTopic.java | 4 - .../persistent/PersistentSubscription.java | 7 +- .../service/persistent/PersistentTopic.java | 4 - .../service/AbstractSubscriptionTest.java | 57 ++++++++++++++ .../broker/service/AbstractTopicTest.java | 75 ++++++++++++++++++ .../pulsar/broker/service/ConsumerTest.java | 76 +++++++++++++++++++ 10 files changed, 277 insertions(+), 19 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java new file mode 100644 index 0000000000000..6a38667055679 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.Optional; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.ToLongFunction; + +public abstract class AbstractSubscription implements Subscription { + protected final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); + protected final LongAdder msgOutFromRemovedConsumer = new LongAdder(); + + public long getMsgOutCounter() { + return msgOutFromRemovedConsumer.longValue() + sumConsumers(Consumer::getMsgOutCounter); + } + + public long getBytesOutCounter() { + return bytesOutFromRemovedConsumers.longValue() + sumConsumers(Consumer::getBytesOutCounter); + } + + private long sumConsumers(ToLongFunction toCounter) { + return Optional.ofNullable(getDispatcher()) + .map(dispatcher -> dispatcher.getConsumers().stream().mapToLong(toCounter).sum()) + .orElse(0L); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 69c51c77d44a5..1307001a72068 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.ToLongFunction; import lombok.Getter; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.lang3.tuple.Pair; @@ -136,6 +137,9 @@ public abstract class AbstractTopic implements Topic { private volatile long lastTopicMaxMessageSizeCheckTimeStamp = 0; private final long topicMaxMessageSizeCheckIntervalMs; + protected final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); + protected final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); + public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; this.brokerService = brokerService; @@ -875,11 +879,20 @@ public long getBytesInCounter() { } public long getMsgOutCounter() { - return getStats(false, false).msgOutCounter; + return msgOutFromRemovedSubscriptions.longValue() + + sumSubscriptions(AbstractSubscription::getMsgOutCounter); } public long getBytesOutCounter() { - return getStats(false, false).bytesOutCounter; + return bytesOutFromRemovedSubscriptions.longValue() + + sumSubscriptions(AbstractSubscription::getBytesOutCounter); + } + + private long sumSubscriptions(ToLongFunction toCounter) { + return getSubscriptions().values().stream() + .map(AbstractSubscription.class::cast) + .mapToLong(toCounter) + .sum(); } public boolean isDeleteWhileInactive() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index f2a4677dbfc55..318f19c51a544 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -761,6 +761,14 @@ public ConsumerStatsImpl getStats() { return stats; } + public long getMsgOutCounter() { + return msgOutCounter.longValue(); + } + + public long getBytesOutCounter() { + return bytesOutCounter.longValue(); + } + public int getUnackedMessages() { return unackedMessages; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index 59fab2df3c683..370402e509df6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -29,6 +29,7 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; @@ -48,7 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NonPersistentSubscription implements Subscription { +public class NonPersistentSubscription extends AbstractSubscription implements Subscription { private final NonPersistentTopic topic; private volatile NonPersistentDispatcher dispatcher; private final String topicName; @@ -66,9 +67,6 @@ public class NonPersistentSubscription implements Subscription { // Timestamp of when this subscription was last seen active private volatile long lastActive; - private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); - private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); - // If isDurable is false(such as a Reader), remove subscription from topic when closing this subscription. private final boolean isDurable; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 1dbe95fb15a30..68740d939e261 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -34,7 +34,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; -import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; @@ -102,9 +101,6 @@ public class NonPersistentTopic extends AbstractTopic implements Topic { AtomicLongFieldUpdater.newUpdater(NonPersistentTopic.class, "entriesAddedCounter"); private volatile long entriesAddedCounter = 0; - private final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); - private final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); - private static final FastThreadLocal threadLocalTopicStats = new FastThreadLocal() { @Override protected TopicStats initialValue() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 9e5758034be3b..2c8c828df94a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -29,7 +29,6 @@ import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.LongAdder; import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; @@ -50,6 +49,7 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; @@ -80,7 +80,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PersistentSubscription implements Subscription { +public class PersistentSubscription extends AbstractSubscription implements Subscription { protected final PersistentTopic topic; protected final ManagedCursor cursor; protected volatile Dispatcher dispatcher; @@ -113,9 +113,6 @@ public class PersistentSubscription implements Subscription { private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; private final PendingAckHandle pendingAckHandle; - private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); - private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); - static { REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 22531e8ed09a4..7a3d4c82fa2bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -44,7 +44,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.LongAdder; import java.util.function.BiFunction; import java.util.stream.Collectors; import lombok.Getter; @@ -222,9 +221,6 @@ protected TopicStatsHelper initialValue() { @Getter protected final TransactionBuffer transactionBuffer; - private final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); - private final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); - // Record the last time a data message (ie: not an internal Pulsar marker) is published on the topic private long lastDataMessagePublishedTimestamp = 0; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java new file mode 100644 index 0000000000000..aaf1f6164a374 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import java.util.List; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class AbstractSubscriptionTest { + private Consumer consumer; + private AbstractSubscription subscription; + + @BeforeMethod + public void beforeMethod() { + Dispatcher dispatcher = mock(Dispatcher.class); + consumer = mock(Consumer.class); + subscription = spy(AbstractSubscription.class); + + when(subscription.getDispatcher()).thenReturn(dispatcher); + when(dispatcher.getConsumers()).thenReturn(List.of(consumer)); + } + + @Test + public void testGetMsgOutCounter() { + subscription.msgOutFromRemovedConsumer.add(1L); + when(consumer.getMsgOutCounter()).thenReturn(2L); + assertEquals(subscription.getMsgOutCounter(), 3L); + } + + @Test + public void testGetBytesOutCounter() { + subscription.bytesOutFromRemovedConsumers.add(1L); + when(consumer.getBytesOutCounter()).thenReturn(2L); + assertEquals(subscription.getBytesOutCounter(), 3L); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java new file mode 100644 index 0000000000000..fb7890dc57f88 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import static org.testng.Assert.assertEquals; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class AbstractTopicTest { + private AbstractSubscription subscription; + private AbstractTopic topic; + + @BeforeMethod + public void beforeMethod() { + BrokerService brokerService = mock(BrokerService.class); + PulsarService pulsarService = mock(PulsarService.class); + ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); + subscription = mock(AbstractSubscription.class); + + when(brokerService.pulsar()).thenReturn(pulsarService); + when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); + when(brokerService.getBacklogQuotaManager()).thenReturn(backlogQuotaManager); + + topic = mock(AbstractTopic.class, withSettings() + .useConstructor("topic", brokerService) + .defaultAnswer(CALLS_REAL_METHODS)); + + ConcurrentOpenHashMap subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + subscriptions.put("subscription", subscription); + when(topic.getSubscriptions()).thenAnswer(invocation -> subscriptions); + } + + @Test + public void testGetMsgOutCounter() { + topic.msgOutFromRemovedSubscriptions.add(1L); + when(subscription.getMsgOutCounter()).thenReturn(2L); + assertEquals(topic.getMsgOutCounter(), 3L); + } + + @Test + public void testGetBytesOutCounter() { + topic.bytesOutFromRemovedSubscriptions.add(1L); + when(subscription.getBytesOutCounter()).thenReturn(2L); + assertEquals(topic.getBytesOutCounter(), 3L); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java new file mode 100644 index 0000000000000..1ad0642d9c5ba --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static java.util.Collections.emptyMap; +import static org.apache.pulsar.client.api.MessageId.latest; +import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive; +import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT; +import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import java.net.SocketAddress; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.api.proto.KeySharedMeta; +import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ConsumerTest { + private Consumer consumer; + private final ConsumerStatsImpl stats = new ConsumerStatsImpl(); + + @BeforeMethod + public void beforeMethod() { + Subscription subscription = mock(Subscription.class); + ServerCnx cnx = mock(ServerCnx.class); + SocketAddress address = mock(SocketAddress.class); + Topic topic = mock(Topic.class); + BrokerService brokerService = mock(BrokerService.class); + PulsarService pulsarService = mock(PulsarService.class); + ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + + when(cnx.clientAddress()).thenReturn(address); + when(subscription.getTopic()).thenReturn(topic); + when(topic.getBrokerService()).thenReturn(brokerService); + when(brokerService.getPulsar()).thenReturn(pulsarService); + when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); + + consumer = + new Consumer(subscription, Exclusive, "topic", 1, 0, "Cons1", true, cnx, "myrole-1", emptyMap(), false, + new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest, DEFAULT_CONSUMER_EPOCH); + } + + @Test + public void testGetMsgOutCounter() { + stats.msgOutCounter = 1L; + consumer.updateStats(stats); + assertEquals(consumer.getMsgOutCounter(), 1L); + } + + @Test + public void testGetBytesOutCounter() { + stats.bytesOutCounter = 1L; + consumer.updateStats(stats); + assertEquals(consumer.getBytesOutCounter(), 1L); + } +} From 0d303c89dfcda16991b1ceb8844645a89a347603 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:17:44 +0800 Subject: [PATCH 627/823] [Transaction] Set TC state is Ready after open MLTransactionMetadataStore completely. (#13957) [Transaction] Set TC state is Ready after open MLTransactionMetadataStore completely. ### Motivation The MLTransactionMetadataStore constructor and openTransactionMetadataStore method are asynchronous. So there may be situations where the store in the Initializing state was put into stores ### Modification Pass in the future to wait for MLTransactionMetadataStore initialization to complete (cherry picked from commit 0fe8ac0d2c1bb909729580e5456b4d57c2a00346) --- .../broker/transaction/TransactionTest.java | 13 +- .../impl/MLTransactionMetadataStore.java | 174 ++++++++++-------- .../MLTransactionMetadataStoreProvider.java | 4 +- .../MLTransactionMetadataStoreTest.java | 38 ++-- 4 files changed, 121 insertions(+), 108 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 10aa86aed2cb3..f9fc52aa547cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -695,9 +695,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ doNothing().when(timeoutTracker).start(); MLTransactionMetadataStore metadataStore1 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), - mlTransactionLog, timeoutTracker, transactionRecoverTracker, - mlTransactionSequenceIdGenerator); - + mlTransactionLog, timeoutTracker, mlTransactionSequenceIdGenerator); + metadataStore1.init(transactionRecoverTracker).get(); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore1.getCoordinatorStats().state, "Ready")); @@ -709,8 +708,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionMetadataStore metadataStore2 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), - mlTransactionLog, timeoutTracker, transactionRecoverTracker, - mlTransactionSequenceIdGenerator); + mlTransactionLog, timeoutTracker, mlTransactionSequenceIdGenerator); + metadataStore2.init(transactionRecoverTracker).get(); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore2.getCoordinatorStats().state, "Ready")); @@ -722,8 +721,8 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionMetadataStore metadataStore3 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), - mlTransactionLog, timeoutTracker, transactionRecoverTracker, - mlTransactionSequenceIdGenerator); + mlTransactionLog, timeoutTracker, mlTransactionSequenceIdGenerator); + metadataStore3.init(transactionRecoverTracker).get(); Awaitility.await().untilAsserted(() -> assertEquals(metadataStore3.getCoordinatorStats().state, "Ready")); } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index 685d57e664e6f..6c88d27cc2298 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -34,6 +34,7 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.Subscription; import org.apache.pulsar.common.policies.data.TransactionCoordinatorStats; @@ -79,7 +80,6 @@ public class MLTransactionMetadataStore public MLTransactionMetadataStore(TransactionCoordinatorID tcID, MLTransactionLogImpl mlTransactionLog, TransactionTimeoutTracker timeoutTracker, - TransactionRecoverTracker recoverTracker, MLTransactionSequenceIdGenerator sequenceIdGenerator) { super(State.None); this.sequenceIdGenerator = sequenceIdGenerator; @@ -96,96 +96,108 @@ public MLTransactionMetadataStore(TransactionCoordinatorID tcID, DefaultThreadFactory threadFactory = new DefaultThreadFactory("transaction_coordinator_" + tcID.toString() + "thread_factory"); this.internalPinnedExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); + } + public CompletableFuture init(TransactionRecoverTracker recoverTracker) { + CompletableFuture completableFuture = new CompletableFuture<>(); if (!changeToInitializingState()) { log.error("Managed ledger transaction metadata store change state error when init it"); - return; - } - - internalPinnedExecutor.execute(() -> transactionLog.replayAsync(new TransactionLogReplayCallback() { - - @Override - public void replayComplete() { - recoverTracker.appendOpenTransactionToTimeoutTracker(); - if (!changeToReadyState()) { - log.error("Managed ledger transaction metadata store change state error when replay complete"); - } else { - recoverTracker.handleCommittingAndAbortingTransaction(); - timeoutTracker.start(); + completableFuture + .completeExceptionally(new TransactionCoordinatorClientException + .CoordinatorNotFoundException("transaction metadata store with tcId " + + tcID.toString() + " change state to Initializing error when init it")); + } else { + internalPinnedExecutor.execute(() -> transactionLog.replayAsync(new TransactionLogReplayCallback() { + + @Override + public void replayComplete() { + recoverTracker.appendOpenTransactionToTimeoutTracker(); + if (!changeToReadyState()) { + log.error("Managed ledger transaction metadata store change state error when replay complete"); + completableFuture + .completeExceptionally(new TransactionCoordinatorClientException + .CoordinatorNotFoundException("transaction metadata store with tcId " + + tcID.toString() + " change state to Ready error when init it")); + + } else { + recoverTracker.handleCommittingAndAbortingTransaction(); + timeoutTracker.start(); + completableFuture.complete(MLTransactionMetadataStore.this); + } } - } - - @Override - public void handleMetadataEntry(Position position, TransactionMetadataEntry transactionMetadataEntry) { - try { + @Override + public void handleMetadataEntry(Position position, TransactionMetadataEntry transactionMetadataEntry) { - TxnID txnID = new TxnID(transactionMetadataEntry.getTxnidMostBits(), + try { + TxnID txnID = new TxnID(transactionMetadataEntry.getTxnidMostBits(), transactionMetadataEntry.getTxnidLeastBits()); - long transactionId = transactionMetadataEntry.getTxnidLeastBits(); - switch (transactionMetadataEntry.getMetadataOp()) { - case NEW: - long txnSequenceId = transactionMetadataEntry.getTxnidLeastBits(); - if (txnMetaMap.containsKey(transactionId)) { - txnMetaMap.get(transactionId).getRight().add(position); - } else { - List positions = new ArrayList<>(); - positions.add(position); - long openTimestamp = transactionMetadataEntry.getStartTime(); - long timeoutAt = transactionMetadataEntry.getTimeoutMs(); - txnMetaMap.put(transactionId, MutablePair.of(new TxnMetaImpl(txnID, - openTimestamp, timeoutAt), positions)); - recoverTracker.handleOpenStatusTransaction(txnSequenceId, - timeoutAt + openTimestamp); - } - break; - case ADD_PARTITION: - if (!txnMetaMap.containsKey(transactionId)) { - transactionLog.deletePosition(Collections.singletonList(position)); - } else { - txnMetaMap.get(transactionId).getLeft() - .addProducedPartitions(transactionMetadataEntry.getPartitionsList()); - txnMetaMap.get(transactionId).getRight().add(position); - } - break; - case ADD_SUBSCRIPTION: - if (!txnMetaMap.containsKey(transactionId)) { - transactionLog.deletePosition(Collections.singletonList(position)); - } else { - txnMetaMap.get(transactionId).getLeft() - .addAckedPartitions(subscriptionToTxnSubscription( - transactionMetadataEntry.getSubscriptionsList())); - txnMetaMap.get(transactionId).getRight().add(position); - } - break; - case UPDATE: - if (!txnMetaMap.containsKey(transactionId)) { - transactionLog.deletePosition(Collections.singletonList(position)); - } else { - TxnStatus newStatus = transactionMetadataEntry.getNewStatus(); - txnMetaMap.get(transactionId).getLeft() - .updateTxnStatus(transactionMetadataEntry.getNewStatus(), - transactionMetadataEntry.getExpectedStatus()); - txnMetaMap.get(transactionId).getRight().add(position); - recoverTracker.updateTransactionStatus(txnID.getLeastSigBits(), newStatus); - if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { - transactionLog.deletePosition(txnMetaMap - .get(transactionId).getRight()).thenAccept(v -> - txnMetaMap.remove(transactionId).getLeft()); + long transactionId = transactionMetadataEntry.getTxnidLeastBits(); + switch (transactionMetadataEntry.getMetadataOp()) { + case NEW: + long txnSequenceId = transactionMetadataEntry.getTxnidLeastBits(); + if (txnMetaMap.containsKey(transactionId)) { + txnMetaMap.get(transactionId).getRight().add(position); + } else { + List positions = new ArrayList<>(); + positions.add(position); + long openTimestamp = transactionMetadataEntry.getStartTime(); + long timeoutAt = transactionMetadataEntry.getTimeoutMs(); + txnMetaMap.put(transactionId, MutablePair.of(new TxnMetaImpl(txnID, + openTimestamp, timeoutAt), positions)); + recoverTracker.handleOpenStatusTransaction(txnSequenceId, + timeoutAt + openTimestamp); } - } - break; - default: - throw new InvalidTxnStatusException("Transaction `" - + txnID + "` load replay metadata operation " - + "from transaction log with unknown operation"); + break; + case ADD_PARTITION: + if (!txnMetaMap.containsKey(transactionId)) { + transactionLog.deletePosition(Collections.singletonList(position)); + } else { + txnMetaMap.get(transactionId).getLeft() + .addProducedPartitions(transactionMetadataEntry.getPartitionsList()); + txnMetaMap.get(transactionId).getRight().add(position); + } + break; + case ADD_SUBSCRIPTION: + if (!txnMetaMap.containsKey(transactionId)) { + transactionLog.deletePosition(Collections.singletonList(position)); + } else { + txnMetaMap.get(transactionId).getLeft() + .addAckedPartitions(subscriptionToTxnSubscription( + transactionMetadataEntry.getSubscriptionsList())); + txnMetaMap.get(transactionId).getRight().add(position); + } + break; + case UPDATE: + if (!txnMetaMap.containsKey(transactionId)) { + transactionLog.deletePosition(Collections.singletonList(position)); + } else { + TxnStatus newStatus = transactionMetadataEntry.getNewStatus(); + txnMetaMap.get(transactionId).getLeft() + .updateTxnStatus(transactionMetadataEntry.getNewStatus(), + transactionMetadataEntry.getExpectedStatus()); + txnMetaMap.get(transactionId).getRight().add(position); + recoverTracker.updateTransactionStatus(txnID.getLeastSigBits(), newStatus); + if (newStatus == TxnStatus.COMMITTED || newStatus == TxnStatus.ABORTED) { + transactionLog.deletePosition(txnMetaMap + .get(transactionId).getRight()).thenAccept(v -> + txnMetaMap.remove(transactionId).getLeft()); + } + } + break; + default: + throw new InvalidTxnStatusException("Transaction `" + + txnID + "` load replay metadata operation " + + "from transaction log with unknown operation"); + } + } catch (InvalidTxnStatusException e) { + transactionLog.deletePosition(Collections.singletonList(position)); + log.error(e.getMessage(), e); } - } catch (InvalidTxnStatusException e) { - transactionLog.deletePosition(Collections.singletonList(position)); - log.error(e.getMessage(), e); } - } - })); + })); + } + return completableFuture; } @Override diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java index f0c32d2648207..22a58ebcc9766 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStoreProvider.java @@ -51,8 +51,8 @@ public CompletableFuture openStore(TransactionCoordina managedLedgerFactory, managedLedgerConfig); // MLTransactionLogInterceptor will init sequenceId and update the sequenceId to managedLedger properties. - return txnLog.initialize().thenApply(__ -> + return txnLog.initialize().thenCompose(__ -> new MLTransactionMetadataStore(transactionCoordinatorId, txnLog, timeoutTracker, - recoverTracker, mlTransactionSequenceIdGenerator)); + mlTransactionSequenceIdGenerator).init(recoverTracker)); } } \ No newline at end of file diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java index a06bf9e6deaae..aafe54e60694e 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java @@ -74,8 +74,9 @@ public void testTransactionOperation() throws Exception { mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); int checkReplayRetryCount = 0; while (true) { checkReplayRetryCount++; @@ -149,8 +150,8 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); TxnID txnID = transactionMetadataStore.newTransaction(20000).get(); @@ -178,8 +179,8 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E mlTransactionLog.initialize().join(); transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); txnID = transactionMetadataStore.newTransaction(100000).get(); @@ -201,10 +202,11 @@ public void testInitTransactionReader() throws Exception { MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); mlTransactionLog.initialize().join(); + MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); int checkReplayRetryCount = 0; while (true) { if (checkReplayRetryCount > 3) { @@ -244,10 +246,11 @@ public void testInitTransactionReader() throws Exception { MLTransactionLogImpl txnLog2 = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig); txnLog2.initialize().join(); + MLTransactionMetadataStore transactionMetadataStoreTest = new MLTransactionMetadataStore(transactionCoordinatorID, - txnLog2, new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + txnLog2, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStoreTest.init(new TransactionRecoverTrackerImpl()).get(); while (true) { if (checkReplayRetryCount > 6) { @@ -315,8 +318,8 @@ public void testDeleteLog() throws Exception { mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); int checkReplayRetryCount = 0; while (true) { if (checkReplayRetryCount > 3) { @@ -382,9 +385,8 @@ public void testRecoverWhenDeleteFromCursor() throws Exception { mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); - + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); @@ -401,8 +403,8 @@ public void testRecoverWhenDeleteFromCursor() throws Exception { mlTransactionLog.initialize().join(); transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); } @@ -423,8 +425,8 @@ public void testManageLedgerWriteFailState() throws Exception { mlTransactionLog.initialize().join(); MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, - new TransactionTimeoutTrackerImpl(), new TransactionRecoverTrackerImpl(), - mlTransactionSequenceIdGenerator); + new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator); + transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); transactionMetadataStore.newTransaction(5000).get(); From 1ee4e29e0c74cadb00419de1bf4b886234331f14 Mon Sep 17 00:00:00 2001 From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com> Date: Sat, 2 Jul 2022 10:39:41 +0800 Subject: [PATCH 628/823] [branch-2.9]add message ack rate (#16146) --- .../pulsar/broker/service/Consumer.java | 39 ++++-- .../pulsar/broker/service/ServerCnx.java | 1 + .../pulsar/broker/service/StreamingStats.java | 1 + .../NonPersistentSubscription.java | 1 + .../nonpersistent/NonPersistentTopic.java | 3 + .../persistent/PersistentSubscription.java | 1 + .../service/persistent/PersistentTopic.java | 3 + .../prometheus/AggregatedConsumerStats.java | 2 + .../prometheus/AggregatedNamespaceStats.java | 2 + .../AggregatedSubscriptionStats.java | 2 + .../prometheus/NamespaceStatsAggregator.java | 2 + .../broker/stats/prometheus/TopicStats.java | 5 + .../broker/stats/ConsumerStatsTest.java | 123 +++++++++++++++++- .../common/policies/data/ConsumerStats.java | 5 + .../policies/data/SubscriptionStats.java | 5 + .../data/stats/ConsumerStatsImpl.java | 6 + .../data/stats/SubscriptionStatsImpl.java | 5 + pulsar-common/src/main/proto/PulsarApi.proto | 3 + 18 files changed, 194 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 318f19c51a544..52a91c5ef3e70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -83,6 +83,7 @@ public class Consumer { private final Rate msgRedeliver; private final LongAdder msgOutCounter; private final LongAdder bytesOutCounter; + private final Rate messageAckRate; private long lastConsumedTimestamp; private long lastAckedTimestamp; @@ -153,6 +154,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.msgRedeliver = new Rate(); this.bytesOutCounter = new LongAdder(); this.msgOutCounter = new LongAdder(); + this.messageAckRate = new Rate(); this.appId = appId; // Ensure we start from compacted view @@ -348,6 +350,8 @@ public void doUnsubscribe(final long requestId) { } public CompletableFuture messageAcked(CommandAck ack) { + CompletableFuture future; + this.lastAckedTimestamp = System.currentTimeMillis(); Map properties = Collections.emptyMap(); if (ack.getPropertiesCount() > 0) { @@ -381,25 +385,32 @@ public CompletableFuture messageAcked(CommandAck ack) { } if (ack.hasTxnidMostBits() && ack.hasTxnidLeastBits()) { List positionsAcked = Collections.singletonList(position); - return transactionCumulativeAcknowledge(ack.getTxnidMostBits(), - ack.getTxnidLeastBits(), positionsAcked); + future = transactionCumulativeAcknowledge(ack.getTxnidMostBits(), + ack.getTxnidLeastBits(), positionsAcked) + .thenApply(v -> 1L); } else { List positionsAcked = Collections.singletonList(position); subscription.acknowledgeMessage(positionsAcked, AckType.Cumulative, properties); - return CompletableFuture.completedFuture(null); + future = CompletableFuture.completedFuture(1L); } } else { if (ack.hasTxnidLeastBits() && ack.hasTxnidMostBits()) { - return individualAckWithTransaction(ack); + future = individualAckWithTransaction(ack); } else { - return individualAckNormal(ack, properties); + future = individualAckNormal(ack, properties); } } + + return future.thenApply(v -> { + this.messageAckRate.recordEvent(v); + return null; + }); } //this method is for individual ack not carry the transaction - private CompletableFuture individualAckNormal(CommandAck ack, Map properties) { + private CompletableFuture individualAckNormal(CommandAck ack, Map properties) { List positionsAcked = new ArrayList<>(); + long totalAckCount = 0; for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); PositionImpl position; @@ -432,10 +443,12 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map completableFuture = new CompletableFuture<>(); - completableFuture.complete(null); + CompletableFuture completableFuture = new CompletableFuture<>(); + completableFuture.complete(totalAckCount); if (isTransactionEnabled() && Subscription.isIndividualAckMode(subType)) { completableFuture.whenComplete((v, e) -> positionsAcked.forEach(position -> { //check if the position can remove from the consumer pending acks. @@ -453,7 +466,7 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map individualAckWithTransaction(CommandAck ack) { + private CompletableFuture individualAckWithTransaction(CommandAck ack) { // Individual ack List> positionsAcked = new ArrayList<>(); if (!isTransactionEnabled()) { @@ -461,6 +474,7 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { new BrokerServiceException.NotAllowedException("Server don't support transaction ack!")); } + LongAdder totalAckCount = new LongAdder(); for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); PositionImpl position; @@ -489,6 +503,8 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { checkCanRemovePendingAcksAndHandle(position, msgId); checkAckValidationError(ack, position); + + totalAckCount.add(ackedCount); } CompletableFuture completableFuture = transactionIndividualAcknowledge(ack.getTxnidMostBits(), @@ -504,7 +520,7 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { } })); } - return completableFuture; + return completableFuture.thenApply(v -> totalAckCount.sum()); } private long getBatchSize(MessageIdData msgId) { @@ -724,8 +740,11 @@ public void updateRates() { msgOut.calculateRate(); chunkedMessageRate.calculateRate(); msgRedeliver.calculateRate(); + messageAckRate.calculateRate(); + stats.msgRateOut = msgOut.getRate(); stats.msgThroughputOut = msgOut.getValueRate(); + stats.messageAckRate = messageAckRate.getValueRate(); stats.msgRateRedeliver = msgRedeliver.getRate(); stats.chunkedMessageRate = chunkedMessageRate.getRate(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 8b56b3a4b3dbd..16cab0a13a917 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -607,6 +607,7 @@ ByteBuf createConsumerStatsResponse(Consumer consumer, long requestId) { .setConnectedSince(consumerStats.getConnectedSince()) .setMsgBacklog(subscription.getNumberOfEntriesInBacklog(false)) .setMsgRateExpired(subscription.getExpiredMessageRate()) + .setMessageAckRate(consumerStats.messageAckRate) .setType(subscription.getTypeString()); return Commands.serializeWithSize(cmd); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StreamingStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StreamingStats.java index 02dcb8233ffae..469c802b76a2c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StreamingStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StreamingStats.java @@ -65,6 +65,7 @@ public static void writeConsumerStats(StatsOutputStream statsStream, CommandSubs statsStream.writePair("msgThroughputOut", stats.msgThroughputOut); statsStream.writePair("msgRateRedeliver", stats.msgRateRedeliver); statsStream.writePair("avgMessagesPerEntry", stats.avgMessagesPerEntry); + statsStream.writePair("messageAckRate", stats.messageAckRate); if (Subscription.isIndividualAckMode(subType)) { statsStream.writePair("unackedMessages", stats.unackedMessages); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index 370402e509df6..749cade53d607 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -451,6 +451,7 @@ public NonPersistentSubscriptionStatsImpl getStats() { ConsumerStatsImpl consumerStats = consumer.getStats(); subStats.consumers.add(consumerStats); subStats.msgRateOut += consumerStats.msgRateOut; + subStats.messageAckRate += consumerStats.messageAckRate; subStats.msgThroughputOut += consumerStats.msgThroughputOut; subStats.bytesOutCounter += consumerStats.bytesOutCounter; subStats.msgOutCounter += consumerStats.msgOutCounter; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 68740d939e261..0c2823a131a8f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -704,6 +704,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats double subMsgRateOut = 0; double subMsgThroughputOut = 0; double subMsgRateRedeliver = 0; + double subMsgAckRate = 0; // Start subscription name & consumers try { @@ -720,6 +721,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats subMsgRateOut += consumerStats.msgRateOut; subMsgThroughputOut += consumerStats.msgThroughputOut; subMsgRateRedeliver += consumerStats.msgRateRedeliver; + subMsgAckRate += consumerStats.messageAckRate; // Populate consumer specific stats here StreamingStats.writeConsumerStats(topicStatsStream, subscription.getType(), consumerStats); @@ -732,6 +734,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog(false)); topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate()); topicStatsStream.writePair("msgRateOut", subMsgRateOut); + topicStatsStream.writePair("messageAckRate", subMsgAckRate); topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut); topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver); topicStatsStream.writePair("type", subscription.getTypeString()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 2c8c828df94a5..93ad2e80a642c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -954,6 +954,7 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri subStats.bytesOutCounter += consumerStats.bytesOutCounter; subStats.msgOutCounter += consumerStats.msgOutCounter; subStats.msgRateRedeliver += consumerStats.msgRateRedeliver; + subStats.messageAckRate += consumerStats.messageAckRate; subStats.chunkedMessageRate += consumerStats.chunkedMessageRate; subStats.unackedMessages += consumerStats.unackedMessages; subStats.lastConsumedTimestamp = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 7a3d4c82fa2bb..0933df87d871a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1791,6 +1791,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats double subMsgRateOut = 0; double subMsgThroughputOut = 0; double subMsgRateRedeliver = 0; + double subMsgAckRate = 0; // Start subscription name & consumers try { @@ -1804,6 +1805,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats ConsumerStatsImpl consumerStats = consumer.getStats(); subMsgRateOut += consumerStats.msgRateOut; + subMsgAckRate += consumerStats.messageAckRate; subMsgThroughputOut += consumerStats.msgThroughputOut; subMsgRateRedeliver += consumerStats.msgRateRedeliver; @@ -1818,6 +1820,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats subscription.getNumberOfEntriesInBacklog(true)); topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate()); topicStatsStream.writePair("msgRateOut", subMsgRateOut); + topicStatsStream.writePair("messageAckRate", subMsgAckRate); topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut); topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver); topicStatsStream.writePair("numberOfEntriesSinceFirstNotAckedMessage", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedConsumerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedConsumerStats.java index 8b6bf7d5c9691..0a4bd317df5de 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedConsumerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedConsumerStats.java @@ -28,6 +28,8 @@ public class AggregatedConsumerStats { public double msgRateOut; + public double msgAckRate; + public double msgThroughputOut; public long availablePermits; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 8bacd0f582de2..5610dbab218e0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -33,6 +33,7 @@ public class AggregatedNamespaceStats { public double throughputIn; public double throughputOut; + public long messageAckRate; public long bytesInCounter; public long msgInCounter; public long bytesOutCounter; @@ -122,6 +123,7 @@ void updateStats(TopicStats stats) { consumerStats.blockedSubscriptionOnUnackedMsgs = v.blockedSubscriptionOnUnackedMsgs; consumerStats.msgRateRedeliver += v.msgRateRedeliver; consumerStats.unackedMessages += v.unackedMessages; + messageAckRate += v.msgAckRate; }); }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java index fb74daf419f59..ffb2f237e03ba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java @@ -36,6 +36,8 @@ public class AggregatedSubscriptionStats { public double msgRateOut; + public double messageAckRate; + public double msgThroughputOut; public long msgDelayed; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 2025c59c1a1f7..0212c3e964a29 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -221,6 +221,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include consumerStats.unackedMessages = conStats.unackedMessages; consumerStats.msgRateRedeliver = conStats.msgRateRedeliver; consumerStats.msgRateOut = conStats.msgRateOut; + consumerStats.msgAckRate = conStats.messageAckRate; consumerStats.msgThroughputOut = conStats.msgThroughputOut; consumerStats.bytesOutCounter = conStats.bytesOutCounter; consumerStats.msgOutCounter = conStats.msgOutCounter; @@ -305,6 +306,7 @@ private static void printNamespaceStats(SimpleTextOutputStream stream, String cl metric(stream, cluster, namespace, "pulsar_rate_out", stats.rateOut); metric(stream, cluster, namespace, "pulsar_throughput_in", stats.throughputIn); metric(stream, cluster, namespace, "pulsar_throughput_out", stats.throughputOut); + metric(stream, cluster, namespace, "pulsar_consumer_msg_ack_rate", stats.messageAckRate); metric(stream, cluster, namespace, "pulsar_in_bytes_total", stats.bytesInCounter); metric(stream, cluster, namespace, "pulsar_in_messages_total", stats.msgInCounter); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index 599c3d82f6fc5..aa46f420acd3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -247,6 +247,8 @@ static void printTopicStats(SimpleTextOutputStream stream, String cluster, Strin subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_out", subsStats.msgRateOut, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_ack_rate", + subsStats.messageAckRate, splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_throughput_out", subsStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, n, "pulsar_out_bytes_total", @@ -281,6 +283,9 @@ static void printTopicStats(SimpleTextOutputStream stream, String cluster, Strin metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, + splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java index 2f702ab89f645..af19e889e9b4c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java @@ -20,14 +20,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.ProducerConsumerBase; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.*; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.policies.data.ConsumerStats; @@ -38,10 +40,15 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.UUID; @Slf4j @Test(groups = "broker") @@ -182,6 +189,7 @@ public void testConsumerStatsOutput() throws Exception { "msgRateOut", "msgThroughputOut", "bytesOutCounter", + "messageAckRate", "msgOutCounter", "msgRateRedeliver", "chunkedMessageRate", @@ -220,4 +228,109 @@ public void testConsumerStatsOutput() throws Exception { consumer.close(); } + + @Test + public void testPersistentTopicMessageAckRateMetricTopicLevel() throws Exception { + String topicName = "persistent://public/default/msg_ack_rate" + UUID.randomUUID(); + testMessageAckRateMetric(topicName, true); + } + + @Test + public void testPersistentTopicMessageAckRateMetricNamespaceLevel() throws Exception { + String topicName = "persistent://public/default/msg_ack_rate" + UUID.randomUUID(); + testMessageAckRateMetric(topicName, false); + } + + private void testMessageAckRateMetric(String topicName, boolean exposeTopicLevelMetrics) + throws Exception { + final int messages = 1000; + String subName = "test_sub"; + CountDownLatch latch = new CountDownLatch(messages); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName) + .enableBatching(true).batchingMaxMessages(10).create(); + + MessageListener listener = (consumer, msg) -> { + try { + consumer.acknowledge(msg); + latch.countDown(); + } catch (PulsarClientException e) { + //ignore + } + }; + @Cleanup + Consumer c1 = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .messageListener(listener) + .subscribe(); + @Cleanup + Consumer c2 = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .messageListener(listener) + .subscribe(); + + String namespace = TopicName.get(topicName).getNamespace(); + + for (int i = 0; i < messages; i++) { + producer.sendAsync(UUID.randomUUID().toString()); + } + producer.flush(); + + latch.await(20, TimeUnit.SECONDS); + TimeUnit.SECONDS.sleep(1); + + Topic topic = pulsar.getBrokerService().getTopic(topicName, false).get().get(); + Subscription subscription = topic.getSubscription(subName); + List consumers = subscription.getConsumers(); + Assert.assertEquals(consumers.size(), 2); + org.apache.pulsar.broker.service.Consumer consumer1 = consumers.get(0); + org.apache.pulsar.broker.service.Consumer consumer2 = consumers.get(1); + consumer1.updateRates(); + consumer2.updateRates(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, exposeTopicLevelMetrics, true, true, output); + String metricStr = output.toString(); + + Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricStr); + Collection ackRateMetric = metricsMap.get("pulsar_consumer_msg_ack_rate"); + + String rateOutMetricName = exposeTopicLevelMetrics ? "pulsar_consumer_msg_rate_out" : "pulsar_rate_out"; + Collection rateOutMetric = metricsMap.get(rateOutMetricName); + Assert.assertTrue(ackRateMetric.size() > 0); + Assert.assertTrue(rateOutMetric.size() > 0); + + if (exposeTopicLevelMetrics) { + String consumer1Name = consumer1.consumerName(); + String consumer2Name = consumer2.consumerName(); + double totalAckRate = ackRateMetric.stream() + .filter(metric -> metric.tags.get("consumer_name").equals(consumer1Name) + || metric.tags.get("consumer_name").equals(consumer2Name)) + .mapToDouble(metric -> metric.value).sum(); + double totalRateOut = rateOutMetric.stream() + .filter(metric -> metric.tags.get("consumer_name").equals(consumer1Name) + || metric.tags.get("consumer_name").equals(consumer2Name)) + .mapToDouble(metric -> metric.value).sum(); + + Assert.assertTrue(totalAckRate > 0D); + Assert.assertTrue(totalRateOut > 0D); + Assert.assertEquals(totalAckRate, totalRateOut, totalRateOut * 0.1D); + } else { + double totalAckRate = ackRateMetric.stream() + .filter(metric -> namespace.equals(metric.tags.get("namespace"))) + .mapToDouble(metric -> metric.value).sum(); + double totalRateOut = rateOutMetric.stream() + .filter(metric -> namespace.equals(metric.tags.get("namespace"))) + .mapToDouble(metric -> metric.value).sum(); + + Assert.assertTrue(totalAckRate > 0D); + Assert.assertTrue(totalRateOut > 0D); + Assert.assertEquals(totalAckRate, totalRateOut, totalRateOut * 0.1D); + } + } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java index 7204af616e2d8..cc421d276c150 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java @@ -40,6 +40,11 @@ public interface ConsumerStats { /** Total rate of messages redelivered by this consumer (msg/s). */ double getMsgRateRedeliver(); + /** + * Total rate of message ack(msg/s). + */ + double getMessageAckRate(); + /** Total chunked messages dispatched. */ double getChunkedMessageRate(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java index 2ce38aafb34d4..1649c6efefe79 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java @@ -43,6 +43,11 @@ public interface SubscriptionStats { /** Chunked message dispatch rate. */ int getChunkedMessageRate(); + /** + * Total rate of message ack(msg/s). + */ + double getMessageAckRate(); + /** Number of messages in the subscription backlog. */ long getMsgBacklog(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java index 47f614889ca10..e4fd0958a34cc 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java @@ -45,6 +45,11 @@ public class ConsumerStatsImpl implements ConsumerStats { /** Total rate of messages redelivered by this consumer (msg/s). */ public double msgRateRedeliver; + /** + * Total rate of message ack(msg/s). + */ + public double messageAckRate; + /** Total chunked messages dispatched. */ public double chunkedMessageRate; @@ -109,6 +114,7 @@ public ConsumerStatsImpl add(ConsumerStatsImpl stats) { this.msgRateRedeliver += stats.msgRateRedeliver; this.availablePermits += stats.availablePermits; this.unackedMessages += stats.unackedMessages; + this.messageAckRate += stats.messageAckRate; this.blockedConsumerOnUnackedMsgs = stats.blockedConsumerOnUnackedMsgs; this.readPositionWhenJoining = stats.readPositionWhenJoining; return this; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index 78781ac32b46b..2a124b9e574fb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -46,6 +46,11 @@ public class SubscriptionStatsImpl implements SubscriptionStats { /** Total rate of messages redelivered on this subscription (msg/s). */ public double msgRateRedeliver; + /** + * Total rate of message ack(msg/s). + */ + public double messageAckRate; + /** Chunked message dispatch rate. */ public int chunkedMessageRate; diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 34ad37b4ee2f6..ec06eae26f645 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -699,6 +699,9 @@ message CommandConsumerStatsResponse { /// Number of messages in the subscription backlog optional uint64 msgBacklog = 15; + + /// Total rate of messages ack. msg/s + optional double messageAckRate = 16; } message CommandGetLastMessageId { From 2bedbab0e6a43c893bf64ad1a7ac6988056c340c Mon Sep 17 00:00:00 2001 From: mattison chao Date: Sat, 2 Jul 2022 10:47:37 +0800 Subject: [PATCH 629/823] Revert "[improve][broker] Optimise msgOutCounter and bytesOutCounter (#16214) (#16286)" This reverts commit 6f12f5ed2ec5ca3c7da74cad02d9d01e1f7d1e66. --- .../broker/service/AbstractSubscription.java | 42 ---------- .../pulsar/broker/service/AbstractTopic.java | 17 +---- .../pulsar/broker/service/Consumer.java | 8 -- .../NonPersistentSubscription.java | 6 +- .../nonpersistent/NonPersistentTopic.java | 4 + .../persistent/PersistentSubscription.java | 7 +- .../service/persistent/PersistentTopic.java | 4 + .../service/AbstractSubscriptionTest.java | 57 -------------- .../broker/service/AbstractTopicTest.java | 75 ------------------ .../pulsar/broker/service/ConsumerTest.java | 76 ------------------- 10 files changed, 19 insertions(+), 277 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java deleted file mode 100644 index 6a38667055679..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service; - -import java.util.Optional; -import java.util.concurrent.atomic.LongAdder; -import java.util.function.ToLongFunction; - -public abstract class AbstractSubscription implements Subscription { - protected final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); - protected final LongAdder msgOutFromRemovedConsumer = new LongAdder(); - - public long getMsgOutCounter() { - return msgOutFromRemovedConsumer.longValue() + sumConsumers(Consumer::getMsgOutCounter); - } - - public long getBytesOutCounter() { - return bytesOutFromRemovedConsumers.longValue() + sumConsumers(Consumer::getBytesOutCounter); - } - - private long sumConsumers(ToLongFunction toCounter) { - return Optional.ofNullable(getDispatcher()) - .map(dispatcher -> dispatcher.getConsumers().stream().mapToLong(toCounter).sum()) - .orElse(0L); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 1307001a72068..69c51c77d44a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.ToLongFunction; import lombok.Getter; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.lang3.tuple.Pair; @@ -137,9 +136,6 @@ public abstract class AbstractTopic implements Topic { private volatile long lastTopicMaxMessageSizeCheckTimeStamp = 0; private final long topicMaxMessageSizeCheckIntervalMs; - protected final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); - protected final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); - public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; this.brokerService = brokerService; @@ -879,20 +875,11 @@ public long getBytesInCounter() { } public long getMsgOutCounter() { - return msgOutFromRemovedSubscriptions.longValue() - + sumSubscriptions(AbstractSubscription::getMsgOutCounter); + return getStats(false, false).msgOutCounter; } public long getBytesOutCounter() { - return bytesOutFromRemovedSubscriptions.longValue() - + sumSubscriptions(AbstractSubscription::getBytesOutCounter); - } - - private long sumSubscriptions(ToLongFunction toCounter) { - return getSubscriptions().values().stream() - .map(AbstractSubscription.class::cast) - .mapToLong(toCounter) - .sum(); + return getStats(false, false).bytesOutCounter; } public boolean isDeleteWhileInactive() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 52a91c5ef3e70..2661d29110c75 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -780,14 +780,6 @@ public ConsumerStatsImpl getStats() { return stats; } - public long getMsgOutCounter() { - return msgOutCounter.longValue(); - } - - public long getBytesOutCounter() { - return bytesOutCounter.longValue(); - } - public int getUnackedMessages() { return unackedMessages; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index 749cade53d607..cb515cdbd9724 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -29,7 +29,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.intercept.BrokerInterceptor; -import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; @@ -49,7 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NonPersistentSubscription extends AbstractSubscription implements Subscription { +public class NonPersistentSubscription implements Subscription { private final NonPersistentTopic topic; private volatile NonPersistentDispatcher dispatcher; private final String topicName; @@ -67,6 +66,9 @@ public class NonPersistentSubscription extends AbstractSubscription implements S // Timestamp of when this subscription was last seen active private volatile long lastActive; + private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); + private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); + // If isDurable is false(such as a Reader), remove subscription from topic when closing this subscription. private final boolean isDurable; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 0c2823a131a8f..3846fc129b091 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -34,6 +34,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; @@ -101,6 +102,9 @@ public class NonPersistentTopic extends AbstractTopic implements Topic { AtomicLongFieldUpdater.newUpdater(NonPersistentTopic.class, "entriesAddedCounter"); private volatile long entriesAddedCounter = 0; + private final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); + private final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); + private static final FastThreadLocal threadLocalTopicStats = new FastThreadLocal() { @Override protected TopicStats initialValue() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 93ad2e80a642c..b119f3ca5d2a3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -29,6 +29,7 @@ import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.LongAdder; import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; @@ -49,7 +50,6 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; -import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; @@ -80,7 +80,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PersistentSubscription extends AbstractSubscription implements Subscription { +public class PersistentSubscription implements Subscription { protected final PersistentTopic topic; protected final ManagedCursor cursor; protected volatile Dispatcher dispatcher; @@ -113,6 +113,9 @@ public class PersistentSubscription extends AbstractSubscription implements Subs private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; private final PendingAckHandle pendingAckHandle; + private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); + private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); + static { REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 0933df87d871a..129333581bae3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -44,6 +44,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; import java.util.function.BiFunction; import java.util.stream.Collectors; import lombok.Getter; @@ -221,6 +222,9 @@ protected TopicStatsHelper initialValue() { @Getter protected final TransactionBuffer transactionBuffer; + private final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); + private final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); + // Record the last time a data message (ie: not an internal Pulsar marker) is published on the topic private long lastDataMessagePublishedTimestamp = 0; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java deleted file mode 100644 index aaf1f6164a374..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import java.util.List; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Test(groups = "broker") -public class AbstractSubscriptionTest { - private Consumer consumer; - private AbstractSubscription subscription; - - @BeforeMethod - public void beforeMethod() { - Dispatcher dispatcher = mock(Dispatcher.class); - consumer = mock(Consumer.class); - subscription = spy(AbstractSubscription.class); - - when(subscription.getDispatcher()).thenReturn(dispatcher); - when(dispatcher.getConsumers()).thenReturn(List.of(consumer)); - } - - @Test - public void testGetMsgOutCounter() { - subscription.msgOutFromRemovedConsumer.add(1L); - when(consumer.getMsgOutCounter()).thenReturn(2L); - assertEquals(subscription.getMsgOutCounter(), 3L); - } - - @Test - public void testGetBytesOutCounter() { - subscription.bytesOutFromRemovedConsumers.add(1L); - when(consumer.getBytesOutCounter()).thenReturn(2L); - assertEquals(subscription.getBytesOutCounter(), 3L); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java deleted file mode 100644 index fb7890dc57f88..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service; - -import static org.mockito.Mockito.CALLS_REAL_METHODS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.withSettings; -import static org.testng.Assert.assertEquals; -import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Test(groups = "broker") -public class AbstractTopicTest { - private AbstractSubscription subscription; - private AbstractTopic topic; - - @BeforeMethod - public void beforeMethod() { - BrokerService brokerService = mock(BrokerService.class); - PulsarService pulsarService = mock(PulsarService.class); - ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); - BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); - subscription = mock(AbstractSubscription.class); - - when(brokerService.pulsar()).thenReturn(pulsarService); - when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); - when(brokerService.getBacklogQuotaManager()).thenReturn(backlogQuotaManager); - - topic = mock(AbstractTopic.class, withSettings() - .useConstructor("topic", brokerService) - .defaultAnswer(CALLS_REAL_METHODS)); - - ConcurrentOpenHashMap subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - subscriptions.put("subscription", subscription); - when(topic.getSubscriptions()).thenAnswer(invocation -> subscriptions); - } - - @Test - public void testGetMsgOutCounter() { - topic.msgOutFromRemovedSubscriptions.add(1L); - when(subscription.getMsgOutCounter()).thenReturn(2L); - assertEquals(topic.getMsgOutCounter(), 3L); - } - - @Test - public void testGetBytesOutCounter() { - topic.bytesOutFromRemovedSubscriptions.add(1L); - when(subscription.getBytesOutCounter()).thenReturn(2L); - assertEquals(topic.getBytesOutCounter(), 3L); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java deleted file mode 100644 index 1ad0642d9c5ba..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service; - -import static java.util.Collections.emptyMap; -import static org.apache.pulsar.client.api.MessageId.latest; -import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive; -import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT; -import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import java.net.SocketAddress; -import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.api.proto.KeySharedMeta; -import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Test(groups = "broker") -public class ConsumerTest { - private Consumer consumer; - private final ConsumerStatsImpl stats = new ConsumerStatsImpl(); - - @BeforeMethod - public void beforeMethod() { - Subscription subscription = mock(Subscription.class); - ServerCnx cnx = mock(ServerCnx.class); - SocketAddress address = mock(SocketAddress.class); - Topic topic = mock(Topic.class); - BrokerService brokerService = mock(BrokerService.class); - PulsarService pulsarService = mock(PulsarService.class); - ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); - - when(cnx.clientAddress()).thenReturn(address); - when(subscription.getTopic()).thenReturn(topic); - when(topic.getBrokerService()).thenReturn(brokerService); - when(brokerService.getPulsar()).thenReturn(pulsarService); - when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); - - consumer = - new Consumer(subscription, Exclusive, "topic", 1, 0, "Cons1", true, cnx, "myrole-1", emptyMap(), false, - new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest, DEFAULT_CONSUMER_EPOCH); - } - - @Test - public void testGetMsgOutCounter() { - stats.msgOutCounter = 1L; - consumer.updateStats(stats); - assertEquals(consumer.getMsgOutCounter(), 1L); - } - - @Test - public void testGetBytesOutCounter() { - stats.bytesOutCounter = 1L; - consumer.updateStats(stats); - assertEquals(consumer.getBytesOutCounter(), 1L); - } -} From 68d30e9dda62bfa90f21d024b5da1547b028ad06 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 24 Jun 2022 00:33:59 +0800 Subject: [PATCH 630/823] [fix][client] Add classLoader field for `SchemaDefinition` (#15915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #15899 ### Motivation Now, don‘t register logical type conversions when use `SchemaDefinition.builder().withJsonDef()` beacase it without classLoader param. See: https://github.com/apache/pulsar/blob/04aa9e8e51869d1621a7e25402a656084eebfc09/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java#L58-L68 We can add the classLoader field for `SchemaDefinition`, user can manually pass a classLoader to register logical type conversions ### Modifications Add classLoader field for `SchemaDefinition` (cherry picked from commit 8434500b6879abc9ab74de6e5b75883e8053fd9c) --- .../client/api/schema/SchemaDefinition.java | 7 ++ .../api/schema/SchemaDefinitionBuilder.java | 9 +++ .../pulsar/client/impl/schema/AvroSchema.java | 5 +- .../schema/SchemaDefinitionBuilderImpl.java | 15 +++- .../impl/schema/SchemaDefinitionImpl.java | 14 +++- .../client/impl/schema/AvroSchemaTest.java | 77 ++++++++++++++++++- .../client/impl/schema/SchemaBuilderTest.java | 8 +- 7 files changed, 124 insertions(+), 11 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java index dffa5e421b202..c0d184f48f24c 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java @@ -76,6 +76,13 @@ static SchemaDefinitionBuilder builder() { */ Class getPojo(); + /** + * Get pojo classLoader. + * + * @return pojo schema + */ + ClassLoader getClassLoader(); + /** * Get supportSchemaVersioning schema definition. * diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java index 61d246674a8dd..97d822b927d20 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java @@ -80,6 +80,15 @@ public interface SchemaDefinitionBuilder { */ SchemaDefinitionBuilder withPojo(Class pojo); + /** + * Set schema of pojo classLoader. + * + * @param classLoader pojo classLoader + * + * @return schema definition builder + */ + SchemaDefinitionBuilder withClassLoader(ClassLoader classLoader); + /** * Set schema of json definition. * diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index cff3ccdf8f689..a5dda082e7c45 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -91,9 +91,12 @@ public static AvroSchema of(SchemaDefinition schemaDefinition) { schemaDefinition.getSchemaWriterOpt().get(), parseSchemaInfo(schemaDefinition, SchemaType.AVRO)); } ClassLoader pojoClassLoader = null; - if (schemaDefinition.getPojo() != null) { + if (schemaDefinition.getClassLoader() != null) { + pojoClassLoader = schemaDefinition.getClassLoader(); + } else if (schemaDefinition.getPojo() != null) { pojoClassLoader = schemaDefinition.getPojo().getClassLoader(); } + return new AvroSchema<>(parseSchemaInfo(schemaDefinition, SchemaType.AVRO), pojoClassLoader); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java index d929c019d194f..80e30f0d64cc4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java @@ -42,6 +42,11 @@ public class SchemaDefinitionBuilderImpl implements SchemaDefinitionBuilder clazz; + /** + * the classLoader definition class. + */ + private ClassLoader classLoader; + /** * The flag of schema type always allow null * @@ -102,6 +107,12 @@ public SchemaDefinitionBuilder withPojo(Class clazz) { return this; } + @Override + public SchemaDefinitionBuilder withClassLoader(ClassLoader classLoader) { + this.classLoader = classLoader; + return this; + } + @Override public SchemaDefinitionBuilder withJsonDef(String jsonDef) { this.jsonDef = jsonDef; @@ -151,8 +162,8 @@ public SchemaDefinition build() { properties.put(ALWAYS_ALLOW_NULL, String.valueOf(this.alwaysAllowNull)); properties.put(JSR310_CONVERSION_ENABLED, String.valueOf(this.jsr310ConversionEnabled)); - return new SchemaDefinitionImpl(clazz, jsonDef, alwaysAllowNull, properties, supportSchemaVersioning, - jsr310ConversionEnabled, reader, writer); + return new SchemaDefinitionImpl(clazz, jsonDef, classLoader, + alwaysAllowNull, properties, supportSchemaVersioning, jsr310ConversionEnabled, reader, writer); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java index 9b026e4dac7d5..1842ab50558b8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java @@ -51,6 +51,8 @@ public class SchemaDefinitionImpl implements SchemaDefinition { private final String jsonDef; + private final ClassLoader classLoader; + private final boolean supportSchemaVersioning; private final boolean jsr310ConversionEnabled; @@ -59,12 +61,15 @@ public class SchemaDefinitionImpl implements SchemaDefinition { private final SchemaWriter writer; - public SchemaDefinitionImpl(Class pojo, String jsonDef, boolean alwaysAllowNull, Map properties, - boolean supportSchemaVersioning, boolean jsr310ConversionEnabled, SchemaReader reader, SchemaWriter writer) { + public SchemaDefinitionImpl(Class pojo, String jsonDef, ClassLoader classLoader, + boolean alwaysAllowNull, Map properties, + boolean supportSchemaVersioning, boolean jsr310ConversionEnabled, + SchemaReader reader, SchemaWriter writer) { this.alwaysAllowNull = alwaysAllowNull; this.properties = properties; this.jsonDef = jsonDef; this.pojo = pojo; + this.classLoader = classLoader; this.supportSchemaVersioning = supportSchemaVersioning; this.jsr310ConversionEnabled = jsr310ConversionEnabled; this.reader = reader; @@ -104,6 +109,11 @@ public Class getPojo() { return pojo; } + @Override + public ClassLoader getClassLoader() { + return this.classLoader; + } + @Override public boolean getSupportSchemaVersioning() { return supportSchemaVersioning; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index d69f8bf66ba7a..2a5040d7815d3 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -36,7 +36,9 @@ import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.UUID; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.SchemaValidationException; @@ -463,21 +465,88 @@ public void testAvroBigDecimal() { @Data - private static class TimestampStruct { + @AllArgsConstructor + @NoArgsConstructor + private static class TimestampPojo { Instant value; } @Test public void testTimestampWithJsr310Conversion() { - AvroSchema schema = AvroSchema.of(TimestampStruct.class); + AvroSchema schema = AvroSchema.of(TimestampPojo.class); Assert.assertEquals( schema.getAvroSchema().getFields().get(0).schema().getTypes().get(1).getLogicalType().getName(), new TimeConversions.TimestampMicrosConversion().getLogicalTypeName()); - AvroSchema schema2 = AvroSchema.of(SchemaDefinition.builder() - .withPojo(TimestampStruct.class).withJSR310ConversionEnabled(true).build()); + AvroSchema schema2 = AvroSchema.of(SchemaDefinition.builder() + .withPojo(TimestampPojo.class).withJSR310ConversionEnabled(true).build()); Assert.assertEquals( schema2.getAvroSchema().getFields().get(0).schema().getTypes().get(1).getLogicalType().getName(), new TimeConversions.TimestampMillisConversion().getLogicalTypeName()); } + + @Test + public void testTimestampWithJsonDef(){ + AvroSchema schemaWithPojo = AvroSchema.of(SchemaDefinition.builder() + .withPojo(TimestampPojo.class) + .withJSR310ConversionEnabled(false).build()); + + TimestampPojo timestampPojo = new TimestampPojo(Instant.parse("2022-06-10T12:38:59.039084Z")); + byte[] encode = schemaWithPojo.encode(timestampPojo); + TimestampPojo decodeWithPojo = schemaWithPojo.decode(encode); + + Assert.assertEquals(decodeWithPojo, timestampPojo); + + String schemaDefinition = new String(schemaWithPojo.schemaInfo.getSchema()); + AvroSchema schemaWithJsonDef = AvroSchema.of(SchemaDefinition.builder() + .withJsonDef(schemaDefinition) + .withClassLoader(TimestampPojo.class.getClassLoader()) + .withJSR310ConversionEnabled(false).build()); + + TimestampPojo decodeWithJson = schemaWithJsonDef.decode(encode); + + Assert.assertEquals(decodeWithJson, decodeWithPojo); + Assert.assertEquals(Instant.class, decodeWithJson.getValue().getClass()); + + AvroSchema schemaWithJsonDefNoClassLoader = AvroSchema.of(SchemaDefinition.builder() + .withJsonDef(schemaDefinition) + .withJSR310ConversionEnabled(false).build()); + + TimestampPojo decodeWithJsonNoClassLoader = schemaWithJsonDefNoClassLoader.decode(encode); + Assert.assertNotEquals(decodeWithJsonNoClassLoader, decodeWithPojo); + Assert.assertNotEquals(Instant.class, decodeWithJsonNoClassLoader.getValue().getClass()); + } + + @Test + public void testTimestampWithJsonDefAndJSR310ConversionEnabled(){ + AvroSchema schemaWithPojo = AvroSchema.of(SchemaDefinition.builder() + .withPojo(TimestampPojo.class) + .withJSR310ConversionEnabled(true).build()); + + TimestampPojo timestampPojo = new TimestampPojo(Instant.parse("2022-06-10T12:38:59.039084Z")); + byte[] encode = schemaWithPojo.encode(timestampPojo); + TimestampPojo decodeWithPojo = schemaWithPojo.decode(encode); + + Assert.assertNotEquals(decodeWithPojo, timestampPojo); + + String schemaDefinition = new String(schemaWithPojo.schemaInfo.getSchema()); + AvroSchema schemaWithJsonDef = AvroSchema.of(SchemaDefinition.builder() + .withJsonDef(schemaDefinition) + .withClassLoader(TimestampPojo.class.getClassLoader()) + .withJSR310ConversionEnabled(true).build()); + + TimestampPojo decodeWithJson = schemaWithJsonDef.decode(encode); + + Assert.assertEquals(decodeWithJson, decodeWithPojo); + Assert.assertEquals(Instant.class, decodeWithJson.getValue().getClass()); + + AvroSchema schemaWithJsonDefNoClassLoader = AvroSchema.of(SchemaDefinition.builder() + .withJsonDef(schemaDefinition) + .withJSR310ConversionEnabled(true).build()); + + TimestampPojo decodeWithJsonNoClassLoader = schemaWithJsonDefNoClassLoader.decode(encode); + Assert.assertNotEquals(decodeWithJsonNoClassLoader, decodeWithPojo); + Assert.assertNotEquals(Instant.class, decodeWithJsonNoClassLoader.getValue().getClass()); + } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java index fa88e144a317f..a1530864c9269 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java @@ -20,11 +20,15 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.assertEquals; - import lombok.Data; import org.apache.avro.reflect.Nullable; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.*; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.GenericRecordBuilder; +import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; +import org.apache.pulsar.client.api.schema.SchemaBuilder; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.reader.MultiVersionAvroReader; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; From e72a51d98de919f8f0f8f6851e8daa6b77a974ce Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Fri, 24 Jun 2022 15:45:26 +0200 Subject: [PATCH 631/823] [fix][tests] TieredStorageConfigurationTests - clear system properties (#15957) (cherry picked from commit bacc9d69c66777879a6418e3d61c546150a5e753) --- .../TieredStorageConfigurationTests.java | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java index bf5e046bf70ab..8370fb9580496 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/TieredStorageConfigurationTests.java @@ -129,19 +129,21 @@ public final void awsS3CredsProviderTest() { // set the aws properties with fake creds so the defaultProviderChain works System.setProperty("aws.accessKeyId", "fakeid1"); System.setProperty("aws.secretKey", "fakekey1"); - Credentials creds1 = config.getProviderCredentials().get(); - assertEquals(creds1.identity, "fakeid1"); - assertEquals(creds1.credential, "fakekey1"); + try { + Credentials creds1 = config.getProviderCredentials().get(); + assertEquals(creds1.identity, "fakeid1"); + assertEquals(creds1.credential, "fakekey1"); - // reset the properties and ensure we get different values by re-evaluating the chain - System.setProperty("aws.accessKeyId", "fakeid2"); - System.setProperty("aws.secretKey", "fakekey2"); - Credentials creds2 = config.getProviderCredentials().get(); - assertEquals(creds2.identity, "fakeid2"); - assertEquals(creds2.credential, "fakekey2"); - - System.clearProperty("aws.accessKeyId"); - System.clearProperty("aws.secretKey"); + // reset the properties and ensure we get different values by re-evaluating the chain + System.setProperty("aws.accessKeyId", "fakeid2"); + System.setProperty("aws.secretKey", "fakekey2"); + Credentials creds2 = config.getProviderCredentials().get(); + assertEquals(creds2.identity, "fakeid2"); + assertEquals(creds2.credential, "fakekey2"); + } finally { + System.clearProperty("aws.accessKeyId"); + System.clearProperty("aws.secretKey"); + } } /** @@ -215,11 +217,15 @@ public void overridePropertiesTest() { map.put("s3ManagedLedgerOffloadRegion", "my-region"); System.setProperty("jclouds.SystemPropertyA", "A"); System.setProperty("jclouds.region", "jclouds-region"); - TieredStorageConfiguration config = new TieredStorageConfiguration(map); - Properties properties = config.getOverrides(); - System.out.println(properties.toString()); - assertEquals(properties.get("jclouds.region"), "jclouds-region"); - assertEquals(config.getServiceEndpoint(), "http://localhost"); - assertEquals(properties.get("jclouds.SystemPropertyA"), "A"); + try { + TieredStorageConfiguration config = new TieredStorageConfiguration(map); + Properties properties = config.getOverrides(); + assertEquals(properties.get("jclouds.region"), "jclouds-region"); + assertEquals(config.getServiceEndpoint(), "http://localhost"); + assertEquals(properties.get("jclouds.SystemPropertyA"), "A"); + } finally { + System.clearProperty("jclouds.SystemPropertyA"); + System.clearProperty("jclouds.region"); + } } } From 713181ec28af254848a18a5303f64f277c8de994 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 15 Jun 2022 21:40:25 +0800 Subject: [PATCH 632/823] [fix][broker] Fix create client with TLS config (#16014) ### Motivation In PulsarService, create a client with an incorrect config. When `tlsEnabled` is `true`, and `brokerClientTlsEnabled` is `false`, users will meet `Failed reason: General OpenSslEngine problem`, due to `tlsTrustCertsFilePath` is incorrect. ### Modifications - Fix check TLS enable - Setup ciphers and protocols - Remove duplicate setTlsTrustCertsFilePath (cherry picked from commit 22057ca0296e4eb6e0c9d41bc10e24bdbdc71efc) --- .../java/org/apache/pulsar/broker/PulsarService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 85f21e0bf8f64..b60c8dc484532 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1340,12 +1340,14 @@ public synchronized PulsarClient getClient() throws PulsarServerException { .filterAndMapProperties(this.getConfiguration().getProperties(), "brokerClient_"); ClientConfigurationData conf = ConfigurationDataUtils.loadData(overrides, initialConf, ClientConfigurationData.class); - conf.setServiceUrl(this.getConfiguration().isTlsEnabled() - ? this.brokerServiceUrlTls : this.brokerServiceUrl); - conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); - conf.setTlsTrustCertsFilePath(this.getConfiguration().getTlsCertificateFilePath()); - if (this.getConfiguration().isBrokerClientTlsEnabled()) { + boolean tlsEnabled = this.getConfiguration().isBrokerClientTlsEnabled(); + conf.setServiceUrl(tlsEnabled ? this.brokerServiceUrlTls : this.brokerServiceUrl); + + if (tlsEnabled) { + conf.setTlsCiphers(this.getConfiguration().getBrokerClientTlsCiphers()); + conf.setTlsProtocols(this.getConfiguration().getBrokerClientTlsProtocols()); + conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); if (this.getConfiguration().isBrokerClientTlsEnabledWithKeyStore()) { conf.setUseKeyStoreTls(true); conf.setTlsTrustStoreType(this.getConfiguration().getBrokerClientTlsTrustStoreType()); From 4fb769e0283b539554f7d9d7e3c8d474cef27bfe Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 13 Jun 2022 15:16:39 +0800 Subject: [PATCH 633/823] [fix][client] Remove producer when close producer command is received (#16028) (cherry picked from commit 5ef895af7d8dec851167e56cdf3e8bec11080f8d) --- .../apache/pulsar/client/impl/ClientCnx.java | 5 ++-- .../pulsar/client/impl/ClientCnxTest.java | 24 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 52d10ad996e86..e8aed1abb5564 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -115,7 +115,8 @@ public class ClientCnx extends PulsarHandler { // LookupRequests that waiting in client side. private final Queue>>> waitingLookupRequests; - private final ConcurrentLongHashMap> producers = + @VisibleForTesting + final ConcurrentLongHashMap> producers = ConcurrentLongHashMap.>newBuilder() .expectedItems(16) .concurrencyLevel(1) @@ -726,7 +727,7 @@ protected void handleError(CommandError error) { protected void handleCloseProducer(CommandCloseProducer closeProducer) { log.info("[{}] Broker notification of Closed producer: {}", remoteAddress, closeProducer.getProducerId()); final long producerId = closeProducer.getProducerId(); - ProducerImpl producer = producers.get(producerId); + ProducerImpl producer = producers.remove(producerId); if (producer != null) { producer.connectionClosed(this); } else { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index a3a00b1b70ec8..6ce4afecd02bd 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.PulsarClientException.BrokerMetadataException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.api.proto.CommandCloseConsumer; +import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.api.proto.CommandError; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.protocol.Commands; @@ -156,7 +157,7 @@ public void testGetLastMessageIdWithError() throws Exception { @Test public void testHandleCloseConsumer() { - ThreadFactory threadFactory = new DefaultThreadFactory("testReceiveErrorAtSendConnectFrameState"); + ThreadFactory threadFactory = new DefaultThreadFactory("testHandleCloseConsumer"); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); ClientConfigurationData conf = new ClientConfigurationData(); ClientCnx cnx = new ClientCnx(conf, eventLoop); @@ -165,11 +166,28 @@ public void testHandleCloseConsumer() { cnx.registerConsumer(consumerId, mock(ConsumerImpl.class)); assertEquals(cnx.consumers.size(), 1); - CommandCloseConsumer closeConsumer = new CommandCloseConsumer() - .setConsumerId(1); + CommandCloseConsumer closeConsumer = new CommandCloseConsumer().setConsumerId(consumerId); cnx.handleCloseConsumer(closeConsumer); assertEquals(cnx.consumers.size(), 0); eventLoop.shutdownGracefully(); } + + @Test + public void testHandleCloseProducer() { + ThreadFactory threadFactory = new DefaultThreadFactory("testHandleCloseProducer"); + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); + ClientConfigurationData conf = new ClientConfigurationData(); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + long producerId = 1; + cnx.registerProducer(producerId, mock(ProducerImpl.class)); + assertEquals(cnx.producers.size(), 1); + + CommandCloseProducer closeProducerCmd = new CommandCloseProducer().setProducerId(producerId); + cnx.handleCloseProducer(closeProducerCmd); + assertEquals(cnx.producers.size(), 0); + + eventLoop.shutdownGracefully(); + } } From e3b376bcbf05be081b49338b470754d28f2df3e8 Mon Sep 17 00:00:00 2001 From: Frank Xiong Date: Tue, 14 Jun 2022 02:20:37 +0800 Subject: [PATCH 634/823] rename pulsar_producer_configuration_set_crypto_failure_action to pulsar_producer_configuration_get_crypto_failure_action (#16031) Fixes #16030 ### Motivation Fix symlink error for function pulsar_producer_configuration_get_crypto_failure_action ### Modifications Rename function name `pulsar_producer_configuration_set_crypto_failure_action` to `pulsar_producer_configuration_get_crypto_failure_action` (cherry picked from commit bff34000385f6faf6dbff4385d0dc562602ac623) --- pulsar-client-cpp/lib/c/c_ProducerConfiguration.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-cpp/lib/c/c_ProducerConfiguration.cc b/pulsar-client-cpp/lib/c/c_ProducerConfiguration.cc index f26f63a593b08..906a4d8230c41 100644 --- a/pulsar-client-cpp/lib/c/c_ProducerConfiguration.cc +++ b/pulsar-client-cpp/lib/c/c_ProducerConfiguration.cc @@ -209,7 +209,7 @@ void pulsar_producer_configuration_set_default_crypto_key_reader(pulsar_producer conf->conf.setCryptoKeyReader(keyReader); } -pulsar_producer_crypto_failure_action pulsar_producer_configuration_set_crypto_failure_action( +pulsar_producer_crypto_failure_action pulsar_producer_configuration_get_crypto_failure_action( pulsar_producer_configuration_t *conf) { return (pulsar_producer_crypto_failure_action)conf->conf.getCryptoFailureAction(); } From 65efa3c98ad231cb50c3bc3074aeb47f760f4524 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 14 Jun 2022 17:03:37 +0800 Subject: [PATCH 635/823] [improve][broker] Avoid reconnection when a partitioned topic was created concurrently (#16043) * [improve][broker] Avoid reconnection when a partitioned topic was created concurrently ### Motivation When a partitioned topic was created concurrently, especially when automatically created by many producers. This case can be reproduced easily by configuring `allowAutoTopicCreationType=non-partitioned` and starting a Pulsar standalone. Then, run the following code: ```java try (PulsarClient client = PulsarClient.builder() .serviceUrl("pulsar://localhost:6650").build()) { for (int i = 0; i < 10; i++) { client.newProducer().topic("topic").createAsync(); } Thread.sleep(1000); } ``` We can see a lot of "Could not get connection while getPartitionedTopicMetadata" warning logs at client side, while there were more warning logs with full stack traces at broker side: ``` 2022-06-14T02:04:20,522+0800 [metadata-store-22-1] WARN org.apache.pulsar.broker.service.ServerCnx - Failed to get Partitioned Metadata [/127.0.0.1:64846] persistent://public/default/topic: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /admin/partitioned-topics/public/default/persistent/topic org.apache.pulsar.metadata.api.MetadataStoreException$AlreadyExistsException: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /admin/partitioned-topics/public/default/persistent/topic ``` It's because when broker handles the partitioned metadata command, it calls `fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync` and will try creating a partitioned topic if it doesn't exist. It's a race condition that if many connections are established during a short time interval and one of them created successfully, the following will fail with the `AlreadyExistsException`. ### Modifications Handles the `MetadataStoreException.AlreadyExistsException` in `unsafeGetPartitionedTopicMetadataAsync`. In this case, invoke `fetchPartitionedTopicMetadataAsync` to get the partitioned metadata again. ### Verifying this change Even if without this patch, the creation of producers could also succeed because they will reconnect to broker again after 100 ms because broker will return a `ServiceNotReady` error in thiss case. The only way to verify this fix is reproducing the bug again with this patch, we can see no reconnection will happen from the logs. * Revert "[improve][broker] Avoid reconnection when a partitioned topic was created concurrently" This reverts commit c259c0fdcfb299e6ed861796f7e2ab50632f9087. * Handle AlreadyExistsException in fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync (cherry picked from commit 2a7a8555c0b0296bcaa6a757a8646b8f65185ac6) --- .../pulsar/broker/service/BrokerService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index b81200f737905..24ef20452c035 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2430,7 +2430,20 @@ public CompletableFuture fetchPartitionedTopicMetadata pulsar.getBrokerService().createDefaultPartitionedTopicAsync(topicName) .thenAccept(md -> future.complete(md)) .exceptionally(ex -> { - future.completeExceptionally(ex); + if (ex.getCause() + instanceof MetadataStoreException.AlreadyExistsException) { + // The partitioned topic might be created concurrently + fetchPartitionedTopicMetadataAsync(topicName) + .whenComplete((metadata2, ex2) -> { + if (ex2 == null) { + future.complete(metadata2); + } else { + future.completeExceptionally(ex2); + } + }); + } else { + future.completeExceptionally(ex); + } return null; }); } else { From 5dd735c75e38dc4ba0953fb5a704dccfa4877745 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 22 Jun 2022 23:34:49 +0800 Subject: [PATCH 636/823] [fix][Java Client] Fix thread safety issue of `LastCumulativeAck` (#16072) ### Motivation There were several issues caused by the thread safe issue of `LastCumulativeAck`, see: - https://github.com/apache/pulsar/pull/10586 - https://github.com/apache/pulsar/pull/12343 The root cause is that `LastCumulativeAck` could be accessed by different threads, especially in `flushAsync` method. But the fields are accessed directly and no thread safety can be guaranteed. In addition, the current `LastCumulativeAck` class was added in https://github.com/apache/pulsar/pull/8996 to hold two object references, but this modification is wrong. Before #8996, there are two CAS operations in `doCumulativeAck` method in case it's called concurretly. Though the composite CAS operation is not atomic. However, after #8996, only CAS operation was performed but it's compared with a `LastCumulativeAck` object, not the two fields (`messageId` and `bitSetRecyclable`). There is another issue that it uses a flag `cumulativeAckFlushRequired` to mark if `lastCumulativeAck` should flush. However, if `flushAsync` was called concurrently, both would send ACK commands to broker. ### Modifications To solve the thread safety issue, this PR move the `LastCumulativeAck` out of the `PersistentAcknowledgmentsGroupingTracker` to disable directly access to the internal fields. Then, the following synchronized methods were added to guarantee the thread safety: - `update`: Guarantee the safe write operations. It also recycles the `BitSetRecyclable` object before assigning new values and indicates itself can be flushed. - `flush`: If it can be flushed, return a thread local `LastCumulativeAck` instance that contains the message ID and the bit set. The bit set is deep copied to avoid the original reference being recycled in another `update` call. In addition, since the `messageId` field is volatile, the `getMessageId` method can always retrieve the latest reference. `LastCumulativeAckTest` is added to verify the sematics above. Based on the new design, we can only maintain a `LastCumulativeAck` field in `PersistentAcknowledgmentsGroupingTracker` and call the related methods in `doCumulativeAck` and `flushAsync`. It also fixes the problem that two concurrent `flushAsync` calls might send the same ACK command twice. (cherry picked from commit 936d6fdc780ea454e72e82b6c7a1885799158d02) --- ...sistentAcknowledgmentsGroupingTracker.java | 141 +++++++++--------- .../client/impl/LastCumulativeAckTest.java | 86 +++++++++++ 2 files changed, 159 insertions(+), 68 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/LastCumulativeAckTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index dc311521083b5..c2206bc7468ab 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; +import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -33,11 +34,8 @@ import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.ReentrantReadWriteLock; - -import io.netty.util.Recycler; -import lombok.NonNull; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Triple; import org.apache.pulsar.client.api.MessageId; @@ -69,17 +67,11 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments private volatile TimedCompletableFuture currentIndividualAckFuture; private volatile TimedCompletableFuture currentCumulativeAckFuture; - private volatile LastCumulativeAck lastCumulativeAck = - LastCumulativeAck.create((MessageIdImpl) MessageIdImpl.earliest, null); - - private volatile boolean cumulativeAckFlushRequired = false; + private final LastCumulativeAck lastCumulativeAck = new LastCumulativeAck(); // When we flush the command, we should ensure current ack request will send correct private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private static final AtomicReferenceFieldUpdater LAST_CUMULATIVE_ACK_UPDATER = AtomicReferenceFieldUpdater - .newUpdater(PersistentAcknowledgmentsGroupingTracker.class, LastCumulativeAck.class, "lastCumulativeAck"); - /** * This is a set of all the individual acks that the application has issued and that were not already sent to * broker. @@ -115,13 +107,13 @@ public PersistentAcknowledgmentsGroupingTracker(ConsumerImpl consumer, Consum * resent after a disconnection and for which the user has already sent an acknowledgement. */ @Override - public boolean isDuplicate(@NonNull MessageId messageId) { - final MessageId messageIdOfLastAck = lastCumulativeAck.messageId; + public boolean isDuplicate(MessageId messageId) { + final MessageIdImpl messageIdOfLastAck = lastCumulativeAck.getMessageId(); if (messageIdOfLastAck != null && messageId.compareTo(messageIdOfLastAck) <= 0) { // Already included in a cumulative ack return true; } else { - return pendingIndividualAcks.contains(messageId); + return pendingIndividualAcks.contains((MessageIdImpl) messageId); } } @@ -369,30 +361,7 @@ private void doIndividualBatchAckAsync(BatchMessageIdImpl batchMessageId) { private void doCumulativeAckAsync(MessageIdImpl msgId, BitSetRecyclable bitSet) { // Handle concurrent updates from different threads - LastCumulativeAck currentCumulativeAck = LastCumulativeAck.create(msgId, bitSet); - while (true) { - LastCumulativeAck lastCumulativeAck = this.lastCumulativeAck; - if (msgId.compareTo(lastCumulativeAck.messageId) > 0) { - if (LAST_CUMULATIVE_ACK_UPDATER.compareAndSet(this, this.lastCumulativeAck, currentCumulativeAck)) { - if (lastCumulativeAck.bitSetRecyclable != null) { - try { - lastCumulativeAck.bitSetRecyclable.recycle(); - } catch (Exception ignore) { - // no-op - } - lastCumulativeAck.bitSetRecyclable = null; - } - lastCumulativeAck.recycle(); - // Successfully updated the last cumulative ack. Next flush iteration will send this to broker. - cumulativeAckFlushRequired = true; - return; - } - } else { - currentCumulativeAck.recycle(); - // message id acknowledging an before the current last cumulative ack - return; - } - } + lastCumulativeAck.update(msgId, bitSet); } private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl batchMessageId, @@ -473,15 +442,15 @@ public void flush() { } private void flushAsync(ClientCnx cnx) { + final LastCumulativeAck lastCumulativeAckToFlush = lastCumulativeAck.flush(); boolean shouldFlush = false; - if (cumulativeAckFlushRequired) { - newMessageAckCommandAndWrite(cnx, consumer.consumerId, lastCumulativeAck.messageId.ledgerId, - lastCumulativeAck.messageId.getEntryId(), lastCumulativeAck.bitSetRecyclable, - AckType.Cumulative, null, Collections.emptyMap(), false, - this.currentCumulativeAckFuture, null); - this.consumer.unAckedChunkedMessageIdSequenceMap.remove(lastCumulativeAck.messageId); + if (lastCumulativeAckToFlush != null) { shouldFlush = true; - cumulativeAckFlushRequired = false; + final MessageIdImpl messageId = lastCumulativeAckToFlush.getMessageId(); + newMessageAckCommandAndWrite(cnx, consumer.consumerId, messageId.getLedgerId(), messageId.getEntryId(), + lastCumulativeAckToFlush.getBitSetRecyclable(), AckType.Cumulative, null, + Collections.emptyMap(), false, this.currentCumulativeAckFuture, null); + this.consumer.unAckedChunkedMessageIdSequenceMap.remove(messageId); } // Flush all individual acks @@ -554,7 +523,7 @@ private void flushAsync(ClientCnx cnx) { @Override public void flushAndClean() { flush(); - lastCumulativeAck = LastCumulativeAck.create((MessageIdImpl) MessageIdImpl.earliest, null); + lastCumulativeAck.reset(); pendingIndividualAcks.clear(); } @@ -657,36 +626,72 @@ private boolean isAckReceiptEnabled(ClientCnx cnx) { return ackReceiptEnabled && cnx != null && Commands.peerSupportsAckReceipt(cnx.getRemoteEndpointProtocolVersion()); } +} - private static class LastCumulativeAck { - private MessageIdImpl messageId; - private BitSetRecyclable bitSetRecyclable; +@Getter +class LastCumulativeAck { - static LastCumulativeAck create(MessageIdImpl messageId, BitSetRecyclable bitSetRecyclable) { - LastCumulativeAck op = RECYCLER.get(); - op.messageId = messageId; - op.bitSetRecyclable = bitSetRecyclable; - return op; - } + // It's used as a returned value by `flush()` to avoid creating a new instance each time `flush()` is called + public static final FastThreadLocal LOCAL_LAST_CUMULATIVE_ACK = + new FastThreadLocal() { - private LastCumulativeAck(Recycler.Handle recyclerHandle) { - this.recyclerHandle = recyclerHandle; - } + @Override + protected LastCumulativeAck initialValue() { + return new LastCumulativeAck(); + } + }; + public static final MessageIdImpl DEFAULT_MESSAGE_ID = (MessageIdImpl) MessageIdImpl.earliest; - void recycle() { - if (bitSetRecyclable != null) { + private volatile MessageIdImpl messageId = DEFAULT_MESSAGE_ID; + private BitSetRecyclable bitSetRecyclable = null; + private boolean flushRequired = false; + + public synchronized void update(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) { + if (messageId.compareTo(this.messageId) > 0) { + if (this.bitSetRecyclable != null && this.bitSetRecyclable != bitSetRecyclable) { this.bitSetRecyclable.recycle(); } - this.messageId = null; - recyclerHandle.recycle(this); + set(messageId, bitSetRecyclable); + flushRequired = true; } + } - private final Recycler.Handle recyclerHandle; - private static final Recycler RECYCLER = new Recycler() { - @Override - protected LastCumulativeAck newObject(Handle handle) { - return new LastCumulativeAck(handle); + public synchronized LastCumulativeAck flush() { + if (flushRequired) { + final LastCumulativeAck localLastCumulativeAck = LOCAL_LAST_CUMULATIVE_ACK.get(); + if (bitSetRecyclable != null) { + localLastCumulativeAck.set(messageId, BitSetRecyclable.valueOf(bitSetRecyclable.toLongArray())); + } else { + localLastCumulativeAck.set(this.messageId, null); } - }; + flushRequired = false; + return localLastCumulativeAck; + } else { + // Return null to indicate nothing to be flushed + return null; + } + } + + public synchronized void reset() { + if (bitSetRecyclable != null) { + bitSetRecyclable.recycle(); + } + messageId = DEFAULT_MESSAGE_ID; + bitSetRecyclable = null; + flushRequired = false; + } + + private synchronized void set(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) { + this.messageId = messageId; + this.bitSetRecyclable = bitSetRecyclable; + } + + @Override + public String toString() { + String s = messageId.toString(); + if (bitSetRecyclable != null) { + s += " (bit set: " + bitSetRecyclable + ")"; + } + return s; } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/LastCumulativeAckTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/LastCumulativeAckTest.java new file mode 100644 index 0000000000000..102ccfc0e07a5 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/LastCumulativeAckTest.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.testng.annotations.Test; + +public class LastCumulativeAckTest { + + @Test + public void testUpdate() { + final LastCumulativeAck lastCumulativeAck = new LastCumulativeAck(); + assertFalse(lastCumulativeAck.isFlushRequired()); + assertEquals(lastCumulativeAck.getMessageId(), LastCumulativeAck.DEFAULT_MESSAGE_ID); + assertNull(lastCumulativeAck.getBitSetRecyclable()); + + final MessageIdImpl messageId1 = new MessageIdImpl(0L, 1L, 10); + final BitSetRecyclable bitSetRecyclable1 = BitSetRecyclable.create(); + bitSetRecyclable1.set(0, 3); + lastCumulativeAck.update(messageId1, bitSetRecyclable1); + assertTrue(lastCumulativeAck.isFlushRequired()); + assertSame(lastCumulativeAck.getMessageId(), messageId1); + assertSame(lastCumulativeAck.getBitSetRecyclable(), bitSetRecyclable1); + + final MessageIdImpl messageId2 = new MessageIdImpl(0L, 2L, 8); + lastCumulativeAck.update(messageId2, bitSetRecyclable1); + // bitSetRecyclable1 is not recycled + assertEquals(bitSetRecyclable1.toString(), "{0, 1, 2}"); + + final BitSetRecyclable bitSetRecyclable2 = BitSetRecyclable.create(); + bitSetRecyclable2.set(0, 2); + + // `update()` only accepts a newer message ID, so this call here has no side effect + lastCumulativeAck.update(messageId2, bitSetRecyclable2); + assertSame(lastCumulativeAck.getBitSetRecyclable(), bitSetRecyclable1); + + final MessageIdImpl messageId3 = new MessageIdImpl(0L, 3L, 9); + lastCumulativeAck.update(messageId3, bitSetRecyclable2); + // bitSetRecyclable1 is recycled because it's replaced in `update` + assertEquals(bitSetRecyclable1.toString(), "{}"); + assertSame(lastCumulativeAck.getMessageId(), messageId3); + assertSame(lastCumulativeAck.getBitSetRecyclable(), bitSetRecyclable2); + bitSetRecyclable2.recycle(); + } + + @Test + public void testFlush() { + final LastCumulativeAck lastCumulativeAck = new LastCumulativeAck(); + assertNull(lastCumulativeAck.flush()); + + final MessageIdImpl messageId = new MessageIdImpl(0L, 1L, 3); + final BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create(); + bitSetRecyclable.set(0, 3); + lastCumulativeAck.update(messageId, bitSetRecyclable); + assertTrue(lastCumulativeAck.isFlushRequired()); + + final LastCumulativeAck lastCumulativeAckToFlush = lastCumulativeAck.flush(); + assertFalse(lastCumulativeAck.isFlushRequired()); + assertSame(lastCumulativeAckToFlush.getMessageId(), messageId); + assertNotSame(lastCumulativeAckToFlush.getBitSetRecyclable(), bitSetRecyclable); + assertEquals(lastCumulativeAckToFlush.getBitSetRecyclable(), bitSetRecyclable); + } + +} From cf73589caede55081505158fdf2747c527672ab1 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 23 Jun 2022 12:13:05 +0800 Subject: [PATCH 637/823] [fix][broker] Fix NPE when get /admin/v2/namespaces/public/default/maxTopicsPerNamespace (#16076) (cherry picked from commit c7d74f39757371dd1b2864602534539f7d8cd4cf) --- .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 3 ++- .../java/org/apache/pulsar/broker/admin/NamespacesTest.java | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index edbe1bfe385b8..caf75a99d1d3e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2559,7 +2559,8 @@ protected OffloadPoliciesImpl internalGetOffloadPolicies() { protected int internalGetMaxTopicsPerNamespace() { validateNamespacePolicyOperation(namespaceName, PolicyName.MAX_TOPICS, PolicyOperation.READ); - return getNamespacePolicies(namespaceName).max_topics_per_namespace; + return getNamespacePolicies(namespaceName).max_topics_per_namespace != null + ? getNamespacePolicies(namespaceName).max_topics_per_namespace : 0; } protected void internalRemoveMaxTopicsPerNamespace() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 69bede376c802..636cbe5980540 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -1502,6 +1502,8 @@ public void testMaxTopicsPerNamespace() throws Exception { admin.tenants().createTenant("testTenant", tenantInfo); admin.namespaces().createNamespace(namespace, Sets.newHashSet("use")); + assertEquals(0, admin.namespaces().getMaxTopicsPerNamespace(namespace)); + admin.namespaces().setMaxTopicsPerNamespace(namespace, 10); assertEquals(10, admin.namespaces().getMaxTopicsPerNamespace(namespace)); From 5234be3d4c1f05d84bb20b4ab4e651aa14ccde3b Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 29 Jun 2022 00:03:21 +0800 Subject: [PATCH 638/823] [improve][broker] Use LinkedHashSet for config items of type Set to preserve elements order (#16138) (cherry picked from commit ca9e4bd6186bdf35280177987ad439492ef5fc55) --- .../main/java/org/apache/pulsar/common/util/FieldParser.java | 5 +++-- .../pulsar/common/util/collections/FieldParserTest.java | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index 0aa19b272a3c5..3fd82f03c696e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -32,6 +32,7 @@ import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -212,7 +213,7 @@ public static void setEmptyValue(String strValue, Field field, T obj) if (field.getType().equals(List.class)) { field.set(obj, Lists.newArrayList()); } else if (field.getType().equals(Set.class)) { - field.set(obj, Sets.newHashSet()); + field.set(obj, new LinkedHashSet<>()); } else if (field.getType().equals(Optional.class)) { field.set(obj, Optional.empty()); } else { @@ -333,7 +334,7 @@ public static Set stringToSet(String val, Class type) { String[] tokens = trim(val).split(","); return Arrays.stream(tokens).map(t -> { return convert(trim(t), type); - }).collect(Collectors.toSet()); + }).collect(Collectors.toCollection(LinkedHashSet::new)); } private static Map stringToMap(String strValue, Class keyType, Class valueType) { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/FieldParserTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/FieldParserTest.java index c8a46cb0a1585..12d579bd8cd77 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/FieldParserTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/FieldParserTest.java @@ -32,6 +32,7 @@ import static org.testng.Assert.fail; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -54,6 +55,10 @@ public void testConversion() { assertEquals(integerToString(1), String.valueOf(1)); assertEquals(stringToList("1,2,3", Integer.class).get(2), Integer.valueOf(3)); assertTrue(stringToSet("1,2,3", Integer.class).contains(3)); + // the order of values should be preserved for a Set configuration item + assertEquals(new ArrayList<>(stringToSet("1,2,3", Integer.class)), Arrays.asList(1, 2, 3)); + assertEquals(new ArrayList<>(stringToSet("2,3,1", Integer.class)), Arrays.asList(2, 3, 1)); + assertEquals(new ArrayList<>(stringToSet("3,2,1", Integer.class)), Arrays.asList(3, 2, 1)); assertEquals(stringToBoolean("true"), Boolean.TRUE); assertEquals(stringToDouble("2.2"), Double.valueOf(2.2)); assertEquals(stringToLong("2"), Long.valueOf(2)); From 47c5b3d6c263580e53b6e6eba6ad464b7293e777 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:33:23 +0800 Subject: [PATCH 639/823] [fix][txn] Fix NPE when ack message with transaction at cnx = null (#16142) Fix https://github.com/apache/pulsar/issues/16124 ## Motivation When a channel is inactive, connectHandler will set the cnx = null and reconnect. At this time, consumers use transaction to ack messages will report NPE. ## Modification Return exception when cnx = null. **Why not use a queue to store operations?** 1. If we use a queue to store op, we need to take care of the timeout of the op. And the lock is required. 2. If the connection time is long or there is a BUG client that has not been connected, the client will crash. (cherry picked from commit 53cc84a580dd747685905e1d11b8e19c0e59a614) --- .../broker/transaction/TransactionTest.java | 41 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 9 +++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index f9fc52aa547cd..dc02313bb1f0d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -51,6 +51,7 @@ import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.util.Bytes; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; @@ -87,6 +88,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.client.impl.transaction.TransactionImpl; @@ -109,6 +111,7 @@ import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; +import org.powermock.reflect.Whitebox; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -996,6 +999,44 @@ public void testConsistencyOfTransactionStatsAtEndTxn() throws Exception { transaction.commit().get(); } + @Test + public void testGetConnectExceptionForAckMsgWhenCnxIsNull() throws Exception { + String topic = NAMESPACE1 + "/testGetConnectExceptionForAckMsgWhenCnxIsNull"; + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.BYTES) + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer() + .topic(topic) + .subscriptionName("sub") + .subscribe(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().value(Bytes.toBytes(i)).send(); + } + ClientCnx cnx = Whitebox.invokeMethod(consumer, "cnx"); + Whitebox.invokeMethod(consumer, "connectionClosed", cnx); + + Message message = consumer.receive(); + Transaction transaction = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + + try { + consumer.acknowledgeAsync(message.getMessageId(), transaction).get(); + fail(); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof PulsarClientException.ConnectException); + } + } + + @Test public void testPendingAckBatchMessageCommit() throws Exception { String topic = NAMESPACE1 + "/testPendingAckBatchMessageCommit"; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 603dae6ecbe58..09f2685eb3ca7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2497,7 +2497,14 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me } else { unAckedMessageTracker.remove(messageId); } - return cnx().newAckForReceipt(cmd, requestId); + ClientCnx cnx = cnx(); + if (cnx == null) { + return FutureUtil.failedFuture(new PulsarClientException + .ConnectException("Failed to ack message [" + messageId + "] " + + "for transaction [" + txnID + "] due to consumer connect fail, consumer state: " + getState())); + } else { + return cnx.newAckForReceipt(cmd, requestId); + } } public Map>> getPossibleSendToDeadLetterTopicMessages() { From 556d3d16c5651b626789f8b97a174ae5085dddb1 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 21 Jun 2022 17:13:00 +0800 Subject: [PATCH 640/823] Fix `messageQueue` release message issue. (#16155) (cherry picked from commit 141c44022a27be2fc07eab9827cfdb168e448953) --- .../pulsar/sql/presto/PulsarRecordCursor.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 195f40fd326e6..e70f574cf29e2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -727,19 +727,17 @@ public boolean isNull(int field) { public void close() { log.info("Closing cursor record"); - if (currentMessage != null) { - currentMessage.release(); - } - - if (messageQueue != null) { - messageQueue.drain(RawMessage::release); - } - if (deserializeEntries != null) { deserializeEntries.close().whenComplete((r, t) -> { if (entryQueue != null) { entryQueue.drain(Entry::release); } + if (messageQueue != null) { + messageQueue.drain(RawMessage::release); + } + if (currentMessage != null) { + currentMessage.release(); + } }); } From 282a0f2f02f07cc6ff21983e8ffa539bfcee9342 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 1 Jul 2022 16:53:31 +0800 Subject: [PATCH 641/823] [fix][function] Ensure bytes is a well-formed UTF-8 byte sequence when decode the `FunctionState` bytes to string (#16199) (cherry picked from commit 172a84624f47bb722e6b419de4e239fbb8ecc154) --- .../worker/rest/api/ComponentImpl.java | 15 ++-- .../integration/io/TestByteStateSource.java | 55 +++++++++++++++ .../functions/PulsarStateTest.java | 70 +++++++++++++++++-- 3 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestByteStateSource.java diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 98994c9745144..bad35e5443e44 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -28,7 +28,7 @@ import static org.apache.pulsar.functions.utils.FunctionCommon.getUniquePackageName; import static org.apache.pulsar.functions.utils.FunctionCommon.isFunctionCodeBuiltin; import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException; - +import com.google.common.base.Utf8; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; @@ -1098,10 +1098,13 @@ public FunctionState getFunctionState(final String tenant, if (kv.isNumber()) { value = new FunctionState(key, null, null, kv.numberValue(), kv.version()); } else { - try { - value = new FunctionState(key, new String(ByteBufUtil.getBytes(kv.value(), kv.value().readerIndex(), kv.value().readableBytes()), UTF_8), null, null, kv.version()); - } catch (Exception e) { - value = new FunctionState(key, null, ByteBufUtil.getBytes(kv.value()), null, kv.version()); + byte[] bytes = ByteBufUtil.getBytes(kv.value()); + if (Utf8.isWellFormed(bytes)) { + value = new FunctionState(key, new String(bytes, UTF_8), + null, null, kv.version()); + } else { + value = new FunctionState( + key, null, bytes, null, kv.version()); } } } @@ -1147,7 +1150,7 @@ public void putFunctionState(final String tenant, log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e); throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } - + if (!key.equals(state.getKey())) { log.error("{}/{}/{} Bad putFunction Request, path key doesn't match key in json", tenant, namespace, functionName); throw new RestException(Status.BAD_REQUEST, "Path key doesn't match key in json"); diff --git a/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestByteStateSource.java b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestByteStateSource.java new file mode 100644 index 0000000000000..4fe382f5ce758 --- /dev/null +++ b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestByteStateSource.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Map; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Source; +import org.apache.pulsar.io.core.SourceContext; + +public class TestByteStateSource implements Source { + + private SourceContext sourceContext; + + public static final String VALUE_BASE64 = "0a8001127e0a172e6576656e74732e437573746f6d65724372656174656412630a243" + + "2336366666263652d623038342d346631352d616565342d326330643135356131666" + + "36312026e311a3700000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000"; + + @Override + public void open(Map config, SourceContext sourceContext) throws Exception { + sourceContext.putState("initial", ByteBuffer.wrap(Base64.getDecoder().decode(VALUE_BASE64))); + this.sourceContext = sourceContext; + } + + @Override + public Record read() throws Exception { + Thread.sleep(50); + ByteBuffer initial = sourceContext.getState("initial"); + sourceContext.putState("now", initial); + return initial::array; + } + + @Override + public void close() throws Exception { + + } +} \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java index c5fb6ecffebff..5b9041b0916c7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java @@ -18,6 +18,16 @@ */ package org.apache.pulsar.tests.integration.functions; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.JAVAJAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.google.common.base.Utf8; +import java.util.Base64; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -40,13 +50,6 @@ import org.awaitility.Awaitility; import org.testng.annotations.Test; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.JAVAJAR; -import static org.apache.pulsar.tests.integration.suites.PulsarTestSuite.retryStrategically; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - /** * State related test cases. */ @@ -58,6 +61,11 @@ public class PulsarStateTest extends PulsarStandaloneTestSuite { public static final String WORDCOUNT_PYTHON_FILE = "wordcount_function.py"; + public static final String VALUE_BASE64 = "0a8001127e0a172e6576656e74732e437573746f6d65724372656174656412630a243" + + "2336366666263652d623038342d346631352d616565342d326330643135356131666" + + "36312026e311a3700000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000"; + @Test(groups = {"python_state", "state", "function", "python_function"}) public void testPythonWordCountFunction() throws Exception { String inputTopicName = "test-wordcount-py-input-" + randomName(8); @@ -184,6 +192,54 @@ public void testSinkState() throws Exception { getSinkInfoNotFound(sinkName); } + @Test(groups = {"java_state", "state", "function", "java_function"}) + public void testBytes2StringNotUTF8() { + byte[] valueBytes = Base64.getDecoder().decode(VALUE_BASE64); + assertFalse(Utf8.isWellFormed(valueBytes)); + assertNotEquals(valueBytes, new String(valueBytes, UTF_8).getBytes(UTF_8)); + } + + @Test(groups = {"java_state", "state", "function", "java_function"}) + public void testSourceByteState() throws Exception { + String outputTopicName = "test-state-source-output-" + randomName(8); + String sourceName = "test-state-source-" + randomName(8); + + submitSourceConnector(sourceName, outputTopicName, "org.apache.pulsar.tests.integration.io.TestByteStateSource", JAVAJAR); + + // get source info + getSourceInfoSuccess(sourceName); + + // get source status + getSourceStatus(sourceName); + + try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(container.getHttpServiceUrl()).build()) { + + Awaitility.await().ignoreExceptions().untilAsserted(() -> { + SourceStatus status = admin.sources().getSourceStatus("public", "default", sourceName); + assertEquals(status.getInstances().size(), 1); + assertTrue(status.getInstances().get(0).getStatus().numWritten > 0); + }); + + { + FunctionState functionState = + admin.functions().getFunctionState("public", "default", sourceName, "initial"); + assertNull(functionState.getStringValue()); + assertEquals(functionState.getByteValue(), Base64.getDecoder().decode(VALUE_BASE64)); + } + + Awaitility.await().ignoreExceptions().untilAsserted(() -> { + FunctionState functionState = admin.functions().getFunctionState("public", "default", sourceName, "now"); + assertNull(functionState.getStringValue()); + assertEquals(functionState.getByteValue(), Base64.getDecoder().decode(VALUE_BASE64)); + }); + } + + // delete source + deleteSource(sourceName); + + getSourceInfoNotFound(sourceName); + } + private void submitSourceConnector(String sourceName, String outputTopicName, String className, From 6b3ed2977ff7b262c52c8603a47596f24efe4de6 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 28 Jun 2022 14:39:00 +0800 Subject: [PATCH 642/823] [fix][broker] Fix NPE when drop backlog for time limit. (#16235) (cherry picked from commit d24d82780fd27a98c6cdbee28d756ee7d419495f) --- .../org/apache/pulsar/broker/service/BacklogQuotaManager.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index 6efc1a4ed9b9a..ee12c3ff7438d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -243,6 +243,10 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); Position oldestPosition = slowestConsumer.getMarkDeletedPosition(); ManagedLedgerInfo.LedgerInfo ledgerInfo = mLedger.getLedgerInfo(oldestPosition.getLedgerId()).get(); + if (ledgerInfo == null) { + slowestConsumer.resetCursor(mLedger.getNextValidPosition((PositionImpl) oldestPosition)); + continue; + } // Timestamp only > 0 if ledger has been closed if (ledgerInfo.getTimestamp() > 0 && currentMillis - ledgerInfo.getTimestamp() > quota.getLimitTime()) { From 15895c3ab0cc9ec2146ec95deddb9cd4c6a0b283 Mon Sep 17 00:00:00 2001 From: Xiaoyu Hou Date: Fri, 1 Jul 2022 10:19:19 +0800 Subject: [PATCH 643/823] [fix][broker]Fix getInternalStats occasional lack of LeaderInfo again (#16238) * Fix getInternalStats occasional lack of LeaderInfo again * Make futures as wildcard (cherry picked from commit a9af98050c0c50d3d9e25f1db50bf2df7584d2ba) --- .../service/persistent/PersistentTopic.java | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 129333581bae3..8c592a7e29cc1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2023,40 +2023,43 @@ public CompletableFuture getInternalStats(boolean stats.state = ml.getState().toString(); stats.ledgers = Lists.newArrayList(); - List> futures = Lists.newArrayList(); + Set> futures = Sets.newConcurrentHashSet(); CompletableFuture> availableBookiesFuture = brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync(); - futures.add(availableBookiesFuture.handle((strings, throwable) -> null)); - availableBookiesFuture.whenComplete((bookies, e) -> { - if (e != null) { - log.error("[{}] Failed to fetch available bookies.", topic, e); - statFuture.completeExceptionally(e); - } else { - ml.getLedgersInfo().forEach((id, li) -> { - LedgerInfo info = new LedgerInfo(); - info.ledgerId = li.getLedgerId(); - info.entries = li.getEntries(); - info.size = li.getSize(); - info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); - stats.ledgers.add(info); - if (includeLedgerMetadata) { - futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> { - if (ex == null) { - info.metadata = lMetadata; - } - return null; - })); - futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> { - if (ex == null) { - info.underReplicated = !bookies.containsAll(ensembles.stream().map(BookieId::toString) - .collect(Collectors.toList())); + futures.add( + availableBookiesFuture + .whenComplete((bookies, e) -> { + if (e != null) { + log.error("[{}] Failed to fetch available bookies.", topic, e); + statFuture.completeExceptionally(e); + } else { + ml.getLedgersInfo().forEach((id, li) -> { + LedgerInfo info = new LedgerInfo(); + info.ledgerId = li.getLedgerId(); + info.entries = li.getEntries(); + info.size = li.getSize(); + info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); + stats.ledgers.add(info); + if (includeLedgerMetadata) { + futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> { + if (ex == null) { + info.metadata = lMetadata; + } + return null; + })); + futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> { + if (ex == null) { + info.underReplicated = + !bookies.containsAll(ensembles.stream().map(BookieId::toString) + .collect(Collectors.toList())); + } + return null; + })); } - return null; - })); + }); } - }); - } - }); + }) + ); // Add ledger info for compacted topic ledger if exist. LedgerInfo info = new LedgerInfo(); @@ -2172,16 +2175,11 @@ public CompletableFuture getInternalStats(boolean } else { schemaStoreLedgersFuture.complete(null); } - schemaStoreLedgersFuture.thenRun(() -> { - if (futures != null) { - FutureUtil.waitForAll(futures).handle((res, ex) -> { - statFuture.complete(stats); - return null; - }); - } else { + schemaStoreLedgersFuture.thenRun(() -> + FutureUtil.waitForAll(futures).handle((res, ex) -> { statFuture.complete(stats); - } - }).exceptionally(e -> { + return null; + })).exceptionally(e -> { statFuture.completeExceptionally(e); return null; }); From c3fd77de35ac3eda3cf0a5093a31f24f035cdfd7 Mon Sep 17 00:00:00 2001 From: lixinyang <84127069+Nicklee007@users.noreply.github.com> Date: Tue, 28 Jun 2022 20:07:38 +0800 Subject: [PATCH 644/823] [fix][broker]fix npe when invoke replaceBookie. (#16239) * fix npe when invoke replaceBookie. * fix npe when invoke replaceBookie. * fix npe when invoke replaceBookie. Co-authored-by: nicklixinyang (cherry picked from commit 0eed84203b23e325ac15d7dc50e1ed6dbdf4fa2a) --- ...IsolatedBookieEnsemblePlacementPolicy.java | 4 +++ ...atedBookieEnsemblePlacementPolicyTest.java | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicy.java b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicy.java index 93202963adbc3..4a50a6f61b380 100644 --- a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicy.java +++ b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicy.java @@ -205,9 +205,13 @@ private static Pair, Set> getIsolationGroup(EnsemblePlacemen String secondaryIsolationGroupString = castToString(properties.getOrDefault(SECONDARY_ISOLATION_BOOKIE_GROUPS, "")); if (!primaryIsolationGroupString.isEmpty()) { pair.setLeft(new HashSet(Arrays.asList(primaryIsolationGroupString.split(",")))); + } else { + pair.setLeft(Collections.emptySet()); } if (!secondaryIsolationGroupString.isEmpty()) { pair.setRight(new HashSet(Arrays.asList(secondaryIsolationGroupString.split(",")))); + } else { + pair.setRight(Collections.emptySet()); } } return pair; diff --git a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java index 02df09b64e28f..9269117361f98 100644 --- a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java @@ -306,6 +306,39 @@ public void testNoIsolationGroup() throws Exception { isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); isolationPolicy.newEnsemble(4, 4, 4, Collections.emptyMap(), new HashSet<>()); + + BookieId bookie1Id = new BookieSocketAddress(BOOKIE1).toBookieId(); + BookieId bookie2Id = new BookieSocketAddress(BOOKIE2).toBookieId(); + BookieId bookie3Id = new BookieSocketAddress(BOOKIE3).toBookieId(); + BookieId bookie4Id = new BookieSocketAddress(BOOKIE4).toBookieId(); + // when we set strictBookieAffinityEnabled=true and some namespace not set ISOLATION_BOOKIE_GROUPS there will set "" by default. + Map placementPolicyProperties1 = new HashMap<>(); + placementPolicyProperties1.put( + IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); + placementPolicyProperties1.put( + IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + EnsemblePlacementPolicyConfig policyConfig = new EnsemblePlacementPolicyConfig( + IsolatedBookieEnsemblePlacementPolicy.class, + placementPolicyProperties1 + ); + Map customMetadata1 = new HashMap<>(); + customMetadata1.put(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG, policyConfig.encode()); + + BookieId replaceBookie1 = isolationPolicy.replaceBookie(3, 3, 3, customMetadata1, + Arrays.asList(bookie1Id,bookie2Id,bookie3Id), bookie3Id, null).getResult(); + assertEquals(replaceBookie1, bookie4Id); + + // when ISOLATION_BOOKIE_GROUPS miss. + Map placementPolicyProperties2 = new HashMap<>(); + EnsemblePlacementPolicyConfig policyConfig2 = new EnsemblePlacementPolicyConfig( + IsolatedBookieEnsemblePlacementPolicy.class, + placementPolicyProperties2 + ); + Map customMetadata2 = new HashMap<>(); + customMetadata2.put(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG, policyConfig.encode()); + BookieId replaceBookie2 = isolationPolicy.replaceBookie(3, 3, 3, customMetadata2, + Arrays.asList(bookie1Id,bookie2Id,bookie3Id), bookie3Id, null).getResult(); + assertEquals(replaceBookie2, bookie4Id); } /** From f02ee1afd07fe17d4c40968b15f8f5203e4a26c8 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 1 Jul 2022 20:05:20 +0800 Subject: [PATCH 645/823] [fix][txn] Fix TopicTransactionBuffer ledger apend marker throw ManagedLedgerAlreadyClosedException (#16265) * [fix][txn] Fix TopicTransactionBuffer ledger apend marker throw ManagedLedgerAlreadyClosedException (cherry picked from commit e616e2cb388d9ff7282653c4976a32e89d5cccae) --- .../buffer/impl/TopicTransactionBuffer.java | 8 ++ .../buffer/TopicTransactionBufferTest.java | 89 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 7bb19d8008695..d0758dc3c5a7d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -315,6 +315,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("Failed to commit for txn {}", txnID, exception); + checkAppendMarkerException(exception); completableFuture.completeExceptionally(new PersistenceException(exception)); } }, null); @@ -361,6 +362,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("Failed to abort for txn {}", txnID, exception); + checkAppendMarkerException(exception); completableFuture.completeExceptionally(new PersistenceException(exception)); } }, null); @@ -375,6 +377,12 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { return completableFuture; } + private void checkAppendMarkerException(ManagedLedgerException exception) { + if (exception instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException) { + topic.getManagedLedger().readyToCreateNewLedger(); + } + } + private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { lowWaterMarks.compute(txnID.getMostSigBits(), (tcId, oldLowWaterMark) -> { if (oldLowWaterMark == null || oldLowWaterMark < lowWaterMark) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java new file mode 100644 index 0000000000000..576ef647248d4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer; + +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; +import org.awaitility.Awaitility; +import org.powermock.reflect.Whitebox; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class TopicTransactionBufferTest extends TransactionTestBase { + + + @BeforeMethod(alwaysRun = true) + protected void setup() throws Exception { + setBrokerCount(1); + setUpBase(1, 16, "persistent://" + NAMESPACE1 + "/test", 0); + + Map stores = + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores(); + Awaitility.await().until(() -> { + if (stores.size() == 16) { + for (TransactionCoordinatorID transactionCoordinatorID : stores.keySet()) { + if (((MLTransactionMetadataStore) stores.get(transactionCoordinatorID)).getState() + != TransactionMetadataStoreState.State.Ready) { + return false; + } + } + return true; + } else { + return false; + } + }); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testTransactionBufferAppendMarkerWriteFailState() throws Exception { + final String topic = "persistent://" + NAMESPACE1 + "/testPendingAckManageLedgerWriteFailState"; + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + + Producer producer = pulsarClient + .newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .enableBatching(false) + .create(); + + producer.newMessage(txn).value("test".getBytes()).send(); + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) + .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); + Whitebox.setInternalState(persistentTopic.getManagedLedger(), "state", ManagedLedgerImpl.State.WriteFailed); + txn.commit().get(); + } +} From 99e4a1933d0dea4d2818cc4b218e820300820fb1 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Tue, 28 Jun 2022 23:33:24 +0800 Subject: [PATCH 646/823] [fix][txn] Fix append txn message is lower than lowWaterMark decrease pendingWriteOps (#16266) (cherry picked from commit af7990dca4720c7f120b2d90ec368cdc82c484e0) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 1 + .../transaction/buffer/TransactionLowWaterMarkTest.java | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 8c592a7e29cc1..932def13db9de 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3083,6 +3083,7 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon throwable = throwable.getCause(); if (throwable instanceof NotAllowedException) { publishContext.completed((NotAllowedException) throwable, -1, -1); + decrementPendingWriteOpsAndCheck(); return null; } else if (!(throwable instanceof ManagedLedgerException)) { throwable = new ManagedLedgerException(throwable); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java index 1f012d7a6b10e..3efdc2473bb21 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -59,6 +60,7 @@ import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; +import org.powermock.reflect.Whitebox; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -320,12 +322,18 @@ public void testTBLowWaterMarkEndToEnd() throws Exception { Field field = TransactionImpl.class.getDeclaredField("state"); field.setAccessible(true); field.set(txn1, TransactionImpl.State.OPEN); + + AtomicLong pendingWriteOps = Whitebox.getInternalState(getPulsarServiceList().get(0) + .getBrokerService().getTopic(TopicName.get(TOPIC).toString(), + false).get().get(), "pendingWriteOps"); try { producer.newMessage(txn1).send(); fail(); } catch (PulsarClientException.NotAllowedException ignore) { // no-op } + + assertEquals(pendingWriteOps.get(), 0); } @Test From f81b2421c6a982992833db36749d09cecef960bc Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 26 Apr 2022 22:11:35 +0800 Subject: [PATCH 647/823] [improve][common] Use `Collection` to instead of `List` parameter type (#15329) ### Motivation We can use `Collection` instead of `List` parameter type in `FutureUtil` for better compatibility. For example when we need to use the values of `Map`: ```java FutureUtil.waitForAll(map.values()); ``` ### Modifications - Use `Collection` instead of `List` parameter type. (cherry picked from commit 0c694cfdc9b11f915f1da86260ad3655c2e99a35) --- .../org/apache/pulsar/common/util/FutureUtil.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index dac204db98e81..d35a6b405b2b4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -20,7 +20,6 @@ import java.time.Duration; import java.util.Collection; -import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -39,17 +38,17 @@ public class FutureUtil { /** - * Return a future that represents the completion of the futures in the provided list. + * Return a future that represents the completion of the futures in the provided Collection. * * @param futures futures to wait for * @return a new CompletableFuture that is completed when all of the given CompletableFutures complete */ - public static CompletableFuture waitForAll(List> futures) { + public static CompletableFuture waitForAll(Collection> futures) { return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } /** - * Return a future that represents the completion of any future in the provided list. + * Return a future that represents the completion of any future in the provided Collection. * * @param futures futures to wait any * @return a new CompletableFuture that is completed when any of the given CompletableFutures complete @@ -102,14 +101,15 @@ public static CompletableFuture> waitForAny(Collection waitForAllAndSupportCancel(List> futures) { + public static CompletableFuture waitForAllAndSupportCancel( + Collection> futures) { CompletableFuture[] futuresArray = futures.toArray(new CompletableFuture[0]); CompletableFuture combinedFuture = CompletableFuture.allOf(futuresArray); whenCancelledOrTimedOut(combinedFuture, () -> { From dc9968a58a8c6140525fec31777e4c1a0de443bb Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sat, 2 Jul 2022 17:12:17 +0800 Subject: [PATCH 648/823] [branch-2.9] Fix cherry-pick issues. (#16346) --- .../ZkIsolatedBookieEnsemblePlacementPolicyTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java index 9269117361f98..04b3b67b35b5e 100644 --- a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZkIsolatedBookieEnsemblePlacementPolicyTest.java @@ -314,11 +314,11 @@ public void testNoIsolationGroup() throws Exception { // when we set strictBookieAffinityEnabled=true and some namespace not set ISOLATION_BOOKIE_GROUPS there will set "" by default. Map placementPolicyProperties1 = new HashMap<>(); placementPolicyProperties1.put( - IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); + ZkIsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); placementPolicyProperties1.put( - IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + ZkIsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); EnsemblePlacementPolicyConfig policyConfig = new EnsemblePlacementPolicyConfig( - IsolatedBookieEnsemblePlacementPolicy.class, + ZkIsolatedBookieEnsemblePlacementPolicy.class, placementPolicyProperties1 ); Map customMetadata1 = new HashMap<>(); @@ -331,7 +331,7 @@ public void testNoIsolationGroup() throws Exception { // when ISOLATION_BOOKIE_GROUPS miss. Map placementPolicyProperties2 = new HashMap<>(); EnsemblePlacementPolicyConfig policyConfig2 = new EnsemblePlacementPolicyConfig( - IsolatedBookieEnsemblePlacementPolicy.class, + ZkIsolatedBookieEnsemblePlacementPolicy.class, placementPolicyProperties2 ); Map customMetadata2 = new HashMap<>(); From d56ae3797c18b1451b58c4b05f60ed23ae321c9f Mon Sep 17 00:00:00 2001 From: Qiang Huang Date: Sat, 2 Jul 2022 21:45:46 +0800 Subject: [PATCH 649/823] [Branch-2.9][Cherry-pick] fix bug: fail to expose managed ledger client stats to prometheus if bookkeeperClientExposeStatsToPrometheus is true #16219 (#16343) --- .../broker/ManagedLedgerClientFactory.java | 5 +-- .../broker/stats/PrometheusMetricsTest.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java index 6a9aa8877e252..174933b9cbd9c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java @@ -84,7 +84,8 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata statsProvider.start(configuration); StatsLogger statsLogger = statsProvider.getStatsLogger("pulsar_managedLedger_client"); - this.defaultBkClient = bookkeeperProvider.create(conf, zkClient, eventLoopGroup, Optional.empty(), null); + this.defaultBkClient = + bookkeeperProvider.create(conf, zkClient, eventLoopGroup, Optional.empty(), null, statsLogger); BookkeeperFactoryForCustomEnsemblePlacementPolicy bkFactory = ( EnsemblePlacementPolicyConfig ensemblePlacementPolicyConfig) -> { @@ -95,7 +96,7 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata try { return bookkeeperProvider.create(conf, zkClient, eventLoopGroup, Optional.ofNullable(ensemblePlacementPolicyConfig.getPolicyClass()), - ensemblePlacementPolicyConfig.getProperties()); + ensemblePlacementPolicyConfig.getProperties(), statsLogger); } catch (Exception e) { log.error("Failed to initialize bk-client for policy {}, properties {}", ensemblePlacementPolicyConfig.getPolicyClass(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 1304d5c7ae2f9..eae58b67c6a83 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -922,6 +922,38 @@ public void testManagedLedgerBookieClientStats() throws Exception { cm = (List) metrics.get("pulsar_managedLedger_client_bookkeeper_ml_workers_task_execution_count"); assertEquals(cm.size(), 0); + + cm = (List) metrics.get( + keyNameBySubstrings(metrics, "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_total_tasks")); + assertEquals(cm.size(), 1); + assertEquals(cm.get(0).tags.get("cluster"), "test"); + + cm = (List) metrics.get(keyNameBySubstrings(metrics, "pulsar_managedLedger_client", + "bookkeeper_ml_scheduler_task_execution_sum")); + assertEquals(cm.size(), 2); + assertEquals(cm.get(0).tags.get("cluster"), "test"); + + cm = (List) metrics.get( + keyNameBySubstrings(metrics, + "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_queue")); + assertEquals(cm.size(), 1); + assertEquals(cm.get(0).tags.get("cluster"), "test"); + } + + private static String keyNameBySubstrings(Multimap metrics, String... substrings) { + for (String key : metrics.keys()) { + boolean found = true; + for (String s : substrings) { + if (!key.contains(s)) { + found = false; + break; + } + } + if (found) { + return key; + } + } + return null; } @Test From c7d990876b0a0bb84a6d596e2a56d821faffa6bc Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sat, 2 Jul 2022 21:46:52 +0800 Subject: [PATCH 650/823] [Branch 2.9] Fix compaction subscription acknowledge Marker msg issue. (#16348) --- .../service/AbstractBaseDispatcher.java | 18 ++-- .../pulsar/compaction/CompactionTest.java | 85 ++++++++++++++++--- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index ba757b529fe0b..95e89af86099e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -22,14 +22,17 @@ import io.netty.buffer.ByteBuf; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.service.persistent.CompactorSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; @@ -128,15 +131,13 @@ public void filterEntriesForConsumer(Optional entryWrapper, int if (Markers.isTxnMarker(msgMetadata)) { // because consumer can receive message is smaller than maxReadPosition, // so this marker is useless for this subscription - subscription.acknowledgeMessage(Collections.singletonList(entry.getPosition()), AckType.Individual, - Collections.emptyMap()); + individualAcknowledgeMessageIfNeeded(entry.getPosition(), Collections.emptyMap()); entries.set(i, null); entry.release(); continue; } else if (((PersistentTopic) subscription.getTopic()) .isTxnAborted(new TxnID(msgMetadata.getTxnidMostBits(), msgMetadata.getTxnidLeastBits()))) { - subscription.acknowledgeMessage(Collections.singletonList(entry.getPosition()), AckType.Individual, - Collections.emptyMap()); + individualAcknowledgeMessageIfNeeded(entry.getPosition(), Collections.emptyMap()); entries.set(i, null); entry.release(); continue; @@ -151,8 +152,7 @@ public void filterEntriesForConsumer(Optional entryWrapper, int entries.set(i, null); entry.release(); - subscription.acknowledgeMessage(Collections.singletonList(pos), AckType.Individual, - Collections.emptyMap()); + individualAcknowledgeMessageIfNeeded(pos, Collections.emptyMap()); continue; } else if (msgMetadata.hasDeliverAtTime() && trackDelayedDelivery(entry.getLedgerId(), entry.getEntryId(), msgMetadata)) { @@ -188,6 +188,12 @@ && trackDelayedDelivery(entry.getLedgerId(), entry.getEntryId(), msgMetadata)) { sendMessageInfo.setTotalChunkedMessages(totalChunkedMessages); } + private void individualAcknowledgeMessageIfNeeded(Position position, Map properties) { + if (!(subscription instanceof CompactorSubscription)) { + subscription.acknowledgeMessage(Collections.singletonList(position), AckType.Individual, properties); + } + } + /** * Determine whether the number of consumers on the subscription reaches the threshold. * @return diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index c9e0d95f7ffdc..ab9e62c6face5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -43,31 +44,30 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + +import io.netty.buffer.ByteBuf; +import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.api.OpenBuilder; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; +import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.CryptoKeyReader; -import org.apache.pulsar.client.api.EncryptionKeyInfo; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.MessageRoutingMode; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.ProducerBuilder; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Reader; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.*; import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -1656,4 +1656,63 @@ public void testReadUnCompacted(boolean batchEnabled) throws PulsarClientExcepti assertNull(none); } } + + @Test(timeOut = 60000) + public void testCompactionWithMarker() throws Exception { + String namespace = "my-property/use/my-ns"; + final TopicName dest = TopicName.get( + BrokerTestUtil.newUniqueName("persistent://" + namespace + "/testWriteMarker")); + admin.topics().createNonPartitionedTopic(dest.toString()); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(dest.toString()) + .subscriptionName("test-compaction-sub") + .subscriptionType(SubscriptionType.Exclusive) + .readCompacted(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Latest) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(dest.toString()) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send("msg-1".getBytes(StandardCharsets.UTF_8)); + Optional topic = pulsar.getBrokerService().getTopic(dest.toString(), true).join(); + Assert.assertTrue(topic.isPresent()); + PersistentTopic persistentTopic = (PersistentTopic) topic.get(); + Random random = new Random(); + for (int i = 0; i < 100; i++) { + int rad = random.nextInt(3); + ByteBuf marker; + if (rad == 0) { + marker = Markers.newTxnCommitMarker(-1L, 0, i); + } else if (rad == 1) { + marker = Markers.newTxnAbortMarker(-1L, 0, i); + } else { + marker = Markers.newReplicatedSubscriptionsSnapshotRequest(UUID.randomUUID().toString(), "r1"); + } + persistentTopic.getManagedLedger().asyncAddEntry(marker, new AsyncCallbacks.AddEntryCallback() { + @Override + public void addComplete(Position position, ByteBuf entryData, Object ctx) { + // + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + // + } + }, null); + marker.release(); + } + producer.send("msg-2".getBytes(StandardCharsets.UTF_8)); + admin.topics().triggerCompaction(dest.toString()); + Awaitility.await() + .atMost(50, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted(() -> { + long ledgerId = admin.topics().getInternalStats(dest.toString()).compactedLedger.ledgerId; + Assert.assertNotEquals(ledgerId, -1L); + }); + } } From bf513e24432d51b079f0a9ad5da699d48c2d58b2 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sat, 2 Jul 2022 22:23:44 +0800 Subject: [PATCH 651/823] [Branch-2.9] Fix passing incorrect authentication data (#16347) --- .../pulsar/broker/service/ServerCnx.java | 47 +- .../service/ServerCnxAuthorizationTest.java | 433 ++++++++++++++++++ .../pulsar/broker/service/ServerCnxTest.java | 2 +- 3 files changed, 474 insertions(+), 8 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 16cab0a13a917..0bbdaca9f747b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -376,19 +376,21 @@ private boolean invalidOriginalPrincipal(String originalPrincipal) { // //// private CompletableFuture isTopicOperationAllowed(TopicName topicName, TopicOperation operation, - AuthenticationDataSource authData) { + AuthenticationDataSource authDataSource, + AuthenticationDataSource originalAuthDataSource) { if (!service.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } CompletableFuture isProxyAuthorizedFuture; if (originalPrincipal != null) { isProxyAuthorizedFuture = service.getAuthorizationService().allowTopicOperationAsync( - topicName, operation, originalPrincipal, authData); + topicName, operation, originalPrincipal, + originalAuthDataSource != null ? originalAuthDataSource : authDataSource); } else { isProxyAuthorizedFuture = CompletableFuture.completedFuture(true); } CompletableFuture isAuthorizedFuture = service.getAuthorizationService().allowTopicOperationAsync( - topicName, operation, authRole, authData); + topicName, operation, authRole, authDataSource); return isProxyAuthorizedFuture.thenCombine(isAuthorizedFuture, (isProxyAuthorized, isAuthorized) -> { if (!isProxyAuthorized) { log.warn("OriginalRole {} is not authorized to perform operation {} on topic {}", @@ -407,7 +409,13 @@ private CompletableFuture isTopicOperationAllowed(TopicName topicName, if (service.isAuthorizationEnabled()) { AuthenticationDataSource authData = new AuthenticationDataSubscription(getAuthenticationData(), subscriptionName); - return isTopicOperationAllowed(topicName, operation, authData); + AuthenticationDataSource authDataSource = + new AuthenticationDataSubscription(authenticationData, subscriptionName); + AuthenticationDataSource originalAuthDataSource = null; + if (originalAuthData != null) { + originalAuthDataSource = new AuthenticationDataSubscription(originalAuthData, subscriptionName); + } + return isTopicOperationAllowed(topicName, operation, authDataSource, originalAuthDataSource); } else { return CompletableFuture.completedFuture(true); } @@ -442,7 +450,7 @@ protected void handleLookup(CommandLookupTopic lookup) { lookupSemaphore.release(); return; } - isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, getAuthenticationData()).thenApply( + isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData).thenApply( isAuthorized -> { if (isAuthorized) { lookupTopicAsync(getBrokerService().pulsar(), topicName, authoritative, @@ -506,7 +514,7 @@ protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata pa lookupSemaphore.release(); return; } - isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, getAuthenticationData()).thenApply( + isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData).thenApply( isAuthorized -> { if (isAuthorized) { unsafeGetPartitionedTopicMetadataAsync(getBrokerService().pulsar(), topicName) @@ -1161,7 +1169,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { } CompletableFuture isAuthorizedFuture = isTopicOperationAllowed( - topicName, TopicOperation.PRODUCE, getAuthenticationData() + topicName, TopicOperation.PRODUCE, authenticationData, originalAuthData ); isAuthorizedFuture.thenApply(isAuthorized -> { if (isAuthorized) { @@ -2837,4 +2845,29 @@ private static void logNamespaceNameAuthException(SocketAddress remoteAddress, S public boolean hasProducers() { return !producers.isEmpty(); } + + @VisibleForTesting + protected String getOriginalPrincipal() { + return originalPrincipal; + } + + @VisibleForTesting + protected AuthenticationDataSource getAuthData() { + return authenticationData; + } + + @VisibleForTesting + protected AuthenticationDataSource getOriginalAuthData() { + return originalAuthData; + } + + @VisibleForTesting + protected AuthenticationState getOriginalAuthState() { + return originalAuthState; + } + + @VisibleForTesting + protected void setAuthRole(String authRole) { + this.authRole = authRole; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java new file mode 100644 index 0000000000000..0d4580d044cd4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java @@ -0,0 +1,433 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.service; + +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import javax.crypto.SecretKey; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.common.util.OrderedExecutor; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; +import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.broker.service.schema.DefaultSchemaRegistryService; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.api.proto.CommandConnect; +import org.apache.pulsar.common.api.proto.CommandLookupTopic; +import org.apache.pulsar.common.api.proto.CommandProducer; +import org.apache.pulsar.common.api.proto.CommandSubscribe; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.ZooKeeper; +import org.mockito.ArgumentMatcher; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ServerCnxAuthorizationTest { + private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + private final String CLIENT_PRINCIPAL = "client"; + private final String PROXY_PRINCIPAL = "proxy"; + private final String CLIENT_TOKEN = Jwts.builder().setSubject(CLIENT_PRINCIPAL).signWith(SECRET_KEY).compact(); + private final String PROXY_TOKEN = Jwts.builder().setSubject(PROXY_PRINCIPAL).signWith(SECRET_KEY).compact(); + + private PulsarService pulsar; + private PulsarResources pulsarResources; + private BrokerService brokerService; + private ServiceConfiguration svcConfig; + + @BeforeMethod(alwaysRun = true) + public void beforeMethod() throws Exception { + EventLoopGroup eventLoopGroup = new NioEventLoopGroup(); + svcConfig = spy(ServiceConfiguration.class); + svcConfig.setKeepAliveIntervalSeconds(0); + svcConfig.setBrokerShutdownTimeoutMs(0L); + svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(1.0d); + svcConfig.setClusterName("pulsar-cluster"); + svcConfig.setSuperUserRoles(Collections.singleton(PROXY_PRINCIPAL)); + svcConfig.setAuthenticationEnabled(true); + svcConfig.setAuthenticationProviders(Sets.newHashSet(AuthenticationProviderToken.class.getName())); + svcConfig.setAuthorizationEnabled(true); + svcConfig.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); + Properties properties = new Properties(); + properties.setProperty("tokenSecretKey", "data:;base64," + + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); + svcConfig.setProperties(properties); + + pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); + doReturn(new DefaultSchemaRegistryService()).when(pulsar).getSchemaRegistryService(); + + doReturn(svcConfig).when(pulsar).getConfiguration(); + doReturn(mock(PulsarResources.class)).when(pulsar).getPulsarResources(); + + ManagedLedgerFactory mlFactoryMock = mock(ManagedLedgerFactory.class); + doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory(); + + ZooKeeper mockZk = createMockZooKeeper(); + OrderedExecutor executor = OrderedExecutor.newBuilder().numThreads(1).build(); + doReturn(createMockBookKeeper(executor)) + .when(pulsar).getBookKeeperClient(); + + MetadataStore store = new ZKMetadataStore(mockZk); + + doReturn(store).when(pulsar).getLocalMetadataStore(); + doReturn(store).when(pulsar).getConfigurationMetadataStore(); + + pulsarResources = spyWithClassAndConstructorArgs(PulsarResources.class, store, store); + doReturn(pulsarResources).when(pulsar).getPulsarResources(); + NamespaceResources namespaceResources = + spyWithClassAndConstructorArgs(NamespaceResources.class, store, store, 30); + doReturn(namespaceResources).when(pulsarResources).getNamespaceResources(); + + TenantResources tenantResources = spyWithClassAndConstructorArgs(TenantResources.class, store, 30); + doReturn(tenantResources).when(pulsarResources).getTenantResources(); + + doReturn(CompletableFuture.completedFuture(Optional.of(TenantInfo.builder().build()))).when(tenantResources) + .getTenantAsync("public"); + + brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); + BrokerInterceptor interceptor = mock(BrokerInterceptor.class); + doReturn(interceptor).when(brokerService).getInterceptor(); + doReturn(brokerService).when(pulsar).getBrokerService(); + doReturn(executor).when(pulsar).getOrderedExecutor(); + } + + @Test + public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws Exception { + doReturn(true).when(svcConfig).isAuthenticateOriginalAuthData(); + + ServerCnx serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + ChannelPipeline channelPipeline = mock(ChannelPipeline.class); + doReturn(channelPipeline).when(channel).pipeline(); + doReturn(null).when(channelPipeline).get(PulsarChannelInitializer.TLS_HANDLER); + + SocketAddress socketAddress = new InetSocketAddress(0); + doReturn(socketAddress).when(channel).remoteAddress(); + doReturn(channel).when(channelHandlerContext).channel(); + channelHandlerContext.channel().remoteAddress(); + serverCnx.channelActive(channelHandlerContext); + + // connect + AuthenticationToken clientAuthenticationToken = new AuthenticationToken(CLIENT_TOKEN); + AuthenticationToken proxyAuthenticationToken = new AuthenticationToken(PROXY_TOKEN); + CommandConnect connect = new CommandConnect(); + connect.setAuthMethodName(proxyAuthenticationToken.getAuthMethodName()); + connect.setAuthData(proxyAuthenticationToken.getAuthData().getCommandData().getBytes(StandardCharsets.UTF_8)); + connect.setClientVersion("test"); + connect.setProtocolVersion(1); + connect.setOriginalPrincipal(CLIENT_PRINCIPAL); + connect.setOriginalAuthData(clientAuthenticationToken.getAuthData().getCommandData()); + connect.setOriginalAuthMethod(clientAuthenticationToken.getAuthMethodName()); + + serverCnx.handleConnect(connect); + assertEquals(serverCnx.getOriginalAuthData().getCommandData(), + clientAuthenticationToken.getAuthData().getCommandData()); + assertEquals(serverCnx.getOriginalAuthState().getAuthRole(), CLIENT_PRINCIPAL); + assertEquals(serverCnx.getOriginalPrincipal(), CLIENT_PRINCIPAL); + assertEquals(serverCnx.getAuthData().getCommandData(), + proxyAuthenticationToken.getAuthData().getCommandData()); + assertEquals(serverCnx.getAuthRole(), PROXY_PRINCIPAL); + assertEquals(serverCnx.getAuthState().getAuthRole(), PROXY_PRINCIPAL); + + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsarResources); + doReturn(authorizationService).when(brokerService).getAuthorizationService(); + + // lookup + CommandLookupTopic commandLookupTopic = new CommandLookupTopic(); + TopicName topicName = TopicName.get("persistent://public/default/test-topic"); + commandLookupTopic.setTopic(topicName.toString()); + commandLookupTopic.setRequestId(1); + serverCnx.handleLookup(commandLookupTopic); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + CLIENT_PRINCIPAL, + serverCnx.getOriginalAuthData()); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + PROXY_PRINCIPAL, + serverCnx.getAuthData()); + + // producer + CommandProducer commandProducer = new CommandProducer(); + commandProducer.setRequestId(1); + commandProducer.setProducerId(1); + commandProducer.setProducerName("test-producer"); + commandProducer.setTopic(topicName.toString()); + serverCnx.handleProducer(commandProducer); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, + CLIENT_PRINCIPAL, + serverCnx.getOriginalAuthData()); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + PROXY_PRINCIPAL, + serverCnx.getAuthData()); + + // consumer + CommandSubscribe commandSubscribe = new CommandSubscribe(); + commandSubscribe.setTopic(topicName.toString()); + commandSubscribe.setRequestId(1); + commandSubscribe.setConsumerId(1); + final String subscriptionName = "test-subscribe"; + commandSubscribe.setSubscription("test-subscribe"); + commandSubscribe.setSubType(CommandSubscribe.SubType.Shared); + serverCnx.handleSubscribe(commandSubscribe); + + verify(authorizationService, times(1)).allowTopicOperationAsync( + eq(topicName), eq(TopicOperation.CONSUME), + eq(CLIENT_PRINCIPAL), argThat(arg -> { + assertTrue(arg instanceof AuthenticationDataSubscription); + try { + assertEquals(arg.getCommandData(), clientAuthenticationToken.getAuthData().getCommandData()); + } catch (PulsarClientException e) { + fail(e.getMessage()); + } + assertEquals(arg.getSubscription(), subscriptionName); + return true; + })); + verify(authorizationService, times(1)).allowTopicOperationAsync( + eq(topicName), eq(TopicOperation.CONSUME), + eq(PROXY_PRINCIPAL), argThat(arg -> { + assertTrue(arg instanceof AuthenticationDataSubscription); + try { + assertEquals(arg.getCommandData(), proxyAuthenticationToken.getAuthData().getCommandData()); + } catch (PulsarClientException e) { + fail(e.getMessage()); + } + assertEquals(arg.getSubscription(), subscriptionName); + return true; + })); + } + + @Test + public void testVerifyOriginalPrincipalWithoutAuthDataForwardedFromProxy() throws Exception { + doReturn(false).when(svcConfig).isAuthenticateOriginalAuthData(); + + ServerCnx serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + ChannelPipeline channelPipeline = mock(ChannelPipeline.class); + doReturn(channelPipeline).when(channel).pipeline(); + doReturn(null).when(channelPipeline).get(PulsarChannelInitializer.TLS_HANDLER); + + SocketAddress socketAddress = new InetSocketAddress(0); + doReturn(socketAddress).when(channel).remoteAddress(); + doReturn(channel).when(channelHandlerContext).channel(); + channelHandlerContext.channel().remoteAddress(); + serverCnx.channelActive(channelHandlerContext); + + // connect + AuthenticationToken proxyAuthenticationToken = new AuthenticationToken(PROXY_TOKEN); + CommandConnect connect = new CommandConnect(); + connect.setAuthMethodName(proxyAuthenticationToken.getAuthMethodName()); + connect.setAuthData(proxyAuthenticationToken.getAuthData().getCommandData().getBytes(StandardCharsets.UTF_8)); + connect.setClientVersion("test"); + connect.setProtocolVersion(1); + connect.setOriginalPrincipal(CLIENT_PRINCIPAL); + serverCnx.handleConnect(connect); + assertNull(serverCnx.getOriginalAuthData()); + assertNull(serverCnx.getOriginalAuthState()); + assertEquals(serverCnx.getOriginalPrincipal(), CLIENT_PRINCIPAL); + assertEquals(serverCnx.getAuthData().getCommandData(), + proxyAuthenticationToken.getAuthData().getCommandData()); + assertEquals(serverCnx.getAuthRole(), PROXY_PRINCIPAL); + assertEquals(serverCnx.getAuthState().getAuthRole(), PROXY_PRINCIPAL); + + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsarResources); + doReturn(authorizationService).when(brokerService).getAuthorizationService(); + + // lookup + CommandLookupTopic commandLookupTopic = new CommandLookupTopic(); + TopicName topicName = TopicName.get("persistent://public/default/test-topic"); + commandLookupTopic.setTopic(topicName.toString()); + commandLookupTopic.setRequestId(1); + serverCnx.handleLookup(commandLookupTopic); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + CLIENT_PRINCIPAL, + serverCnx.getAuthData()); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + PROXY_PRINCIPAL, + serverCnx.getAuthData()); + + // producer + CommandProducer commandProducer = new CommandProducer(); + commandProducer.setRequestId(1); + commandProducer.setProducerId(1); + commandProducer.setProducerName("test-producer"); + commandProducer.setTopic(topicName.toString()); + serverCnx.handleProducer(commandProducer); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, + CLIENT_PRINCIPAL, + serverCnx.getAuthData()); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + PROXY_PRINCIPAL, + serverCnx.getAuthData()); + + // consumer + CommandSubscribe commandSubscribe = new CommandSubscribe(); + commandSubscribe.setTopic(topicName.toString()); + commandSubscribe.setRequestId(1); + commandSubscribe.setConsumerId(1); + final String subscriptionName = "test-subscribe"; + commandSubscribe.setSubscription("test-subscribe"); + commandSubscribe.setSubType(CommandSubscribe.SubType.Shared); + serverCnx.handleSubscribe(commandSubscribe); + + ArgumentMatcher authenticationDataSourceArgumentMatcher = arg -> { + assertTrue(arg instanceof AuthenticationDataSubscription); + try { + assertEquals(arg.getCommandData(), proxyAuthenticationToken.getAuthData().getCommandData()); + } catch (PulsarClientException e) { + fail(e.getMessage()); + } + assertEquals(arg.getSubscription(), subscriptionName); + return true; + }; + + verify(authorizationService, times(1)).allowTopicOperationAsync( + eq(topicName), eq(TopicOperation.CONSUME), + eq(CLIENT_PRINCIPAL), argThat(authenticationDataSourceArgumentMatcher)); + verify(authorizationService, times(1)).allowTopicOperationAsync( + eq(topicName), eq(TopicOperation.CONSUME), + eq(PROXY_PRINCIPAL), argThat(authenticationDataSourceArgumentMatcher)); + } + + @Test + public void testVerifyAuthRoleAndAuthDataFromDirectConnectionBroker() throws Exception { + ServerCnx serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + + ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + ChannelPipeline channelPipeline = mock(ChannelPipeline.class); + doReturn(channelPipeline).when(channel).pipeline(); + doReturn(null).when(channelPipeline).get(PulsarChannelInitializer.TLS_HANDLER); + + SocketAddress socketAddress = new InetSocketAddress(0); + doReturn(socketAddress).when(channel).remoteAddress(); + doReturn(channel).when(channelHandlerContext).channel(); + channelHandlerContext.channel().remoteAddress(); + serverCnx.channelActive(channelHandlerContext); + + // connect + AuthenticationToken clientAuthenticationToken = new AuthenticationToken(CLIENT_TOKEN); + CommandConnect connect = new CommandConnect(); + connect.setAuthMethodName(clientAuthenticationToken.getAuthMethodName()); + connect.setAuthData(clientAuthenticationToken.getAuthData().getCommandData().getBytes(StandardCharsets.UTF_8)); + connect.setClientVersion("test"); + connect.setProtocolVersion(1); + serverCnx.handleConnect(connect); + assertNull(serverCnx.getOriginalAuthData()); + assertNull(serverCnx.getOriginalAuthState()); + assertNull(serverCnx.getOriginalPrincipal()); + assertEquals(serverCnx.getAuthData().getCommandData(), + clientAuthenticationToken.getAuthData().getCommandData()); + assertEquals(serverCnx.getAuthRole(), CLIENT_PRINCIPAL); + assertEquals(serverCnx.getAuthState().getAuthRole(), CLIENT_PRINCIPAL); + + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsarResources); + doReturn(authorizationService).when(brokerService).getAuthorizationService(); + + // lookup + CommandLookupTopic commandLookupTopic = new CommandLookupTopic(); + TopicName topicName = TopicName.get("persistent://public/default/test-topic"); + commandLookupTopic.setTopic(topicName.toString()); + commandLookupTopic.setRequestId(1); + serverCnx.handleLookup(commandLookupTopic); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.LOOKUP, + CLIENT_PRINCIPAL, + serverCnx.getAuthData()); + + // producer + CommandProducer commandProducer = new CommandProducer(); + commandProducer.setRequestId(1); + commandProducer.setProducerId(1); + commandProducer.setProducerName("test-producer"); + commandProducer.setTopic(topicName.toString()); + serverCnx.handleProducer(commandProducer); + verify(authorizationService, times(1)).allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, + CLIENT_PRINCIPAL, + serverCnx.getAuthData()); + + // consumer + CommandSubscribe commandSubscribe = new CommandSubscribe(); + commandSubscribe.setTopic(topicName.toString()); + commandSubscribe.setRequestId(1); + commandSubscribe.setConsumerId(1); + final String subscriptionName = "test-subscribe"; + commandSubscribe.setSubscription("test-subscribe"); + commandSubscribe.setSubType(CommandSubscribe.SubType.Shared); + serverCnx.handleSubscribe(commandSubscribe); + + verify(authorizationService, times(1)).allowTopicOperationAsync( + eq(topicName), eq(TopicOperation.CONSUME), + eq(CLIENT_PRINCIPAL), argThat(arg -> { + assertTrue(arg instanceof AuthenticationDataSubscription); + try { + assertEquals(arg.getCommandData(), clientAuthenticationToken.getAuthData().getCommandData()); + } catch (PulsarClientException e) { + fail(e.getMessage()); + } + assertEquals(arg.getSubscription(), subscriptionName); + return true; + })); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index be77a50eea52d..5615a28a45772 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -1550,7 +1550,7 @@ protected void resetChannel() throws Exception { channel.close().get(); } serverCnx = new ServerCnx(pulsar); - serverCnx.authRole = ""; + serverCnx.setAuthRole(""); channel = new EmbeddedChannel(new LengthFieldBasedFrameDecoder( MaxMessageSize, 0, From 1038a745c352e1adda1e0b9f62d5fab23dce6b17 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 2 Jul 2022 17:02:07 -0700 Subject: [PATCH 652/823] Fixed deadlock when checking topic ownership (#16310) * Fixed deadlock when checking topic ownership * Fixed mocked test * Fixed ServerCnxTest * Fixed testProducerOnNotOwnedTopic --- .../broker/namespace/NamespaceService.java | 9 +++++ .../pulsar/broker/service/BrokerService.java | 33 ++++++++++++------- .../broker/service/PersistentTopicTest.java | 1 + .../pulsar/broker/service/ServerCnxTest.java | 4 ++- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 4d910cb901dfc..fa4cd16ee111f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1008,6 +1008,15 @@ public boolean isServiceUnitActive(TopicName topicName) { } } + public CompletableFuture isServiceUnitActiveAsync(TopicName topicName) { + Optional> res = ownershipCache.getOwnedBundleAsync(getBundle(topicName)); + if (!res.isPresent()) { + return CompletableFuture.completedFuture(false); + } + + return res.get().thenApply(ob -> ob != null && ob.isActive()); + } + private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception { return ownershipCache.getOwnedBundle(getFullBundle(fqnn)) != null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 24ef20452c035..bc6dbb73750b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1279,7 +1279,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); if (topicLoadSemaphore.tryAcquire()) { - createPersistentTopic(topic, createIfMissing, topicFuture); + checkOwnershipAndCreatePersistentTopic(topic, createIfMissing, topicFuture); topicFuture.handle((persistentTopic, ex) -> { // release permit and process pending topic topicLoadSemaphore.release(); @@ -1300,19 +1300,30 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S return topicFuture; } - private void createPersistentTopic(final String topic, boolean createIfMissing, + private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean createIfMissing, CompletableFuture> topicFuture) { + TopicName topicName = TopicName.get(topic); + pulsar.getNamespaceService().isServiceUnitActiveAsync(topicName) + .thenAccept(isActive -> { + if (isActive) { + createPersistentTopic(topic, createIfMissing, topicFuture); + } else { + // namespace is being unloaded + String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic); + log.warn(msg); + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg)); + } + }).exceptionally(ex -> { + topicFuture.completeExceptionally(ex); + return null; + }); + } + private void createPersistentTopic(final String topic, boolean createIfMissing, + CompletableFuture> topicFuture) { final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); TopicName topicName = TopicName.get(topic); - if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) { - // namespace is being unloaded - String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic); - log.warn(msg); - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg)); - return; - } if (isTransactionSystemTopic(topicName)) { String msg = String.format("Can not create transaction system topic %s", topic); @@ -2386,7 +2397,7 @@ private void createPendingLoadTopic() { CompletableFuture> pendingFuture = pendingTopic.getRight(); final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); final boolean acquiredPermit = topicLoadSemaphore.tryAcquire(); - createPersistentTopic(topic, true, pendingFuture); + checkOwnershipAndCreatePersistentTopic(topic, true, pendingFuture); pendingFuture.handle((persistentTopic, ex) -> { // release permit and process next pending topic if (acquiredPermit) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 260042513996e..3e527e8135a9e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -229,6 +229,7 @@ public void setup() throws Exception { doReturn(nsSvc).when(pulsar).getNamespaceService(); doReturn(true).when(nsSvc).isServiceUnitOwned(any()); doReturn(true).when(nsSvc).isServiceUnitActive(any()); + doReturn(CompletableFuture.completedFuture(true)).when(nsSvc).isServiceUnitActiveAsync(any()); doReturn(CompletableFuture.completedFuture(true)).when(nsSvc).checkTopicOwnership(any()); setupMLAsyncCallbackMocks(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 5615a28a45772..be9ed72cd1474 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -198,6 +198,7 @@ public void setup() throws Exception { doReturn(namespaceService).when(pulsar).getNamespaceService(); doReturn(true).when(namespaceService).isServiceUnitOwned(any()); doReturn(true).when(namespaceService).isServiceUnitActive(any()); + doReturn(CompletableFuture.completedFuture(true)).when(namespaceService).isServiceUnitActiveAsync(any()); doReturn(CompletableFuture.completedFuture(true)).when(namespaceService).checkTopicOwnership(any()); setupMLAsyncCallbackMocks(); @@ -463,7 +464,8 @@ public void testProducerOnNotOwnedTopic() throws Exception { setChannelConnected(); // Force the case where the broker doesn't own any topic - doReturn(false).when(namespaceService).isServiceUnitActive(any(TopicName.class)); + doReturn(CompletableFuture.completedFuture(false)).when(namespaceService) + .isServiceUnitActiveAsync(any(TopicName.class)); // test PRODUCER failure case ByteBuf clientCommand = Commands.newProducer(nonOwnedTopicName, 1 /* producer id */, 1 /* request id */, From 69830a6662070b4655c1b554f5e7035d87700547 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sun, 3 Jul 2022 12:00:46 +0800 Subject: [PATCH 653/823] [Branch-2.9][transaction] Cmd-Subscribe and Cmd-Producer will not succeed even after 100 retries. (#16349) --- .../buffer/impl/TopicTransactionBuffer.java | 11 +- .../impl/MLPendingAckReplyCallBack.java | 7 +- .../broker/transaction/TransactionTest.java | 180 ++++++++++++++++++ 3 files changed, 194 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index d0758dc3c5a7d..3dc9ff88c2028 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer.impl; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.util.Timeout; import io.netty.util.Timer; @@ -131,7 +132,12 @@ public void recoverComplete() { maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } if (!changeToReadyState()) { - log.error("[{}]Transaction buffer recover fail", topic.getName()); + log.error("[{}]Transaction buffer recover fail, current state: {}", + topic.getName(), getState()); + transactionBufferFuture.completeExceptionally + (new BrokerServiceException.ServiceUnitNotReadyException( + "Transaction buffer recover failed to change the status to Ready," + + "current state is: " + getState())); } else { timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); @@ -566,7 +572,8 @@ public void run(Timeout timeout) { // we store the maxReadPosition from snapshot then open the non-durable cursor by this topic's manageLedger. // the non-durable cursor will read to lastConfirmedEntry. - static class TopicTransactionBufferRecover implements Runnable { + @VisibleForTesting + public static class TopicTransactionBufferRecover implements Runnable { private final PersistentTopic topic; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java index 728e7f0476b34..53de549f69fcd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; @@ -53,8 +54,10 @@ public void replayComplete() { log.info("Topic name : [{}], SubName : [{}] pending ack handle cache request success!", pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); } else { - log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail!", - pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail! current state: {}", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName(), pendingAckHandle.state); + replayFailed(new BrokerServiceException.ServiceUnitNotReadyException("Failed" + + " to change PendingAckHandle state to Ready, current state is : " + pendingAckHandle.state)); } pendingAckHandle.handleCacheRequest(); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index dc02313bb1f0d..72181ee53b711 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -26,8 +26,10 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -37,6 +39,7 @@ import io.netty.util.Timeout; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.List; @@ -46,9 +49,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.util.Bytes; @@ -62,16 +69,29 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.broker.service.BacklogQuotaManager; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TransactionBufferSnapshotService; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.buffer.matadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckReplyCallBack; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; @@ -90,6 +110,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.events.EventType; @@ -111,6 +132,8 @@ import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.awaitility.Awaitility; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.powermock.reflect.Whitebox; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -1089,4 +1112,161 @@ public void testPendingAckBatchMessageCommit() throws Exception { // repeat ack the second message, can ack successful consumer.acknowledgeAsync(messageId, txn3).get(); } + + /** + * When change pending ack handle state failure, exceptionally complete cmd-subscribe. + * see: https://github.com/apache/pulsar/pull/16248. + */ + @Test + public void testPendingAckReplayChangeStateError() throws InterruptedException, TimeoutException { + AtomicInteger atomicInteger = new AtomicInteger(1); + // Create Executor + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + // Mock serviceConfiguration. + ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + when(serviceConfiguration.isTransactionCoordinatorEnabled()).thenReturn(true); + // Mock executorProvider. + ExecutorProvider executorProvider = mock(ExecutorProvider.class); + when(executorProvider.getExecutor()).thenReturn(executorService); + when(executorProvider.getExecutor(any(Object.class))).thenReturn(executorService); + // Mock pendingAckStore. + PendingAckStore pendingAckStore = mock(PendingAckStore.class); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + executorService.execute(()->{ + PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) invocation.getArguments()[0]; + pendingAckHandle.close(); + MLPendingAckReplyCallBack mlPendingAckReplyCallBack + = new MLPendingAckReplyCallBack(pendingAckHandle); + mlPendingAckReplyCallBack.replayComplete(); + }); + return null; + } + }).when(pendingAckStore).replayAsync(any(), any()); + // Mock executorProvider. + TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class); + when(pendingAckStoreProvider.checkInitializedBefore(any())) + .thenReturn(CompletableFuture.completedFuture(true)); + when(pendingAckStoreProvider.newPendingAckStore(any())) + .thenReturn(CompletableFuture.completedFuture(pendingAckStore)); + // Mock pulsar. + PulsarService pulsar = mock(PulsarService.class); + when(pulsar.getConfig()).thenReturn(serviceConfiguration); + when(pulsar.getTransactionExecutorProvider()).thenReturn(executorProvider); + when(pulsar.getTransactionPendingAckStoreProvider()).thenReturn(pendingAckStoreProvider); + // Mock brokerService. + BrokerService brokerService = mock(BrokerService.class); + when(brokerService.getPulsar()).thenReturn(pulsar); + // Mock topic. + PersistentTopic topic = mock(PersistentTopic.class); + when(topic.getBrokerService()).thenReturn(brokerService); + when(topic.getName()).thenReturn("topic-a"); + // Mock cursor for subscription. + ManagedCursor cursor_subscription = mock(ManagedCursor.class); + doThrow(new RuntimeException("1")).when(cursor_subscription).updateLastActive(); + // Create subscription. + String subscriptionName = "sub-a"; + boolean replicated = false; + PersistentSubscription persistentSubscription = new PersistentSubscription(topic, subscriptionName, + cursor_subscription, replicated); + org.apache.pulsar.broker.service.Consumer consumer = mock(org.apache.pulsar.broker.service.Consumer.class); + try { + CompletableFuture addConsumerFuture = persistentSubscription.addConsumer(consumer); + addConsumerFuture.get(5, TimeUnit.SECONDS); + fail("Expect failure by PendingAckHandle closed, but success"); + } catch (ExecutionException executionException){ + Throwable t = executionException.getCause(); + Assert.assertTrue(t instanceof BrokerServiceException.ServiceUnitNotReadyException); + } + } + + /** + * When change TB state failure, exceptionally complete cmd-producer. + * see: https://github.com/apache/pulsar/pull/16248. + */ + @Test + public void testTBRecoverChangeStateError() throws InterruptedException, TimeoutException { + final AtomicReference persistentTopic = new AtomicReference(); + AtomicInteger atomicInteger = new AtomicInteger(1); + // Create Executor + ScheduledExecutorService executorService_recover = mock(ScheduledExecutorService.class); + // Mock serviceConfiguration. + ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + when(serviceConfiguration.isEnableReplicatedSubscriptions()).thenReturn(false); + when(serviceConfiguration.isTransactionCoordinatorEnabled()).thenReturn(true); + // Mock executorProvider. + ExecutorProvider executorProvider = mock(ExecutorProvider.class); + when(executorProvider.getExecutor(any(Object.class))).thenReturn(executorService_recover); + // Mock pendingAckStore. + PendingAckStore pendingAckStore = mock(PendingAckStore.class); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + new Thread(() -> { + TopicTransactionBuffer.TopicTransactionBufferRecover recover + = (TopicTransactionBuffer.TopicTransactionBufferRecover)invocation.getArguments()[0]; + TopicTransactionBufferRecoverCallBack callBack + = Whitebox.getInternalState(recover, "callBack");; + try { + persistentTopic.get().getTransactionBuffer().closeAsync().get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + callBack.recoverComplete(); + }).start(); + return null; + } + }).when(executorService_recover).execute(any()); + // Mock executorProvider. + TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class); + when(pendingAckStoreProvider.checkInitializedBefore(any())) + .thenReturn(CompletableFuture.completedFuture(true)); + when(pendingAckStoreProvider.newPendingAckStore(any())) + .thenReturn(CompletableFuture.completedFuture(pendingAckStore)); + // Mock TransactionBufferSnapshotService + TransactionBufferSnapshotService transactionBufferSnapshotService + = mock(TransactionBufferSnapshotService.class); + SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class); + when(writer.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + when(transactionBufferSnapshotService.createWriter(any())) + .thenReturn(CompletableFuture.completedFuture(writer)); + // Mock pulsar. + PulsarService pulsar = mock(PulsarService.class); + when(pulsar.getConfiguration()).thenReturn(serviceConfiguration); + when(pulsar.getConfig()).thenReturn(serviceConfiguration); + when(pulsar.getTransactionExecutorProvider()).thenReturn(executorProvider); + when(pulsar.getTransactionBufferSnapshotService()).thenReturn(transactionBufferSnapshotService); + PulsarResources pulsarResources = mock(PulsarResources.class); + when(pulsar.getPulsarResources()).thenReturn(pulsarResources); + NamespaceResources nsResources = mock(NamespaceResources.class); + when(pulsarResources.getNamespaceResources()).thenReturn(nsResources); + TopicTransactionBufferProvider topicTransactionBufferProvider = new TopicTransactionBufferProvider(); + when(pulsar.getTransactionBufferProvider()).thenReturn(topicTransactionBufferProvider); + // Mock BacklogQuotaManager + BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); + // Mock brokerService. + BrokerService brokerService = mock(BrokerService.class); + when(brokerService.getPulsar()).thenReturn(pulsar); + when(brokerService.pulsar()).thenReturn(pulsar); + when(brokerService.getBacklogQuotaManager()).thenReturn(backlogQuotaManager); + // Mock managedLedger. + ManagedLedgerImpl managedLedger = mock(ManagedLedgerImpl.class); + ManagedCursorContainer managedCursors = new ManagedCursorContainer(); + when(managedLedger.getCursors()).thenReturn(managedCursors); + PositionImpl position = PositionImpl.earliest; + when(managedLedger.getLastConfirmedEntry()).thenReturn(position); + // Create topic. + persistentTopic.set(new PersistentTopic("topic-a", managedLedger, brokerService)); + try { + // Do check. + persistentTopic.get().checkIfTransactionBufferRecoverCompletely(true).get(5, TimeUnit.SECONDS); + fail("Expect failure by TB closed, but it is finished."); + } catch (ExecutionException executionException){ + Throwable t = executionException.getCause(); + Assert.assertTrue(t instanceof BrokerServiceException.ServiceUnitNotReadyException); + } + } } From 51d658435120de750f925ad024fd60b7ee951f5e Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Thu, 7 Jul 2022 01:54:22 +0800 Subject: [PATCH 654/823] [fix][broker] Fix RawReader out of order (#16390) * fix RawReader out ouf order * address comments * tune code --- .../java/org/apache/pulsar/client/impl/RawReaderImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 7c75f09977376..bf9fbec59f669 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -210,9 +210,10 @@ void messageReceived(MessageIdData messageId, int redeliveryCount, log.debug("[{}][{}] Received raw message: {}/{}/{}", topic, subscription, messageId.getEntryId(), messageId.getLedgerId(), messageId.getPartition()); } + incomingRawMessages.add( - new RawMessageAndCnx(new RawMessageImpl(messageId, headersAndPayload), cnx)); - tryCompletePending(); + new RawMessageAndCnx(new RawMessageImpl(messageId, headersAndPayload), cnx)); + internalPinnedExecutor.execute(this::tryCompletePending); } } From 0735f6be652bae0a8868489c16c55ad0487e6994 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 11 Jul 2022 11:41:51 +0800 Subject: [PATCH 655/823] [Branch-2.9][Cherry-pick] Fix get non-persistent topics issue in Namespaces. (#16517) * Fix get non-persistent topics issue in Namespaces. (#16170) --- .../broker/admin/impl/NamespacesBase.java | 44 +++++++++++++++++++ .../pulsar/broker/admin/v1/Namespaces.java | 10 ++--- .../pulsar/broker/admin/v2/Namespaces.java | 10 ++--- .../pulsar/broker/web/PulsarWebResource.java | 23 ++++++++++ 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index caf75a99d1d3e..2be1e32ba3201 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -48,6 +48,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.UriBuilder; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang.mutable.MutableObject; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; @@ -61,6 +62,7 @@ import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.naming.NamedEntity; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; @@ -163,6 +165,48 @@ protected void internalDeleteNamespace(AsyncResponse asyncResponse, boolean auth } } + protected CompletableFuture> internalGetListOfTopics(Policies policies, + CommandGetTopicsOfNamespace.Mode mode) { + switch (mode) { + case ALL: + return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName) + .thenCombine(internalGetNonPersistentTopics(policies), + (persistentTopics, nonPersistentTopics) -> + ListUtils.union(persistentTopics, nonPersistentTopics)); + case NON_PERSISTENT: + return internalGetNonPersistentTopics(policies); + case PERSISTENT: + default: + return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName); + } + } + + protected CompletableFuture> internalGetNonPersistentTopics(Policies policies) { + final List>> futures = Lists.newArrayList(); + final List boundaries = policies.bundles.getBoundaries(); + for (int i = 0; i < boundaries.size() - 1; i++) { + final String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); + try { + futures.add(pulsar().getAdminClient().topics() + .getListInBundleAsync(namespaceName.toString(), bundle)); + } catch (PulsarServerException e) { + throw new RestException(e); + } + } + return FutureUtil.waitForAll(futures) + .thenApply(__ -> { + final List topics = Lists.newArrayList(); + for (int i = 0; i < futures.size(); i++) { + List topicList = futures.get(i).join(); + if (topicList != null) { + topics.addAll(topicList); + } + } + return topics.stream().filter(name -> !TopicName.get(name).isPersistent()) + .collect(Collectors.toList()); + }); + } + @SuppressWarnings("deprecation") protected void internalDeleteNamespace(AsyncResponse asyncResponse, boolean authoritative) { validateTenantOperation(namespaceName.getTenant(), TenantOperation.DELETE_NAMESPACE); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index afb6174c5508e..ab8ca2d65f140 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -129,12 +129,10 @@ public void getTopics(@PathParam("property") String property, @QueryParam("mode") @DefaultValue("PERSISTENT") Mode mode, @Suspended AsyncResponse asyncResponse) { validateNamespaceName(property, cluster, namespace); - validateNamespaceOperation(NamespaceName.get(property, namespace), NamespaceOperation.GET_TOPICS); - - // Validate that namespace exists, throws 404 if it doesn't exist - getNamespacePolicies(namespaceName); - - pulsar().getNamespaceService().getListOfTopics(namespaceName, mode) + validateNamespaceOperationAsync(NamespaceName.get(property, namespace), NamespaceOperation.GET_TOPICS) + // Validate that namespace exists, throws 404 if it doesn't exist + .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenCompose(policies -> internalGetListOfTopics(policies, mode)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { log.error("Failed to get topics list for namespace {}", namespaceName, ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 4fe4cf315607a..89efd8f83de8d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -99,12 +99,10 @@ public void getTopics(@PathParam("tenant") String tenant, @QueryParam("mode") @DefaultValue("PERSISTENT") Mode mode, @Suspended AsyncResponse asyncResponse) { validateNamespaceName(tenant, namespace); - validateNamespaceOperation(NamespaceName.get(tenant, namespace), NamespaceOperation.GET_TOPICS); - - // Validate that namespace exists, throws 404 if it doesn't exist - getNamespacePolicies(namespaceName); - - pulsar().getNamespaceService().getListOfTopics(namespaceName, mode) + validateNamespaceOperationAsync(NamespaceName.get(tenant, namespace), NamespaceOperation.GET_TOPICS) + // Validate that namespace exists, throws 404 if it doesn't exist + .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenCompose(policies -> internalGetListOfTopics(policies, mode)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { log.error("Failed to get topics list for namespace {}", namespaceName, ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 20652d14a539c..da9a3f060abd2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -946,6 +946,29 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) { } } + public CompletableFuture validateNamespaceOperationAsync(NamespaceName namespaceName, + NamespaceOperation operation) { + if (pulsar().getConfiguration().isAuthenticationEnabled() + && pulsar().getBrokerService().isAuthorizationEnabled()) { + if (!isClientAuthenticated(clientAppId())) { + return FutureUtil.failedFuture( + new RestException(Status.FORBIDDEN, "Need to authenticate to perform the request")); + } + + return pulsar().getBrokerService().getAuthorizationService() + .allowNamespaceOperationAsync(namespaceName, operation, originalPrincipal(), + clientAppId(), clientAuthData()) + .thenAccept(isAuthorized -> { + if (!isAuthorized) { + throw new RestException(Status.FORBIDDEN, + String.format("Unauthorized to validateNamespaceOperation for" + + " operation [%s] on namespace [%s]", operation.toString(), namespaceName)); + } + }); + } + return CompletableFuture.completedFuture(null); + } + public void validateNamespacePolicyOperation(NamespaceName namespaceName, PolicyName policy, PolicyOperation operation) { From 3cea10a5533df5f346558b28ebc1cfa4628e76c9 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Mon, 11 Jul 2022 18:20:33 +0800 Subject: [PATCH 656/823] [fix][txn] Allow producer enable send timeout in transaction (#16519) (cherry picked from commit bbf2a47867ff54327c1f2940e72f08a44a5dc5f7) --- .../client/impl/TransactionEndToEndTest.java | 42 ++++++++++++++++++- .../pulsar/client/impl/ProducerBase.java | 5 --- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index b4f43f8e83839..d3e8674925c61 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -19,6 +19,10 @@ package org.apache.pulsar.client.impl; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyObject; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -33,10 +37,10 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; - +import io.netty.channel.ChannelHandlerContext; +import io.netty.util.concurrent.EventExecutor; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; - import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -1061,4 +1065,38 @@ public void testTxnTimeOutInClient() throws Exception{ .InvalidTxnStatusException); } } + + @Test + public void testSendTxnMessageTimeout() throws Exception { + String topic = NAMESPACE1 + "/testSendTxnMessageTimeout"; + @Cleanup + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer() + .topic(topic) + .sendTimeout(1, TimeUnit.SECONDS) + .create(); + + Transaction transaction = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + + // mock cnx, send message can't receive response + ClientCnx cnx = mock(ClientCnx.class); + ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); + doReturn(channelHandlerContext).when(cnx).ctx(); + EventExecutor eventExecutor = mock(EventExecutor.class); + doReturn(eventExecutor).when(channelHandlerContext).executor(); + CompletableFuture completableFuture = new CompletableFuture<>(); + completableFuture.complete(new ProducerResponse("test", 1, + "1".getBytes(), Optional.of(30L))); + doReturn(completableFuture).when(cnx).sendRequestWithId(anyObject(), anyLong()); + producer.getConnectionHandler().setClientCnx(cnx); + + + try { + // send message with txn use mock cnx, will not receive send response + producer.newMessage(transaction).value("Hello Pulsar!".getBytes()).send(); + fail(); + } catch (PulsarClientException ex) { + assertTrue(ex instanceof PulsarClientException.TimeoutException); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java index 180319034daea..053fb529596e7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java @@ -92,11 +92,6 @@ public TypedMessageBuilder newMessage(Schema schema) { public TypedMessageBuilder newMessage(Transaction txn) { checkArgument(txn instanceof TransactionImpl); - // check the producer has proper settings to send transactional messages - if (conf.getSendTimeoutMs() > 0) { - throw new IllegalArgumentException("Only producers disabled sendTimeout are allowed to" - + " produce transactional messages"); - } return new TypedMessageBuilderImpl<>(this, schema, (TransactionImpl) txn); } From ab1d708e6bb7df0a71458f3d1e64105aceba3dee Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Tue, 7 Jun 2022 08:52:40 +0800 Subject: [PATCH 657/823] Support shrink for TripleLongPriorityQueue (#15936) * Support shrinkage in TripleLongPriorityQueue * Add unit test * Remove unused code * style * Address comments --- .../collections/TripleLongPriorityQueue.java | 59 +++++++++++++++++-- .../TripleLongPriorityQueueTest.java | 35 +++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java index 1d8d909beae79..487c2284cffdf 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java @@ -19,6 +19,7 @@ package org.apache.pulsar.common.util.collections; import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.buffer.PooledByteBufAllocator; @@ -31,16 +32,30 @@ public class TripleLongPriorityQueue implements AutoCloseable { private static final int SIZE_OF_LONG = 8; private static final int DEFAULT_INITIAL_CAPACITY = 16; + private static final float DEFAULT_SHRINK_FACTOR = 0.5f; // Each item is composed of 3 longs private static final int ITEMS_COUNT = 3; private static final int TUPLE_SIZE = ITEMS_COUNT * SIZE_OF_LONG; - private final ByteBuf buffer; + /** + * Reserve 10% of the capacity when shrinking to avoid frequent expansion and shrinkage. + */ + private static final float RESERVATION_FACTOR = 0.9f; + + private ByteBuf buffer; + + private final int initialCapacity; private int capacity; private int size; + /** + * When size < capacity * shrinkFactor, may trigger shrinking. + */ + private final float shrinkFactor; + + private float shrinkThreshold; /** * Create a new priority queue with default initial capacity. @@ -49,14 +64,22 @@ public TripleLongPriorityQueue() { this(DEFAULT_INITIAL_CAPACITY); } + public TripleLongPriorityQueue(int initialCapacity, float shrinkFactor) { + checkArgument(shrinkFactor > 0); + this.initialCapacity = initialCapacity; + this.capacity = initialCapacity; + this.shrinkThreshold = this.capacity * shrinkFactor; + this.buffer = PooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity * TUPLE_SIZE); + this.size = 0; + this.shrinkFactor = shrinkFactor; + } + /** * Create a new priority queue with a given initial capacity. * @param initialCapacity */ public TripleLongPriorityQueue(int initialCapacity) { - capacity = initialCapacity; - buffer = PooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity * ITEMS_COUNT * SIZE_OF_LONG); - size = 0; + this(initialCapacity, DEFAULT_SHRINK_FACTOR); } /** @@ -122,6 +145,7 @@ public void pop() { swap(0, size - 1); size--; siftDown(0); + shrinkCapacity(); } /** @@ -144,14 +168,36 @@ public int size() { public void clear() { this.buffer.clear(); this.size = 0; + shrinkCapacity(); } private void increaseCapacity() { // For bigger sizes, increase by 50% this.capacity += (capacity <= 256 ? capacity : capacity / 2); + this.shrinkThreshold = this.capacity * shrinkFactor; buffer.capacity(this.capacity * TUPLE_SIZE); } + private void shrinkCapacity() { + if (capacity > initialCapacity && size < shrinkThreshold) { + int decreasingSize = (int) (capacity * shrinkFactor * RESERVATION_FACTOR); + if (decreasingSize <= 0) { + return; + } + if (capacity - decreasingSize <= initialCapacity) { + this.capacity = initialCapacity; + } else { + this.capacity = capacity - decreasingSize; + } + this.shrinkThreshold = this.capacity * shrinkFactor; + + ByteBuf newBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(this.capacity * TUPLE_SIZE); + buffer.getBytes(0, newBuffer, size * TUPLE_SIZE); + buffer.release(); + this.buffer = newBuffer; + } + } + private void siftUp(int idx) { while (idx > 0) { int parentIdx = (idx - 1) / 2; @@ -229,4 +275,9 @@ private void swap(int idx1, int idx2) { buffer.setLong(i2 + 1 * SIZE_OF_LONG, tmp2); buffer.setLong(i2 + 2 * SIZE_OF_LONG, tmp3); } + + @VisibleForTesting + ByteBuf getBuffer() { + return buffer; + } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java index 4cb1027e0a9ff..bd3aef86ad122 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java @@ -135,4 +135,39 @@ public void testCompareWithSamePrefix() { pq.close(); } + + @Test + public void testShrink() throws Exception { + int initialCapacity = 20; + int tupleSize = 3 * 8; + TripleLongPriorityQueue pq = new TripleLongPriorityQueue(initialCapacity, 0.5f); + pq.add(0, 0, 0); + assertEquals(pq.size(), 1); + assertEquals(pq.getBuffer().capacity(), initialCapacity * tupleSize); + + // Scale out to capacity * 2 + triggerScaleOut(initialCapacity, pq); + int scaleCapacity = initialCapacity * 2; + assertEquals(pq.getBuffer().capacity(), scaleCapacity * tupleSize); + // Trigger shrinking + for (int i = 0; i < initialCapacity / 2 + 1; i++) { + pq.pop(); + } + int capacity = scaleCapacity - (int)(scaleCapacity * 0.5f * 0.9f); + assertEquals(pq.getBuffer().capacity(), capacity * tupleSize); + // Scale out to capacity * 2 + triggerScaleOut(initialCapacity, pq); + scaleCapacity = capacity * 2; + assertEquals(pq.getBuffer().capacity(), scaleCapacity * tupleSize); + // Trigger shrinking + pq.clear(); + capacity = scaleCapacity - (int)(scaleCapacity * 0.5f * 0.9f); + assertEquals(pq.getBuffer().capacity(), capacity * tupleSize); + } + + private void triggerScaleOut(int initialCapacity, TripleLongPriorityQueue pq) { + for (long i = 0; i < initialCapacity + 1; i++) { + pq.add(i, i, i); + } + } } From f305c99dbe8d8ff973a2c6cca74a1e048831ea2b Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 12 Jul 2022 16:12:57 -0700 Subject: [PATCH 658/823] [fix][broker] Fixed error when delayed messages trackers state grows to >1.5GB (#16490) * Fixed error when delayed messages trackers state grows to >1.5GB * Fixed spotbugs issues * Fixed javadocs * In the constructor, ensure all segments after the first one are of max size * Use poll to figure out where the test is stuck * Added SegmentedLongArray specific unit test * Removed unused imports --- ...sistentDispatcherFailoverConsumerTest.java | 26 ++- .../util/collections/SegmentedLongArray.java | 128 ++++++++++++ .../collections/TripleLongPriorityQueue.java | 187 ++++++++---------- .../collections/SegmentedLongArrayTest.java | 103 ++++++++++ .../TripleLongPriorityQueueTest.java | 41 +++- 5 files changed, 369 insertions(+), 116 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index 663a1023543d4..40f58c9a67fbe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -32,6 +32,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertSame; @@ -49,6 +50,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.bookkeeper.common.util.OrderedExecutor; @@ -358,7 +360,8 @@ public void testAddRemoveConsumer() throws Exception { // 4. Verify active consumer assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); // get the notified with who is the leader - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 1, true); verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer1)); @@ -370,7 +373,8 @@ public void testAddRemoveConsumer() throws Exception { assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); assertEquals(3, consumers.size()); // get notified with who is the leader - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 2, false); verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer1)); verify(consumer2, times(1)).notifyActiveConsumerChange(same(consumer1)); @@ -385,13 +389,17 @@ public void testAddRemoveConsumer() throws Exception { assertEquals(4, consumers.size()); // all consumers will receive notifications - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 0, true); - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 1, false); - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 1, false); - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 2, false); verify(consumer0, times(1)).notifyActiveConsumerChange(same(consumer0)); verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer1)); @@ -417,9 +425,11 @@ public void testAddRemoveConsumer() throws Exception { assertEquals(2, consumers.size()); // the remaining consumers will receive notifications - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 1, true); - change = consumerChanges.take(); + change = consumerChanges.poll(10, TimeUnit.SECONDS); + assertNotNull(change); verifyActiveConsumerChange(change, 1, true); // 10. Attempt to remove already removed consumer diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java new file mode 100644 index 0000000000000..dc4d1c4908c48 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.concurrent.NotThreadSafe; +import lombok.Getter; + +@NotThreadSafe +public class SegmentedLongArray implements AutoCloseable { + + private static final int SIZE_OF_LONG = 8; + + private static final int MAX_SEGMENT_SIZE = 2 * 1024 * 1024; // 2M longs -> 16 MB + private final List buffers = new ArrayList<>(); + + @Getter + private final long initialCapacity; + + @Getter + private long capacity; + + public SegmentedLongArray(long initialCapacity) { + long remainingToAdd = initialCapacity; + + // Add first segment + int sizeToAdd = (int) Math.min(remainingToAdd, MAX_SEGMENT_SIZE); + ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(sizeToAdd * SIZE_OF_LONG); + buffer.writerIndex(sizeToAdd * SIZE_OF_LONG); + buffers.add(buffer); + remainingToAdd -= sizeToAdd; + + // Add the remaining segments, all at full segment size, if necessary + while (remainingToAdd > 0) { + buffer = PooledByteBufAllocator.DEFAULT.directBuffer(MAX_SEGMENT_SIZE * SIZE_OF_LONG); + buffer.writerIndex(MAX_SEGMENT_SIZE * SIZE_OF_LONG); + buffers.add(buffer); + remainingToAdd -= MAX_SEGMENT_SIZE; + } + + this.initialCapacity = initialCapacity; + this.capacity = this.initialCapacity; + } + + public void writeLong(long offset, long value) { + int bufferIdx = (int) (offset / MAX_SEGMENT_SIZE); + int internalIdx = (int) (offset % MAX_SEGMENT_SIZE); + buffers.get(bufferIdx).setLong(internalIdx * SIZE_OF_LONG, value); + } + + public long readLong(long offset) { + int bufferIdx = (int) (offset / MAX_SEGMENT_SIZE); + int internalIdx = (int) (offset % MAX_SEGMENT_SIZE); + return buffers.get(bufferIdx).getLong(internalIdx * SIZE_OF_LONG); + } + + public void increaseCapacity() { + if (capacity < MAX_SEGMENT_SIZE) { + // Resize the current buffer to bigger capacity + capacity += (capacity <= 256 ? capacity : capacity / 2); + capacity = Math.min(capacity, MAX_SEGMENT_SIZE); + buffers.get(0).capacity((int) this.capacity * SIZE_OF_LONG); + buffers.get(0).writerIndex((int) this.capacity * SIZE_OF_LONG); + } else { + // Let's add 1 mode buffer to the list + int bufferSize = MAX_SEGMENT_SIZE * SIZE_OF_LONG; + ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(bufferSize, bufferSize); + buffer.writerIndex(bufferSize); + buffers.add(buffer); + capacity += MAX_SEGMENT_SIZE; + } + } + + public void shrink(long newCapacity) { + if (newCapacity >= capacity || newCapacity < initialCapacity) { + return; + } + + long sizeToReduce = capacity - newCapacity; + while (sizeToReduce >= MAX_SEGMENT_SIZE && buffers.size() > 1) { + ByteBuf b = buffers.remove(buffers.size() - 1); + b.release(); + capacity -= MAX_SEGMENT_SIZE; + sizeToReduce -= MAX_SEGMENT_SIZE; + } + + if (buffers.size() == 1 && sizeToReduce > 0) { + // We should also reduce the capacity of the first buffer + capacity -= sizeToReduce; + ByteBuf oldBuffer = buffers.get(0); + ByteBuf newBuffer = PooledByteBufAllocator.DEFAULT.directBuffer((int) capacity * SIZE_OF_LONG); + oldBuffer.getBytes(0, newBuffer, (int) capacity * SIZE_OF_LONG); + oldBuffer.release(); + buffers.set(0, newBuffer); + } + } + + @Override + public void close() { + buffers.forEach(ByteBuf::release); + } + + /** + * The amount of memory used to back the array of longs. + */ + public long bytesCapacity() { + return capacity * SIZE_OF_LONG; + } +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java index 487c2284cffdf..50288247c643c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java @@ -19,9 +19,6 @@ package org.apache.pulsar.common.util.collections; import static com.google.common.base.Preconditions.checkArgument; -import com.google.common.annotations.VisibleForTesting; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; /** * Provides a priority-queue implementation specialized on items composed by 3 longs. @@ -29,33 +26,28 @@ *

    This class is not thread safe and the items are stored in direct memory. */ public class TripleLongPriorityQueue implements AutoCloseable { - - private static final int SIZE_OF_LONG = 8; private static final int DEFAULT_INITIAL_CAPACITY = 16; private static final float DEFAULT_SHRINK_FACTOR = 0.5f; // Each item is composed of 3 longs private static final int ITEMS_COUNT = 3; - private static final int TUPLE_SIZE = ITEMS_COUNT * SIZE_OF_LONG; - /** * Reserve 10% of the capacity when shrinking to avoid frequent expansion and shrinkage. */ private static final float RESERVATION_FACTOR = 0.9f; - private ByteBuf buffer; + private final SegmentedLongArray array; - private final int initialCapacity; + // Count of how many (long,long,long) tuples are currently inserted + private long tuplesCount; - private int capacity; - private int size; /** * When size < capacity * shrinkFactor, may trigger shrinking. */ private final float shrinkFactor; - private float shrinkThreshold; + private long shrinkThreshold; /** * Create a new priority queue with default initial capacity. @@ -64,13 +56,12 @@ public TripleLongPriorityQueue() { this(DEFAULT_INITIAL_CAPACITY); } - public TripleLongPriorityQueue(int initialCapacity, float shrinkFactor) { + public TripleLongPriorityQueue(long initialCapacity, float shrinkFactor) { + checkArgument(initialCapacity > 0); checkArgument(shrinkFactor > 0); - this.initialCapacity = initialCapacity; - this.capacity = initialCapacity; - this.shrinkThreshold = this.capacity * shrinkFactor; - this.buffer = PooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity * TUPLE_SIZE); - this.size = 0; + this.array = new SegmentedLongArray(initialCapacity * ITEMS_COUNT); + this.tuplesCount = 0; + this.shrinkThreshold = (long) (initialCapacity * shrinkFactor); this.shrinkFactor = shrinkFactor; } @@ -87,7 +78,7 @@ public TripleLongPriorityQueue(int initialCapacity) { */ @Override public void close() { - buffer.release(); + array.close(); } /** @@ -98,13 +89,14 @@ public void close() { * @param n3 */ public void add(long n1, long n2, long n3) { - if (size == capacity) { - increaseCapacity(); + long arrayIdx = tuplesCount * ITEMS_COUNT; + if ((arrayIdx + 2) >= array.getCapacity()) { + array.increaseCapacity(); } - put(size, n1, n2, n3); - siftUp(size); - ++size; + put(tuplesCount, n1, n2, n3); + siftUp(tuplesCount); + ++tuplesCount; } /** @@ -113,8 +105,8 @@ public void add(long n1, long n2, long n3) { *

    The tuple will not be extracted */ public long peekN1() { - checkArgument(size != 0); - return buffer.getLong(0); + checkArgument(tuplesCount != 0); + return array.readLong(0); } /** @@ -123,8 +115,8 @@ public long peekN1() { *

    The tuple will not be extracted */ public long peekN2() { - checkArgument(size != 0); - return buffer.getLong(0 + 1 * SIZE_OF_LONG); + checkArgument(tuplesCount != 0); + return array.readLong(1); } /** @@ -133,17 +125,17 @@ public long peekN2() { *

    The tuple will not be extracted */ public long peekN3() { - checkArgument(size != 0); - return buffer.getLong(0 + 2 * SIZE_OF_LONG); + checkArgument(tuplesCount != 0); + return array.readLong(2); } /** * Removes the first item from the queue. */ public void pop() { - checkArgument(size != 0); - swap(0, size - 1); - size--; + checkArgument(tuplesCount != 0); + swap(0, tuplesCount - 1); + tuplesCount--; siftDown(0); shrinkCapacity(); } @@ -152,132 +144,125 @@ public void pop() { * Returns whether the priority queue is empty. */ public boolean isEmpty() { - return size == 0; + return tuplesCount == 0; } /** * Returns the number of tuples in the priority queue. */ - public int size() { - return size; + public long size() { + return tuplesCount; + } + + /** + * The amount of memory used to back the priority queue. + */ + public long bytesCapacity() { + return array.bytesCapacity(); } /** * Clear all items. */ public void clear() { - this.buffer.clear(); - this.size = 0; + this.tuplesCount = 0; shrinkCapacity(); } - private void increaseCapacity() { - // For bigger sizes, increase by 50% - this.capacity += (capacity <= 256 ? capacity : capacity / 2); - this.shrinkThreshold = this.capacity * shrinkFactor; - buffer.capacity(this.capacity * TUPLE_SIZE); - } - private void shrinkCapacity() { - if (capacity > initialCapacity && size < shrinkThreshold) { - int decreasingSize = (int) (capacity * shrinkFactor * RESERVATION_FACTOR); - if (decreasingSize <= 0) { + if (tuplesCount <= shrinkThreshold && array.getCapacity() > array.getInitialCapacity()) { + long sizeToShrink = (long) (array.getCapacity() * shrinkFactor * RESERVATION_FACTOR); + if (sizeToShrink == 0) { return; } - if (capacity - decreasingSize <= initialCapacity) { - this.capacity = initialCapacity; + + long newCapacity; + if (array.getCapacity() - sizeToShrink <= array.getInitialCapacity()) { + newCapacity = array.getInitialCapacity(); } else { - this.capacity = capacity - decreasingSize; + newCapacity = array.getCapacity() - sizeToShrink; } - this.shrinkThreshold = this.capacity * shrinkFactor; - ByteBuf newBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(this.capacity * TUPLE_SIZE); - buffer.getBytes(0, newBuffer, size * TUPLE_SIZE); - buffer.release(); - this.buffer = newBuffer; + array.shrink(newCapacity); + this.shrinkThreshold = (long) (array.getCapacity() / (double) ITEMS_COUNT * shrinkFactor); } } - private void siftUp(int idx) { - while (idx > 0) { - int parentIdx = (idx - 1) / 2; - if (compare(idx, parentIdx) >= 0) { + private void siftUp(long tupleIdx) { + while (tupleIdx > 0) { + long parentIdx = (tupleIdx - 1) / 2; + if (compare(tupleIdx, parentIdx) >= 0) { break; } - swap(idx, parentIdx); - idx = parentIdx; + swap(tupleIdx, parentIdx); + tupleIdx = parentIdx; } } - private void siftDown(int idx) { - int half = size / 2; - while (idx < half) { - int left = 2 * idx + 1; - int right = 2 * idx + 2; + private void siftDown(long tupleIdx) { + long half = tuplesCount / 2; + while (tupleIdx < half) { + long left = 2 * tupleIdx + 1; + long right = 2 * tupleIdx + 2; - int swapIdx = idx; + long swapIdx = tupleIdx; - if (compare(idx, left) > 0) { + if (compare(tupleIdx, left) > 0) { swapIdx = left; } - if (right < size && compare(swapIdx, right) > 0) { + if (right < tuplesCount && compare(swapIdx, right) > 0) { swapIdx = right; } - if (swapIdx == idx) { + if (swapIdx == tupleIdx) { return; } - swap(idx, swapIdx); - idx = swapIdx; + swap(tupleIdx, swapIdx); + tupleIdx = swapIdx; } } - private void put(int idx, long n1, long n2, long n3) { - int i = idx * TUPLE_SIZE; - buffer.setLong(i, n1); - buffer.setLong(i + 1 * SIZE_OF_LONG, n2); - buffer.setLong(i + 2 * SIZE_OF_LONG, n3); + private void put(long tupleIdx, long n1, long n2, long n3) { + long idx = tupleIdx * ITEMS_COUNT; + array.writeLong(idx, n1); + array.writeLong(idx + 1, n2); + array.writeLong(idx + 2, n3); } - private int compare(int idx1, int idx2) { - int i1 = idx1 * TUPLE_SIZE; - int i2 = idx2 * TUPLE_SIZE; + private int compare(long tupleIdx1, long tupleIdx2) { + long idx1 = tupleIdx1 * ITEMS_COUNT; + long idx2 = tupleIdx2 * ITEMS_COUNT; - int c1 = Long.compare(buffer.getLong(i1), buffer.getLong(i2)); + int c1 = Long.compare(array.readLong(idx1), array.readLong(idx2)); if (c1 != 0) { return c1; } - int c2 = Long.compare(buffer.getLong(i1 + SIZE_OF_LONG), buffer.getLong(i2 + SIZE_OF_LONG)); + int c2 = Long.compare(array.readLong(idx1 + 1), array.readLong(idx2 + 1)); if (c2 != 0) { return c2; } - return Long.compare(buffer.getLong(i1 + 2 * SIZE_OF_LONG), buffer.getLong(i2 + 2 * SIZE_OF_LONG)); + return Long.compare(array.readLong(idx1 + 2), array.readLong(idx2 + 2)); } - private void swap(int idx1, int idx2) { - int i1 = idx1 * TUPLE_SIZE; - int i2 = idx2 * TUPLE_SIZE; + private void swap(long tupleIdx1, long tupleIdx2) { + long idx1 = tupleIdx1 * ITEMS_COUNT; + long idx2 = tupleIdx2 * ITEMS_COUNT; - long tmp1 = buffer.getLong(i1); - long tmp2 = buffer.getLong(i1 + 1 * SIZE_OF_LONG); - long tmp3 = buffer.getLong(i1 + 2 * SIZE_OF_LONG); + long tmp1 = array.readLong(idx1); + long tmp2 = array.readLong(idx1 + 1); + long tmp3 = array.readLong(idx1 + 2); - buffer.setLong(i1, buffer.getLong(i2)); - buffer.setLong(i1 + 1 * SIZE_OF_LONG, buffer.getLong(i2 + 1 * SIZE_OF_LONG)); - buffer.setLong(i1 + 2 * SIZE_OF_LONG, buffer.getLong(i2 + 2 * SIZE_OF_LONG)); - - buffer.setLong(i2, tmp1); - buffer.setLong(i2 + 1 * SIZE_OF_LONG, tmp2); - buffer.setLong(i2 + 2 * SIZE_OF_LONG, tmp3); - } + array.writeLong(idx1, array.readLong(idx2)); + array.writeLong(idx1 + 1, array.readLong(idx2 + 1)); + array.writeLong(idx1 + 2, array.readLong(idx2 + 2)); - @VisibleForTesting - ByteBuf getBuffer() { - return buffer; + array.writeLong(idx2, tmp1); + array.writeLong(idx2 + 1, tmp2); + array.writeLong(idx2 + 2, tmp3); } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java new file mode 100644 index 0000000000000..efb86fd4f9d48 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; +import lombok.Cleanup; +import org.testng.annotations.Test; + +public class SegmentedLongArrayTest { + + @Test + public void testArray() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(4); + assertEquals(a.getCapacity(), 4); + assertEquals(a.bytesCapacity(), 4 * 8); + assertEquals(a.getInitialCapacity(), 4); + + a.writeLong(0, 0); + a.writeLong(1, 1); + a.writeLong(2, 2); + a.writeLong(3, Long.MAX_VALUE); + + try { + a.writeLong(4, Long.MIN_VALUE); + fail("should have failed"); + } catch (IndexOutOfBoundsException e) { + // Expected + } + + a.increaseCapacity(); + + a.writeLong(4, Long.MIN_VALUE); + + assertEquals(a.getCapacity(), 8); + assertEquals(a.bytesCapacity(), 8 * 8); + assertEquals(a.getInitialCapacity(), 4); + + assertEquals(a.readLong(0), 0); + assertEquals(a.readLong(1), 1); + assertEquals(a.readLong(2), 2); + assertEquals(a.readLong(3), Long.MAX_VALUE); + assertEquals(a.readLong(4), Long.MIN_VALUE); + + a.shrink(5); + assertEquals(a.getCapacity(), 5); + assertEquals(a.bytesCapacity(), 5 * 8); + assertEquals(a.getInitialCapacity(), 4); + } + + @Test + public void testLargeArray() { + long initialCap = 3 * 1024 * 1024; + + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(initialCap); + assertEquals(a.getCapacity(), initialCap); + assertEquals(a.bytesCapacity(), initialCap * 8); + assertEquals(a.getInitialCapacity(), initialCap); + + long baseOffset = initialCap - 100; + + a.writeLong(baseOffset, 0); + a.writeLong(baseOffset + 1, 1); + a.writeLong(baseOffset + 2, 2); + a.writeLong(baseOffset + 3, Long.MAX_VALUE); + a.writeLong(baseOffset + 4, Long.MIN_VALUE); + + a.increaseCapacity(); + + assertEquals(a.getCapacity(), 5 * 1024 * 1024); + assertEquals(a.bytesCapacity(), 5 * 1024 * 1024 * 8); + assertEquals(a.getInitialCapacity(), initialCap); + + assertEquals(a.readLong(baseOffset), 0); + assertEquals(a.readLong(baseOffset + 1), 1); + assertEquals(a.readLong(baseOffset + 2), 2); + assertEquals(a.readLong(baseOffset + 3), Long.MAX_VALUE); + assertEquals(a.readLong(baseOffset + 4), Long.MIN_VALUE); + + a.shrink(initialCap); + assertEquals(a.getCapacity(), initialCap); + assertEquals(a.bytesCapacity(), initialCap * 8); + assertEquals(a.getInitialCapacity(), initialCap); + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java index bd3aef86ad122..de3b1adb7929d 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java @@ -54,6 +54,35 @@ public void testQueue() { pq.close(); } + @Test + public void testLargeQueue() { + TripleLongPriorityQueue pq = new TripleLongPriorityQueue(); + assertEquals(pq.size(), 0); + + final int N = 3_000_000; + + for (int i = N; i > 0; i--) { + pq.add(i, i * 2L, i * 3L); + } + + assertEquals(pq.size(), N); + assertFalse(pq.isEmpty()); + + for (int i = 1; i <= N; i++) { + assertEquals(pq.peekN1(), i); + assertEquals(pq.peekN2(), i * 2); + assertEquals(pq.peekN3(), i * 3); + + pq.pop(); + + assertEquals(pq.size(), N - i); + } + + pq.clear(); + pq.close(); + } + + @Test public void testCheckForEmpty() { TripleLongPriorityQueue pq = new TripleLongPriorityQueue(); @@ -143,26 +172,24 @@ public void testShrink() throws Exception { TripleLongPriorityQueue pq = new TripleLongPriorityQueue(initialCapacity, 0.5f); pq.add(0, 0, 0); assertEquals(pq.size(), 1); - assertEquals(pq.getBuffer().capacity(), initialCapacity * tupleSize); + assertEquals(pq.bytesCapacity(), initialCapacity * tupleSize); // Scale out to capacity * 2 triggerScaleOut(initialCapacity, pq); int scaleCapacity = initialCapacity * 2; - assertEquals(pq.getBuffer().capacity(), scaleCapacity * tupleSize); + assertEquals(pq.bytesCapacity(), scaleCapacity * tupleSize); // Trigger shrinking - for (int i = 0; i < initialCapacity / 2 + 1; i++) { + for (int i = 0; i < initialCapacity / 2 + 2; i++) { pq.pop(); } - int capacity = scaleCapacity - (int)(scaleCapacity * 0.5f * 0.9f); - assertEquals(pq.getBuffer().capacity(), capacity * tupleSize); + int capacity = scaleCapacity - (int)((scaleCapacity ) * 0.5f * 0.9f); + assertTrue(pq.bytesCapacity() < scaleCapacity * tupleSize); // Scale out to capacity * 2 triggerScaleOut(initialCapacity, pq); scaleCapacity = capacity * 2; - assertEquals(pq.getBuffer().capacity(), scaleCapacity * tupleSize); // Trigger shrinking pq.clear(); capacity = scaleCapacity - (int)(scaleCapacity * 0.5f * 0.9f); - assertEquals(pq.getBuffer().capacity(), capacity * tupleSize); } private void triggerScaleOut(int initialCapacity, TripleLongPriorityQueue pq) { From 54b52887197c9115a022c5fd74065edf9c44f0a8 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Tue, 12 Jul 2022 14:36:08 +0800 Subject: [PATCH 659/823] cherry-pick [fix][txn] fix pattern sub filter transaction system topic (#16533) --- .../pulsar/broker/service/ServerCnx.java | 3 + .../pendingack/PendingAckPersistentTest.java | 55 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 0bbdaca9f747b..02aa9e93c5718 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1882,6 +1882,9 @@ protected void handleGetTopicsOfNamespace(CommandGetTopicsOfNamespace commandGet if (isAuthorized) { getBrokerService().pulsar().getNamespaceService().getListOfTopics(namespaceName, mode) .thenAccept(topics -> { + topics = topics.stream() + .filter(topic -> !PulsarService.isTransactionSystemTopic(TopicName.get(topic))) + .collect(Collectors.toList()); if (log.isDebugEnabled()) { log.debug( "[{}] Received CommandGetTopicsOfNamespace for namespace [//{}] by {}, size:{}", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index bd22ff423a956..b85f4efd133a3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -645,4 +645,59 @@ public void testTransactionConflictExceptionWhenAckBatchMessage() throws Excepti } } + @Test + public void testGetSubPatternTopicFilterTxnInternalTopic() throws Exception { + String topic = TopicName.get(TopicDomain.persistent.toString(), + NamespaceName.get(NAMESPACE1), "testGetSubPatternTopicFilterTxnInternalTopic").toString(); + + int partition = 3; + admin.topics().createPartitionedTopic(topic, partition); + + String subscriptionName = "sub"; + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false) + .topic(topic).create(); + + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .topic(topic) + .subscribe(); + + for (int i = 0; i < partition; i++) { + producer.send("test"); + } + + // creat pending ack managedLedger + for (int i = 0; i < partition; i++) { + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build() + .get(); + consumer.acknowledgeAsync(consumer.receive().getMessageId(), transaction); + transaction.commit().get(); + } + + consumer.close(); + + @Cleanup + Consumer patternConsumer = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName("patternSub") + .subscriptionType(SubscriptionType.Shared) + .topicsPattern("persistent://" + NAMESPACE1 + "/.*") + .subscribe(); + + for (int i = 0; i < partition; i++) { + producer.send("test" + i); + } + + // can use pattern sub consume + for (int i = 0; i < partition; i++) { + patternConsumer.acknowledgeAsync(patternConsumer.receive().getMessageId()); + } + patternConsumer.close(); + producer.close(); + } } From 87f64d05f64b18a724d64251862014fbbedd562e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 13 Jul 2022 01:16:01 +0200 Subject: [PATCH 660/823] [fix][security] Upgrade to Jetty to 9.4.48.v20220622 to get rid of CVE-2022-2047 (#16520) * [fix][security] Upgrade to Jetty to 9.4.48.v20220622 to get rid of CVE-2022-2047 * suppress CVE-2022-2191 - false positive * Revert "suppress CVE-2022-2191 - false positive" This reverts commit ab4601f43093c88ae97b03af9d90518fea174768. (cherry picked from commit 6872ac332f392790688dfd971e232240bd9f4978) --- .../server/src/assemble/LICENSE.bin.txt | 38 +++++++++---------- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 32 ++++++++-------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 3418ff92248da..6a228e7ea3f50 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -430,25 +430,25 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar * Jetty - - org.eclipse.jetty-jetty-client-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-continuation-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-http-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-io-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-proxy-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-security-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-server-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-servlets-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar - - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.43.v20210629.jar - - org.eclipse.jetty.websocket-websocket-api-9.4.43.v20210629.jar - - org.eclipse.jetty.websocket-websocket-client-9.4.43.v20210629.jar - - org.eclipse.jetty.websocket-websocket-common-9.4.43.v20210629.jar - - org.eclipse.jetty.websocket-websocket-server-9.4.43.v20210629.jar - - org.eclipse.jetty.websocket-websocket-servlet-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.43.v20210629.jar - - org.eclipse.jetty-jetty-alpn-server-9.4.43.v20210629.jar + - org.eclipse.jetty-jetty-client-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-continuation-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-http-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-io-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-proxy-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-security-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-server-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-servlet-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-servlets-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-util-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-util-ajax-9.4.48.v20220622.jar + - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.48.v20220622.jar + - org.eclipse.jetty.websocket-websocket-api-9.4.48.v20220622.jar + - org.eclipse.jetty.websocket-websocket-client-9.4.48.v20220622.jar + - org.eclipse.jetty.websocket-websocket-common-9.4.48.v20220622.jar + - org.eclipse.jetty.websocket-websocket-server-9.4.48.v20220622.jar + - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar * SnakeYaml -- org.yaml-snakeyaml-1.30.jar * RocksDB - org.rocksdb-rocksdbjni-6.10.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar diff --git a/pom.xml b/pom.xml index da348ec8f75e0..14aa41c09a3ed 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ flexible messaging model and an intuitive client API. 5.1.0 4.1.77.Final 2.0.52.Final - 9.4.43.v20210629 + 9.4.48.v20220622 2.5.2 2.34 1.10.9 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d137a9a809875..a8d4ba8942066 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -259,22 +259,22 @@ The Apache Software License, Version 2.0 * Joda Time - joda-time-2.10.5.jar * Jetty - - http2-client-9.4.43.v20210629.jar - - http2-common-9.4.43.v20210629.jar - - http2-hpack-9.4.43.v20210629.jar - - http2-http-client-transport-9.4.43.v20210629.jar - - jetty-alpn-client-9.4.43.v20210629.jar - - http2-server-9.4.43.v20210629.jar - - jetty-alpn-java-client-9.4.43.v20210629.jar - - jetty-client-9.4.43.v20210629.jar - - jetty-http-9.4.43.v20210629.jar - - jetty-io-9.4.43.v20210629.jar - - jetty-jmx-9.4.43.v20210629.jar - - jetty-security-9.4.43.v20210629.jar - - jetty-server-9.4.43.v20210629.jar - - jetty-servlet-9.4.43.v20210629.jar - - jetty-util-9.4.43.v20210629.jar - - jetty-util-ajax-9.4.43.v20210629.jar + - http2-client-9.4.48.v20220622.jar + - http2-common-9.4.48.v20220622.jar + - http2-hpack-9.4.48.v20220622.jar + - http2-http-client-transport-9.4.48.v20220622.jar + - jetty-alpn-client-9.4.48.v20220622.jar + - http2-server-9.4.48.v20220622.jar + - jetty-alpn-java-client-9.4.48.v20220622.jar + - jetty-client-9.4.48.v20220622.jar + - jetty-http-9.4.48.v20220622.jar + - jetty-io-9.4.48.v20220622.jar + - jetty-jmx-9.4.48.v20220622.jar + - jetty-security-9.4.48.v20220622.jar + - jetty-server-9.4.48.v20220622.jar + - jetty-servlet-9.4.48.v20220622.jar + - jetty-util-9.4.48.v20220622.jar + - jetty-util-ajax-9.4.48.v20220622.jar * Apache BVal - bval-jsr-2.0.0.jar * Bytecode From 91bebddf78aa0766b65c7860f27076c8799c9acd Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 13 Jul 2022 21:16:41 +0800 Subject: [PATCH 661/823] Fix: Make DeadLetterPolicy deserializable (#16513) (cherry picked from commit abe4350d46c7a4dc82b17f3c552d3d69c3a878f9) --- .../client/api/DeadLetterTopicTest.java | 29 ++++++++++++++----- .../pulsar/client/api/DeadLetterPolicy.java | 4 +++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java index 645c7674b00c5..22b224990edb6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java @@ -18,19 +18,26 @@ */ package org.apache.pulsar.client.api; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - +import lombok.Cleanup; import lombok.Data; import org.apache.avro.reflect.Nullable; import org.apache.pulsar.client.api.schema.GenericRecord; -import lombok.Cleanup; +import org.apache.pulsar.client.impl.ConsumerBuilderImpl; import org.apache.pulsar.client.util.RetryMessageUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,12 +45,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - @Test(groups = "flaky") public class DeadLetterTopicTest extends ProducerConsumerBase { @@ -615,4 +616,16 @@ public void testDeadLetterTopicUnderPartitionedTopicWithKeyShareType() throws Ex checkConsumer.close(); } + + @Test + public void testDeadLetterPolicyDeserialize() { + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING); + DeadLetterPolicy policy = + DeadLetterPolicy.builder().deadLetterTopic("a").retryLetterTopic("a") + .maxRedeliverCount(1).build(); + consumerBuilder.deadLetterPolicy(policy); + Map config = new HashMap<>(); + consumerBuilder.loadConf(config); + assertEquals(((ConsumerBuilderImpl)consumerBuilder).getConf().getDeadLetterPolicy(), policy); + } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DeadLetterPolicy.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DeadLetterPolicy.java index 279a5c508f5a1..91a53c99409a1 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DeadLetterPolicy.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DeadLetterPolicy.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.client.api; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; @@ -30,6 +32,8 @@ */ @Builder @Data +@NoArgsConstructor +@AllArgsConstructor @InterfaceAudience.Public @InterfaceStability.Stable public class DeadLetterPolicy { From 720585e4b0cc4f64339242fd1d25f14637070c37 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 6 Jul 2022 23:21:24 +0800 Subject: [PATCH 662/823] [fix][C++ client] Fix the close of Client might stuck or return a wrong result (#16285) Fixes https://github.com/apache/pulsar/issues/15976 ### Motivation Currently even if the producer, consumer, or reader failed to create, it would still be added to the producers or consumers in `Client`. `Client::close` first closes the internal producers and consumers, if the producers or consumers to close include failed producers or consumers, `Client::close` would return `ResultAlreadyClosed`. Even worse, closing a failed partitioned producer might stuck. It also makes the Python test `test_listener_name_client` flaky because `client.close()` will throw an exception if the underlying `Client::close` call in C++ client doesn't return `ResultOk`. ### Modifications - Only adding the created producer or consumer to the internal list of `Client` after the creation succeeded. - Add `ClientTest.testWrongListener` to verify when producer, consumer, reader failed to create, the internal producer list and consumer list are both empty. And `client.close()` would return `ResultOk`. (cherry picked from commit e23d312c04da1d82d35f9e2faf8a446f8e8a4eeb) --- pulsar-client-cpp/lib/ClientImpl.cc | 41 ++++++++++++++++----------- pulsar-client-cpp/lib/ReaderImpl.cc | 21 ++++++++------ pulsar-client-cpp/lib/ReaderImpl.h | 4 +-- pulsar-client-cpp/tests/ClientTest.cc | 32 +++++++++++++++++++++ 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/pulsar-client-cpp/lib/ClientImpl.cc b/pulsar-client-cpp/lib/ClientImpl.cc index dd00e3b9d4539..7c9a3a18f4a84 100644 --- a/pulsar-client-cpp/lib/ClientImpl.cc +++ b/pulsar-client-cpp/lib/ClientImpl.cc @@ -181,9 +181,6 @@ void ClientImpl::handleCreateProducer(const Result result, const LookupDataResul producer->getProducerCreatedFuture().addListener( std::bind(&ClientImpl::handleProducerCreated, shared_from_this(), std::placeholders::_1, std::placeholders::_2, callback, producer)); - Lock lock(mutex_); - producers_.push_back(producer); - lock.unlock(); producer->start(); } else { LOG_ERROR("Error Checking/Getting Partition Metadata while creating producer on " @@ -194,7 +191,14 @@ void ClientImpl::handleCreateProducer(const Result result, const LookupDataResul void ClientImpl::handleProducerCreated(Result result, ProducerImplBaseWeakPtr producerBaseWeakPtr, CreateProducerCallback callback, ProducerImplBasePtr producer) { - callback(result, Producer(producer)); + if (result == ResultOk) { + Lock lock(mutex_); + producers_.push_back(producer); + lock.unlock(); + callback(result, Producer(producer)); + } else { + callback(result, {}); + } } void ClientImpl::createReaderAsync(const std::string& topic, const MessageId& startMessageId, @@ -237,10 +241,13 @@ void ClientImpl::handleReaderMetadataLookup(const Result result, const LookupDat ReaderImplPtr reader = std::make_shared(shared_from_this(), topicName->toString(), conf, getListenerExecutorProvider()->get(), callback); - reader->start(startMessageId); - - Lock lock(mutex_); - consumers_.push_back(reader->getConsumer()); + ConsumerImplBasePtr consumer = reader->getConsumer().lock(); + auto self = shared_from_this(); + reader->start(startMessageId, [this, self](const ConsumerImplBaseWeakPtr& weakConsumerPtr) { + Lock lock(mutex_); + consumers_.push_back(weakConsumerPtr); + lock.unlock(); + }); } void ClientImpl::subscribeWithRegexAsync(const std::string& regexPattern, const std::string& subscriptionName, @@ -287,9 +294,6 @@ void ClientImpl::createPatternMultiTopicsConsumer(const Result result, const Nam consumer->getConsumerCreatedFuture().addListener( std::bind(&ClientImpl::handleConsumerCreated, shared_from_this(), std::placeholders::_1, std::placeholders::_2, callback, consumer)); - Lock lock(mutex_); - consumers_.push_back(consumer); - lock.unlock(); consumer->start(); } else { LOG_ERROR("Error Getting topicsOfNameSpace while createPatternMultiTopicsConsumer: " << result); @@ -313,6 +317,7 @@ void ClientImpl::subscribeAsync(const std::vector& topics, const st return; } } + lock.unlock(); if (topicNamePtr) { std::string randomName = generateRandomName(); @@ -327,8 +332,6 @@ void ClientImpl::subscribeAsync(const std::vector& topics, const st consumer->getConsumerCreatedFuture().addListener(std::bind(&ClientImpl::handleConsumerCreated, shared_from_this(), std::placeholders::_1, std::placeholders::_2, callback, consumer)); - consumers_.push_back(consumer); - lock.unlock(); consumer->start(); } @@ -385,9 +388,6 @@ void ClientImpl::handleSubscribe(const Result result, const LookupDataResultPtr consumer->getConsumerCreatedFuture().addListener( std::bind(&ClientImpl::handleConsumerCreated, shared_from_this(), std::placeholders::_1, std::placeholders::_2, callback, consumer)); - Lock lock(mutex_); - consumers_.push_back(consumer); - lock.unlock(); consumer->start(); } else { LOG_ERROR("Error Checking/Getting Partition Metadata while Subscribing on " << topicName->toString() @@ -398,7 +398,14 @@ void ClientImpl::handleSubscribe(const Result result, const LookupDataResultPtr void ClientImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, SubscribeCallback callback, ConsumerImplBasePtr consumer) { - callback(result, Consumer(consumer)); + if (result == ResultOk) { + Lock lock(mutex_); + consumers_.push_back(consumer); + lock.unlock(); + callback(result, Consumer(consumer)); + } else { + callback(result, {}); + } } Future ClientImpl::getConnection(const std::string& topic) { diff --git a/pulsar-client-cpp/lib/ReaderImpl.cc b/pulsar-client-cpp/lib/ReaderImpl.cc index 9401c12011983..c660c01ab2a16 100644 --- a/pulsar-client-cpp/lib/ReaderImpl.cc +++ b/pulsar-client-cpp/lib/ReaderImpl.cc @@ -35,7 +35,8 @@ ReaderImpl::ReaderImpl(const ClientImplPtr client, const std::string& topic, con const ExecutorServicePtr listenerExecutor, ReaderCallback readerCreatedCallback) : topic_(topic), client_(client), readerConf_(conf), readerCreatedCallback_(readerCreatedCallback) {} -void ReaderImpl::start(const MessageId& startMessageId) { +void ReaderImpl::start(const MessageId& startMessageId, + std::function callback) { ConsumerConfiguration consumerConf; consumerConf.setConsumerType(ConsumerExclusive); consumerConf.setReceiverQueueSize(readerConf_.getReceiverQueueSize()); @@ -79,19 +80,21 @@ void ReaderImpl::start(const MessageId& startMessageId) { client_.lock(), topic_, subscription, consumerConf, ExecutorServicePtr(), false, NonPartitioned, Commands::SubscriptionModeNonDurable, Optional::of(startMessageId)); consumer_->setPartitionIndex(TopicName::getPartitionIndex(topic_)); - consumer_->getConsumerCreatedFuture().addListener(std::bind(&ReaderImpl::handleConsumerCreated, - shared_from_this(), std::placeholders::_1, - std::placeholders::_2)); + auto self = shared_from_this(); + consumer_->getConsumerCreatedFuture().addListener( + [this, self, callback](Result result, const ConsumerImplBaseWeakPtr& weakConsumerPtr) { + if (result == ResultOk) { + readerCreatedCallback_(result, Reader(self)); + callback(weakConsumerPtr); + } else { + readerCreatedCallback_(result, {}); + } + }); consumer_->start(); } const std::string& ReaderImpl::getTopic() const { return consumer_->getTopic(); } -void ReaderImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumer) { - auto self = shared_from_this(); - readerCreatedCallback_(result, Reader(self)); -} - Result ReaderImpl::readNext(Message& msg) { Result res = consumer_->receive(msg); acknowledgeIfNecessary(res, msg); diff --git a/pulsar-client-cpp/lib/ReaderImpl.h b/pulsar-client-cpp/lib/ReaderImpl.h index 6de6c02e460ed..b0d8a6bc40a21 100644 --- a/pulsar-client-cpp/lib/ReaderImpl.h +++ b/pulsar-client-cpp/lib/ReaderImpl.h @@ -42,7 +42,7 @@ class PULSAR_PUBLIC ReaderImpl : public std::enable_shared_from_this ReaderImpl(const ClientImplPtr client, const std::string& topic, const ReaderConfiguration& conf, const ExecutorServicePtr listenerExecutor, ReaderCallback readerCreatedCallback); - void start(const MessageId& startMessageId); + void start(const MessageId& startMessageId, std::function callback); const std::string& getTopic() const; @@ -65,8 +65,6 @@ class PULSAR_PUBLIC ReaderImpl : public std::enable_shared_from_this bool isConnected() const; private: - void handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumer); - void messageListener(Consumer consumer, const Message& msg); void acknowledgeIfNecessary(Result result, const Message& msg); diff --git a/pulsar-client-cpp/tests/ClientTest.cc b/pulsar-client-cpp/tests/ClientTest.cc index 364e170f8969c..dd073fb6bfc77 100644 --- a/pulsar-client-cpp/tests/ClientTest.cc +++ b/pulsar-client-cpp/tests/ClientTest.cc @@ -223,3 +223,35 @@ TEST(ClientTest, testReferenceCount) { ASSERT_EQ(readerWeakPtr.use_count(), 0); client.close(); } + +TEST(ClientTest, testWrongListener) { + const std::string topic = "client-test-wrong-listener-" + std::to_string(time(nullptr)); + auto httpCode = makePutRequest( + "http://localhost:8080/admin/v2/persistent/public/default/" + topic + "/partitions", "3"); + LOG_INFO("create " << topic << ": " << httpCode); + + Client client(lookupUrl, ClientConfiguration().setListenerName("test")); + Producer producer; + ASSERT_EQ(ResultServiceUnitNotReady, client.createProducer(topic, producer)); + ASSERT_EQ(ResultProducerNotInitialized, producer.close()); + + Consumer consumer; + ASSERT_EQ(ResultServiceUnitNotReady, client.subscribe(topic, "sub", consumer)); + ASSERT_EQ(ResultConsumerNotInitialized, consumer.close()); + + ASSERT_EQ(PulsarFriend::getProducers(client).size(), 0); + ASSERT_EQ(PulsarFriend::getConsumers(client).size(), 0); + ASSERT_EQ(ResultOk, client.close()); + + // The connection will be closed when the consumer failed, we must recreate the Client. Otherwise, the + // creation of Reader would fail with ResultConnectError. + client = Client(lookupUrl, ClientConfiguration().setListenerName("test")); + + // Currently Reader can only read a non-partitioned topic in C++ client + Reader reader; + ASSERT_EQ(ResultServiceUnitNotReady, + client.createReader(topic + "-partition-0", MessageId::earliest(), {}, reader)); + ASSERT_EQ(ResultConsumerNotInitialized, reader.close()); + ASSERT_EQ(PulsarFriend::getConsumers(client).size(), 0); + ASSERT_EQ(ResultOk, client.close()); +} From d5139b906271a7bf9980f42b8e11ae4f5aaec793 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Thu, 14 Jul 2022 16:21:28 +0800 Subject: [PATCH 663/823] [fix][pulsar-broker] Fix RawReader hasMessageAvailable returns true when no messages (#16443) * [fix][pulsar-broker] Fix RawReader hasMessageAvailable returns true when no messages --- *Motivation* The RawReader hasMessageAvailable will return true when all the messages have been consumed. And that will cause the readNextAsync blocked and the process never recovered. In the ConsumerImpl, we update the lastDequeuedMessageId in the messageProcess. The messageReceived method rewrites by the RawReader, we should update the lastDequeuedMessageId in the RawReader as well. * Fix the hasMessageAvaiable in batch messages (cherry picked from commit 1d174db872914afbafb03607a29d7a2396d87a33) --- .../pulsar/client/impl/RawReaderImpl.java | 3 ++ .../pulsar/client/impl/RawReaderTest.java | 52 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index bf9fbec59f669..30944289dc3a8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -154,6 +154,9 @@ void tryCompletePending() { messageAndCnx.msg.close(); closeAsync(); } + MessageIdData messageId = messageAndCnx.msg.getMessageIdData(); + lastDequeuedMessageId = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), + messageId.getPartition(), numMsg - 1); ClientCnx currentCnx = cnx(); if (currentCnx == messageAndCnx.cnx) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java index de9eb80a26c14..75aa3ee594131 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java @@ -107,6 +107,58 @@ public static String extractKey(RawMessage m) { return msgMetadata.getPartitionKey(); } + @Test + public void testHasMessageAvailableWithoutBatch() throws Exception { + int numKeys = 10; + String topic = "persistent://my-property/my-ns/my-raw-topic"; + Set keys = publishMessages(topic, numKeys); + RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); + while (true) { + boolean hasMsg = reader.hasMessageAvailableAsync().get(); + if (hasMsg && keys.isEmpty()) { + Assert.fail("HasMessageAvailable shows still has message when there is no message"); + } + if (hasMsg) { + try (RawMessage m = reader.readNextAsync().get()) { + Assert.assertTrue(keys.remove(extractKey(m))); + } + } else { + break; + } + } + Assert.assertTrue(keys.isEmpty()); + } + + @Test + public void testHasMessageAvailableWithBatch() throws Exception { + int numKeys = 20; + String topic = "persistent://my-property/my-ns/my-raw-topic"; + Set keys = publishMessages(topic, numKeys, true); + RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); + int messageCount = 0; + while (true) { + boolean hasMsg = reader.hasMessageAvailableAsync().get(); + if (hasMsg && (messageCount == numKeys)) { + Assert.fail("HasMessageAvailable shows still has message when there is no message"); + } + if (hasMsg) { + try (RawMessage m = reader.readNextAsync().get()) { + MessageMetadata meta = Commands.parseMessageMetadata(m.getHeadersAndPayload()); + messageCount += meta.getNumMessagesInBatch(); + RawBatchConverter.extractIdsAndKeysAndSize(m).forEach(batchInfo -> { + String key = batchInfo.getMiddle(); + Assert.assertTrue(keys.remove(key)); + }); + + } + } else { + break; + } + } + Assert.assertEquals(messageCount, numKeys); + Assert.assertTrue(keys.isEmpty()); + } + @Test public void testRawReader() throws Exception { int numKeys = 10; From 98fc43cc2f7c22d1b49678f3de22f8c483d62e97 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 8 Jul 2022 10:04:21 +0800 Subject: [PATCH 664/823] [fix][broker] Fix setManagedLedgerOffloadedReadPriority not work. (#16436) (cherry picked from commit 52f7f1357c7a2c57f3f26321da0fc11d103bf6ca) --- .../pulsar/common/naming/ServiceConfigurationTest.java | 3 +++ .../pulsar/common/policies/data/OffloadPoliciesImpl.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java index 078ad61249f26..80baa7c2f8e24 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; +import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.testng.annotations.Test; @Test(groups = "broker-naming") @@ -61,6 +62,8 @@ public void testInit() throws Exception { assertEquals(config.getMaxMessagePublishBufferSizeInMB(), -1); assertEquals(config.getManagedLedgerDataReadPriority(), "bookkeeper-first"); assertEquals(config.getBacklogQuotaDefaultLimitGB(), 0.05); + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(config.getProperties()); + assertEquals(offloadPolicies.getManagedLedgerOffloadedReadPriority().getValue(), "bookkeeper-first"); } @Test diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java index e1da6a1c73741..3d920902d8071 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java @@ -254,6 +254,11 @@ public void compatibleWithBrokerConfigFile(Properties properties) { setManagedLedgerOffloadDeletionLagInMillis( Long.parseLong(properties.getProperty(DELETION_LAG_NAME_IN_CONF_FILE))); } + + if (properties.containsKey("managedLedgerDataReadPriority")) { + setManagedLedgerOffloadedReadPriority( + OffloadedReadPriority.fromString(properties.getProperty("managedLedgerDataReadPriority"))); + } } public boolean driverSupported() { From 9961a116e1b3b448b9e7fdd8ddf85c5fbe33d0e6 Mon Sep 17 00:00:00 2001 From: LinChen <1572139390@qq.com> Date: Fri, 8 Jul 2022 21:36:12 +0800 Subject: [PATCH 665/823] [fix][broker] fix No such ledger exception (#16420) * fix No such ledger exception * move currentLedgerEntries and currentLedgerSize * move ledgers.put to operationComplete * fix ledgers.put * use ledgersTmp to write zookkeeper,after sucess,update ldgers * update updateLedgersListAfterRollover * put lh to ledgersTmp * extend buildManagedLedgerInfo * getManagedLedgerInfo(newLedger) after get metadataMutex (cherry picked from commit 2e5fbbc09af28d1ba4d159ee0cdb71ec7735bd29) --- .../mledger/impl/ManagedLedgerImpl.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index c79b8a9f8f700..0f879b057a9cf 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1417,11 +1417,7 @@ public synchronized void createComplete(int rc, final LedgerHandle lh, Object ct lastLedgerCreationFailureTimestamp = clock.millis(); } else { log.info("[{}] Created new ledger {}", name, lh.getId()); - ledgers.put(lh.getId(), LedgerInfo.newBuilder().setLedgerId(lh.getId()).setTimestamp(0).build()); - currentLedger = lh; - currentLedgerEntries = 0; - currentLedgerSize = 0; - + LedgerInfo newLedger = LedgerInfo.newBuilder().setLedgerId(lh.getId()).setTimestamp(0).build(); final MetaStoreCallback cb = new MetaStoreCallback() { @Override public void operationComplete(Void v, Stat stat) { @@ -1429,6 +1425,10 @@ public void operationComplete(Void v, Stat stat) { log.debug("[{}] Updating of ledgers list after create complete. version={}", name, stat); } ledgersStat = stat; + ledgers.put(lh.getId(), newLedger); + currentLedger = lh; + currentLedgerEntries = 0; + currentLedgerSize = 0; metadataMutex.unlock(); updateLedgersIdsComplete(stat); synchronized (ManagedLedgerImpl.this) { @@ -1443,8 +1443,6 @@ public void operationComplete(Void v, Stat stat) { @Override public void operationFailed(MetaStoreException e) { log.warn("[{}] Error updating meta data with the new list of ledgers: {}", name, e.getMessage()); - // Remove the ledger, since we failed to update the list - ledgers.remove(lh.getId()); mbean.startDataLedgerDeleteOp(); bookKeeper.asyncDeleteLedger(lh.getId(), (rc1, ctx1) -> { mbean.endDataLedgerDeleteOp(); @@ -1479,21 +1477,22 @@ public void operationFailed(MetaStoreException e) { } }; - updateLedgersListAfterRollover(cb); + updateLedgersListAfterRollover(cb, newLedger); } } - - private void updateLedgersListAfterRollover(MetaStoreCallback callback) { + private void updateLedgersListAfterRollover(MetaStoreCallback callback, LedgerInfo newLedger) { if (!metadataMutex.tryLock()) { // Defer update for later - scheduledExecutor.schedule(() -> updateLedgersListAfterRollover(callback), 100, TimeUnit.MILLISECONDS); + scheduledExecutor.schedule(() -> updateLedgersListAfterRollover(callback, newLedger), + 100, TimeUnit.MILLISECONDS); return; } if (log.isDebugEnabled()) { log.debug("[{}] Updating ledgers ids with new ledger. version={}", name, ledgersStat); } - store.asyncUpdateLedgerIds(name, getManagedLedgerInfo(), ledgersStat, callback); + ManagedLedgerInfo mlInfo = getManagedLedgerInfo(newLedger); + store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, callback); } public synchronized void updateLedgersIdsComplete(Stat stat) { @@ -3481,8 +3480,17 @@ private ManagedLedgerInfo getManagedLedgerInfo() { return buildManagedLedgerInfo(ledgers); } + private ManagedLedgerInfo getManagedLedgerInfo(LedgerInfo newLedger) { + ManagedLedgerInfo.Builder mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()) + .addLedgerInfo(newLedger); + return buildManagedLedgerInfo(mlInfo); + } private ManagedLedgerInfo buildManagedLedgerInfo(Map ledgers) { ManagedLedgerInfo.Builder mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()); + return buildManagedLedgerInfo(mlInfo); + } + + private ManagedLedgerInfo buildManagedLedgerInfo(ManagedLedgerInfo.Builder mlInfo) { if (state == State.Terminated) { mlInfo.setTerminatedPosition(NestedPositionInfo.newBuilder().setLedgerId(lastConfirmedEntry.getLedgerId()) .setEntryId(lastConfirmedEntry.getEntryId())); From ab38b1a9f85f83f1b5bb9fafef5020e29cb1f187 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 12 Jul 2022 15:02:06 +0800 Subject: [PATCH 666/823] [fix][broker] Expose timestamp field for SchemaData&SchemaInfo (#16380) Fixes #16379 Miss the timestamp field when constructing SchemaData from `SchemaRegistryFormat.SchemaInfo`, so timestamp value 0 gets the scheme. See: https://github.com/apache/pulsar/blob/95b984694bd3c4f1eadf4e6198de033f9b517128/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java#L621-L629 * Expose timestamp field for SchemaData * Add timestamp field for SchemaInfo (cherry picked from commit bd150c0c264371893b99b4b8c050f90cb7b9863b) --- .../schema/SchemaRegistryServiceImpl.java | 1 + .../broker/admin/AdminApiSchemaTest.java | 18 +++++++- .../pulsar/client/api/SimpleSchemaTest.java | 44 ++++++++++--------- .../org/apache/pulsar/schema/SchemaTest.java | 9 ++-- .../client/admin/internal/SchemasImpl.java | 1 + .../PulsarClientImplementationBinding.java | 3 +- .../pulsar/common/schema/SchemaInfo.java | 13 +++++- ...PulsarClientImplementationBindingImpl.java | 5 ++- .../impl/schema/RecordSchemaBuilderImpl.java | 1 + .../client/impl/schema/SchemaInfoTest.java | 16 ++++--- .../client/impl/schema/SchemaInfoImpl.java | 5 +++ 11 files changed, 81 insertions(+), 35 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index e1c9b13e60f0d..ff94aa32f2b8c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -507,6 +507,7 @@ static SchemaData schemaInfoToSchema(SchemaRegistryFormat.SchemaInfo info) { .user(info.getUser()) .type(convertToDomainType(info.getType())) .data(info.getSchema().toByteArray()) + .timestamp(info.getTimestamp()) .isDeleted(info.getDeleted()) .props(toMap(info.getPropsList())) .build(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index 64e4251cb01c2..b4654afb6a811 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; @@ -41,6 +42,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.impl.schema.StringSchema; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; @@ -169,11 +171,13 @@ private void testSchemaInfoApi(Schema schema, SchemaInfo readSi = admin.schemas().getSchemaInfo(topicName); log.info("Read schema of topic {} : {}", topicName, readSi); + ((SchemaInfoImpl)readSi).setTimestamp(0); assertEquals(readSi, si); readSi = admin.schemas().getSchemaInfo(topicName + "-partition-0"); log.info("Read schema of topic {} : {}", topicName, readSi); + ((SchemaInfoImpl)readSi).setTimestamp(0); assertEquals(readSi, si); } @@ -222,12 +226,14 @@ private void testSchemaInfoWithVersionApi(Schema schema, SchemaInfoWithVersion readSi = admin.schemas().getSchemaInfoWithVersion(topicName); log.info("Read schema of topic {} : {}", topicName, readSi); + ((SchemaInfoImpl)readSi.getSchemaInfo()).setTimestamp(0); assertEquals(readSi.getSchemaInfo(), si); assertEquals(readSi.getVersion(), 0); readSi = admin.schemas().getSchemaInfoWithVersion(topicName + "-partition-0"); log.info("Read schema of topic {} : {}", topicName, readSi); + ((SchemaInfoImpl)readSi.getSchemaInfo()).setTimestamp(0); assertEquals(readSi.getSchemaInfo(), si); assertEquals(readSi.getVersion(), 0); @@ -239,11 +245,19 @@ public void createKeyValueSchema(ApiVersion version) throws Exception { "test"); String topicName = "persistent://"+namespace + "/test-key-value-schema"; Schema keyValueSchema = Schema.KeyValue(Schema.AVRO(Foo.class), Schema.AVRO(Foo.class)); - admin.schemas().createSchema(topicName, - keyValueSchema.getSchemaInfo()); + admin.schemas().createSchema(topicName, keyValueSchema.getSchemaInfo()); SchemaInfo schemaInfo = admin.schemas().getSchemaInfo(topicName); + long timestamp = schemaInfo.getTimestamp(); + assertNotEquals(keyValueSchema.getSchemaInfo().getTimestamp(), timestamp); + assertNotEquals(0, timestamp); + + ((SchemaInfoImpl)keyValueSchema.getSchemaInfo()).setTimestamp(schemaInfo.getTimestamp()); assertEquals(keyValueSchema.getSchemaInfo(), schemaInfo); + + admin.schemas().createSchema(topicName, keyValueSchema.getSchemaInfo()); + SchemaInfo schemaInfo2 = admin.schemas().getSchemaInfo(topicName); + assertEquals(timestamp, schemaInfo2.getTimestamp()); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index 983a7f341e06d..cb8b8728ceee8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -18,12 +18,6 @@ */ package org.apache.pulsar.client.api; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Cleanup; -import lombok.Data; -import lombok.NoArgsConstructor; - import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; @@ -31,13 +25,22 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Cleanup; +import lombok.Data; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.avro.reflect.ReflectData; import org.apache.avro.Schema.Parser; -import org.apache.pulsar.client.impl.MessageImpl; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; -import org.apache.pulsar.common.schema.LongSchemaVersion; +import org.apache.avro.reflect.ReflectData; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClientException.IncompatibleSchemaException; import org.apache.pulsar.client.api.PulsarClientException.InvalidMessageException; @@ -45,12 +48,16 @@ import org.apache.pulsar.client.impl.BinaryProtoLookupService; import org.apache.pulsar.client.impl.HttpLookupService; import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; +import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.impl.schema.reader.AvroReader; import org.apache.pulsar.client.impl.schema.writer.AvroWriter; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.common.schema.LongSchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.testng.Assert; @@ -60,15 +67,6 @@ import org.testng.annotations.Factory; import org.testng.annotations.Test; -import java.io.ByteArrayInputStream; -import java.io.EOFException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - @Test(groups = "broker-api") @Slf4j public class SimpleSchemaTest extends ProducerConsumerBase { @@ -455,6 +453,9 @@ public void newProducerForMessageSchemaOnTopicInitialWithNoSchema() throws Excep } List allSchemas = admin.schemas().getAllSchemas(topic); + allSchemas.forEach(schemaInfo -> { + ((SchemaInfoImpl)schemaInfo).setTimestamp(0); + }); Assert.assertEquals(allSchemas, Arrays.asList(v1Schema.getSchemaInfo(), v2Schema.getSchemaInfo())); } @@ -493,6 +494,9 @@ public void newNativeAvroProducerForMessageSchemaOnTopicInitialWithNoSchema() th } List allSchemas = admin.schemas().getAllSchemas(topic); + allSchemas.forEach(schemaInfo -> { + ((SchemaInfoImpl)schemaInfo).setTimestamp(0); + }); Assert.assertEquals(allSchemas, Arrays.asList(v1Schema.getSchemaInfo(), v2Schema.getSchemaInfo())); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 682d6a52b7897..1a0921a8aea5f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -20,7 +20,6 @@ import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; import static org.apache.pulsar.schema.compatibility.SchemaCompatibilityCheckTest.randomName; - import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -28,9 +27,6 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import static org.testng.internal.junit.ArrayAsserts.assertArrayEquals; - -import lombok.EqualsAndHashCode; -import org.apache.avro.Schema.Parser; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Sets; import java.io.ByteArrayInputStream; @@ -45,7 +41,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import lombok.Cleanup; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.Schema.Parser; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -850,6 +848,9 @@ public void testProducerMultipleSchemaMessages() throws Exception { producer.newMessage(Schema.NATIVE_AVRO(personThreeSchemaAvroNative)).value(content).send(); List allSchemas = admin.schemas().getAllSchemas(topic); + allSchemas.forEach(schemaInfo -> { + ((SchemaInfoImpl)schemaInfo).setTimestamp(0); + }); Assert.assertEquals(allSchemas.size(), 5); Assert.assertEquals(allSchemas.get(0), Schema.STRING.getSchemaInfo()); Assert.assertEquals(allSchemas.get(1), Schema.JSON(Schemas.PersonThree.class).getSchemaInfo()); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java index 54e731e1b979b..ac485ef425e55 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java @@ -451,6 +451,7 @@ static SchemaInfo convertGetSchemaResponseToSchemaInfo(TopicName tn, return SchemaInfo.builder() .schema(schema) .type(response.getType()) + .timestamp(response.getTimestamp()) .properties(response.getProperties()) .name(tn.getLocalName()) .build(); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java index c8c300cc2eb1b..75cd7dc1feec9 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java @@ -252,5 +252,6 @@ static byte[] getBytes(ByteBuffer byteBuffer) { return array; } - SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, Map propertiesValue); + SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, long timestamp, + Map propertiesValue); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java index e05b0d24cb946..077730253119a 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java @@ -49,6 +49,11 @@ public interface SchemaInfo { */ Map getProperties(); + /** + * The created time of schema. + */ + long getTimestamp(); + String getSchemaDefinition(); static SchemaInfoBuilder builder() { @@ -61,6 +66,7 @@ class SchemaInfoBuilder { private SchemaType type; private Map properties; private boolean propertiesSet; + private long timestamp; SchemaInfoBuilder() { } @@ -86,6 +92,11 @@ public SchemaInfoBuilder properties(Map properties) { return this; } + public SchemaInfoBuilder timestamp(long timestamp) { + this.timestamp = timestamp; + return this; + } + public SchemaInfo build() { Map propertiesValue = this.properties; if (!this.propertiesSet) { @@ -93,7 +104,7 @@ public SchemaInfo build() { } return DefaultImplementation .getDefaultImplementation() - .newSchemaInfoImpl(name, schema, type, propertiesValue); + .newSchemaInfoImpl(name, schema, type, timestamp, propertiesValue); } } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java index c7f3af9227907..2747d39a7357e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java @@ -356,7 +356,8 @@ public MessagePayloadFactory newDefaultMessagePayloadFactory() { return new MessagePayloadFactoryImpl(); } - public SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, Map propertiesValue) { - return new SchemaInfoImpl(name, schema, type, propertiesValue); + public SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, long timestamp, + Map propertiesValue) { + return new SchemaInfoImpl(name, schema, type, timestamp, propertiesValue); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/RecordSchemaBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/RecordSchemaBuilderImpl.java index 0fda7d52b0dbe..5854a80716f63 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/RecordSchemaBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/RecordSchemaBuilderImpl.java @@ -109,6 +109,7 @@ public SchemaInfo build(SchemaType schemaType) { name, baseSchema.toString().getBytes(UTF_8), schemaType, + System.currentTimeMillis(), properties ); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java index 719704c5c6941..7ed5406b76e94 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.client.impl.schema; +import static org.testng.Assert.assertEquals; +import java.util.HashMap; +import java.util.Map; + import com.google.common.collect.Maps; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.schema.KeyValueEncodingType; @@ -26,11 +30,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.util.HashMap; -import java.util.Map; - -import static org.testng.Assert.assertEquals; - /** * Unit test {@link org.apache.pulsar.common.schema.SchemaInfo}. */ @@ -40,6 +39,7 @@ public class SchemaInfoTest { + " \"name\": \"INT32\",\n" + " \"schema\": \"\",\n" + " \"type\": \"INT32\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {}\n" + "}"; @@ -47,6 +47,7 @@ public class SchemaInfoTest { + " \"name\": \"String\",\n" + " \"schema\": \"\",\n" + " \"type\": \"STRING\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {}\n" + "}"; @@ -64,6 +65,7 @@ public class SchemaInfoTest { + " ]\n" + " },\n" + " \"type\": \"JSON\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {\n" + " \"__alwaysAllowNull\": \"true\",\n" + " \"__jsr310ConversionEnabled\": \"false\",\n" @@ -136,6 +138,7 @@ public class SchemaInfoTest { + " ]\n" + " },\n" + " \"type\": \"AVRO\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {\n" + " \"__alwaysAllowNull\": \"false\",\n" + " \"__jsr310ConversionEnabled\": \"false\",\n" @@ -211,6 +214,7 @@ public class SchemaInfoTest { + " ]\n" + " },\n" + " \"type\": \"AVRO\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {\n" + " \"__alwaysAllowNull\": \"false\",\n" + " \"__jsr310ConversionEnabled\": \"false\",\n" @@ -233,6 +237,7 @@ public class SchemaInfoTest { + " ]\n" + " },\n" + " \"type\": \"JSON\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {\n" + " \"__alwaysAllowNull\": \"true\",\n" + " \"__jsr310ConversionEnabled\": \"false\",\n" @@ -243,6 +248,7 @@ public class SchemaInfoTest { + " }\n" + " },\n" + " \"type\": \"KEY_VALUE\",\n" + + " \"timestamp\": 0,\n" + " \"properties\": {\n" + " \"key.schema.name\": \"\",\n" + " \"key.schema.properties\": \"{\\\"__alwaysAllowNull\\\":\\\"false\\\",\\\"__jsr310ConversionEnabled\\\":\\\"false\\\",\\\"foo1\\\":\\\"foo-value1\\\",\\\"foo2\\\":\\\"foo-value2\\\",\\\"foo3\\\":\\\"foo-value3\\\"}\",\n" diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaInfoImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaInfoImpl.java index ca8b6ccf9bd28..d67dc5f29e1c8 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaInfoImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaInfoImpl.java @@ -59,6 +59,11 @@ public class SchemaInfoImpl implements SchemaInfo { */ private SchemaType type; + /** + * The created time of schema. + */ + private long timestamp; + /** * Additional properties of the schema definition (implementation defined). */ From 7e17d7a2f31df3e835c32c663636390cd936cffb Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 7 Jul 2022 21:27:13 +0800 Subject: [PATCH 667/823] [improve][broker] Recycle OpReadEntry in some corner cases (#16399) ### Motivation `ManagedCursorImpl` maintains a field `waitingReadOp` as the cache of the `OpReadEntry` created in `asyncReadEntriesOrWait` when there are no more entries to read. However, there are two cases that the created `OpReadEntry` are not recycled: 1. `asyncReadEntriesOrWait` is called repeatedly when `waitingReadOp` is not null and there are no more entries. The new created `OpReadEntry` cannot pass the CAS check but it's not recycled. 2. `cancelPendingReadRequest` is called. The `waitingReadOp` is just set with null and the previous reference is not recycled. ### Modifications For the two cases described above, recycle the `OpReadEntry` objects. ### Verifying this change Add `testOpReadEntryRecycle` to reproduce the corner cases and verify the count of `recycle()` calls. (cherry picked from commit 6cec62e89629c258f9458a726ff8ba3b644788b7) --- .../mledger/impl/ManagedCursorImpl.java | 7 ++- .../mledger/impl/ManagedCursorTest.java | 62 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 604e3a4f3d89b..90ab8218b3088 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -796,6 +796,7 @@ public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntrie ctx, maxPosition); if (!WAITING_READ_OP_UPDATER.compareAndSet(this, null, op)) { + op.recycle(); callback.readEntriesFailed(new ManagedLedgerException("We can only have a single waiting callback"), ctx); return; @@ -878,7 +879,11 @@ public boolean cancelPendingReadRequest() { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Cancel pending read request", ledger.getName(), name); } - return WAITING_READ_OP_UPDATER.getAndSet(this, null) != null; + final OpReadEntry op = WAITING_READ_OP_UPDATER.getAndSet(this, null); + if (op != null) { + op.recycle(); + } + return op != null; } public boolean hasPendingReadRequest() { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 074a55db50091..aa7a7730c5ebc 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.eq; @@ -42,10 +43,12 @@ import java.util.BitSet; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -57,6 +60,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.stream.Collectors; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -93,6 +97,8 @@ import org.apache.pulsar.metadata.api.Stat; import org.apache.pulsar.common.api.proto.IntRange; import org.awaitility.Awaitility; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; @@ -3752,5 +3758,61 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { Awaitility.await().untilAsserted(() -> assertTrue(flag.get())); } + @Test + public void testOpReadEntryRecycle() throws Exception { + final Map opReadEntryToRecycleCount = new ConcurrentHashMap<>(); + final Supplier createOpReadEntry = () -> { + final OpReadEntry mockedOpReadEntry = mock(OpReadEntry.class); + doAnswer(__ -> opReadEntryToRecycleCount.computeIfAbsent(mockedOpReadEntry, + ignored -> new AtomicInteger(0)).getAndIncrement() + ).when(mockedOpReadEntry).recycle(); + return mockedOpReadEntry; + }; + + @Cleanup final MockedStatic mockedStaticOpReadEntry = Mockito.mockStatic(OpReadEntry.class); + mockedStaticOpReadEntry.when(() -> OpReadEntry.create(any(), any(), anyInt(), any(), any(), any())) + .thenAnswer(__ -> createOpReadEntry.get()); + + final ManagedLedgerConfig ledgerConfig = new ManagedLedgerConfig(); + ledgerConfig.setNewEntriesCheckDelayInMillis(10); + final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", ledgerConfig); + final ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("my_cursor"); + final List exceptions = new ArrayList<>(); + final AtomicBoolean readEntriesSuccess = new AtomicBoolean(false); + final ReadEntriesCallback callback = new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntriesSuccess.set(true); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + exceptions.add(exception); + } + }; + + final int numReadRequests = 3; + for (int i = 0; i < numReadRequests; i++) { + cursor.asyncReadEntriesOrWait(1, callback, null, new PositionImpl(0, 0)); + } + Awaitility.await().atMost(Duration.ofSeconds(1)) + .untilAsserted(() -> assertEquals(ledger.waitingCursors.size(), 1)); + assertTrue(cursor.cancelPendingReadRequest()); + + ledger.addEntry(new byte[1]); + Awaitility.await().atMost(Duration.ofSeconds(1)) + .untilAsserted(() -> assertTrue(ledger.waitingCursors.isEmpty())); + assertFalse(readEntriesSuccess.get()); + + assertEquals(exceptions.size(), numReadRequests - 1); + exceptions.forEach(e -> assertEquals(e.getMessage(), "We can only have a single waiting callback")); + assertEquals(opReadEntryToRecycleCount.size(), 3); + assertEquals(opReadEntryToRecycleCount.entrySet().stream() + .map(Map.Entry::getValue) + .map(AtomicInteger::get) + .collect(Collectors.toList()), + Arrays.asList(1, 1, 1)); + } + private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class); } From e0141bc19bd62028238aecb09c642126682c15f7 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 13 Jul 2022 13:13:34 +0800 Subject: [PATCH 668/823] [fix][broker] Skip reading more entries for a pending read with no more entries (#16400) ### Motivation Related issue: https://github.com/streamnative/kop/issues/1379 KoP uses reader on a single partition of a compacted topic and we observed a lot of logs like: > Error reading entries at 928511:1 : We can only have a single waiting callback It happened on a `ManagedCursorImpl` when `hasMoreEntries` returns false, `asyncReadEntriesOrWait` is called for multiple times before `cancelPendingReadRequest` or new messages arrived. ### Modifications Throw a `ConcurrentWaitCallbackException` instead of a raw `ManagedLedgerException` when there are more wait callbacks. Then check this exception type and skip the following steps in `PersistentDispatcherSingleActiveConsumer#internalReadEntriesFailed`. (cherry picked from commit 5ec4e3d0cf977f7473cb4be1272699b11ad9e4a6) --- .../apache/bookkeeper/mledger/ManagedLedgerException.java | 7 +++++++ .../apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 3 +-- .../PersistentDispatcherSingleActiveConsumer.java | 7 +++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java index 26cc9e1265925..7046ba481937a 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java @@ -182,6 +182,13 @@ public ManagedLedgerFactoryClosedException(Throwable e) { } } + public static class ConcurrentWaitCallbackException extends ManagedLedgerException { + + public ConcurrentWaitCallbackException() { + super("We can only have a single waiting callback"); + } + } + @Override public synchronized Throwable fillInStackTrace() { // Disable stack traces to be filled in diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 90ab8218b3088..ddd4408e29e9a 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -797,8 +797,7 @@ public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntrie if (!WAITING_READ_OP_UPDATER.compareAndSet(this, null, op)) { op.recycle(); - callback.readEntriesFailed(new ManagedLedgerException("We can only have a single waiting callback"), - ctx); + callback.readEntriesFailed(new ManagedLedgerException.ConcurrentWaitCallbackException(), ctx); return; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 8d0adcf0ddb41..d59b84b570e25 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -30,6 +30,7 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerException.ConcurrentWaitCallbackException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -452,6 +453,12 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep havePendingRead = false; Consumer c = (Consumer) ctx; + if (exception instanceof ConcurrentWaitCallbackException) { + // At most one pending read request is allowed when there are no more entries, we should not trigger more + // read operations in this case and just wait the existing read operation completes. + return; + } + long waitTimeMillis = readFailureBackoff.next(); if (exception instanceof NoMoreEntriesToReadException) { From c121d0008bf8befdb02ad1c40315e2b190da61d3 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Fri, 15 Jul 2022 18:03:29 +0800 Subject: [PATCH 669/823] [improve][client] Add message key if exists to deadLetter messages (#16615) (cherry picked from commit 8c528c4dc427c8456b3bdff8a35b273bba1d31c3) --- .../client/api/DeadLetterTopicTest.java | 60 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 10 +++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java index 22b224990edb6..9dfe3cf9aca65 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java @@ -63,6 +63,66 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testDeadLetterTopicWithMessageKey() throws Exception { + final String topic = "persistent://my-property/my-ns/dead-letter-topic"; + + final int maxRedeliveryCount = 1; + + final int sendMessages = 100; + + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(1, TimeUnit.SECONDS) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) + .topic("persistent://my-property/my-ns/dead-letter-topic-my-subscription-DLQ") + .subscriptionName("my-subscription") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .create(); + + for (int i = 0; i < sendMessages; i++) { + producer.newMessage() + .key("test-key") + .value(String.format("Hello Pulsar [%d]", i).getBytes()) + .send(); + } + + producer.close(); + + int totalReceived = 0; + do { + Message message = consumer.receive(); + log.info("consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + totalReceived++; + } while (totalReceived < sendMessages * (maxRedeliveryCount + 1)); + + int totalInDeadLetter = 0; + do { + Message message = deadLetterConsumer.receive(); + assertEquals(message.getKey(), "test-key"); + log.info("dead letter consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + deadLetterConsumer.acknowledge(message); + totalInDeadLetter++; + } while (totalInDeadLetter < sendMessages); + + deadLetterConsumer.close(); + consumer.close(); + } + + @Test(groups = "quarantine") public void testDeadLetterTopic() throws Exception { final String topic = "persistent://my-property/my-ns/dead-letter-topic"; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 09f2685eb3ca7..46a6264ae73f1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1815,10 +1815,14 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId) for (MessageImpl message : finalDeadLetterMessages) { String originMessageIdStr = getOriginMessageIdStr(message); String originTopicNameStr = getOriginTopicNameStr(message); - producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) + TypedMessageBuilder typedMessageBuilderNew = + producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) .value(message.getData()) - .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)) - .sendAsync() + .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); + if (message.hasKey()) { + typedMessageBuilderNew.key(message.getKey()); + } + typedMessageBuilderNew.sendAsync() .thenAccept(messageIdInDLQ -> { possibleSendToDeadLetterTopicMessages.remove(finalMessageId); acknowledgeAsync(finalMessageId).whenComplete((v, ex) -> { From 8ca3941c00e30f4c9de25c2fb02da7cfe9d94828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 14 Jul 2022 10:17:11 +0200 Subject: [PATCH 670/823] [fix][test][branch-2.10] Fix test TransactionEndToEndTest#testSendTxnMessageTimeout (only release branches) (#16570) ### Motivation `TransactionEndToEndTest#testSendTxnMessageTimeout` fails on releases branch after https://github.com/apache/pulsar/pull/16519 has been cherry-picked. ``` java.lang.AssertionError: expected [true] but found [false] at org.testng.Assert.fail(Assert.java:99) at org.testng.Assert.failNotEquals(Assert.java:1037) at org.testng.Assert.assertTrue(Assert.java:45) at org.testng.Assert.assertTrue(Assert.java:55) at org.apache.pulsar.client.impl.TransactionEndToEndTest.testSendTxnMessageTimeout(TransactionEndToEndTest.java:1099) ``` The reason is that the mock setup must be slightly different since the `ProducerImpl` is not exactly the same between master and branch-2.10. --- .../apache/pulsar/client/impl/TransactionEndToEndTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index d3e8674925c61..3c0fe8f8e521f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -23,6 +23,7 @@ import static org.mockito.ArgumentMatchers.anyObject; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -37,7 +38,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.DefaultEventLoop; import io.netty.util.concurrent.EventExecutor; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -1080,7 +1083,10 @@ public void testSendTxnMessageTimeout() throws Exception { // mock cnx, send message can't receive response ClientCnx cnx = mock(ClientCnx.class); + Channel channel = mock(Channel.class); + doReturn(spy(DefaultEventLoop.class)).when(channel).eventLoop(); ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); + doReturn(channel).when(channelHandlerContext).channel(); doReturn(channelHandlerContext).when(cnx).ctx(); EventExecutor eventExecutor = mock(EventExecutor.class); doReturn(eventExecutor).when(channelHandlerContext).executor(); From 5bc399de8cad0a97673cd9415bd54c679f461221 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 16 Jun 2022 00:40:24 -0500 Subject: [PATCH 671/823] [broker] Add config to allow deliverAt time to be strictly honored (#16068) * [broker] Add config to allow deliverAt time to be strictly honored * Fix checkstyle error (this is what happens why you change names last minute) * Improve documentation; add private final modifiers The current implementation for `InMemoryDelayedDeliveryTracker` allows messages to deliver early when their `deliverAt` time is within `tickTimeMillis` from now. This is an optimization that ensures messages deliver around the `deliverAt` time. However, some use cases require that messages do not deliver before the `deliverAt` time. (Note that the client api includes a `deliverAfter` method that implies messages won't deliver before some duration of time.) In order to support this alternative implementation, this PR adds a broker configuration named `isDelayedDeliveryDeliverAtTimeStrict`. When true, messages will only deliver when the `deliverAt` time is greater than or equal to `now`. Note that a tradeoff here is that messages will be later than the `deliverAt` time. There are two factors that will determine how late messages will get to consumers. The first is the topic's `DelayedDeliveryTickTimeMillis` and the second is the broker's `delayedDeliveryTickTimeMillis`. The first will determine how frequently a timer will be scheduled to deliver delayed messages. The second is used to determine the tick time of the `HashedWheelTimer`, and as a result, can compound with the topic's delay to make a message deliver even later. * Add broker config named `isDelayedDeliveryDeliverAtTimeStrict`. This config defaults to `false` to maintain the original behavior. * Update the `InMemoryDelayedDeliveryTracker#addMessage` method so that it will return false when `deliverAt <= getCutoffTime()` instead of just `deliverAt <= getCutoffTime()`. * Update documentation in several places. * Implement `InMemoryDelayedDeliveryTracker#getCutoffTime` method that returns the right cutoff time based on the value of `isDelayedDeliveryDeliverAtTimeStrict`. This is the core logical change. * Update `InMemoryDelayedDeliveryTracker#updateTimer` so that it will not schedule a tick to run sooner that the most recent tick run plus the `tickTimeMillis`. This will ensure the timer is not run too frequently. It is also backwards compatible since the existing feature will deliver any messages that were within now plus the `tickTimeMillis`. * Add new tests to cover the new configuration. New tests are added as part of this change. This is a new feature that maintains backwards compatibility. --- conf/broker.conf | 10 ++ .../pulsar/broker/ServiceConfiguration.java | 13 +- .../InMemoryDelayedDeliveryTracker.java | 83 ++++++---- ...InMemoryDelayedDeliveryTrackerFactory.java | 6 +- .../delayed/InMemoryDeliveryTrackerTest.java | 149 ++++++++++++++++-- site2/docs/concepts-messaging.md | 2 + 6 files changed, 224 insertions(+), 39 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index db8b618fff8f7..3c4e4b4524f2f 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -487,9 +487,19 @@ delayedDeliveryEnabled=true # Control the tick time for when retrying on delayed delivery, # affecting the accuracy of the delivery time compared to the scheduled time. +# Note that this time is used to configure the HashedWheelTimer's tick time for the +# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory). # Default is 1 second. delayedDeliveryTickTimeMillis=1000 +# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether +# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt +# time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index +# for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time +# has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the +# delayedDeliveryTickTimeMillis. +isDelayedDeliveryDeliverAtTimeStrict=false + # Whether to enable acknowledge of batch local index. acknowledgmentAtBatchIndexLevelEnabled=false diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index bbdbc31b3f16c..47e47a71d3a4b 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -274,9 +274,20 @@ public class ServiceConfiguration implements PulsarConfiguration { private String delayedDeliveryTrackerFactoryClassName = "org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTrackerFactory"; @FieldContext(category = CATEGORY_SERVER, doc = "Control the tick time for when retrying on delayed delivery, " - + " affecting the accuracy of the delivery time compared to the scheduled time. Default is 1 second.") + + "affecting the accuracy of the delivery time compared to the scheduled time. Default is 1 second. " + + "Note that this time is used to configure the HashedWheelTimer's tick time for the " + + "InMemoryDelayedDeliveryTrackerFactory.") private long delayedDeliveryTickTimeMillis = 1000; + @FieldContext(category = CATEGORY_SERVER, doc = "When using the InMemoryDelayedDeliveryTrackerFactory (the default " + + "DelayedDeliverTrackerFactory), whether the deliverAt time is strictly followed. When false (default), " + + "messages may be sent to consumers before the deliverAt time by as much as the tickTimeMillis. This can " + + "reduce the overhead on the broker of maintaining the delayed index for a potentially very short time " + + "period. When true, messages will not be sent to consumer until the deliverAt time has passed, and they " + + "may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the " + + "delayedDeliveryTickTimeMillis.") + private boolean isDelayedDeliveryDeliverAtTimeStrict = false; + @FieldContext(category = CATEGORY_SERVER, doc = "Whether to enable the acknowledge of batch local index") private boolean acknowledgmentAtBatchIndexLevelEnabled = false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 80ec2185e567b..540c9976a4820 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -46,39 +46,55 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // Timestamp at which the timeout is currently set private long currentTimeoutTarget; + // Last time the TimerTask was triggered for this class + private long lastTickRun; + private long tickTimeMillis; private final Clock clock; - InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis) { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC()); + private final boolean isDelayedDeliveryDeliverAtTimeStrict; + + InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, + boolean isDelayedDeliveryDeliverAtTimeStrict) { + this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); } InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, - long tickTimeMillis, Clock clock) { + long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict) { this.dispatcher = dispatcher; this.timer = timer; this.tickTimeMillis = tickTimeMillis; this.clock = clock; + this.isDelayedDeliveryDeliverAtTimeStrict = isDelayedDeliveryDeliverAtTimeStrict; + } + + /** + * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is false, we allow for early delivery by as much as the + * {@link #tickTimeMillis} because it is a slight optimization to let messages skip going back into the delay + * tracker for a brief amount of time when we're already trying to dispatch to the consumer. + * + * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is true, we use the current time to determine when messages + * can be delivered. As a consequence, there are two delays that will affect delivery. The first is the + * {@link #tickTimeMillis} and the second is the {@link Timer}'s granularity. + * + * @return the cutoff time to determine whether a message is ready to deliver to the consumer + */ + private long getCutoffTime() { + return isDelayedDeliveryDeliverAtTimeStrict ? clock.millis() : clock.millis() + tickTimeMillis; } @Override - public boolean addMessage(long ledgerId, long entryId, long deliveryAt) { - long now = clock.millis(); + public boolean addMessage(long ledgerId, long entryId, long deliverAt) { if (log.isDebugEnabled()) { log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, - deliveryAt - now); + deliverAt - clock.millis()); } - if (deliveryAt < (now + tickTimeMillis)) { - // It's already about time to deliver this message. We add the buffer of - // `tickTimeMillis` because messages can be extracted from the tracker - // slightly before the expiration time. We don't want the messages to - // go back into the delay tracker (for a brief amount of time) when we're - // trying to dispatch to the consumer. + if (deliverAt <= getCutoffTime()) { return false; } - priorityQueue.add(deliveryAt, ledgerId, entryId); + priorityQueue.add(deliverAt, ledgerId, entryId); updateTimer(); return true; } @@ -88,11 +104,8 @@ public boolean addMessage(long ledgerId, long entryId, long deliveryAt) { */ @Override public boolean hasMessageAvailable() { - // Avoid the TimerTask run before reach the timeout. - long cutOffTime = clock.millis() + tickTimeMillis; - boolean hasMessageAvailable = !priorityQueue.isEmpty() && priorityQueue.peekN1() <= cutOffTime; + boolean hasMessageAvailable = !priorityQueue.isEmpty() && priorityQueue.peekN1() <= getCutoffTime(); if (!hasMessageAvailable) { - // prevent the first delay message later than cutoffTime updateTimer(); } return hasMessageAvailable; @@ -105,11 +118,7 @@ public boolean hasMessageAvailable() { public Set getScheduledMessages(int maxMessages) { int n = maxMessages; Set positions = new TreeSet<>(); - long now = clock.millis(); - // Pick all the messages that will be ready within the tick time period. - // This is to avoid keeping rescheduling the timer for each message at - // very short delay - long cutoffTime = now + tickTimeMillis; + long cutoffTime = getCutoffTime(); while (n > 0 && !priorityQueue.isEmpty()) { long timestamp = priorityQueue.peekN1(); @@ -149,6 +158,17 @@ public long getNumberOfDelayedMessages() { return priorityQueue.size(); } + /** + * Update the scheduled timer task such that: + * 1. If there are no delayed messages, return and do not schedule a timer task. + * 2. If the next message in the queue has the same deliverAt time as the timer task, return and leave existing + * timer task in place. + * 3. If the deliverAt time for the next delayed message has already passed (i.e. the delay is negative), return + * without scheduling a timer task since the subscription is backlogged. + * 4. Else, schedule a timer task where the delay is the greater of these two: the next message's deliverAt time or + * the last tick time plus the tickTimeMillis (to ensure we do not schedule the task more frequently than the + * tickTimeMillis). + */ private void updateTimer() { if (priorityQueue.isEmpty()) { if (timeout != null) { @@ -169,10 +189,8 @@ private void updateTimer() { timeout.cancel(); } - long delayMillis = timestamp - clock.millis(); - if (log.isDebugEnabled()) { - log.debug("[{}] Start timer in {} millis", dispatcher.getName(), delayMillis); - } + long now = clock.millis(); + long delayMillis = timestamp - now; if (delayMillis < 0) { // There are messages that are already ready to be delivered. If @@ -184,8 +202,18 @@ private void updateTimer() { return; } + // Compute the earliest time that we schedule the timer to run. + long remainingTickDelayMillis = lastTickRun + tickTimeMillis - now; + long calculatedDelayMillis = Math.max(delayMillis, remainingTickDelayMillis); + + if (log.isDebugEnabled()) { + log.debug("[{}] Start timer in {} millis", dispatcher.getName(), calculatedDelayMillis); + } + + // Even though we may delay longer than this timestamp because of the tick delay, we still track the + // current timeout with reference to the next message's timestamp. currentTimeoutTarget = timestamp; - timeout = timer.newTimeout(this, delayMillis, TimeUnit.MILLISECONDS); + timeout = timer.newTimeout(this, calculatedDelayMillis, TimeUnit.MILLISECONDS); } @Override @@ -198,6 +226,7 @@ public void run(Timeout timeout) throws Exception { } synchronized (dispatcher) { + lastTickRun = clock.millis(); currentTimeoutTarget = -1; timeout = null; dispatcher.readMoreEntries(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java index b1a9b2633699c..5c04a6d53b257 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java @@ -31,16 +31,20 @@ public class InMemoryDelayedDeliveryTrackerFactory implements DelayedDeliveryTra private long tickTimeMillis; + private boolean isDelayedDeliveryDeliverAtTimeStrict; + @Override public void initialize(ServiceConfiguration config) { this.timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-delayed-delivery"), config.getDelayedDeliveryTickTimeMillis(), TimeUnit.MILLISECONDS); this.tickTimeMillis = config.getDelayedDeliveryTickTimeMillis(); + this.isDelayedDeliveryDeliverAtTimeStrict = config.isDelayedDeliveryDeliverAtTimeStrict(); } @Override public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { - return new InMemoryDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis); + return new InMemoryDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, + isDelayedDeliveryDeliverAtTimeStrict); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index d7b304d8a0c24..f44f61a67f9ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -21,13 +21,16 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; @@ -40,27 +43,38 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import io.netty.util.concurrent.DefaultThreadFactory; import lombok.Cleanup; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @Test(groups = "broker") public class InMemoryDeliveryTrackerTest { + // Create a single shared timer for the test. + private final Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), + 500, TimeUnit.MILLISECONDS); + + @AfterClass(alwaysRun = true) + public void cleanup() { + timer.stop(); + } + @Test public void test() throws Exception { PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - Timer timer = mock(Timer.class); - AtomicLong clockTime = new AtomicLong(); Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock); + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + false); assertFalse(tracker.hasMessageAvailable()); @@ -131,7 +145,8 @@ public void testWithTimer() throws Exception { }); @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock); + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + false); assertTrue(tasks.isEmpty()); assertTrue(tracker.addMessage(2, 2, 20)); @@ -160,29 +175,143 @@ public void testWithTimer() throws Exception { /** * Adding a message that is about to expire within the tick time should lead - * to a rejection from the tracker. + * to a rejection from the tracker when isDelayedDeliveryDeliverAtTimeStrict is false. */ @Test public void testAddWithinTickTime() { PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); - Timer timer = mock(Timer.class); - AtomicLong clockTime = new AtomicLong(); Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); @Cleanup - InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock); + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, + false); clockTime.set(0); assertFalse(tracker.addMessage(1, 1, 10)); assertFalse(tracker.addMessage(2, 2, 99)); - assertTrue(tracker.addMessage(3, 3, 100)); - assertTrue(tracker.addMessage(4, 4, 200)); + assertFalse(tracker.addMessage(3, 3, 100)); + assertTrue(tracker.addMessage(4, 4, 101)); + assertTrue(tracker.addMessage(5, 5, 200)); assertEquals(tracker.getNumberOfDelayedMessages(), 2); } + public void testAddMessageWithStrictDelay() { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, + true); + + clockTime.set(10); + + // Verify behavior for the less than, equal to, and greater than deliverAt times. + assertFalse(tracker.addMessage(1, 1, 9)); + assertFalse(tracker.addMessage(4, 4, 10)); + assertTrue(tracker.addMessage(1, 1, 11)); + + assertEquals(tracker.getNumberOfDelayedMessages(), 1); + assertFalse(tracker.hasMessageAvailable()); + } + + /** + * In this test, the deliverAt time is after now, but the deliverAt time is too early to run another tick, so the + * tickTimeMillis determines the delay. + */ + public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict() throws Exception { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, + 1000, clock, true); + + // Set clock time, then run tracker to inherit clock time as the last tick time. + clockTime.set(10000); + Timeout timeout = mock(Timeout.class); + when(timeout.isCancelled()).then(x -> false); + tracker.run(timeout); + verify(dispatcher, times(1)).readMoreEntries(); + + // Add a message that has a delivery time just after the previous run. It will get delivered based on the + // tick delay plus the last tick run. + assertTrue(tracker.addMessage(1, 1, 10001)); + + // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has + // passed where it would have been triggered if the tick time was doing the triggering. + Thread.sleep(600); + verify(dispatcher, times(1)).readMoreEntries(); + + // Not wait for the message delivery to get triggered. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + } + + /** + * In this test, the deliverAt time is after now, but before the (tickTimeMillis + now). Because there wasn't a + * recent tick run, the deliverAt time determines the delay. + */ + public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict() { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + // Use a large tick time to show that the message will get delivered earlier because there wasn't + // a previous tick run. + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, + 100000, clock, true); + + clockTime.set(500000); + + assertTrue(tracker.addMessage(1, 1, 500005)); + + // Wait long enough for the runnable to run, but not longer than the tick time. The point is that the delivery + // should get scheduled early when the tick duration has passed since the last tick. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + } + + /** + * In this test, the deliverAt time is after now plus tickTimeMillis, so the tickTimeMillis determines the delay. + */ + public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict() throws Exception { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, + 500, clock, true); + + clockTime.set(0); + + assertTrue(tracker.addMessage(1, 1, 2000)); + + // Wait longer than the tick time plus the HashedWheelTimer's tick time to ensure that enough time has + // passed where it would have been triggered if the tick time was doing the triggering. + Thread.sleep(1000); + verifyNoInteractions(dispatcher); + + // Not wait for the message delivery to get triggered. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> verify(dispatcher).readMoreEntries()); + } } diff --git a/site2/docs/concepts-messaging.md b/site2/docs/concepts-messaging.md index 3e488b1acfbaf..abb68624f8348 100644 --- a/site2/docs/concepts-messaging.md +++ b/site2/docs/concepts-messaging.md @@ -596,6 +596,8 @@ delayedDeliveryEnabled=true # Control the ticking time for the retry of delayed message delivery, # affecting the accuracy of the delivery time compared to the scheduled time. +# Note that this time is used to configure the HashedWheelTimer's tick time for the +# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory). # Default is 1 second. delayedDeliveryTickTimeMillis=1000 ``` From ee1d80a2d60568621f64a67813312d542f073812 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 15 Jul 2022 09:01:41 -0700 Subject: [PATCH 672/823] Avoid tracking the delays of all the message when we detect that they are fixed (#16609) * Avoid tracking the delays of all the message when we detect that they are fixed * Use tick time to avoid clock skews across different producers --- .../delayed/DelayedDeliveryTracker.java | 5 + .../InMemoryDelayedDeliveryTracker.java | 47 +++++++- .../service/AbstractBaseDispatcher.java | 3 +- ...PersistentDispatcherMultipleConsumers.java | 17 ++- .../delayed/InMemoryDeliveryTrackerTest.java | 113 ++++++++++++++++++ 5 files changed, 179 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java index 2fbd9a51d4ab4..35853d3599b0f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java @@ -55,6 +55,11 @@ public interface DelayedDeliveryTracker extends AutoCloseable { */ Set getScheduledMessages(int maxMessages); + /** + * Tells whether the dispatcher should pause any message deliveries, until the DelayedDeliveryTracker has + * more messages available. + */ + boolean shouldPauseAllDeliveries(); /** * Reset tick time use zk policies cache. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 540c9976a4820..38ca138d0337b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -55,6 +55,20 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T private final boolean isDelayedDeliveryDeliverAtTimeStrict; + // If we detect that all messages have fixed delay time, such that the delivery is + // always going to be in FIFO order, then we can avoid pulling all the messages in + // tracker. Instead, we use the lookahead for detection and pause the read from + // the cursor if the delays are fixed. + public static final long DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES = 50_000; + + // This is the timestamp of the message with the highest delivery time + // If new added messages are lower than this, it means the delivery is requested + // to be out-of-order. It gets reset to 0, once the tracker is emptied. + private long highestDeliveryTimeTracked = 0; + + // Track whether we have seen all messages with fixed delay so far. + private boolean messagesHaveFixedDelay = true; + InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict) { this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); @@ -86,16 +100,28 @@ private long getCutoffTime() { @Override public boolean addMessage(long ledgerId, long entryId, long deliverAt) { + if (deliverAt < 0 || deliverAt <= getCutoffTime()) { + messagesHaveFixedDelay = false; + return false; + } + if (log.isDebugEnabled()) { log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, deliverAt - clock.millis()); } - if (deliverAt <= getCutoffTime()) { - return false; - } + priorityQueue.add(deliverAt, ledgerId, entryId); updateTimer(); + + // Check that new delivery time comes after the current highest, or at + // least within a single tick time interval of 1 second. + if (deliverAt < (highestDeliveryTimeTracked - tickTimeMillis)) { + messagesHaveFixedDelay = false; + } + + highestDeliveryTimeTracked = Math.max(highestDeliveryTimeTracked, deliverAt); + return true; } @@ -137,6 +163,13 @@ public Set getScheduledMessages(int maxMessages) { if (log.isDebugEnabled()) { log.debug("[{}] Get scheduled messages - found {}", dispatcher.getName(), positions.size()); } + + if (priorityQueue.isEmpty()) { + // Reset to initial state + highestDeliveryTimeTracked = 0; + messagesHaveFixedDelay = true; + } + updateTimer(); return positions; } @@ -240,4 +273,12 @@ public void close() { timeout.cancel(); } } + + @Override + public boolean shouldPauseAllDeliveries() { + // Pause deliveries if we know all delays are fixed within the lookahead window + return messagesHaveFixedDelay + && priorityQueue.size() >= DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES + && !hasMessageAvailable(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 95e89af86099e..3a3b542b4659f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -154,8 +154,7 @@ public void filterEntriesForConsumer(Optional entryWrapper, int entry.release(); individualAcknowledgeMessageIfNeeded(pos, Collections.emptyMap()); continue; - } else if (msgMetadata.hasDeliverAtTime() - && trackDelayedDelivery(entry.getLedgerId(), entry.getEntryId(), msgMetadata)) { + } else if (trackDelayedDelivery(entry.getLedgerId(), entry.getEntryId(), msgMetadata)) { // The message is marked for delayed delivery. Ignore for now. entries.set(i, null); entry.release(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 782c597a81e15..03cb72e3e61fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -224,6 +224,10 @@ public synchronized void consumerFlow(Consumer consumer, int additionalNumberOfM } public synchronized void readMoreEntries() { + if (shouldPauseDeliveryForDelayTracker()) { + return; + } + // totalAvailablePermits may be updated by other threads int firstAvailableConsumerPermits = getFirstAvailableConsumerPermits(); int currentTotalAvailablePermits = Math.max(totalAvailablePermits, firstAvailableConsumerPermits); @@ -835,13 +839,20 @@ public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata synchronized (this) { if (!delayedDeliveryTracker.isPresent()) { + if (!msgMetadata.hasDeliverAtTime()) { + // No need to initialize the tracker here + return false; + } + // Initialize the tracker the first time we need to use it delayedDeliveryTracker = Optional .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this)); } delayedDeliveryTracker.get().resetTickTime(topic.getDelayedDeliveryTickTimeMillis()); - return delayedDeliveryTracker.get().addMessage(ledgerId, entryId, msgMetadata.getDeliverAtTime()); + + long deliverAtTime = msgMetadata.hasDeliverAtTime() ? msgMetadata.getDeliverAtTime() : -1L; + return delayedDeliveryTracker.get().addMessage(ledgerId, entryId, deliverAtTime); } } @@ -856,6 +867,10 @@ protected synchronized Set getMessagesToReplayNow(int maxMessagesT } } + protected synchronized boolean shouldPauseDeliveryForDelayTracker() { + return delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().shouldPauseAllDeliveries(); + } + @Override public synchronized long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index f44f61a67f9ca..db2db6cc1dbb0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -314,4 +314,117 @@ public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict() throws Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> verify(dispatcher).readMoreEntries()); } + + @Test + public void testWithFixedDelays() throws Exception { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true); + + assertFalse(tracker.hasMessageAvailable()); + + assertTrue(tracker.addMessage(1, 1, 10)); + assertTrue(tracker.addMessage(2, 2, 20)); + assertTrue(tracker.addMessage(3, 3, 30)); + assertTrue(tracker.addMessage(4, 4, 40)); + assertTrue(tracker.addMessage(5, 5, 50)); + + assertFalse(tracker.hasMessageAvailable()); + assertEquals(tracker.getNumberOfDelayedMessages(), 5); + assertFalse(tracker.shouldPauseAllDeliveries()); + + for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + assertTrue(tracker.addMessage(i, i, i * 10)); + } + + assertTrue(tracker.shouldPauseAllDeliveries()); + + clockTime.set(InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES * 10); + + tracker.getScheduledMessages(100); + assertFalse(tracker.shouldPauseAllDeliveries()); + + // Empty the tracker + int removed = 0; + do { + removed = tracker.getScheduledMessages(100).size(); + } while (removed > 0); + + assertFalse(tracker.shouldPauseAllDeliveries()); + } + + @Test + public void testWithMixedDelays() throws Exception { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true); + + assertFalse(tracker.hasMessageAvailable()); + + assertTrue(tracker.addMessage(1, 1, 10)); + assertTrue(tracker.addMessage(2, 2, 20)); + assertTrue(tracker.addMessage(3, 3, 30)); + assertTrue(tracker.addMessage(4, 4, 40)); + assertTrue(tracker.addMessage(5, 5, 50)); + + assertFalse(tracker.shouldPauseAllDeliveries()); + + for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + assertTrue(tracker.addMessage(i, i, i * 10)); + } + + assertTrue(tracker.shouldPauseAllDeliveries()); + + // Add message with earlier delivery time + assertTrue(tracker.addMessage(5, 5, 5)); + + assertFalse(tracker.shouldPauseAllDeliveries()); + } + + @Test + public void testWithNoDelays() throws Exception { + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + @Cleanup + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true); + + assertFalse(tracker.hasMessageAvailable()); + + assertTrue(tracker.addMessage(1, 1, 10)); + assertTrue(tracker.addMessage(2, 2, 20)); + assertTrue(tracker.addMessage(3, 3, 30)); + assertTrue(tracker.addMessage(4, 4, 40)); + assertTrue(tracker.addMessage(5, 5, 50)); + + assertFalse(tracker.shouldPauseAllDeliveries()); + + for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + assertTrue(tracker.addMessage(i, i, i * 10)); + } + + assertTrue(tracker.shouldPauseAllDeliveries()); + + // Add message with no-delay + assertFalse(tracker.addMessage(5, 5, -1L)); + + assertFalse(tracker.shouldPauseAllDeliveries()); + } + } From 2d7a82f1d728cacd51dd39450ee6e7e37e10909b Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sat, 16 Jul 2022 08:37:01 +0800 Subject: [PATCH 673/823] [branch-2.9] Fix `MockStatic` method caused the exception (#16628) --- .../mledger/impl/ManagedCursorTest.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index aa7a7730c5ebc..db351aa2a74f6 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -97,18 +97,30 @@ import org.apache.pulsar.metadata.api.Stat; import org.apache.pulsar.common.api.proto.IntRange; import org.awaitility.Awaitility; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; +import org.testng.IObjectFactory; import org.testng.annotations.DataProvider; +import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; +@PrepareForTest({ + OpReadEntry.class +}) +@PowerMockIgnore({"org.apache.logging.log4j.*"}) public class ManagedCursorTest extends MockedBookKeeperTestCase { + @ObjectFactory + public IObjectFactory getObjectFactory() { + return new org.powermock.modules.testng.PowerMockObjectFactory(); + } + private static final Charset Encoding = Charsets.UTF_8; @DataProvider(name = "useOpenRangeSet") @@ -3769,8 +3781,8 @@ public void testOpReadEntryRecycle() throws Exception { return mockedOpReadEntry; }; - @Cleanup final MockedStatic mockedStaticOpReadEntry = Mockito.mockStatic(OpReadEntry.class); - mockedStaticOpReadEntry.when(() -> OpReadEntry.create(any(), any(), anyInt(), any(), any(), any())) + PowerMockito.mockStatic(OpReadEntry.class); + PowerMockito.when(OpReadEntry.create(any(), any(), anyInt(), any(), any(), any())) .thenAnswer(__ -> createOpReadEntry.get()); final ManagedLedgerConfig ledgerConfig = new ManagedLedgerConfig(); From 0bca713536701d422e689ba3abb3fed6480f921e Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 18 Jul 2022 17:43:17 -0700 Subject: [PATCH 674/823] Fixed deadlock in key-shared dispatcher (#16660) --- ...tStickyKeyDispatcherMultipleConsumers.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 21c98b9ab3399..881d3db1a81e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -391,13 +391,19 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en } @Override - public synchronized void markDeletePositionMoveForward() { - if (recentlyJoinedConsumers != null && !recentlyJoinedConsumers.isEmpty() - && removeConsumersFromRecentJoinedConsumers()) { - // After we process acks, we need to check whether the mark-delete position was advanced and we can finally - // read more messages. It's safe to call readMoreEntries() multiple times. - readMoreEntries(); - } + public void markDeletePositionMoveForward() { + // Execute the notification in different thread to avoid a mutex chain here + // from the delete operation that was completed + topic.getBrokerService().getTopicOrderedExecutor().execute(() -> { + synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { + if (recentlyJoinedConsumers != null && !recentlyJoinedConsumers.isEmpty() + && removeConsumersFromRecentJoinedConsumers()) { + // After we process acks, we need to check whether the mark-delete position was advanced and we + // can finally read more messages. It's safe to call readMoreEntries() multiple times. + readMoreEntries(); + } + } + }); } private boolean removeConsumersFromRecentJoinedConsumers() { From f02853bc6c7bd8bd996d19d0fa912423a56f2850 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 13 Jul 2022 16:47:09 +0800 Subject: [PATCH 675/823] [improve][security] Add load multiple certificates support in TrustManagerProxy (#14798) Signed-off-by: Zixuan Liu (cherry picked from commit 35037cdebd602980d7eb79f6a5f29ecf62e6dd88) --- .../pulsar/common/util/TrustManagerProxy.java | 30 +++++----- .../common/util/TrustManagerProxyTest.java | 55 +++++++++++++++++++ .../src/test/resources/ca/multiple-ca.pem | 36 ++++++++++++ .../src/test/resources/ca/single-ca.pem | 19 +++++++ 4 files changed, 123 insertions(+), 17 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java create mode 100644 pulsar-common/src/test/resources/ca/multiple-ca.pem create mode 100644 pulsar-common/src/test/resources/ca/single-ca.pem diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/TrustManagerProxy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/TrustManagerProxy.java index 7edbbb47da197..64e9545fe0481 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/TrustManagerProxy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/TrustManagerProxy.java @@ -18,16 +18,14 @@ */ package org.apache.pulsar.common.util; -import java.io.FileInputStream; -import java.io.FileNotFoundException; +import io.netty.handler.ssl.SslContext; import java.io.IOException; -import java.io.InputStream; import java.net.Socket; +import java.security.KeyManagementException; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; -import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -36,7 +34,6 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedTrustManager; -import io.netty.handler.ssl.SslContext; import lombok.extern.slf4j.Slf4j; /** @@ -47,13 +44,13 @@ public class TrustManagerProxy extends X509ExtendedTrustManager { private volatile X509ExtendedTrustManager trustManager; - private FileModifiedTimeUpdater certFile; + private final FileModifiedTimeUpdater certFile; public TrustManagerProxy(String caCertFile, int refreshDurationSec, ScheduledExecutorService executor) { this.certFile = new FileModifiedTimeUpdater(caCertFile); try { updateTrustManager(); - } catch (IOException | CertificateException e) { + } catch (KeyManagementException | IOException | CertificateException e) { log.warn("Failed to load cert {}, {}", certFile, e.getMessage()); throw new IllegalArgumentException(e); } catch (NoSuchAlgorithmException | KeyStoreException e) { @@ -73,19 +70,18 @@ private void updateTrustManagerSafely() { } private void updateTrustManager() throws CertificateException, KeyStoreException, NoSuchAlgorithmException, - FileNotFoundException, IOException { - CertificateFactory factory = CertificateFactory.getInstance("X.509"); - try (InputStream inputStream = new FileInputStream(certFile.getFileName())) { - X509Certificate certificate = (X509Certificate) factory.generateCertificate(inputStream); + IOException, KeyManagementException { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null); + X509Certificate[] certificates = SecurityUtility.loadCertificatesFromPemFile(certFile.getFileName()); + for (X509Certificate certificate : certificates) { String alias = certificate.getSubjectX500Principal().getName(); - KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(null); keyStore.setCertificateEntry(alias, certificate); - final TrustManagerFactory trustManagerFactory = TrustManagerFactory - .getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keyStore); - trustManager = (X509ExtendedTrustManager) trustManagerFactory.getTrustManagers()[0]; } + final TrustManagerFactory trustManagerFactory = TrustManagerFactory + .getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + trustManager = (X509ExtendedTrustManager) trustManagerFactory.getTrustManagers()[0]; } @Override diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java new file mode 100644 index 0000000000000..33163f6e6a568 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import com.google.common.io.Resources; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TrustManagerProxyTest { + @DataProvider(name = "caDataProvider") + public static Object[][] caDataProvider() { + return new Object[][]{ + {"ca/multiple-ca.pem", 2}, + {"ca/single-ca.pem", 1} + }; + } + + @Test(dataProvider = "caDataProvider") + public void testLoadCA(String path, int count) { + String caPath = Resources.getResource(path).getPath(); + + ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); + try { + TrustManagerProxy trustManagerProxy = + new TrustManagerProxy(caPath, 120, scheduledExecutor); + X509Certificate[] x509Certificates = trustManagerProxy.getAcceptedIssuers(); + assertNotNull(x509Certificates); + assertEquals(Arrays.stream(x509Certificates).count(), count); + } finally { + scheduledExecutor.shutdown(); + } + } +} diff --git a/pulsar-common/src/test/resources/ca/multiple-ca.pem b/pulsar-common/src/test/resources/ca/multiple-ca.pem new file mode 100644 index 0000000000000..15f136a695391 --- /dev/null +++ b/pulsar-common/src/test/resources/ca/multiple-ca.pem @@ -0,0 +1,36 @@ +-----BEGIN CERTIFICATE----- +MIIC8jCCAdqgAwIBAgIUPnoDe05/dkrbpa2vpmnp45e6/4UwDQYJKoZIhvcNAQEL +BQAwEDEOMAwGA1UEAxMFdGVzdDEwIBcNMjIwMzIyMDkxOTAwWhgPMjEyMjAzMjQw +OTE5MDBaMBAxDjAMBgNVBAMTBXRlc3QxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAvbd4hZAwrgShqrA6g7QKyXQw/4TvskSBT411XCu9+ZIvG4tEafUJ +CjuHv6AEAt8XN9DSNzVB64q/cOczskaOMa/MQd6Qe+peAiqUsFyu6vucQyCWOLz6 +iuvjPyhuIL5ZYbh17CtXNZOn50BYzq95K4vcAvNUxq/HAGnAm2HegSujq4IMaVpU +gBE3OinUf6patbGqDDuPRUy/gw3I/+xkQcP9RxZqmbsvc6tw6ZpejBdCunCF9hxH +p1V70AqNlxUo7H2w7O7gSDU17gzq8kYoyyiJSLS4Wh1nDscpCQykcxYtS+Agb4VZ +GOYxWMyIBvhRHXLfPVSaYRKw5t5cVy4GjQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMC +AQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQULlE62PqE/wu4x4BmUmnE5Xfw +8rowDQYJKoZIhvcNAQELBQADggEBAEVd7o4Gdm9jpsIFuq8879q3XTmdVvLFL5/6 +g/AddloVeyznd6J1vP4n/4fcIskJ084SD1g8FXG21hOb4vQR06E1qWYhxgJJs7fz +kY3nInbmEWba4Sg7dHXL1KnKOCkhq25UlFF2sMI5BSyKwwAi1R7PKdbTFXZuwFL+ +bKJIvegh+jawlFi1LbSjYYRTy4GgSE8f8/T0xVjqNjdxBnExEkV/dklUJgxck1b1 +K7fRAeoB65tpO/jvYeoQGu1kJkUNmbJu8k1TbBJQm9AxR6OaE+ZKFCf3U5wqH3Ff +hvrO1utY5yvTUnZ2EMTBytH5cGQ+8zW9tm00AB8eimCb4360l/8= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC+jCCAeKgAwIBAgIUEyTKRqhwRp7VqaWPm50zo+CWrt0wDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MCAXDTIyMDMyMTA5NTkwMFoYDzIxMjIw +MzIzMDk1OTAwWjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDrVbbVOJwhdFzi5hueWKwqHQ8OKtWup8lsOIQXbpc0 +T3/FPclF7Qo481SFBUH9Kg+kXJvxYS/sy/9VnoCq/UaUpaOZ6DQTM09bS9b1LOLM +EjXv9sMJ82ipQiwG/MOCGtuDHV++Hmf1lMej0pULL6WpBUhbIYWauiUWLlgLzc1u +v4JZcO/AuBl+tli49Af1ODGWQ4kJYESv27IDU0Jv+/HyE4fmm82vJEqAwjnjxmek +vsFpBvVK6dPUpTJ4hmG4pRrs4MzyxWBGi4PlWhka0LoT8pJ7gcykABToj3gt4Dmz +vVS1LoPq5ph5XgqE+8OHlIMaYIrG6fSxzFXQTyzp27pNAgMBAAGjQjBAMA4GA1Ud +DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR26y6V2jSvt5Dh +n2yaI0YW/nK2aTANBgkqhkiG9w0BAQsFAAOCAQEAxThI6kyDeKajNcnYP6urFO/d +7j/Yvhm1m/xsbg3Ou0iwJqygiJ+IC+jsVzA/tZE1TSX/Yn2KkGdc6vtZVTqESkSi +Gjxp36M1mhLKr/s4pspzSB+8pIOnhOBO2hcZ31DuWASv4AGpIT6XnuoK0KWaJvo8 +Dwbv1D89m5E2WickT4G/QLtbd05Ens/5BrrWW9Lt3f1IxffRWuTBdM7D7a/fF3zF +PpMWCAwmDeDwB9fbyBMtXo+Hd+R1YoeO5X5f0F4HO6VcVo+AkUNxs7FETYAiAQXn +yUYS/bCWHY6eeb67siCLtt4FprkYHt4SQHwKU1V4YmoEE7O/YN9IBEEvKVJZSg== +-----END CERTIFICATE----- diff --git a/pulsar-common/src/test/resources/ca/single-ca.pem b/pulsar-common/src/test/resources/ca/single-ca.pem new file mode 100644 index 0000000000000..2cf2d06f97a67 --- /dev/null +++ b/pulsar-common/src/test/resources/ca/single-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC8jCCAdqgAwIBAgIUPnoDe05/dkrbpa2vpmnp45e6/4UwDQYJKoZIhvcNAQEL +BQAwEDEOMAwGA1UEAxMFdGVzdDEwIBcNMjIwMzIyMDkxOTAwWhgPMjEyMjAzMjQw +OTE5MDBaMBAxDjAMBgNVBAMTBXRlc3QxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAvbd4hZAwrgShqrA6g7QKyXQw/4TvskSBT411XCu9+ZIvG4tEafUJ +CjuHv6AEAt8XN9DSNzVB64q/cOczskaOMa/MQd6Qe+peAiqUsFyu6vucQyCWOLz6 +iuvjPyhuIL5ZYbh17CtXNZOn50BYzq95K4vcAvNUxq/HAGnAm2HegSujq4IMaVpU +gBE3OinUf6patbGqDDuPRUy/gw3I/+xkQcP9RxZqmbsvc6tw6ZpejBdCunCF9hxH +p1V70AqNlxUo7H2w7O7gSDU17gzq8kYoyyiJSLS4Wh1nDscpCQykcxYtS+Agb4VZ +GOYxWMyIBvhRHXLfPVSaYRKw5t5cVy4GjQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMC +AQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQULlE62PqE/wu4x4BmUmnE5Xfw +8rowDQYJKoZIhvcNAQELBQADggEBAEVd7o4Gdm9jpsIFuq8879q3XTmdVvLFL5/6 +g/AddloVeyznd6J1vP4n/4fcIskJ084SD1g8FXG21hOb4vQR06E1qWYhxgJJs7fz +kY3nInbmEWba4Sg7dHXL1KnKOCkhq25UlFF2sMI5BSyKwwAi1R7PKdbTFXZuwFL+ +bKJIvegh+jawlFi1LbSjYYRTy4GgSE8f8/T0xVjqNjdxBnExEkV/dklUJgxck1b1 +K7fRAeoB65tpO/jvYeoQGu1kJkUNmbJu8k1TbBJQm9AxR6OaE+ZKFCf3U5wqH3Ff +hvrO1utY5yvTUnZ2EMTBytH5cGQ+8zW9tm00AB8eimCb4360l/8= +-----END CERTIFICATE----- + From ad5d8a16bc136906e2b2c1c7d896ac44441299eb Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 21 Jul 2022 15:05:49 +0800 Subject: [PATCH 676/823] [fix][proxy] Do not preserve host when forwarding admin requests. (#16342) (cherry picked from commit e8ee8aff448e6d0c329e99c8bda47c3b45228880) --- .../java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java | 1 - .../apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java | 1 - .../proxy/server/SuperUserAuthedAdminProxyHandlerTest.java | 1 - .../pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java | 1 - 4 files changed, 4 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index c966868bcbed8..e358ffef37c93 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -235,7 +235,6 @@ public static void addWebServerHandlers(WebServer server, AdminProxyHandler adminProxyHandler = new AdminProxyHandler(config, discoveryProvider); ServletHolder servletHolder = new ServletHolder(adminProxyHandler); - servletHolder.setInitParameter("preserveHost", "true"); server.addServlet("/admin", servletHolder); server.addServlet("/lookup", servletHolder); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java index 88de4b37469fb..8c7d4fa9198ea 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java @@ -110,7 +110,6 @@ protected void setup() throws Exception { doReturn(report).when(discoveryProvider).nextBroker(); ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider)); - servletHolder.setInitParameter("preserveHost", "true"); webServer.addServlet("/admin", servletHolder); webServer.addServlet("/lookup", servletHolder); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java index 342df28c31079..b044469b4f2dc 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java @@ -107,7 +107,6 @@ protected void setup() throws Exception { doReturn(report).when(discoveryProvider).nextBroker(); ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider)); - servletHolder.setInitParameter("preserveHost", "true"); webServer.addServlet("/admin", servletHolder); webServer.addServlet("/lookup", servletHolder); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java index ee18b60a11e5f..b7535b756ac97 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java @@ -83,7 +83,6 @@ protected void setup() throws Exception { discoveryProvider = spy(new BrokerDiscoveryProvider(proxyConfig, resource)); adminProxyHandler = new AdminProxyWrapper(proxyConfig, discoveryProvider); ServletHolder servletHolder = new ServletHolder(adminProxyHandler); - servletHolder.setInitParameter("preserveHost", "true"); webServer.addServlet("/admin", servletHolder); webServer.addServlet("/lookup", servletHolder); From 7a9bd8f492b4f029df6d096619df67f6b6018796 Mon Sep 17 00:00:00 2001 From: LinChen <1572139390@qq.com> Date: Mon, 18 Jul 2022 22:13:06 +0800 Subject: [PATCH 677/823] [fix][broker] The configuration loadBalancerNamespaceMaximumBundles is invalid (#16552) (cherry picked from commit 5698b08d57f5497b355aa61ac33e7f1303f1ca8e) --- .../loadbalance/impl/BundleSplitterTask.java | 12 ++++- .../impl/BundleSplitterTaskTest.java | 52 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java index fa48618bd205c..f910c2fe7729f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -39,12 +40,16 @@ public class BundleSplitterTask implements BundleSplitStrategy { private static final Logger log = LoggerFactory.getLogger(BundleSplitStrategy.class); private final Set bundleCache; + private final Map namespaceBundleCount; + + /** * Construct a BundleSplitterTask. * */ public BundleSplitterTask() { bundleCache = new HashSet<>(); + namespaceBundleCount = new HashMap<>(); } /** @@ -61,12 +66,14 @@ public BundleSplitterTask() { @Override public Set findBundlesToSplit(final LoadData loadData, final PulsarService pulsar) { bundleCache.clear(); + namespaceBundleCount.clear(); final ServiceConfiguration conf = pulsar.getConfiguration(); int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles(); long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics(); long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions(); long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate(); long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI; + loadData.getBrokerData().forEach((broker, brokerData) -> { LocalBrokerData localData = brokerData.getLocalData(); for (final Map.Entry entry : localData.getLastStats().entrySet()) { @@ -90,8 +97,11 @@ public Set findBundlesToSplit(final LoadData loadData, final PulsarServi try { final int bundleCount = pulsar.getNamespaceService() .getBundleCount(NamespaceName.get(namespace)); - if (bundleCount < maxBundleCount) { + if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0)) + < maxBundleCount) { bundleCache.add(bundle); + int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0); + namespaceBundleCount.put(namespace, bundleNum + 1); } else { log.warn( "Could not split namespace bundle {} because namespace {} has too many bundles: {}", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java index 7480989bbb586..9ff266ba96ce0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java @@ -25,6 +25,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.TimeAverageMessageData; import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; @@ -94,6 +95,57 @@ public void testSplitTaskWhenTopicJustOne() { Assert.assertEquals(bundlesToSplit.size(), 0); } + @Test + public void testLoadBalancerNamespaceMaximumBundles() throws Exception { + pulsar.getConfiguration().setLoadBalancerNamespaceMaximumBundles(3); + + final BundleSplitterTask bundleSplitterTask = new BundleSplitterTask(); + LoadData loadData = new LoadData(); + + LocalBrokerData brokerData = new LocalBrokerData(); + Map lastStats = new HashMap<>(); + final NamespaceBundleStats namespaceBundleStats = new NamespaceBundleStats(); + namespaceBundleStats.topics = 5; + lastStats.put("ten/ns/0x00000000_0x20000000", namespaceBundleStats); + + final NamespaceBundleStats namespaceBundleStats2 = new NamespaceBundleStats(); + namespaceBundleStats2.topics = 5; + lastStats.put("ten/ns/0x20000000_0x40000000", namespaceBundleStats2); + + final NamespaceBundleStats namespaceBundleStats3 = new NamespaceBundleStats(); + namespaceBundleStats3.topics = 5; + lastStats.put("ten/ns/0x40000000_0x60000000", namespaceBundleStats3); + + brokerData.setLastStats(lastStats); + loadData.getBrokerData().put("broker", new BrokerData(brokerData)); + + BundleData bundleData1 = new BundleData(); + TimeAverageMessageData averageMessageData1 = new TimeAverageMessageData(); + averageMessageData1.setMsgRateIn(pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate() * 2); + averageMessageData1.setMsgRateOut(1); + bundleData1.setLongTermData(averageMessageData1); + loadData.getBundleData().put("ten/ns/0x00000000_0x20000000", bundleData1); + + BundleData bundleData2 = new BundleData(); + TimeAverageMessageData averageMessageData2 = new TimeAverageMessageData(); + averageMessageData2.setMsgRateIn(pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate() * 2); + averageMessageData2.setMsgRateOut(1); + bundleData2.setLongTermData(averageMessageData2); + loadData.getBundleData().put("ten/ns/0x20000000_0x40000000", bundleData2); + + BundleData bundleData3 = new BundleData(); + TimeAverageMessageData averageMessageData3 = new TimeAverageMessageData(); + averageMessageData3.setMsgRateIn(pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate() * 2); + averageMessageData3.setMsgRateOut(1); + bundleData3.setLongTermData(averageMessageData3); + loadData.getBundleData().put("ten/ns/0x40000000_0x60000000", bundleData3); + + int currentBundleCount = pulsar.getNamespaceService().getBundleCount(NamespaceName.get("ten/ns")); + final Set bundlesToSplit = bundleSplitterTask.findBundlesToSplit(loadData, pulsar); + Assert.assertEquals(bundlesToSplit.size() + currentBundleCount, + pulsar.getConfiguration().getLoadBalancerNamespaceMaximumBundles()); + } + @AfterMethod(alwaysRun = true) void shutdown() throws Exception { From 9d206584d4bf40ecb05a5991c78a40a78ad4fbdc Mon Sep 17 00:00:00 2001 From: Xiaoyu Hou Date: Mon, 25 Jul 2022 10:42:35 +0800 Subject: [PATCH 678/823] Fix newLookup TooManyRequestsException message (#16594) (cherry picked from commit 595bb55b8537ccb0da12dc7741a0e21fb30c23aa) --- .../main/java/org/apache/pulsar/client/impl/ClientCnx.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index e8aed1abb5564..2d7055eaff903 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -777,8 +777,9 @@ public CompletableFuture newLookup(ByteBuf request, long reque log.debug("{} Failed to add lookup-request into waiting queue", requestId); } future.completeExceptionally(new PulsarClientException.TooManyRequestsException(String.format( - "Requests number out of config: There are {%s} lookup requests outstanding and {%s} requests pending.", - pendingLookupRequestSemaphore.availablePermits(), + "Requests number out of config: There are {%s} lookup requests outstanding and {%s} requests" + + " pending.", + pendingLookupRequestSemaphore.getQueueLength(), waitingLookupRequests.size()))); } } From f30f348ccf2d197d48ae6caefeff07dff1ff87e5 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 20 Jul 2022 10:51:35 +0800 Subject: [PATCH 679/823] [fix][broker] Fix consumer does not abide by the max unacks limitation for Shared subscription (#16670) (cherry picked from commit 42fe0603518be7db7a14802eb4274b6ea22b0c9a) --- ...PersistentDispatcherMultipleConsumers.java | 3 ++ .../api/SimpleProducerConsumerTest.java | 40 +++++++++++++++++++ .../api/v1/V1_ProducerConsumerTest.java | 2 +- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 03cb72e3e61fb..3f33995c6a215 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -526,6 +526,9 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { // round-robin dispatch batch size for this consumer int availablePermits = c.isWritable() ? c.getAvailablePermits() : 1; + if (c.getMaxUnackedMessages() > 0) { + availablePermits = Math.min(availablePermits, c.getMaxUnackedMessages() - c.getUnackedMessages()); + } if (log.isDebugEnabled() && !c.isWritable()) { log.debug("[{}-{}] consumer is not writable. dispatching only 1 message to {}; " + "availablePermits are {}", topic.getName(), name, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 24302a16f8e13..57f82136636cf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -1596,6 +1596,46 @@ public void testConsumerBlockingWithUnAckedMessagesMultipleIteration(boolean ack } } + @Test(dataProvider = "ackReceiptEnabled") + public void testMaxUnAckMessagesLowerThanPermits(boolean ackReceiptEnabled) throws PulsarClientException { + final int maxUnacks = 10; + pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnacks); + final String topic = "persistent://my-property/my-ns/testMaxUnAckMessagesLowerThanPermits"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic).subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(ackReceiptEnabled) + .acknowledgmentGroupTime(0, TimeUnit.SECONDS) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false) + .topic(topic) + .create(); + + final int messages = 1000; + for (int i = 0; i < messages; i++) { + producer.sendAsync("Message - " + i); + } + producer.flush(); + List receives = new ArrayList<>(); + for (int i = 0; i < maxUnacks; i++) { + Message received = consumer.receive(); + log.info("Received message {} with message ID {}", received.getValue(), received.getMessageId()); + receives.add(received.getMessageId()); + } + assertNull(consumer.receive(3, TimeUnit.SECONDS)); + consumer.acknowledge(receives); + for (int i = 0; i < messages - maxUnacks; i++) { + Message received = consumer.receive(); + log.info("Received message {} with message ID {}", received.getValue(), received.getMessageId()); + consumer.acknowledge(received); + } + } + /** * Verify: Consumer1 which doesn't send ack will not impact Consumer2 which sends ack for consumed message. * diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java index 55c120592e1fc..e4cb941c650ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java @@ -1708,7 +1708,7 @@ public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhile } // client should not receive all produced messages and should be blocked due to unack-messages - assertEquals(messages1.size(), receiverQueueSize); + assertEquals(messages1.size(), unAckedMessagesBufferSize); Set redeliveryMessages = messages1.stream().map(m -> { return (MessageIdImpl) m.getMessageId(); }).collect(Collectors.toSet()); From 120242477e800924f537acd3c64070ae04a190de Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 22 Jul 2022 15:24:40 +0800 Subject: [PATCH 680/823] [fix][broker] Fix consumer does not abide by the max unacks limitation for Key_Shared subscription (#16718) (cherry picked from commit fd9c418147503ec226f2c87e187a58af6f601b7d) --- ...entStickyKeyDispatcherMultipleConsumers.java | 6 +++++- .../client/api/SimpleProducerConsumerTest.java | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 881d3db1a81e6..886be363c4120 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -228,7 +228,11 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { Consumer consumer = current.getKey(); List entriesWithSameKey = current.getValue(); int entriesWithSameKeyCount = entriesWithSameKey.size(); - final int availablePermits = consumer == null ? 0 : Math.max(consumer.getAvailablePermits(), 0); + int availablePermits = consumer == null ? 0 : Math.max(consumer.getAvailablePermits(), 0); + if (consumer != null && consumer.getMaxUnackedMessages() > 0) { + availablePermits = Math.min(availablePermits, + consumer.getMaxUnackedMessages() - consumer.getUnackedMessages()); + } int maxMessagesForC = Math.min(entriesWithSameKeyCount, availablePermits); int messagesForC = getRestrictedMaxEntriesForConsumer(consumer, entriesWithSameKey, maxMessagesForC, readType, consumerStickyKeyHashesMap.get(consumer)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 57f82136636cf..555191ae59988 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -149,6 +149,16 @@ public Object[][] ackReceiptEnabled() { return new Object[][] { { true }, { false } }; } + @DataProvider(name = "ackReceiptEnabledAndSubscriptionTypes") + public Object[][] ackReceiptEnabledAndSubscriptionTypes() { + return new Object[][] { + {true, SubscriptionType.Shared}, + {true, SubscriptionType.Key_Shared}, + {false, SubscriptionType.Shared}, + {false, SubscriptionType.Key_Shared}, + }; + } + @AfterMethod(alwaysRun = true) @Override protected void cleanup() throws Exception { @@ -1596,8 +1606,9 @@ public void testConsumerBlockingWithUnAckedMessagesMultipleIteration(boolean ack } } - @Test(dataProvider = "ackReceiptEnabled") - public void testMaxUnAckMessagesLowerThanPermits(boolean ackReceiptEnabled) throws PulsarClientException { + @Test(dataProvider = "ackReceiptEnabledAndSubscriptionTypes") + public void testMaxUnAckMessagesLowerThanPermits(boolean ackReceiptEnabled, SubscriptionType subType) + throws PulsarClientException { final int maxUnacks = 10; pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnacks); final String topic = "persistent://my-property/my-ns/testMaxUnAckMessagesLowerThanPermits"; @@ -1605,7 +1616,7 @@ public void testMaxUnAckMessagesLowerThanPermits(boolean ackReceiptEnabled) thro @Cleanup Consumer consumer = pulsarClient.newConsumer(Schema.STRING) .topic(topic).subscriptionName("sub") - .subscriptionType(SubscriptionType.Shared) + .subscriptionType(subType) .isAckReceiptEnabled(ackReceiptEnabled) .acknowledgmentGroupTime(0, TimeUnit.SECONDS) .subscribe(); From 01dbf180942f0ba09ab6c4b297bf4addcc45e417 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 20 Jul 2022 23:11:44 +0800 Subject: [PATCH 681/823] [fix][broker] Retry to delete the namespace if new topics created during the namespace deletion (#16676) (cherry picked from commit 9077a73b30ea1ad0b8fa2f0cda589797ffcece2e) --- .../broker/admin/impl/NamespacesBase.java | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 2be1e32ba3201..56544a56106f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -347,8 +347,15 @@ protected void internalDeleteNamespace(AsyncResponse asyncResponse, boolean auth asyncResponse.resume(Response.noContent().build()); }) .exceptionally(ex -> { - log.error("[{}] Failed to remove namespace {}", clientAppId(), namespaceName, ex.getCause()); - resumeAsyncResponseExceptionally(asyncResponse, ex.getCause()); + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.error("[{}] Failed to remove namespace {}", clientAppId(), namespaceName, cause); + if (cause instanceof PulsarAdminException.ConflictException) { + log.info("[{}] There are new topics created during the namespace deletion, " + + "retry to delete the namespace again.", namespaceName); + pulsar().getExecutor().execute(() -> internalDeleteNamespace(asyncResponse, authoritative)); + } else { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } return null; }); } @@ -517,15 +524,18 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo FutureUtil.waitForAll(futures).thenCompose(__ -> internalClearZkSources()).handle((result, exception) -> { if (exception != null) { - if (exception.getCause() instanceof PulsarAdminException) { - asyncResponse.resume(new RestException((PulsarAdminException) exception.getCause())); - return null; + Throwable cause = FutureUtil.unwrapCompletionException(exception); + if (cause instanceof PulsarAdminException.ConflictException) { + log.info("[{}] There are new topics created during the namespace deletion, " + + "retry to force delete the namespace again.", namespaceName); + pulsar().getExecutor().execute(() -> + internalDeleteNamespaceForcefully(asyncResponse, authoritative)); } else { log.error("[{}] Failed to remove forcefully owned namespace {}", - clientAppId(), namespaceName, exception); - asyncResponse.resume(new RestException(exception.getCause())); - return null; + clientAppId(), namespaceName, cause); + asyncResponse.resume(new RestException(cause)); } + return null; } asyncResponse.resume(Response.noContent().build()); From ca643cafc12650f914c647480314b9630cf94fbc Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 25 Jul 2022 21:13:26 +0800 Subject: [PATCH 682/823] Fix branch-2.9 compatible issue. (#16779) --- .../apache/pulsar/broker/admin/AdminResource.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index fef7abf6d0711..415ab2f97d107 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -746,14 +746,15 @@ private CompletableFuture provisionPartitionedTopicPath(AsyncResponse asyn } protected void resumeAsyncResponseExceptionally(AsyncResponse asyncResponse, Throwable throwable) { - if (throwable instanceof WebApplicationException) { - asyncResponse.resume(throwable); - } else if (throwable instanceof BrokerServiceException.NotAllowedException) { - asyncResponse.resume(new RestException(Status.CONFLICT, throwable)); - } else if (throwable instanceof PulsarAdminException) { - asyncResponse.resume(new RestException(((PulsarAdminException) throwable))); + Throwable realCause = FutureUtil.unwrapCompletionException(throwable); + if (realCause instanceof WebApplicationException) { + asyncResponse.resume(realCause); + } else if (realCause instanceof BrokerServiceException.NotAllowedException) { + asyncResponse.resume(new RestException(Status.CONFLICT, realCause)); + } else if (realCause instanceof PulsarAdminException) { + asyncResponse.resume(new RestException(((PulsarAdminException) realCause))); } else { - asyncResponse.resume(new RestException(throwable)); + asyncResponse.resume(new RestException(realCause)); } } From a16f03e93a75b597f926d0beddf08cf1fedb948d Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Mon, 25 Jul 2022 16:44:41 +0000 Subject: [PATCH 683/823] [improve][connector] add reader config to `pulsar-io-debezium` and `pulsar-io-kafka-connect-adaptor` (#16675) (cherry picked from commit 9c199761387dcad97316f482b9ceeaf1980a3334) --- .../pulsar/io/common/IOConfigUtils.java | 24 +++++++--- .../io/debezium/PulsarDatabaseHistory.java | 48 ++++++++++++++----- .../debezium/PulsarDatabaseHistoryTest.java | 30 ++++++++++-- pulsar-io/kafka-connect-adaptor/pom.xml | 6 +++ .../connect/PulsarKafkaWorkerConfig.java | 15 +++++- .../connect/PulsarOffsetBackingStore.java | 11 +++++ .../connect/PulsarOffsetBackingStoreTest.java | 24 ++++++++-- 7 files changed, 133 insertions(+), 25 deletions(-) diff --git a/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java b/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java index efb72e82567f9..bfa3cfd1cdb7d 100644 --- a/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java +++ b/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java @@ -18,18 +18,21 @@ */ package org.apache.pulsar.io.common; +import static org.apache.commons.lang.StringUtils.isBlank; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.Reflections; -import org.apache.pulsar.io.core.SinkContext; -import org.apache.pulsar.io.core.SourceContext; -import org.apache.pulsar.io.core.annotations.FieldDoc; - import java.lang.annotation.Annotation; import java.lang.reflect.Field; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.Reflections; +import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.io.core.SourceContext; +import org.apache.pulsar.io.core.annotations.FieldDoc; @Slf4j public class IOConfigUtils { @@ -41,6 +44,15 @@ public static T loadWithSecrets(Map map, Class clazz, Sin return loadWithSecrets(map, clazz, secretName -> sinkContext.getSecret(secretName)); } + public static Map loadConfigFromJsonString(String config) throws JsonProcessingException { + if (!isBlank(config)) { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(config, new TypeReference>() { + }); + } else { + return Collections.emptyMap(); + } + } private static T loadWithSecrets(Map map, Class clazz, Function secretsGetter) { Map configs = new HashMap<>(map); diff --git a/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java b/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java index c97e101a39716..00a0408873f24 100644 --- a/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java +++ b/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java @@ -19,6 +19,9 @@ package org.apache.pulsar.io.debezium; import static org.apache.commons.lang.StringUtils.isBlank; +import static org.apache.pulsar.io.common.IOConfigUtils.loadConfigFromJsonString; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.annotations.VisibleForTesting; import io.debezium.annotation.ThreadSafe; import io.debezium.config.Configuration; import io.debezium.config.Field; @@ -30,6 +33,8 @@ import io.debezium.relational.history.HistoryRecord; import io.debezium.relational.history.HistoryRecordComparator; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; @@ -77,14 +82,26 @@ public final class PulsarDatabaseHistory extends AbstractDatabaseHistory { .withDescription("Pulsar client builder") .withValidation(Field::isOptional); - public static Field.Set ALL_FIELDS = Field.setOf( + public static final Field READER_CONFIG = Field.create(CONFIGURATION_FIELD_PREFIX_STRING + "pulsar.reader.config") + .withDisplayName("Extra configs of the reader") + .withType(Type.STRING) + .withWidth(Width.LONG) + .withImportance(Importance.HIGH) + .withDescription("The configs of the reader for the database schema history topic, " + + "in the form of a JSON string with key-value pairs") + .withDefault((String) null) + .withValidation(Field::isOptional); + + public static final Field.Set ALL_FIELDS = Field.setOf( TOPIC, SERVICE_URL, CLIENT_BUILDER, - DatabaseHistory.NAME); + DatabaseHistory.NAME, + READER_CONFIG); private final DocumentReader reader = DocumentReader.defaultReader(); private String topicName; + private Map readerConfigMap = new HashMap<>(); private String dbHistoryName; private ClientBuilder clientBuilder; private volatile PulsarClient pulsarClient; @@ -102,6 +119,12 @@ public void configure( + getClass().getSimpleName() + "; check the logs for details"); } this.topicName = config.getString(TOPIC); + try { + this.readerConfigMap = loadConfigFromJsonString(config.getString(READER_CONFIG)); + } catch (JsonProcessingException exception) { + log.warn("The provided reader configs are invalid, " + + "will not passing any extra config to the reader builder.", exception); + } String clientBuilderBase64Encoded = config.getString(CLIENT_BUILDER); if (isBlank(clientBuilderBase64Encoded) && isBlank(config.getString(SERVICE_URL))) { @@ -209,11 +232,7 @@ public void stop() { @Override protected void recoverRecords(Consumer records) { setupClientIfNeeded(); - try (Reader historyReader = pulsarClient.newReader(Schema.STRING) - .topic(topicName) - .startMessageId(MessageId.earliest) - .create() - ) { + try (Reader historyReader = createHistoryReader()) { log.info("Scanning the database history topic '{}'", topicName); // Read all messages in the topic ... @@ -256,11 +275,7 @@ protected void recoverRecords(Consumer records) { @Override public boolean exists() { setupClientIfNeeded(); - try (Reader historyReader = pulsarClient.newReader(Schema.STRING) - .topic(topicName) - .startMessageId(MessageId.earliest) - .create() - ) { + try (Reader historyReader = createHistoryReader()) { return historyReader.hasMessageAvailable(); } catch (IOException e) { log.error("Encountered issues on checking existence of database history", e); @@ -280,4 +295,13 @@ public String toString() { } return "Pulsar topic"; } + + @VisibleForTesting + Reader createHistoryReader() throws PulsarClientException { + return pulsarClient.newReader(Schema.STRING) + .topic(topicName) + .startMessageId(MessageId.earliest) + .loadConf(readerConfigMap) + .create(); + } } diff --git a/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java b/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java index 04334da5e435b..081cfdcc5435a 100644 --- a/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java +++ b/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import io.debezium.config.Configuration; import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser; @@ -34,12 +35,14 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; import java.util.Base64; +import java.util.List; import java.util.Map; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -74,7 +77,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWithClientBuilder) throws Exception { + private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWithClientBuilder, boolean testWithReaderConfig) throws Exception { Configuration.Builder configBuidler = Configuration.create() .with(PulsarDatabaseHistory.TOPIC, topicName) .with(DatabaseHistory.NAME, "my-db-history") @@ -93,6 +96,10 @@ private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWit configBuidler.with(PulsarDatabaseHistory.SERVICE_URL, brokerUrl.toString()); } + if (testWithReaderConfig) { + configBuidler.with(PulsarDatabaseHistory.READER_CONFIG, "{\"subscriptionName\":\"my-subscription\"}"); + } + // Start up the history ... history.configure(configBuidler.build(), null, DatabaseHistoryListener.NOOP, true); history.start(); @@ -122,8 +129,8 @@ private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWit // Now record schema changes, which writes out to kafka but doesn't actually change the Tables ... setLogPosition(10); ddl = "CREATE TABLE foo ( first VARCHAR(22) NOT NULL ); \n" + - "CREATE TABLE customers ( id INTEGER NOT NULL PRIMARY KEY, name VARCHAR(100) NOT NULL ); \n" + - "CREATE TABLE products ( productId INTEGER NOT NULL PRIMARY KEY, description VARCHAR(255) NOT NULL ); \n"; + "CREATE TABLE customers ( id INTEGER NOT NULL PRIMARY KEY, name VARCHAR(100) NOT NULL ); \n" + + "CREATE TABLE products ( productId INTEGER NOT NULL PRIMARY KEY, description VARCHAR(255) NOT NULL ); \n"; history.record(source, position, "db1", ddl); // Parse the DDL statement 3x and each time update a different Tables object ... @@ -181,6 +188,10 @@ private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWit assertEquals(recoveredTables, tables3); } + private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWithClientBuilder) throws Exception { + testHistoryTopicContent(skipUnparseableDDL, testWithClientBuilder, false); + } + protected void setLogPosition(int index) { this.position = Collect.hashMapOf("filename", "my-txn-file.log", "position", index); @@ -239,4 +250,17 @@ public void testExists() throws Exception { // dummytopic should not exist yet assertFalse(history.exists()); } + + @Test + public void testSubscriptionName() throws Exception { + testHistoryTopicContent(true, false, true); + assertTrue(history.exists()); + try (Reader ignored = history.createHistoryReader()) { + List subscriptions = admin.topics().getSubscriptions(topicName); + assertEquals(subscriptions.size(), 1); + assertTrue(subscriptions.contains("my-subscription")); + } catch (Exception e) { + fail("Failed to create history reader"); + } + } } diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index f43933dfc838c..334ba9255c489 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -38,6 +38,12 @@ ${project.version} + + ${project.groupId} + pulsar-io-common + ${project.version} + + org.apache.kafka kafka_${scala.binary.version} diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaWorkerConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaWorkerConfig.java index a6cc72517fe66..bf66b6fc98ae4 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaWorkerConfig.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaWorkerConfig.java @@ -43,6 +43,14 @@ public class PulsarKafkaWorkerConfig extends WorkerConfig { public static final String TOPIC_NAMESPACE_CONFIG = "topic.namespace"; private static final String TOPIC_NAMESPACE_CONFIG_DOC = "namespace of topic name to store the output topics"; + /** + * offset.storage.reader.config. + */ + public static final String OFFSET_STORAGE_READER_CONFIG = "offset.storage.reader.config"; + private static final String OFFSET_STORAGE_READER_CONFIG_DOC = "The configs of the reader for the " + + "kafka connector offsets topic, in the form of a JSON string with key-value pairs"; + + static { CONFIG = new ConfigDef() .define(OFFSET_STORAGE_TOPIC_CONFIG, @@ -53,7 +61,12 @@ public class PulsarKafkaWorkerConfig extends WorkerConfig { Type.STRING, "public/default", Importance.HIGH, - TOPIC_NAMESPACE_CONFIG_DOC); + TOPIC_NAMESPACE_CONFIG_DOC) + .define(OFFSET_STORAGE_READER_CONFIG, + Type.STRING, + null, + Importance.HIGH, + OFFSET_STORAGE_READER_CONFIG_DOC); } public PulsarKafkaWorkerConfig(Map props) { diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java index 86905ad8990e3..d2f5aeef72a79 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStore.java @@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.lang.StringUtils.isBlank; +import static org.apache.pulsar.io.common.IOConfigUtils.loadConfigFromJsonString; +import com.fasterxml.jackson.core.JsonProcessingException; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; @@ -54,6 +56,7 @@ public class PulsarOffsetBackingStore implements OffsetBackingStore { private final Map data = new ConcurrentHashMap<>(); private PulsarClient client; private String topic; + private Map readerConfigMap = new HashMap<>(); private Producer producer; private Reader reader; private volatile CompletableFuture outstandingReadToEnd = null; @@ -67,6 +70,13 @@ public PulsarOffsetBackingStore(PulsarClient client) { public void configure(WorkerConfig workerConfig) { this.topic = workerConfig.getString(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG); checkArgument(!isBlank(topic), "Offset storage topic must be specified"); + try { + this.readerConfigMap = loadConfigFromJsonString( + workerConfig.getString(PulsarKafkaWorkerConfig.OFFSET_STORAGE_READER_CONFIG)); + } catch (JsonProcessingException exception) { + log.warn("The provided reader configs are invalid, " + + "will not passing any extra config to the reader builder.", exception); + } log.info("Configure offset backing store on pulsar topic {}", topic); } @@ -148,6 +158,7 @@ public void start() { reader = client.newReader(Schema.BYTES) .topic(topic) .startMessageId(MessageId.earliest) + .loadConf(readerConfigMap) .create(); log.info("Successfully created reader to replay updates from topic {}", topic); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStoreTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStoreTest.java index bb2eced011f52..7b71ba1a13f98 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStoreTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarOffsetBackingStoreTest.java @@ -64,13 +64,10 @@ protected void setup() throws Exception { this.topicName = "persistent://my-property/my-ns/offset-topic"; this.defaultProps.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, topicName); - this.distributedConfig = new PulsarKafkaWorkerConfig(this.defaultProps); this.client = PulsarClient.builder() .serviceUrl(brokerUrl.toString()) .build(); this.offsetBackingStore = new PulsarOffsetBackingStore(client); - this.offsetBackingStore.configure(distributedConfig); - this.offsetBackingStore.start(); } @AfterMethod(alwaysRun = true) @@ -84,8 +81,19 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + private void testOffsetBackingStore(boolean testWithReaderConfig) throws Exception { + if (testWithReaderConfig) { + this.defaultProps.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_READER_CONFIG, + "{\"subscriptionName\":\"my-subscription\"}"); + } + this.distributedConfig = new PulsarKafkaWorkerConfig(this.defaultProps); + this.offsetBackingStore.configure(distributedConfig); + this.offsetBackingStore.start(); + } + @Test public void testGetFromEmpty() throws Exception { + testOffsetBackingStore(false); assertTrue(offsetBackingStore.get( Arrays.asList(ByteBuffer.wrap("empty-key".getBytes(UTF_8))) ).get().isEmpty()); @@ -93,11 +101,13 @@ public void testGetFromEmpty() throws Exception { @Test public void testGetSet() throws Exception { + testOffsetBackingStore(false); testGetSet(false); } @Test public void testGetSetCallback() throws Exception { + testOffsetBackingStore(false); testGetSet(true); } @@ -139,4 +149,12 @@ private void testGetSet(boolean testCallback) throws Exception { assertEquals(new String(valData, UTF_8), "test-val-" + idx); }); } + + @Test + public void testWithReaderConfig() throws Exception { + testOffsetBackingStore(true); + testGetSet(false); + List subscriptions = admin.topics().getSubscriptions(topicName); + assertTrue(subscriptions.contains("my-subscription")); + } } From 157056764dd6a9170f80cf89c93c07ec188d4bb5 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 27 Jul 2022 07:46:33 +0800 Subject: [PATCH 684/823] [fix][common] Add back FutureUtil#waitForAll and FutureUtil#waitForAny methods with List parameter (#16794) --- .../apache/pulsar/common/util/FutureUtil.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index d35a6b405b2b4..51c6087558ab8 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -20,6 +20,7 @@ import java.time.Duration; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -37,6 +38,19 @@ */ public class FutureUtil { + /** + * Return a future that represents the completion of the futures in the provided List. + * This method with the List parameter is needed to keep compatibility with external + * applications that are compiled with Pulsar < 2.10.0. + * + * @param futures futures to wait for + * @return a new CompletableFuture that is completed when all of the given CompletableFutures complete + */ + @Deprecated + public static CompletableFuture waitForAll(List> futures) { + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + /** * Return a future that represents the completion of the futures in the provided Collection. * @@ -47,6 +61,19 @@ public static CompletableFuture waitForAll(Collection waitForAny(List> futures) { + return CompletableFuture.anyOf(futures.toArray(new CompletableFuture[0])); + } + /** * Return a future that represents the completion of any future in the provided Collection. * From 28dc04bbb0f044bb50703be674c6395e4ee9a1f9 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 28 Jul 2022 16:43:39 -0500 Subject: [PATCH 685/823] [Broker] Expose topic level storage write and read rate metrics (#16855) ### Motivation We collect metrics on storage read/write rate, but we do not expose it at the topic level. We already expose it at the namespace level. ### Modifications * Expose `pulsar_storage_read_rate` and `pulsar_storage_write_rate` in topic level metrics ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: This change adds two new metrics. ### Documentation - [x] `doc-not-needed` These two metrics are already documented for topic level stats: https://pulsar.apache.org/docs/next/reference-metrics. (cherry picked from commit 23b52e47f227e0ee5fc9516083c59af4ad2d19f3) --- .../org/apache/pulsar/broker/stats/prometheus/TopicStats.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index aa46f420acd3b..e6e5883847df2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -134,6 +134,10 @@ static void printTopicStats(SimpleTextOutputStream stream, String cluster, Strin stats.managedLedgerStats.storageLogicalSize, splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, "pulsar_msg_backlog", stats.msgBacklog, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_rate", + stats.managedLedgerStats.storageWriteRate, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, + splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, splitTopicAndPartitionIndexLabel); metric(stream, cluster, namespace, topic, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, From 085678aab22f4d3e8d3282e8a1331006dc16b341 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 12 Jul 2022 18:58:26 -0500 Subject: [PATCH 686/823] Update/fix Swagger Annotation for param: authoritative (#16222) * Update/fix Swagger Annotation for param: authoritative * Fix Checkstyle (cherry picked from commit b4ef4a3f4b752749277ae460d7e0739cf32672bc) --- .../broker/admin/v1/PersistentTopics.java | 10 +- .../broker/admin/v2/NonPersistentTopics.java | 10 +- .../broker/admin/v2/PersistentTopics.java | 207 +++++++++--------- 3 files changed, 114 insertions(+), 113 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index fcb2ba8305730..f3b5e6b56a318 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -194,7 +194,7 @@ public void createNonPartitionedTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateNamespaceName(tenant, cluster, namespace); validateTopicName(tenant, cluster, namespace, encodedTopic); @@ -229,7 +229,7 @@ public void createNonPartitionedTopic( public void updatePartitionedTopic(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("updateLocalTopicOnly") @DefaultValue("false") boolean updateLocalTopicOnly, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("force") @DefaultValue("false") boolean force, int numPartitions) { @@ -530,7 +530,7 @@ public void expireTopicMessages( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Subscription to be Expiry messages on") @PathParam("subName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(name = "messageId", value = "messageId to reset back to (ledgerId:entryId)") ResetCursorData resetCursorData) { @@ -832,7 +832,7 @@ public void getLastMessageId( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, cluster, namespace, encodedTopic); @@ -867,7 +867,7 @@ public void setReplicatedSubscriptionStatus( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Name of subscription", required = true) @PathParam("subName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Whether to enable replicated subscription", required = true) boolean enabled) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index 38b1d725a703e..18f3a2dc1c9b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -94,7 +94,7 @@ public PartitionedTopicMetadata getPartitionedMetadata( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Is check configuration required to automatically create topic") @QueryParam("checkAllowAutoCreation") @DefaultValue("false") boolean checkAllowAutoCreation) { @@ -152,7 +152,7 @@ public PersistentTopicInternalStats getInternalStats( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("metadata") @DefaultValue("false") boolean metadata) { validateTopicName(tenant, namespace, encodedTopic); @@ -231,7 +231,7 @@ public void getPartitionedStats( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Get per partition stats") @QueryParam("perPartition") @DefaultValue("true") boolean perPartition, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "If return precise backlog or imprecise backlog") @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog, @@ -340,7 +340,7 @@ public void unloadTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -517,7 +517,7 @@ public void truncateTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative){ asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED.getStatusCode(), "unsupport truncate")); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 3b2ba8d670ce6..8e178da303c9c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -263,7 +263,7 @@ public void createNonPartitionedTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); @@ -283,7 +283,7 @@ public void getOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -304,7 +304,7 @@ public void setOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Offload policies for the specified topic") OffloadPoliciesImpl offloadPolicies) { validateTopicName(tenant, namespace, encodedTopic); @@ -326,7 +326,7 @@ public void removeOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -349,7 +349,7 @@ public void getMaxUnackedMessagesOnConsumer(@Suspended final AsyncResponse async @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -370,7 +370,7 @@ public void setMaxUnackedMessagesOnConsumer( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Max unacked messages on consumer policies for the specified topic") Integer maxUnackedNum) { @@ -393,7 +393,7 @@ public void deleteMaxUnackedMessagesOnConsumer(@Suspended final AsyncResponse as @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -415,7 +415,7 @@ public void getDeduplicationSnapshotInterval(@Suspended final AsyncResponse asyn @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -442,7 +442,7 @@ public void setDeduplicationSnapshotInterval( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Interval to take deduplication snapshot for the specified topic") Integer interval, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -463,7 +463,7 @@ public void deleteDeduplicationSnapshotInterval(@Suspended final AsyncResponse a @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -486,7 +486,7 @@ public void getInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -506,7 +506,7 @@ public void setInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "inactive topic policies for the specified topic") InactiveTopicPolicies inactiveTopicPolicies) { @@ -529,7 +529,7 @@ public void deleteInactiveTopicPolicies(@Suspended final AsyncResponse asyncResp @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -552,7 +552,7 @@ public void getMaxUnackedMessagesOnSubscription(@Suspended final AsyncResponse a @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -574,7 +574,7 @@ public void setMaxUnackedMessagesOnSubscription( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Max unacked messages on subscription policies for the specified topic") Integer maxUnackedNum) { @@ -600,7 +600,7 @@ public void deleteMaxUnackedMessagesOnSubscription(@Suspended final AsyncRespons @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperation(topicName, PolicyName.MAX_UNACKED, PolicyOperation.WRITE); @@ -624,7 +624,7 @@ public void getDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRespo @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -646,7 +646,7 @@ public void setDelayedDeliveryPolicies( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Delayed delivery policies for the specified topic") DelayedDeliveryPolicies deliveryPolicies) { @@ -673,7 +673,7 @@ public void deleteDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRe @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); validatePoliciesReadOnlyAccess(); @@ -725,7 +725,7 @@ public void updatePartitionedTopic( @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, @QueryParam("updateLocalTopicOnly") @DefaultValue("false") boolean updateLocalTopicOnly, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("force") @DefaultValue("false") boolean force, @ApiParam(value = "The number of partitions for the topic", @@ -786,7 +786,7 @@ public PartitionedTopicMetadata getPartitionedMetadata( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Is check configuration required to automatically create topic") @QueryParam("checkAllowAutoCreation") @DefaultValue("false") boolean checkAllowAutoCreation) { @@ -818,7 +818,7 @@ public void deletePartitionedTopic( @ApiParam(value = "Stop all producer/consumer/replicator and delete topic forcefully", defaultValue = "false", type = "boolean") @QueryParam("force") @DefaultValue("false") boolean force, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Delete the topic's schema storage") @QueryParam("deleteSchema") @DefaultValue("false") boolean deleteSchema) { @@ -851,7 +851,7 @@ public void unloadTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -886,7 +886,7 @@ public void deleteTopic( @ApiParam(value = "Stop all producer/consumer/replicator and delete topic forcefully", defaultValue = "false", type = "boolean") @QueryParam("force") @DefaultValue("false") boolean force, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Delete the topic's schema storage") @QueryParam("deleteSchema") @DefaultValue("false") boolean deleteSchema) { @@ -914,7 +914,7 @@ public void getSubscriptions( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -944,7 +944,7 @@ public TopicStats getStats( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "If return precise backlog or imprecise backlog") @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog, @@ -973,7 +973,7 @@ public PersistentTopicInternalStats getInternalStats( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("metadata") @DefaultValue("false") boolean metadata) { validateTopicName(tenant, namespace, encodedTopic); @@ -995,7 +995,7 @@ public void getManagedLedgerInfo( @PathParam("tenant") String tenant, @ApiParam(value = "Specify the namespace", required = true) @PathParam("namespace") String namespace, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @@ -1026,7 +1026,7 @@ public void getPartitionedStats( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Get per partition stats") @QueryParam("perPartition") @DefaultValue("true") boolean perPartition, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "If return precise backlog or imprecise backlog") @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog, @@ -1063,7 +1063,7 @@ public void getPartitionedStatsInternal( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1102,7 +1102,7 @@ public void deleteSubscription( @ApiParam(value = "Disconnect and close all consumers and delete subscription forcefully", defaultValue = "false", type = "boolean") @QueryParam("force") @DefaultValue("false") boolean force, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1139,7 +1139,7 @@ public void skipAllMessages( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Name of subscription") @PathParam("subName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1174,7 +1174,7 @@ public void skipMessages( @PathParam("subName") String encodedSubName, @ApiParam(value = "The number of messages to skip", defaultValue = "0") @PathParam("numMessages") int numMessages, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); internalSkipMessages(decode(encodedSubName), numMessages, authoritative); @@ -1204,7 +1204,7 @@ public void expireTopicMessages( @PathParam("subName") String encodedSubName, @ApiParam(value = "Expires beyond the specified number of seconds", defaultValue = "0") @PathParam("expireTimeInSeconds") int expireTimeInSeconds, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1239,7 +1239,7 @@ public void expireTopicMessages( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Subscription to be Expiry messages on") @PathParam("subName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(name = "messageId", value = "messageId to reset back to (ledgerId:entryId)") ResetCursorData resetCursorData) { @@ -1279,7 +1279,7 @@ public void expireMessagesForAllSubscriptions( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Expires beyond the specified number of seconds", defaultValue = "0") @PathParam("expireTimeInSeconds") int expireTimeInSeconds, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1315,7 +1315,7 @@ public void createSubscription( @PathParam("topic") @Encoded String topic, @ApiParam(value = "Subscription to create position on", required = true) @PathParam("subscriptionName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(name = "messageId", value = "messageId where to create the subscription. " + "It can be 'latest', 'earliest' or (ledgerId:entryId)", @@ -1368,7 +1368,7 @@ public void resetCursor( @PathParam("subName") String encodedSubName, @ApiParam(value = "the timestamp to reset back") @PathParam("timestamp") long timestamp, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1404,7 +1404,7 @@ public void resetCursorOnPosition( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(name = "subName", value = "Subscription to reset position on", required = true) @PathParam("subName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(name = "messageId", value = "messageId to reset back to (ledgerId:entryId)") ResetCursorData resetCursorData) { @@ -1443,7 +1443,7 @@ public Response peekNthMessage( @PathParam("subName") String encodedSubName, @ApiParam(value = "The number of messages (default 1)", defaultValue = "1") @PathParam("messagePosition") int messagePosition, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); return internalPeekNthMessage(decode(encodedSubName), messagePosition, authoritative); @@ -1475,7 +1475,7 @@ public Response examineMessage( @QueryParam("initialPosition") String initialPosition, @ApiParam(value = "The position of messages (default 1)", defaultValue = "1") @QueryParam("messagePosition") long messagePosition, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); return internalExamineMessage(initialPosition, messagePosition, authoritative); @@ -1506,7 +1506,7 @@ public void getMessageById( @PathParam("ledgerId") long ledgerId, @ApiParam(value = "The entry id", required = true) @PathParam("entryId") long entryId, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -1541,7 +1541,7 @@ public void getMessageIdByTimestamp( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Specify the timestamp", required = true) @PathParam("timestamp") long timestamp, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); internalGetMessageIdByTimestamp(timestamp, authoritative) @@ -1574,7 +1574,7 @@ public PersistentOfflineTopicStats getBacklog( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); return internalGetBacklog(authoritative); @@ -1597,7 +1597,7 @@ public void getBacklogSizeByMessageId( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, MessageIdImpl messageId) { validateTopicName(tenant, namespace, encodedTopic); internalGetBacklogSizeByMessageId(asyncResponse, messageId, authoritative); @@ -1616,7 +1616,7 @@ public void getBacklogQuotaMap( @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1642,7 +1642,7 @@ public void setBacklogQuota( @Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("backlogQuotaType") BacklogQuotaType backlogQuotaType, BacklogQuotaImpl backlogQuota) { validateTopicName(tenant, namespace, encodedTopic); @@ -1667,7 +1667,7 @@ public void removeBacklogQuota(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("backlogQuotaType") BacklogQuotaType backlogQuotaType, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1691,7 +1691,7 @@ public void getMessageTTL(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1727,7 +1727,7 @@ public void setMessageTTL(@Suspended final AsyncResponse asyncResponse, @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "TTL in seconds for the specified namespace", required = true) @QueryParam("messageTTL") Integer messageTTL, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1753,7 +1753,7 @@ public void removeMessageTTL(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1778,7 +1778,7 @@ public void getDeduplication(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1802,7 +1802,7 @@ public void setDeduplication( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "DeduplicationEnabled policies for the specified topic") Boolean enabled) { @@ -1828,7 +1828,7 @@ public void removeDeduplication(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1853,7 +1853,7 @@ public void getRetention(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1878,7 +1878,7 @@ public void setRetention(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Retention policies for the specified namespace") RetentionPolicies retention) { validateTopicName(tenant, namespace, encodedTopic); @@ -1914,7 +1914,7 @@ public void removeRetention(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1945,7 +1945,7 @@ public void getPersistence(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -1970,7 +1970,7 @@ public void setPersistence(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Bookkeeper persistence policies for specified topic") PersistencePolicies persistencePolicies) { @@ -2007,7 +2007,7 @@ public void removePersistence(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2037,7 +2037,7 @@ public void getMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2063,7 +2063,7 @@ public void setMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max subscriptions of the topic") int maxSubscriptionsPerTopic) { validateTopicName(tenant, namespace, encodedTopic); @@ -2093,7 +2093,7 @@ public void removeMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncR @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2122,7 +2122,7 @@ public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2147,7 +2147,7 @@ public void setReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Replicator dispatch rate of the topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); @@ -2177,7 +2177,7 @@ public void removeReplicatorDispatchRate(@Suspended final AsyncResponse asyncRes @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2206,7 +2206,7 @@ public void getMaxProducers(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2231,7 +2231,7 @@ public void setMaxProducers(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max producers of the topic") int maxProducers) { validateTopicName(tenant, namespace, encodedTopic); @@ -2263,7 +2263,7 @@ public void removeMaxProducers(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2294,7 +2294,7 @@ public void getMaxConsumers(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2319,7 +2319,7 @@ public void setMaxConsumers(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max consumers of the topic") int maxConsumers) { validateTopicName(tenant, namespace, encodedTopic); @@ -2351,7 +2351,7 @@ public void removeMaxConsumers(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2381,7 +2381,7 @@ public void getMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2408,7 +2408,7 @@ public void setMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max message size of the topic") int maxMessageSize) { validateTopicName(tenant, namespace, encodedTopic); @@ -2440,7 +2440,7 @@ public void removeMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2481,7 +2481,7 @@ public MessageId terminate( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validatePersistentTopicName(tenant, namespace, encodedTopic); return internalTerminate(authoritative); @@ -2507,7 +2507,8 @@ public void terminatePartitionedTopic(@Suspended final AsyncResponse asyncRespon @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker." + + " For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); internalTerminatePartitionedTopic(asyncResponse, authoritative); @@ -2535,7 +2536,7 @@ public void compact( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -2567,7 +2568,7 @@ public LongRunningProcessStatus compactionStatus( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); return internalCompactionStatus(authoritative); @@ -2595,7 +2596,7 @@ public void triggerOffload( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, MessageIdImpl messageId) { if (messageId == null) { @@ -2625,7 +2626,7 @@ public OffloadProcessStatus offloadStatus( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); return internalOffloadStatus(authoritative); @@ -2652,7 +2653,7 @@ public void getLastMessageId( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(tenant, namespace, encodedTopic); @@ -2675,7 +2676,7 @@ public void getDispatchRate(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2699,7 +2700,7 @@ public void setDispatchRate(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Dispatch rate for the specified topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); @@ -2735,7 +2736,7 @@ public void removeDispatchRate(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2767,7 +2768,7 @@ public void getSubscriptionDispatchRate(@Suspended final AsyncResponse asyncResp @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2792,7 +2793,7 @@ public void setSubscriptionDispatchRate( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Subscription message dispatch rate for the specified topic") DispatchRateImpl dispatchRate) { @@ -2829,7 +2830,7 @@ public void removeSubscriptionDispatchRate(@Suspended final AsyncResponse asyncR @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2861,7 +2862,7 @@ public void getCompactionThreshold(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2885,7 +2886,7 @@ public void setCompactionThreshold(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Dispatch rate for the specified topic") long compactionThreshold) { validateTopicName(tenant, namespace, encodedTopic); @@ -2921,7 +2922,7 @@ public void removeCompactionThreshold(@Suspended final AsyncResponse asyncRespon @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2952,7 +2953,7 @@ public void getMaxConsumersPerSubscription(@Suspended final AsyncResponse asyncR @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -2978,7 +2979,7 @@ public void setMaxConsumersPerSubscription( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Dispatch rate for the specified topic") int maxConsumersPerSubscription) { validateTopicName(tenant, namespace, encodedTopic); @@ -3014,7 +3015,7 @@ public void removeMaxConsumersPerSubscription(@Suspended final AsyncResponse asy @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -3046,7 +3047,7 @@ public void getPublishRate(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -3071,7 +3072,7 @@ public void setPublishRate(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Dispatch rate for the specified topic") PublishRate publishRate) { validateTopicName(tenant, namespace, encodedTopic); @@ -3107,7 +3108,7 @@ public void removePublishRate(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -3138,7 +3139,7 @@ public void getSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncResp @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -3165,7 +3166,7 @@ public void setSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncResp @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Enable sub types for the specified topic") Set subscriptionTypesEnabled) { @@ -3203,7 +3204,7 @@ public void getSubscribeRate(@Suspended final AsyncResponse asyncResponse, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("applied") boolean applied, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); preValidation(authoritative) @@ -3227,7 +3228,7 @@ public void setSubscribeRate( @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Subscribe rate for the specified topic") SubscribeRate subscribeRate) { validateTopicName(tenant, namespace, encodedTopic); @@ -3301,7 +3302,7 @@ public void truncateTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative){ validateTopicName(tenant, namespace, encodedTopic); internalTruncateTopic(asyncResponse, authoritative); @@ -3331,7 +3332,7 @@ public void setReplicatedSubscriptionStatus( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Name of subscription", required = true) @PathParam("subName") String encodedSubName, - @ApiParam(value = "Is authentication required to perform this operation") + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Whether to enable replicated subscription", required = true) boolean enabled) { From 9339fd4d98a4af971924d7406df8301e8c9c31b0 Mon Sep 17 00:00:00 2001 From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com> Date: Thu, 4 Aug 2022 12:23:28 +0800 Subject: [PATCH 687/823] [broker][monitoring][fix] fix `pulsar_subscription_msg_ack_rate` (#16866) --- .../stats/prometheus/NamespaceStatsAggregator.java | 1 + .../apache/pulsar/broker/stats/ConsumerStatsTest.java | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 0212c3e964a29..e70f092af80a3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -196,6 +196,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include subsStats.unackedMessages += cStats.unackedMessages; subsStats.msgRateRedeliver += cStats.msgRateRedeliver; subsStats.msgRateOut += cStats.msgRateOut; + subsStats.messageAckRate += cStats.messageAckRate; subsStats.msgThroughputOut += cStats.msgThroughputOut; subsStats.bytesOutCounter += cStats.bytesOutCounter; subsStats.msgOutCounter += cStats.msgOutCounter; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java index af19e889e9b4c..24220625f9141 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java @@ -300,6 +300,8 @@ private void testMessageAckRateMetric(String topicName, boolean exposeTopicLevel Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricStr); Collection ackRateMetric = metricsMap.get("pulsar_consumer_msg_ack_rate"); + Collection subAckRateMetrics = metricsMap.get("pulsar_subscription_msg_ack_rate"); + String rateOutMetricName = exposeTopicLevelMetrics ? "pulsar_consumer_msg_rate_out" : "pulsar_rate_out"; Collection rateOutMetric = metricsMap.get(rateOutMetricName); Assert.assertTrue(ackRateMetric.size() > 0); @@ -316,9 +318,16 @@ private void testMessageAckRateMetric(String topicName, boolean exposeTopicLevel .filter(metric -> metric.tags.get("consumer_name").equals(consumer1Name) || metric.tags.get("consumer_name").equals(consumer2Name)) .mapToDouble(metric -> metric.value).sum(); + double subAckRate = subAckRateMetrics + .stream() + .filter(m -> m.tags.get("subscription").equals(subName)) + .mapToDouble(m -> m.value) + .sum(); + Assert.assertEquals(subAckRateMetrics.size(), 1); Assert.assertTrue(totalAckRate > 0D); Assert.assertTrue(totalRateOut > 0D); + Assert.assertEquals(totalAckRate, subAckRate, 0.1D * totalAckRate); Assert.assertEquals(totalAckRate, totalRateOut, totalRateOut * 0.1D); } else { double totalAckRate = ackRateMetric.stream() From 2e761b8a0ddfc6a0097f85f2f326f41aefb43a44 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 3 Aug 2022 09:56:21 +0800 Subject: [PATCH 688/823] [fix][client-c++] Close `messages_` when PartitionedConsumer is closed (#16887) (cherry picked from commit 7c8cd7bdefbe277af07768b5d2fdb01809bf9404) --- pulsar-client-cpp/lib/ConsumerImpl.cc | 4 ++ .../lib/MultiTopicsConsumerImpl.cc | 4 ++ .../lib/PartitionedConsumerImpl.cc | 7 +++ pulsar-client-cpp/tests/ConsumerTest.cc | 43 +++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/pulsar-client-cpp/lib/ConsumerImpl.cc b/pulsar-client-cpp/lib/ConsumerImpl.cc index 84583d521cc3f..598d2c1ae19ab 100644 --- a/pulsar-client-cpp/lib/ConsumerImpl.cc +++ b/pulsar-client-cpp/lib/ConsumerImpl.cc @@ -721,6 +721,10 @@ Result ConsumerImpl::receiveHelper(Message& msg, int timeout) { messageProcessed(msg); return ResultOk; } else { + Lock lock(mutex_); + if (state_ != Ready) { + return ResultAlreadyClosed; + } return ResultTimeout; } } diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc index 0ae86d5879a4f..361a863d247d5 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc @@ -493,6 +493,10 @@ Result MultiTopicsConsumerImpl::receive(Message& msg, int timeout) { unAckedMessageTrackerPtr_->add(msg.getMessageId()); return ResultOk; } else { + lock.lock(); + if (state_ != Ready) { + return ResultAlreadyClosed; + } return ResultTimeout; } } diff --git a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc index e43b5090e43f7..48f1e4a398bd7 100644 --- a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc +++ b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc @@ -106,6 +106,10 @@ Result PartitionedConsumerImpl::receive(Message& msg, int timeout) { unAckedMessageTrackerPtr_->add(msg.getMessageId()); return ResultOk; } else { + lock.lock(); + if (state_ != Ready) { + return ResultAlreadyClosed; + } return ResultTimeout; } } @@ -428,6 +432,9 @@ void PartitionedConsumerImpl::messageReceived(Consumer consumer, const Message& void PartitionedConsumerImpl::failPendingReceiveCallback() { Message msg; + + messages_.close(); + Lock lock(pendingReceiveMutex_); while (!pendingReceives_.empty()) { ReceiveCallback callback = pendingReceives_.front(); diff --git a/pulsar-client-cpp/tests/ConsumerTest.cc b/pulsar-client-cpp/tests/ConsumerTest.cc index b61c15a886630..adb94a4ad138d 100644 --- a/pulsar-client-cpp/tests/ConsumerTest.cc +++ b/pulsar-client-cpp/tests/ConsumerTest.cc @@ -720,4 +720,47 @@ TEST(ConsumerTest, testIsConnected) { ASSERT_FALSE(consumer.isConnected()); } +TEST(ConsumerTest, testPartitionsWithCloseUnblock) { + Client client(lookupUrl); + const std::string partitionedTopic = "testPartitionsWithCloseUnblock" + std::to_string(time(nullptr)); + constexpr int numPartitions = 2; + + int res = + makePutRequest(adminUrl + "admin/v2/persistent/public/default/" + partitionedTopic + "/partitions", + std::to_string(numPartitions)); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + + Consumer consumer; + ConsumerConfiguration consumerConfig; + ASSERT_EQ(ResultOk, client.subscribe(partitionedTopic, "SubscriptionName", consumerConfig, consumer)); + + // send messages + ProducerConfiguration producerConfig; + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(partitionedTopic, producerConfig, producer)); + Message msg = MessageBuilder().setContent("message").build(); + ASSERT_EQ(ResultOk, producer.send(msg)); + + producer.close(); + + // receive message on another thread + pulsar::Latch latch(1); + auto thread = std::thread([&]() { + Message msg; + ASSERT_EQ(ResultOk, consumer.receive(msg, 10 * 1000)); + consumer.acknowledge(msg.getMessageId()); + ASSERT_EQ(ResultAlreadyClosed, consumer.receive(msg, 10 * 1000)); + latch.countdown(); + }); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + + consumer.close(); + + bool wasUnblocked = latch.wait(std::chrono::milliseconds(100)); + + ASSERT_TRUE(wasUnblocked); + thread.join(); +} + } // namespace pulsar From ccb6140190203e51e68e17a9bfe3a758b2fda883 Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Fri, 29 Jul 2022 22:30:23 +0800 Subject: [PATCH 689/823] fix PatternTopicsChangedListener blocked when topic removed (#16842) (cherry picked from commit 2d6b534ef72824ffb5b2f76568ca8346bbc058c6) --- .../pulsar/client/impl/MultiTopicsConsumerImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 6bfbb71c72c00..aad4c62273492 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -1109,7 +1109,9 @@ public CompletableFuture unsubscribeAsync(String topicName) { }); removeTopic(topicName); - ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { + ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + } unsubscribeFuture.complete(null); log.info("[{}] [{}] [{}] Unsubscribed Topics Consumer, allTopicPartitionsNumber: {}", @@ -1157,7 +1159,9 @@ public CompletableFuture removeConsumerAsync(String topicName) { }); removeTopic(topicName); - ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { + ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + } unsubscribeFuture.complete(null); log.info("[{}] [{}] [{}] Removed Topics Consumer, allTopicPartitionsNumber: {}", From 29b23acd510dda129cf6eca84ad673fd67e37c73 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 29 Jul 2022 11:21:11 +0800 Subject: [PATCH 690/823] [fix][broker] ManagedCursor: mark delete no callback when create meta-ledger fail (#16841) (cherry picked from commit 5faac761372c5b0848924d6607a8da77f209679c) --- .../org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index ddd4408e29e9a..5fb4cb356ff09 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1799,8 +1799,10 @@ protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map Date: Fri, 5 Aug 2022 11:12:35 +0800 Subject: [PATCH 691/823] Revert "[fix][client-c++] Close `messages_` when PartitionedConsumer is closed (#16887)" This reverts commit 2e761b8a0ddfc6a0097f85f2f326f41aefb43a44. --- pulsar-client-cpp/lib/ConsumerImpl.cc | 4 -- .../lib/MultiTopicsConsumerImpl.cc | 4 -- .../lib/PartitionedConsumerImpl.cc | 7 --- pulsar-client-cpp/tests/ConsumerTest.cc | 43 ------------------- 4 files changed, 58 deletions(-) diff --git a/pulsar-client-cpp/lib/ConsumerImpl.cc b/pulsar-client-cpp/lib/ConsumerImpl.cc index 598d2c1ae19ab..84583d521cc3f 100644 --- a/pulsar-client-cpp/lib/ConsumerImpl.cc +++ b/pulsar-client-cpp/lib/ConsumerImpl.cc @@ -721,10 +721,6 @@ Result ConsumerImpl::receiveHelper(Message& msg, int timeout) { messageProcessed(msg); return ResultOk; } else { - Lock lock(mutex_); - if (state_ != Ready) { - return ResultAlreadyClosed; - } return ResultTimeout; } } diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc index 361a863d247d5..0ae86d5879a4f 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc @@ -493,10 +493,6 @@ Result MultiTopicsConsumerImpl::receive(Message& msg, int timeout) { unAckedMessageTrackerPtr_->add(msg.getMessageId()); return ResultOk; } else { - lock.lock(); - if (state_ != Ready) { - return ResultAlreadyClosed; - } return ResultTimeout; } } diff --git a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc index 48f1e4a398bd7..e43b5090e43f7 100644 --- a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc +++ b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc @@ -106,10 +106,6 @@ Result PartitionedConsumerImpl::receive(Message& msg, int timeout) { unAckedMessageTrackerPtr_->add(msg.getMessageId()); return ResultOk; } else { - lock.lock(); - if (state_ != Ready) { - return ResultAlreadyClosed; - } return ResultTimeout; } } @@ -432,9 +428,6 @@ void PartitionedConsumerImpl::messageReceived(Consumer consumer, const Message& void PartitionedConsumerImpl::failPendingReceiveCallback() { Message msg; - - messages_.close(); - Lock lock(pendingReceiveMutex_); while (!pendingReceives_.empty()) { ReceiveCallback callback = pendingReceives_.front(); diff --git a/pulsar-client-cpp/tests/ConsumerTest.cc b/pulsar-client-cpp/tests/ConsumerTest.cc index adb94a4ad138d..b61c15a886630 100644 --- a/pulsar-client-cpp/tests/ConsumerTest.cc +++ b/pulsar-client-cpp/tests/ConsumerTest.cc @@ -720,47 +720,4 @@ TEST(ConsumerTest, testIsConnected) { ASSERT_FALSE(consumer.isConnected()); } -TEST(ConsumerTest, testPartitionsWithCloseUnblock) { - Client client(lookupUrl); - const std::string partitionedTopic = "testPartitionsWithCloseUnblock" + std::to_string(time(nullptr)); - constexpr int numPartitions = 2; - - int res = - makePutRequest(adminUrl + "admin/v2/persistent/public/default/" + partitionedTopic + "/partitions", - std::to_string(numPartitions)); - ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; - - Consumer consumer; - ConsumerConfiguration consumerConfig; - ASSERT_EQ(ResultOk, client.subscribe(partitionedTopic, "SubscriptionName", consumerConfig, consumer)); - - // send messages - ProducerConfiguration producerConfig; - Producer producer; - ASSERT_EQ(ResultOk, client.createProducer(partitionedTopic, producerConfig, producer)); - Message msg = MessageBuilder().setContent("message").build(); - ASSERT_EQ(ResultOk, producer.send(msg)); - - producer.close(); - - // receive message on another thread - pulsar::Latch latch(1); - auto thread = std::thread([&]() { - Message msg; - ASSERT_EQ(ResultOk, consumer.receive(msg, 10 * 1000)); - consumer.acknowledge(msg.getMessageId()); - ASSERT_EQ(ResultAlreadyClosed, consumer.receive(msg, 10 * 1000)); - latch.countdown(); - }); - - std::this_thread::sleep_for(std::chrono::seconds(1)); - - consumer.close(); - - bool wasUnblocked = latch.wait(std::chrono::milliseconds(100)); - - ASSERT_TRUE(wasUnblocked); - thread.join(); -} - } // namespace pulsar From 752225a9ec240eda338c73001c032383ff26986c Mon Sep 17 00:00:00 2001 From: lixinyang <84127069+Nicklee007@users.noreply.github.com> Date: Mon, 8 Aug 2022 10:22:44 +0800 Subject: [PATCH 692/823] cherry pick #16837 to branch 2.9 (#16971) --- .../client/impl/ProducerMemoryLimitTest.java | 29 ++++++++++++ .../client/impl/ProducerSemaphoreTest.java | 47 ++++++++++++++++++- .../pulsar/client/impl/ProducerImpl.java | 5 +- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java index 264ec306413cd..77e3ee811a714 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java @@ -69,6 +69,35 @@ public void testProducerTimeoutMemoryRelease() throws Exception { } + @Test(timeOut = 10_000) + public void testProducerBatchSendTimeoutMemoryRelease() throws Exception { + initClientWithMemoryLimit(); + @Cleanup + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer() + .topic("testProducerMemoryLimit") + .sendTimeout(2, TimeUnit.SECONDS) + .maxPendingMessages(0) + .enableBatching(true) + .batchingMaxPublishDelay(3000, TimeUnit.MILLISECONDS) + .batchingMaxBytes(12) + .create(); + this.stopBroker(); + try { + producer.newMessage().value("memory-test".getBytes(StandardCharsets.UTF_8)).sendAsync(); + try { + producer.newMessage().value("memory-test".getBytes(StandardCharsets.UTF_8)).sendAsync().get(); + } catch (Exception e) { + throw PulsarClientException.unwrap(e); + } + + throw new IllegalStateException("can not reach here"); + } catch (PulsarClientException.TimeoutException ex) { + PulsarClientImpl clientImpl = (PulsarClientImpl) this.pulsarClient; + final MemoryLimitController memoryLimitController = clientImpl.getMemoryLimitController(); + Assert.assertEquals(memoryLimitController.currentUsage(), 0); + } + } + @Test(timeOut = 10_000) public void testProducerCloseMemoryRelease() throws Exception { initClientWithMemoryLimit(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java index 0c7f4b137e791..181e9d05d9c69 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.client.impl; +import static org.mockito.ArgumentMatchers.any; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.ProducerConsumerBase; @@ -25,6 +29,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.util.FutureUtil; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -36,8 +41,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Test(groups = "broker-impl") public class ProducerSemaphoreTest extends ProducerConsumerBase { @@ -206,4 +209,44 @@ public void testEnsureNotBlockOnThePendingQueue() throws Exception { Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); Assert.assertFalse(producer.isErrorStat()); } + + @Test(timeOut = 10_000) + public void testBatchMessageSendTimeoutProducerSemaphoreRelease() throws Exception { + final int pendingQueueSize = 10; + @Cleanup + ProducerImpl producer = + (ProducerImpl) pulsarClient.newProducer() + .topic("testProducerSemaphoreRelease") + .sendTimeout(2, TimeUnit.SECONDS) + .maxPendingMessages(pendingQueueSize) + .enableBatching(true) + .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS) + .batchingMaxBytes(15) + .create(); + this.stopBroker(); + try { + ProducerImpl spyProducer = Mockito.spy(producer); + // Make the pendingMessages not empty + spyProducer.newMessage().value("semaphore-test".getBytes(StandardCharsets.UTF_8)).sendAsync(); + spyProducer.newMessage().value("semaphore-test".getBytes(StandardCharsets.UTF_8)).sendAsync(); + + Field batchMessageContainerField = ProducerImpl.class.getDeclaredField("batchMessageContainer"); + batchMessageContainerField.setAccessible(true); + BatchMessageContainerImpl batchMessageContainer = + (BatchMessageContainerImpl) batchMessageContainerField.get(spyProducer); + batchMessageContainer.setProducer(spyProducer); + Mockito.doThrow(new PulsarClientException.CryptoException("crypto error")).when(spyProducer) + .encryptMessage(any(), any()); + + try { + spyProducer.newMessage().value("memory-test".getBytes(StandardCharsets.UTF_8)).sendAsync().get(); + } catch (Exception e) { + throw PulsarClientException.unwrap(e); + } + + throw new IllegalStateException("can not reach here"); + } catch (PulsarClientException.TimeoutException ex) { + Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index d944bf3acbb72..53578b0eb3bad 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1864,8 +1864,10 @@ private void failPendingBatchMessages(PulsarClientException ex) { return; } final int numMessagesInBatch = batchMessageContainer.getNumMessagesInBatch(); + final long currentBatchSize = batchMessageContainer.getCurrentBatchSize(); batchMessageContainer.discard(ex); semaphoreRelease(numMessagesInBatch); + client.getMemoryLimitController().releaseMemory(currentBatchSize); } @Override @@ -1910,10 +1912,7 @@ private void batchMessageAndSend() { for (OpSendMsg opSendMsg : opSendMsgs) { processOpSendMsg(opSendMsg); } - } catch (PulsarClientException e) { - semaphoreRelease(batchMessageContainer.getNumMessagesInBatch()); } catch (Throwable t) { - semaphoreRelease(batchMessageContainer.getNumMessagesInBatch()); log.warn("[{}] [{}] error while create opSendMsg by batch message container", topic, producerName, t); } } From 41c7dca8c938752702f9fe004ef99bb159e468f4 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Mon, 8 Aug 2022 15:18:50 +0100 Subject: [PATCH 693/823] [improve][broker] Optimise msgOutCounter and bytesOutCounter (#16771) --- .../broker/service/AbstractSubscription.java | 42 +++++++++++ .../pulsar/broker/service/AbstractTopic.java | 17 ++++- .../pulsar/broker/service/Consumer.java | 8 ++ .../NonPersistentSubscription.java | 7 +- .../nonpersistent/NonPersistentTopic.java | 4 - .../persistent/PersistentSubscription.java | 7 +- .../service/persistent/PersistentTopic.java | 4 - .../service/AbstractSubscriptionTest.java | 58 ++++++++++++++ .../broker/service/AbstractTopicTest.java | 75 +++++++++++++++++++ .../pulsar/broker/service/ConsumerTest.java | 75 +++++++++++++++++++ 10 files changed, 277 insertions(+), 20 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java new file mode 100644 index 0000000000000..6a38667055679 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.Optional; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.ToLongFunction; + +public abstract class AbstractSubscription implements Subscription { + protected final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); + protected final LongAdder msgOutFromRemovedConsumer = new LongAdder(); + + public long getMsgOutCounter() { + return msgOutFromRemovedConsumer.longValue() + sumConsumers(Consumer::getMsgOutCounter); + } + + public long getBytesOutCounter() { + return bytesOutFromRemovedConsumers.longValue() + sumConsumers(Consumer::getBytesOutCounter); + } + + private long sumConsumers(ToLongFunction toCounter) { + return Optional.ofNullable(getDispatcher()) + .map(dispatcher -> dispatcher.getConsumers().stream().mapToLong(toCounter).sum()) + .orElse(0L); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 69c51c77d44a5..1307001a72068 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.ToLongFunction; import lombok.Getter; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.lang3.tuple.Pair; @@ -136,6 +137,9 @@ public abstract class AbstractTopic implements Topic { private volatile long lastTopicMaxMessageSizeCheckTimeStamp = 0; private final long topicMaxMessageSizeCheckIntervalMs; + protected final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); + protected final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); + public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; this.brokerService = brokerService; @@ -875,11 +879,20 @@ public long getBytesInCounter() { } public long getMsgOutCounter() { - return getStats(false, false).msgOutCounter; + return msgOutFromRemovedSubscriptions.longValue() + + sumSubscriptions(AbstractSubscription::getMsgOutCounter); } public long getBytesOutCounter() { - return getStats(false, false).bytesOutCounter; + return bytesOutFromRemovedSubscriptions.longValue() + + sumSubscriptions(AbstractSubscription::getBytesOutCounter); + } + + private long sumSubscriptions(ToLongFunction toCounter) { + return getSubscriptions().values().stream() + .map(AbstractSubscription.class::cast) + .mapToLong(toCounter) + .sum(); } public boolean isDeleteWhileInactive() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 2661d29110c75..52a91c5ef3e70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -780,6 +780,14 @@ public ConsumerStatsImpl getStats() { return stats; } + public long getMsgOutCounter() { + return msgOutCounter.longValue(); + } + + public long getBytesOutCounter() { + return bytesOutCounter.longValue(); + } + public int getUnackedMessages() { return unackedMessages; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index cb515cdbd9724..b8a7df36af28e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -24,11 +24,11 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; @@ -48,7 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NonPersistentSubscription implements Subscription { +public class NonPersistentSubscription extends AbstractSubscription implements Subscription { private final NonPersistentTopic topic; private volatile NonPersistentDispatcher dispatcher; private final String topicName; @@ -66,9 +66,6 @@ public class NonPersistentSubscription implements Subscription { // Timestamp of when this subscription was last seen active private volatile long lastActive; - private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); - private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); - // If isDurable is false(such as a Reader), remove subscription from topic when closing this subscription. private final boolean isDurable; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 3846fc129b091..0c2823a131a8f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -34,7 +34,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; -import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; @@ -102,9 +101,6 @@ public class NonPersistentTopic extends AbstractTopic implements Topic { AtomicLongFieldUpdater.newUpdater(NonPersistentTopic.class, "entriesAddedCounter"); private volatile long entriesAddedCounter = 0; - private final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); - private final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); - private static final FastThreadLocal threadLocalTopicStats = new FastThreadLocal() { @Override protected TopicStats initialValue() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index b119f3ca5d2a3..93ad2e80a642c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -29,7 +29,6 @@ import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.LongAdder; import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; @@ -50,6 +49,7 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; @@ -80,7 +80,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PersistentSubscription implements Subscription { +public class PersistentSubscription extends AbstractSubscription implements Subscription { protected final PersistentTopic topic; protected final ManagedCursor cursor; protected volatile Dispatcher dispatcher; @@ -113,9 +113,6 @@ public class PersistentSubscription implements Subscription { private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; private final PendingAckHandle pendingAckHandle; - private final LongAdder bytesOutFromRemovedConsumers = new LongAdder(); - private final LongAdder msgOutFromRemovedConsumer = new LongAdder(); - static { REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 932def13db9de..5f2fd5ea1d9af 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -44,7 +44,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.LongAdder; import java.util.function.BiFunction; import java.util.stream.Collectors; import lombok.Getter; @@ -222,9 +221,6 @@ protected TopicStatsHelper initialValue() { @Getter protected final TransactionBuffer transactionBuffer; - private final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); - private final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); - // Record the last time a data message (ie: not an internal Pulsar marker) is published on the topic private long lastDataMessagePublishedTimestamp = 0; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java new file mode 100644 index 0000000000000..fbc2ecf8059c4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractSubscriptionTest.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import java.util.Collections; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class AbstractSubscriptionTest { + private Consumer consumer; + private AbstractSubscription subscription; + + @BeforeMethod + public void beforeMethod() { + Dispatcher dispatcher = mock(Dispatcher.class); + consumer = mock(Consumer.class); + subscription = spy(AbstractSubscription.class); + + when(subscription.getDispatcher()).thenReturn(dispatcher); + when(dispatcher.getConsumers()).thenReturn(Collections.singletonList(consumer)); + } + + @Test + public void testGetMsgOutCounter() { + subscription.msgOutFromRemovedConsumer.add(1L); + when(consumer.getMsgOutCounter()).thenReturn(2L); + assertEquals(subscription.getMsgOutCounter(), 3L); + } + + @Test + public void testGetBytesOutCounter() { + subscription.bytesOutFromRemovedConsumers.add(1L); + when(consumer.getBytesOutCounter()).thenReturn(2L); + assertEquals(subscription.getBytesOutCounter(), 3L); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java new file mode 100644 index 0000000000000..fb7890dc57f88 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import static org.testng.Assert.assertEquals; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class AbstractTopicTest { + private AbstractSubscription subscription; + private AbstractTopic topic; + + @BeforeMethod + public void beforeMethod() { + BrokerService brokerService = mock(BrokerService.class); + PulsarService pulsarService = mock(PulsarService.class); + ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); + subscription = mock(AbstractSubscription.class); + + when(brokerService.pulsar()).thenReturn(pulsarService); + when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); + when(brokerService.getBacklogQuotaManager()).thenReturn(backlogQuotaManager); + + topic = mock(AbstractTopic.class, withSettings() + .useConstructor("topic", brokerService) + .defaultAnswer(CALLS_REAL_METHODS)); + + ConcurrentOpenHashMap subscriptions = + ConcurrentOpenHashMap.newBuilder() + .expectedItems(16) + .concurrencyLevel(1) + .build(); + subscriptions.put("subscription", subscription); + when(topic.getSubscriptions()).thenAnswer(invocation -> subscriptions); + } + + @Test + public void testGetMsgOutCounter() { + topic.msgOutFromRemovedSubscriptions.add(1L); + when(subscription.getMsgOutCounter()).thenReturn(2L); + assertEquals(topic.getMsgOutCounter(), 3L); + } + + @Test + public void testGetBytesOutCounter() { + topic.bytesOutFromRemovedSubscriptions.add(1L); + when(subscription.getBytesOutCounter()).thenReturn(2L); + assertEquals(topic.getBytesOutCounter(), 3L); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java new file mode 100644 index 0000000000000..65dc4336a0ece --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static java.util.Collections.emptyMap; +import static org.apache.pulsar.client.api.MessageId.latest; +import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive; +import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import java.net.SocketAddress; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.api.proto.CommandSubscribe; +import org.apache.pulsar.common.api.proto.KeySharedMeta; +import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ConsumerTest { + private Consumer consumer; + private final ConsumerStatsImpl stats = new ConsumerStatsImpl(); + + @BeforeMethod + public void beforeMethod() { + Subscription subscription = mock(Subscription.class); + ServerCnx cnx = mock(ServerCnx.class); + SocketAddress address = mock(SocketAddress.class); + Topic topic = mock(Topic.class); + BrokerService brokerService = mock(BrokerService.class); + PulsarService pulsarService = mock(PulsarService.class); + ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + + when(cnx.clientAddress()).thenReturn(address); + when(subscription.getTopic()).thenReturn(topic); + when(topic.getBrokerService()).thenReturn(brokerService); + when(brokerService.getPulsar()).thenReturn(pulsarService); + when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); + + consumer = new Consumer(subscription, Exclusive, "topic", 1, 0, "Cons1", 1, cnx, "myrole-1", emptyMap(), false, + CommandSubscribe.InitialPosition.Earliest, new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest); + } + + @Test + public void testGetMsgOutCounter() { + stats.msgOutCounter = 1L; + consumer.updateStats(stats); + assertEquals(consumer.getMsgOutCounter(), 1L); + } + + @Test + public void testGetBytesOutCounter() { + stats.bytesOutCounter = 1L; + consumer.updateStats(stats); + assertEquals(consumer.getBytesOutCounter(), 1L); + } +} From f8ce46e9fd918d78c1261eccc72ae384b2b1e9f4 Mon Sep 17 00:00:00 2001 From: lixinyang <84127069+Nicklee007@users.noreply.github.com> Date: Tue, 9 Aug 2022 09:10:52 +0800 Subject: [PATCH 694/823] Fix MaxQueueSize semaphore release leak in createOpSendMsg (#16958) --- .../client/impl/ProducerSemaphoreTest.java | 33 +++++++++++++++++++ .../impl/BatchMessageContainerImpl.java | 1 + 2 files changed, 34 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java index 181e9d05d9c69..de858c8d2bdd8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java @@ -58,6 +58,39 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Test(timeOut = 10_000) + public void testProducerSemaphoreInvalidMessage() throws Exception { + final int pendingQueueSize = 100; + + @Cleanup + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer() + .topic("testProducerSemaphoreAcquire") + .maxPendingMessages(pendingQueueSize) + .enableBatching(true) + .create(); + + this.stopBroker(); + + Field maxMessageSizeFiled = ClientCnx.class.getDeclaredField("maxMessageSize"); + maxMessageSizeFiled.setAccessible(true); + maxMessageSizeFiled.set(null, 2); + + try { + producer.send("semaphore-test".getBytes(StandardCharsets.UTF_8)); + Assert.fail("can not reach here"); + } catch (PulsarClientException.InvalidMessageException ex) { + Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + } + + producer.conf.setBatchingEnabled(false); + try { + producer.send("semaphore-test".getBytes(StandardCharsets.UTF_8)); + Assert.fail("can not reach here"); + } catch (PulsarClientException.InvalidMessageException ex) { + Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); + } + } + @Test(timeOut = 30000) public void testProducerSemaphoreAcquireAndRelease() throws PulsarClientException, ExecutionException, InterruptedException { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 996875a7131fa..e0ab2d942ca15 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -199,6 +199,7 @@ public boolean isMultiBatches() { public OpSendMsg createOpSendMsg() throws IOException { ByteBuf encryptedPayload = producer.encryptMessage(messageMetadata, getCompressedBatchMetadataAndPayload()); if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { + producer.semaphoreRelease(messages.size()); discard(new PulsarClientException.InvalidMessageException( "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); return null; From 05d35d8d81ffb962ee303483865c97ba0652ddb3 Mon Sep 17 00:00:00 2001 From: Dezhi LIiu <33149602+liudezhi2098@users.noreply.github.com> Date: Tue, 9 Aug 2022 10:50:28 +0800 Subject: [PATCH 695/823] [Branch 2.9][fix][broker] Upgrade log4j2 version to 2.18.0 (#16884) (#16995) * Upgrade log4j2 version to 2.18.0 Co-authored-by: liudezhi --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 8 ++-- pom.xml | 10 +++- .../EnvironmentBasedSecretsProviderTest.java | 41 ++-------------- .../node/TestEnvVarResolverProperties.java | 47 ++++++++++--------- 5 files changed, 43 insertions(+), 65 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index e35e7bab1da4d..7bb238b716778 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -39,7 +39,7 @@ 1.8 1.8 3.0.0-M3 - 2.17.1 + 2.18.0 1.7.32 7.3.0 3.11 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 6a228e7ea3f50..669fe881b6fa0 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -390,10 +390,10 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.17.1.jar - - org.apache.logging.log4j-log4j-core-2.17.1.jar - - org.apache.logging.log4j-log4j-slf4j-impl-2.17.1.jar - - org.apache.logging.log4j-log4j-web-2.17.1.jar + - org.apache.logging.log4j-log4j-api-2.18.0.jar + - org.apache.logging.log4j-log4j-core-2.18.0.jar + - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar + - org.apache.logging.log4j-log4j-web-2.18.0.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - org.apache.bookkeeper-bookkeeper-common-4.14.5.jar diff --git a/pom.xml b/pom.xml index 14aa41c09a3ed..64be13ca11d7e 100644 --- a/pom.xml +++ b/pom.xml @@ -120,7 +120,7 @@ flexible messaging model and an intuitive client API. 6.10.2 1.7.32 3.2.2 - 2.17.1 + 2.18.0 1.69 1.0.2 2.13.2 @@ -137,6 +137,7 @@ flexible messaging model and an intuitive client API. 0.19.0 ${grpc.version} 2.8.9 + 1.2.1 0.8.3 2.2.0 3.6.0 @@ -1279,6 +1280,13 @@ flexible messaging model and an intuitive client API. test + + com.github.stefanbirkner + system-lambda + ${system-lambda.version} + test + + org.powermock powermock-module-testng diff --git a/pulsar-functions/secrets/src/test/java/org/apache/pulsar/functions/secretsprovider/EnvironmentBasedSecretsProviderTest.java b/pulsar-functions/secrets/src/test/java/org/apache/pulsar/functions/secretsprovider/EnvironmentBasedSecretsProviderTest.java index 1fc76b337a5fd..22ef2dd9e6093 100644 --- a/pulsar-functions/secrets/src/test/java/org/apache/pulsar/functions/secretsprovider/EnvironmentBasedSecretsProviderTest.java +++ b/pulsar-functions/secrets/src/test/java/org/apache/pulsar/functions/secretsprovider/EnvironmentBasedSecretsProviderTest.java @@ -21,10 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; - -import java.lang.reflect.Field; -import java.util.Map; - +import com.github.stefanbirkner.systemlambda.SystemLambda; import org.testng.annotations.Test; public class EnvironmentBasedSecretsProviderTest { @@ -32,38 +29,8 @@ public class EnvironmentBasedSecretsProviderTest { public void testConfigValidation() throws Exception { EnvironmentBasedSecretsProvider provider = new EnvironmentBasedSecretsProvider(); assertNull(provider.provideSecret("mySecretName", "Ignored")); - injectEnvironmentVariable("mySecretName", "SecretValue"); - assertEquals(provider.provideSecret("mySecretName", "Ignored"), "SecretValue"); - } - - private static void injectEnvironmentVariable(String key, String value) - throws Exception { - - Class processEnvironment = Class.forName("java.lang.ProcessEnvironment"); - - Field unmodifiableMapField = getAccessibleField(processEnvironment, "theUnmodifiableEnvironment"); - Object unmodifiableMap = unmodifiableMapField.get(null); - injectIntoUnmodifiableMap(key, value, unmodifiableMap); - - Field mapField = getAccessibleField(processEnvironment, "theEnvironment"); - Map map = (Map) mapField.get(null); - map.put(key, value); - } - - private static Field getAccessibleField(Class clazz, String fieldName) - throws NoSuchFieldException { - - Field field = clazz.getDeclaredField(fieldName); - field.setAccessible(true); - return field; - } - - private static void injectIntoUnmodifiableMap(String key, String value, Object map) - throws ReflectiveOperationException { - - Class unmodifiableMap = Class.forName("java.util.Collections$UnmodifiableMap"); - Field field = getAccessibleField(unmodifiableMap, "m"); - Object obj = field.get(map); - ((Map) obj).put(key, value); + SystemLambda.withEnvironmentVariable("mySecretName", "SecretValue").execute(() -> { + assertEquals(provider.provideSecret("mySecretName", "Ignored"), "SecretValue"); + }); } } diff --git a/pulsar-io/flume/src/test/java/org/apache/pulsar/io/flume/node/TestEnvVarResolverProperties.java b/pulsar-io/flume/src/test/java/org/apache/pulsar/io/flume/node/TestEnvVarResolverProperties.java index 4440acf2f1250..adfe9142d71c1 100644 --- a/pulsar-io/flume/src/test/java/org/apache/pulsar/io/flume/node/TestEnvVarResolverProperties.java +++ b/pulsar-io/flume/src/test/java/org/apache/pulsar/io/flume/node/TestEnvVarResolverProperties.java @@ -18,15 +18,16 @@ */ package org.apache.pulsar.io.flume.node; +import static org.testng.Assert.assertEquals; +import com.github.stefanbirkner.systemlambda.SystemLambda; import java.io.File; -import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.contrib.java.lang.system.EnvironmentVariables; public final class TestEnvVarResolverProperties { - private static final File TESTFILE = new File( + private static final File TEST_FILE = new File( TestEnvVarResolverProperties.class.getClassLoader() .getResource("flume-conf-with-envvars.properties").getFile()); @@ -36,34 +37,36 @@ public final class TestEnvVarResolverProperties { @Before public void setUp() { - provider = new PropertiesFileConfigurationProvider("a1", TESTFILE); + provider = new PropertiesFileConfigurationProvider("a1", TEST_FILE); } @Test - public void resolveEnvVar() { - environmentVariables.set("VARNAME", "varvalue"); - String resolved = EnvVarResolverProperties.resolveEnvVars("padding ${VARNAME} padding"); - Assert.assertEquals("padding varvalue padding", resolved); + public void resolveEnvVar() throws Exception { + SystemLambda.withEnvironmentVariable("VARNAME", "varvalue").execute(() -> { + String resolved = EnvVarResolverProperties.resolveEnvVars("padding ${VARNAME} padding"); + assertEquals(resolved, "padding varvalue padding"); + }); } @Test - public void resolveEnvVars() { - environmentVariables.set("VARNAME1", "varvalue1"); - environmentVariables.set("VARNAME2", "varvalue2"); - String resolved = EnvVarResolverProperties - .resolveEnvVars("padding ${VARNAME1} ${VARNAME2} padding"); - Assert.assertEquals("padding varvalue1 varvalue2 padding", resolved); + public void resolveEnvVars() throws Exception { + SystemLambda.withEnvironmentVariable("VARNAME1", "varvalue1") + .and("VARNAME2", "varvalue2") + .execute(() -> { + String resolved = EnvVarResolverProperties.resolveEnvVars( + "padding ${VARNAME1} ${VARNAME2} padding"); + assertEquals(resolved, "padding varvalue1 varvalue2 padding"); + }); } @Test - public void getProperty() { - String NC_PORT = "6667"; - environmentVariables.set("NC_PORT", NC_PORT); - System.setProperty("propertiesImplementation", - "org.apache.pulsar.io.flume.node.EnvVarResolverProperties"); - - Assert.assertEquals(NC_PORT, provider.getFlumeConfiguration() - .getConfigurationFor("a1") - .getSourceContext().get("r1").getParameters().get("port")); + public void getProperty() throws Exception { + SystemLambda.withEnvironmentVariable("NC_PORT", "6667").execute(() -> { + System.setProperty("propertiesImplementation", + "org.apache.pulsar.io.flume.node.EnvVarResolverProperties"); + assertEquals(provider.getFlumeConfiguration() + .getConfigurationFor("a1") + .getSourceContext().get("r1").getParameters().get("port"), "6667"); + }); } } From 2c6db60abeec33f1e37fdd906f9994e267ad180b Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 9 Aug 2022 19:47:26 +0800 Subject: [PATCH 696/823] [improve][authentication] Support for get token from HTTP params (#16987) --- .../AuthenticationDataHttps.java | 3 +- .../AuthenticationProviderToken.java | 50 +++++++++----- .../AuthenticationProviderTokenTest.java | 66 ++++++++++++++++--- 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttps.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttps.java index 4e1d33b5ec5e7..0c262f5c62056 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttps.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataHttps.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.authentication; import java.security.cert.X509Certificate; - import javax.servlet.http.HttpServletRequest; public class AuthenticationDataHttps extends AuthenticationDataHttp { @@ -27,7 +26,7 @@ public class AuthenticationDataHttps extends AuthenticationDataHttp { protected final X509Certificate[] certificates; public AuthenticationDataHttps(HttpServletRequest request) { - super(request); + super(new AuthenticationProviderToken.HttpServletRequestWrapper(request)); certificates = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate"); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java index 21bda4c97bf46..dccd7bbb2b724 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java @@ -19,35 +19,32 @@ package org.apache.pulsar.broker.authentication; import static java.nio.charset.StandardCharsets.UTF_8; - +import com.google.common.annotations.VisibleForTesting; +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.ExpiredJwtException; +import io.jsonwebtoken.Jwt; +import io.jsonwebtoken.JwtException; +import io.jsonwebtoken.JwtParser; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.RequiredTypeException; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.security.SignatureException; +import io.prometheus.client.Counter; +import io.prometheus.client.Histogram; import java.io.IOException; import java.net.SocketAddress; import java.security.Key; - import java.util.Date; import java.util.List; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; - -import com.google.common.annotations.VisibleForTesting; -import io.jsonwebtoken.ExpiredJwtException; -import io.jsonwebtoken.RequiredTypeException; -import io.jsonwebtoken.JwtParser; -import io.prometheus.client.Counter; -import io.prometheus.client.Histogram; +import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.common.api.AuthData; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jwt; -import io.jsonwebtoken.JwtException; -import io.jsonwebtoken.Jwts; -import io.jsonwebtoken.SignatureAlgorithm; -import io.jsonwebtoken.security.SignatureException; - public class AuthenticationProviderToken implements AuthenticationProvider { static final String HTTP_HEADER_NAME = "Authorization"; @@ -368,4 +365,25 @@ public boolean isExpired() { return expiration < System.currentTimeMillis(); } } + public static final class HttpServletRequestWrapper extends javax.servlet.http.HttpServletRequestWrapper { + private final HttpServletRequest request; + + public HttpServletRequestWrapper(HttpServletRequest request) { + super(request); + this.request = request; + } + + @Override + public String getHeader(String name) { + // The browser javascript WebSocket client couldn't add the auth param to the request header, use the + // query param `token` to transport the auth token for the browser javascript WebSocket client. + if (name.equals(HTTP_HEADER_NAME) && request.getHeader(HTTP_HEADER_NAME) == null) { + String token = request.getParameter(TOKEN); + if (token != null) { + return !token.startsWith(HTTP_HEADER_VALUE_PREFIX) ? HTTP_HEADER_VALUE_PREFIX + token : token; + } + } + return super.getHeader(name); + } + } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java index b05ad4ca4839d..7d6404c3a32a6 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java @@ -18,12 +18,13 @@ */ package org.apache.pulsar.broker.authentication; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.google.common.collect.Lists; import io.jsonwebtoken.Claims; import io.jsonwebtoken.Jwt; @@ -32,26 +33,24 @@ import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.io.Decoders; import io.jsonwebtoken.security.Keys; -import java.security.Key; -import java.util.Arrays; -import java.util.List; -import lombok.Cleanup; - import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.Key; import java.security.KeyPair; import java.security.PrivateKey; import java.sql.Date; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; - import javax.crypto.SecretKey; import javax.naming.AuthenticationException; - +import javax.servlet.http.HttpServletRequest; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.common.api.AuthData; @@ -848,4 +847,55 @@ public String getCommandData() { assertEquals(subject, SUBJECT); provider.close(); } + + @Test + public void testTokenFromHttpParams() throws Exception { + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + @Cleanup + AuthenticationProviderToken provider = new AuthenticationProviderToken(); + + Properties properties = new Properties(); + properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, + AuthTokenUtils.encodeKeyBase64(secretKey)); + + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setProperties(properties); + provider.initialize(conf); + + String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); + HttpServletRequest servletRequest = mock(HttpServletRequest.class); + doReturn(token).when(servletRequest).getParameter("token"); + doReturn(null).when(servletRequest).getHeader("Authorization"); + doReturn("127.0.0.1").when(servletRequest).getRemoteAddr(); + doReturn(0).when(servletRequest).getRemotePort(); + + AuthenticationDataHttps authenticationDataHttps = new AuthenticationDataHttps(servletRequest); + provider.authenticate(authenticationDataHttps); + } + + @Test + public void testTokenFromHttpHeaders() throws Exception { + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + @Cleanup + AuthenticationProviderToken provider = new AuthenticationProviderToken(); + + Properties properties = new Properties(); + properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, + AuthTokenUtils.encodeKeyBase64(secretKey)); + + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setProperties(properties); + provider.initialize(conf); + + String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); + HttpServletRequest servletRequest = mock(HttpServletRequest.class); + doReturn("Bearer " + token).when(servletRequest).getHeader("Authorization"); + doReturn("127.0.0.1").when(servletRequest).getRemoteAddr(); + doReturn(0).when(servletRequest).getRemotePort(); + + AuthenticationDataHttps authenticationDataHttps = new AuthenticationDataHttps(servletRequest); + provider.authenticate(authenticationDataHttps); + } } From ba24b40a1f9e9eb3c394e7180149532161829198 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 5 Jul 2022 14:48:34 +0800 Subject: [PATCH 697/823] Exclude the Netty Reactive Stream from asynchttpclient (#16312) * Exclude the Netty Reactive Stream from asynchttpclient --- *Motivation* We upgrade the Netty Reactive Stream in the PR #15990, but the asynchttpclient still uses it. We should use our project dependency to address the CVE. * Add the related dependency to the sub module (cherry picked from commit f9e89edee9ccb88c3656443b1cf6ffbb0aa1ac55) --- pom.xml | 4 ++++ pulsar-client-tools/pom.xml | 4 ++++ pulsar-client/pom.xml | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/pom.xml b/pom.xml index 64be13ca11d7e..b17952cfc3c84 100644 --- a/pom.xml +++ b/pom.xml @@ -272,6 +272,10 @@ flexible messaging model and an intuitive client API. io.netty * + + com.typesafe.netty + netty-reactive-streams + diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index c20458d5129d8..d74731449db7b 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -72,6 +72,10 @@ org.asynchttpclient async-http-client + + com.typesafe.netty + netty-reactive-streams + org.apache.commons commons-lang3 diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 233b4f9e5b403..ebac828acb1f1 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -94,6 +94,11 @@ async-http-client + + com.typesafe.netty + netty-reactive-streams + + org.slf4j slf4j-api From 0cc42d9130d6245a3b278ff183cb72be4c1a4927 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 25 Jul 2022 17:57:19 +0800 Subject: [PATCH 698/823] [fix][authorization] Fix multiple roles authorization (#16645) (cherry picked from commit d8483d48cb21e8e99fd56c786e5198f7fe7135f6) --- .../MultiRolesTokenAuthorizationProvider.java | 86 ++++++- .../PulsarAuthorizationProvider.java | 3 +- ...tiRolesTokenAuthorizationProviderTest.java | 231 ++++++++++++++++++ 3 files changed, 307 insertions(+), 13 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index b8f46a5248352..d72c951c8894c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -26,9 +26,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; @@ -38,9 +41,12 @@ import org.apache.pulsar.common.policies.data.NamespaceOperation; import org.apache.pulsar.common.policies.data.PolicyName; import org.apache.pulsar.common.policies.data.PolicyOperation; +import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantOperation; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,56 +85,112 @@ public void initialize(ServiceConfiguration conf, PulsarResources pulsarResource super.initialize(conf, pulsarResources); } - private List getRoles(AuthenticationDataSource authData) { + @Override + public CompletableFuture isSuperUser(String role, AuthenticationDataSource authenticationData, + ServiceConfiguration serviceConfiguration) { + Set roles = getRoles(authenticationData); + if (roles.isEmpty()) { + return CompletableFuture.completedFuture(false); + } + Set superUserRoles = serviceConfiguration.getSuperUserRoles(); + if (superUserRoles.isEmpty()) { + return CompletableFuture.completedFuture(false); + } + + return CompletableFuture.completedFuture(roles.stream().anyMatch(superUserRoles::contains)); + } + + @Override + public CompletableFuture validateTenantAdminAccess(String tenantName, String role, + AuthenticationDataSource authData) { + return isSuperUser(role, authData, conf) + .thenCompose(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } + Set roles = getRoles(authData); + if (roles.isEmpty()) { + return CompletableFuture.completedFuture(false); + } + + return pulsarResources.getTenantResources() + .getTenantAsync(tenantName) + .thenCompose(op -> { + if (op.isPresent()) { + TenantInfo tenantInfo = op.get(); + if (tenantInfo.getAdminRoles() == null || tenantInfo.getAdminRoles().isEmpty()) { + return CompletableFuture.completedFuture(false); + } + + return CompletableFuture.completedFuture(roles.stream() + .anyMatch(n -> tenantInfo.getAdminRoles().contains(n))); + } else { + throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); + } + }).exceptionally(ex -> { + Throwable cause = ex.getCause(); + if (cause instanceof MetadataStoreException.NotFoundException) { + log.warn("Failed to get tenant info data for non existing tenant {}", tenantName); + throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); + } + log.error("Failed to get tenant {}", tenantName, cause); + throw new RestException(cause); + }); + }); + } + + private Set getRoles(AuthenticationDataSource authData) { String token = null; if (authData.hasDataFromCommand()) { // Authenticate Pulsar binary connection token = authData.getCommandData(); if (StringUtils.isBlank(token)) { - return Collections.emptyList(); + return Collections.emptySet(); } } else if (authData.hasDataFromHttp()) { // The format here should be compliant to RFC-6750 // (https://tools.ietf.org/html/rfc6750#section-2.1). Eg: Authorization: Bearer xxxxxxxxxxxxx String httpHeaderValue = authData.getHttpHeader(HTTP_HEADER_NAME); if (httpHeaderValue == null || !httpHeaderValue.startsWith(HTTP_HEADER_VALUE_PREFIX)) { - return Collections.emptyList(); + return Collections.emptySet(); } // Remove prefix token = httpHeaderValue.substring(HTTP_HEADER_VALUE_PREFIX.length()); } - if (token == null) - return Collections.emptyList(); + if (token == null) { + return Collections.emptySet(); + } String[] splitToken = token.split("\\."); if (splitToken.length < 2) { log.warn("Unable to extract additional roles from JWT token"); - return Collections.emptyList(); + return Collections.emptySet(); } String unsignedToken = splitToken[0] + "." + splitToken[1] + "."; Jwt jwt = parser.parseClaimsJwt(unsignedToken); try { - return Collections.singletonList(jwt.getBody().get(roleClaim, String.class)); + return new HashSet<>(Collections.singletonList(jwt.getBody().get(roleClaim, String.class))); } catch (RequiredTypeException requiredTypeException) { try { List list = jwt.getBody().get(roleClaim, List.class); if (list != null) { - return list; + return new HashSet(list); } } catch (RequiredTypeException requiredTypeException1) { - return Collections.emptyList(); + return Collections.emptySet(); } } - return Collections.emptyList(); + return Collections.emptySet(); } - public CompletableFuture authorize(AuthenticationDataSource authenticationData, Function> authorizeFunc) { - List roles = getRoles(authenticationData); + public CompletableFuture authorize(AuthenticationDataSource authenticationData, Function> authorizeFunc) { + Set roles = getRoles(authenticationData); if (roles.isEmpty()) { return CompletableFuture.completedFuture(false); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index 097464bfb5f3d..b753d2ed6346c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -59,7 +59,8 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider { private static final Logger log = LoggerFactory.getLogger(PulsarAuthorizationProvider.class); public ServiceConfiguration conf; - private PulsarResources pulsarResources; + + protected PulsarResources pulsarResources; public PulsarAuthorizationProvider() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java new file mode 100644 index 0000000000000..12d7c71358bbd --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import javax.crypto.SecretKey; +import lombok.Cleanup; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.authorization.MultiRolesTokenAuthorizationProvider; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class MultiRolesTokenAuthorizationProviderTest extends MockedPulsarServiceBaseTest { + + private final SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + private final String superUserToken; + private final String normalUserToken; + + public MultiRolesTokenAuthorizationProviderTest() { + Map claims = new HashMap<>(); + Set roles = new HashSet<>(); + roles.add("user1"); + roles.add("superUser"); + claims.put("roles", roles); + superUserToken = Jwts.builder() + .setClaims(claims) + .signWith(secretKey) + .compact(); + + roles = new HashSet<>(); + roles.add("normalUser"); + roles.add("user2"); + roles.add("user5"); + claims.put("roles", roles); + normalUserToken = Jwts.builder() + .setClaims(claims) + .signWith(secretKey) + .compact(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(true); + + Set superUserRoles = new HashSet<>(); + superUserRoles.add("superUser"); + conf.setSuperUserRoles(superUserRoles); + + Properties properties = new Properties(); + properties.setProperty("tokenSecretKey", + "data:;base64," + Base64.getEncoder().encodeToString(secretKey.getEncoded())); + properties.setProperty("tokenAuthClaim", "roles"); + conf.setProperties(properties); + + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters(superUserToken); + + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + conf.setAuthenticationProviders(providers); + conf.setAuthorizationProvider(MultiRolesTokenAuthorizationProvider.class.getName()); + + conf.setClusterName(configClusterName); + conf.setNumExecutorThreadPoolSize(5); + } + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + + admin.clusters().createCluster(configClusterName, + ClusterData.builder() + .brokerServiceUrl(brokerUrl.toString()) + .serviceUrl(getPulsar().getWebServiceAddress()) + .build() + ); + } + + @BeforeClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + clientBuilder.authentication(new AuthenticationToken(superUserToken)); + } + + @Override + protected void customizeNewPulsarAdminBuilder(PulsarAdminBuilder pulsarAdminBuilder) { + pulsarAdminBuilder.authentication(new AuthenticationToken(superUserToken)); + } + + private PulsarAdmin newPulsarAdmin(String token) throws PulsarClientException { + return PulsarAdmin.builder() + .serviceHttpUrl(pulsar.getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .requestTimeout(3, TimeUnit.SECONDS) + .build(); + } + + private PulsarClient newPulsarClient(String token) throws PulsarClientException { + return PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .authentication(new AuthenticationToken(token)) + .operationTimeout(3, TimeUnit.SECONDS) + .build(); + } + + @Test + public void testAdminRequestWithSuperUserToken() throws Exception { + String tenant = "superuser-admin-tenant"; + @Cleanup + PulsarAdmin admin = newPulsarAdmin(superUserToken); + admin.tenants().createTenant(tenant, TenantInfo.builder() + .allowedClusters(Sets.newHashSet(configClusterName)).build()); + String namespace = "superuser-admin-namespace"; + admin.namespaces().createNamespace(tenant + "/" + namespace); + admin.brokers().getAllDynamicConfigurations(); + admin.tenants().getTenants(); + admin.topics().getList(tenant + "/" + namespace); + } + + @Test + public void testProduceAndConsumeWithSuperUserToken() throws Exception { + String tenant = "superuser-client-tenant"; + @Cleanup + PulsarAdmin admin = newPulsarAdmin(superUserToken); + admin.tenants().createTenant(tenant, TenantInfo.builder() + .allowedClusters(Sets.newHashSet(configClusterName)).build()); + String namespace = "superuser-client-namespace"; + admin.namespaces().createNamespace(tenant + "/" + namespace); + String topic = tenant + "/" + namespace + "/" + "test-topic"; + + @Cleanup + PulsarClient client = newPulsarClient(superUserToken); + @Cleanup + Producer producer = client.newProducer().topic(topic).create(); + byte[] body = "hello".getBytes(StandardCharsets.UTF_8); + producer.send(body); + + @Cleanup + Consumer consumer = client.newConsumer().topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test") + .subscribe(); + Message message = consumer.receive(3, TimeUnit.SECONDS); + assertNotNull(message); + assertEquals(message.getData(), body); + } + + @Test + public void testAdminRequestWithNormalUserToken() throws Exception { + String tenant = "normaluser-admin-tenant"; + @Cleanup + PulsarAdmin admin = newPulsarAdmin(normalUserToken); + + assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> admin.tenants().createTenant(tenant, TenantInfo.builder() + .allowedClusters(Sets.newHashSet(configClusterName)).build())); + } + + @Test + public void testProduceAndConsumeWithNormalUserToken() throws Exception { + String tenant = "normaluser-client-tenant"; + @Cleanup + PulsarAdmin admin = newPulsarAdmin(superUserToken); + admin.tenants().createTenant(tenant, TenantInfo.builder() + .allowedClusters(Sets.newHashSet(configClusterName)).build()); + String namespace = "normaluser-client-namespace"; + admin.namespaces().createNamespace(tenant + "/" + namespace); + String topic = tenant + "/" + namespace + "/" + "test-topic"; + + @Cleanup + PulsarClient client = newPulsarClient(normalUserToken); + assertThrows(PulsarClientException.AuthorizationException.class, () -> { + @Cleanup + Producer ignored = client.newProducer().topic(topic).create(); + }); + + assertThrows(PulsarClientException.AuthorizationException.class, () -> { + @Cleanup + Consumer ignored = client.newConsumer().topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test") + .subscribe(); + }); + } +} From 6bdffab17dbe7e2a007fba7d6f0114f72a3ee4d3 Mon Sep 17 00:00:00 2001 From: Bonan Hou Date: Mon, 25 Jul 2022 21:14:39 +0800 Subject: [PATCH 699/823] add artifactSet to pom.xml for pulsar-functions-local-runner (#16565) (cherry picked from commit 52d8fe03160648f14dc196edaa87921064a1b2b2) --- pulsar-functions/localrun-shaded/pom.xml | 66 ++++++++++++++++++++---- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index c6f1957f88cff..9d081892cbf56 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -83,7 +83,7 @@ @@ -106,6 +106,40 @@ + + + org.apache.pulsar:* + org.apache.bookkeeper:* + commons-*:* + org.apache.commons:* + com.fasterxml.jackson.*:* + io.netty:* + com.google.*:* + javax.servlet:* + org.reactivestreams:reactive-streams + org.apache.commons:* + io.swagger:* + org.yaml:snakeyaml + io.perfmark:* + io.prometheus:* + io.prometheus.jmx:* + javax.ws.rs:* + org.tukaani:xz + com.github.zafarkhaja:java-semver + net.java.dev.jna:* + org.apache.zookeeper:* + com.thoughtworks.paranamer:paranamer + jline:* + org.rocksdb:* + org.eclipse.jetty*:* + org.apache.avro:avro + com.beust:* + net.jodah:* + io.airlift:* + com.yahoo.datasketches:* + io.netty.resolver:* + + org.apache.pulsar:pulsar-client-original @@ -225,10 +259,10 @@ org.apache.pulsar.functions.runtime.shaded.org.apache.distributedlog - + org.inferred org.apache.pulsar.functions.runtime.shaded.org.inferred @@ -246,10 +280,10 @@ org.apache.pulsar.functions.runtime.shaded.dlshade - + net.java.dev.jna org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna @@ -270,6 +304,10 @@ io.prometheus org.apache.pulsar.functions.runtime.shaded.io.prometheus + + io.prometheus.jmx + org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx + org.apache.zookeeper org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper @@ -347,18 +385,22 @@ org.apache.pulsar.functions.runtime.shaded.avo.shaded - com.yahoo - org.apache.pulsar.functions.runtime.shaded.com.yahoo + com.yahoo.datasketches + org.apache.pulsar.shaded.com.yahoo.datasketches + + + com.yahoo.sketches + org.apache.pulsar.shaded.com.yahoo.sketches com.beust org.apache.pulsar.functions.runtime.shaded.com.beust - + org.hamcrest org.apache.pulsar.functions.runtime.shaded.org.hamcrest @@ -382,7 +424,11 @@ --> org.asynchttpclient - org.apache.pulsar.shade.org.asynchttpclient + org.apache.pulsar.functions.runtime.shaded.org.asynchttpclient + + + io.airlift + org.apache.pulsar.functions.runtime.shaded.io.airlift rename-netty-native-libs.sh diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 07a5d9b2eac17..1b37233c813d2 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -32,6 +32,11 @@ pulsar-io-canal Pulsar IO :: Canal + + 5.3.19 + 1.1.5 + + ${project.groupId} @@ -73,4 +78,4 @@ - \ No newline at end of file + From 960c8a81c3406405a79f148ae71e3dde1a12d555 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 18 Aug 2022 11:21:14 +0800 Subject: [PATCH 738/823] [fix][broker] Fix schema does not replicate successfully (#17049) But there is a mistake that the returned schema state is incorrect. https://github.com/apache/pulsar/blob/e826d849ceef9d6aef28569ad57950bba90dfff1/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java#L765-L770 Because the replicator used MessageImpl will not have the schema. And this will cause the producer to skip the schema upload. https://github.com/apache/pulsar/blob/e826d849ceef9d6aef28569ad57950bba90dfff1/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java#L2147-L2149 We should remove https://github.com/apache/pulsar/blob/e826d849ceef9d6aef28569ad57950bba90dfff1/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java#L766-L768 To return the correct schema state. And then we should also provide the correct schema hash. If the message is used by the replicator, the schema hash should be based on the replicator schema. Otherwise, it should use based on the schema of the message. - Fixed the incorrect returned schema state - Provide the method for getting schema hash for MessageImpl (cherry picked from commit 7689133adfd930a50c2690ecca1f2068cafa8bcb) --- build/run_unit_group.sh | 2 +- .../pulsar/broker/service/ReplicatorTest.java | 30 +++++++++++-------- .../pulsar/client/impl/MessageImpl.java | 12 ++++++-- .../pulsar/client/impl/ProducerImpl.java | 9 ++---- .../impl/MultiTopicsConsumerImplTest.java | 2 +- .../common/protocol/schema/SchemaHash.java | 7 ++++- 6 files changed, 37 insertions(+), 25 deletions(-) diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 2d99e84eff325..8bb84679fd944 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -117,7 +117,7 @@ function other() { **/ManagedLedgerTest.java, **/TestPulsarKeyValueSchemaHandler.java, **/PrimitiveSchemaTest.java, - BlobStoreManagedLedgerOffloaderTest.java' + BlobStoreManagedLedgerOffloaderTest.java' -DtestReuseFork=false $MVN_TEST_COMMAND -pl managed-ledger -Dinclude='**/ManagedLedgerTest.java, **/OffloadersCacheTest.java' diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 488fbfef0235c..e78539f985d95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -69,6 +69,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -88,7 +89,6 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ReplicatorStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.schema.Schemas; @@ -390,19 +390,24 @@ public void testReplicationWithSchema() throws Exception { final String subName = "my-sub"; @Cleanup - Producer producer1 = client1.newProducer(Schema.AVRO(Schemas.PersonOne.class)) - .topic(topic.toString()) - .create(); - @Cleanup - Producer producer2 = client2.newProducer(Schema.AVRO(Schemas.PersonOne.class)) - .topic(topic.toString()) - .create(); - @Cleanup - Producer producer3 = client3.newProducer(Schema.AVRO(Schemas.PersonOne.class)) + Producer producer = client1.newProducer(Schema.AVRO(Schemas.PersonOne.class)) .topic(topic.toString()) .create(); - List> producers = Lists.newArrayList(producer1, producer2, producer3); + admin1.topics().createSubscription(topic.toString(), subName, MessageId.earliest); + admin2.topics().createSubscription(topic.toString(), subName, MessageId.earliest); + admin3.topics().createSubscription(topic.toString(), subName, MessageId.earliest); + + + for (int i = 0; i < 10; i++) { + producer.send(new Schemas.PersonOne(i)); + } + + Awaitility.await().untilAsserted(() -> { + assertTrue(admin1.topics().getInternalStats(topic.toString()).schemaLedgers.size() > 0); + assertTrue(admin2.topics().getInternalStats(topic.toString()).schemaLedgers.size() > 0); + assertTrue(admin3.topics().getInternalStats(topic.toString()).schemaLedgers.size() > 0); + }); @Cleanup Consumer consumer1 = client1.newConsumer(Schema.AVRO(Schemas.PersonOne.class)) @@ -422,8 +427,7 @@ public void testReplicationWithSchema() throws Exception { .subscriptionName(subName) .subscribe(); - for (int i = 0; i < 3; i++) { - producers.get(i).send(new Schemas.PersonOne(i)); + for (int i = 0; i < 10; i++) { Message msg1 = consumer1.receive(); Message msg2 = consumer2.receive(); Message msg3 = consumer3.receive(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java index fd00475de71f3..acdf73bed3f82 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java @@ -54,6 +54,7 @@ import org.apache.pulsar.common.api.proto.SingleMessageMetadata; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; +import org.apache.pulsar.common.protocol.schema.SchemaHash; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -66,6 +67,8 @@ public class MessageImpl implements Message { private ByteBuf payload; private Schema schema; + + private SchemaHash schemaHash; private SchemaInfo schemaInfoForReplicator; private SchemaState schemaState = SchemaState.None; private Optional encryptionCtx = Optional.empty(); @@ -92,6 +95,7 @@ public static MessageImpl create(MessageMetadata msgMetadata, ByteBuffer msg.payload = Unpooled.wrappedBuffer(payload); msg.properties = null; msg.schema = schema; + msg.schemaHash = SchemaHash.of(schema); msg.uncompressedSize = payload.remaining(); return msg; } @@ -439,9 +443,14 @@ public SchemaInfo getSchemaInfo() { return schema.getSchemaInfo(); } + public SchemaHash getSchemaHash() { + return schemaHash == null ? SchemaHash.of(new byte[0], null) : schemaHash; + } + public void setSchemaInfoForReplicator(SchemaInfo schemaInfo) { if (msgMetadata.hasReplicatedFrom()) { this.schemaInfoForReplicator = schemaInfo; + this.schemaHash = SchemaHash.of(schemaInfo); } else { throw new IllegalArgumentException("Only allowed to set schemaInfoForReplicator for a replicated message."); } @@ -761,9 +770,6 @@ int getUncompressedSize() { } SchemaState getSchemaState() { - if (getSchemaInfo() == null) { - return SchemaState.Ready; - } return schemaState; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 0a0ba549650cb..711f8e38dd3bd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -637,8 +637,7 @@ private boolean populateMessageSchema(MessageImpl msg, SendCallback callback) { completeCallbackAndReleaseSemaphore(msg.getUncompressedSize(), callback, e); return false; } - SchemaHash schemaHash = SchemaHash.of(msg.getSchemaInternal()); - byte[] schemaVersion = schemaCache.get(schemaHash); + byte[] schemaVersion = schemaCache.get(msg.getSchemaHash()); if (schemaVersion != null) { msgMetadataBuilder.setSchemaVersion(schemaVersion); msg.setSchemaState(MessageImpl.SchemaState.Ready); @@ -647,8 +646,7 @@ private boolean populateMessageSchema(MessageImpl msg, SendCallback callback) { } private boolean rePopulateMessageSchema(MessageImpl msg) { - SchemaHash schemaHash = SchemaHash.of(msg.getSchemaInternal()); - byte[] schemaVersion = schemaCache.get(schemaHash); + byte[] schemaVersion = schemaCache.get(msg.getSchemaHash()); if (schemaVersion == null) { return false; } @@ -679,8 +677,7 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call // case, we should not cache the schema version so that the schema version of the message metadata will // be null, instead of an empty array. if (v.length != 0) { - SchemaHash schemaHash = SchemaHash.of(msg.getSchemaInternal()); - schemaCache.putIfAbsent(schemaHash, v); + schemaCache.putIfAbsent(msg.getSchemaHash(), v); msg.getMessageBuilder().setSchemaVersion(v); } msg.setSchemaState(MessageImpl.SchemaState.Ready); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index faa621cae2193..38e668072281f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -124,7 +124,7 @@ public void testGetStats() throws Exception { // // Code under tests is using CompletableFutures. Theses may hang indefinitely if code is broken. // That's why a test timeout is defined. - @Test(timeOut = 5000) + @Test(timeOut = 10000) public void testParallelSubscribeAsync() throws Exception { String topicName = "parallel-subscribe-async-topic"; MultiTopicsConsumerImpl impl = createMultiTopicsConsumer(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java index 40220e6047a3b..8bbc18fbb703c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java @@ -54,7 +54,12 @@ public static SchemaHash of(SchemaData schemaData) { return of(schemaData.getData(), schemaData.getType()); } - private static SchemaHash of(byte[] schemaBytes, SchemaType schemaType) { + public static SchemaHash of(SchemaInfo schemaInfo) { + return of(schemaInfo == null ? new byte[0] : schemaInfo.getSchema(), + schemaInfo == null ? null : schemaInfo.getType()); + } + + public static SchemaHash of(byte[] schemaBytes, SchemaType schemaType) { return new SchemaHash(hashFunction.hashBytes(schemaBytes), schemaType); } From 8d293a2b5fcfe44c78fc40022ab40e11387e2fd4 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 19 Aug 2022 09:17:48 +0800 Subject: [PATCH 739/823] [fix][broker] Fix out of order data replication (#17154) * [fix][broker] Fix out of order data replication The schema replication will break the data replication order while fetching the schema from the local cluster. https://github.com/apache/pulsar/blob/8a6ecd7d4c9399bb7ce5a224ca854e4a71db79b1/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java#L366-L369 The method getSchemaInfo() is an async method that will reverse the order in which messages are written. Added a new state for replicator `fetchSchemaInProgress` which means the replicator had detected a new schema that needed to fetch the schema info from the local cluster. During the schema fetching, the replicator should pause the data replicator and resume after the schema has been fetched. (cherry picked from commit 39c1ee1aebc06c85b7dcb203b8cfa16fe035ae27) --- .../persistent/PersistentReplicator.java | 50 ++++++++++++++---- .../pulsar/broker/service/ReplicatorTest.java | 51 ++++++++++++++----- 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 1db685ccbec34..c10b70df6a87b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -104,6 +104,8 @@ public class PersistentReplicator extends AbstractReplicator private final ReplicatorStatsImpl stats = new ReplicatorStatsImpl(); + private volatile boolean fetchSchemaInProgress = false; + public PersistentReplicator(PersistentTopic topic, ManagedCursor cursor, String localCluster, String remoteCluster, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { @@ -219,6 +221,11 @@ private int getAvailablePermits() { } protected void readMoreEntries() { + if (fetchSchemaInProgress) { + log.info("[{}][{} -> {}] Skip the reading due to new detected schema", + topicName, localCluster, remoteCluster); + return; + } int availablePermits = getAvailablePermits(); if (availablePermits > 0) { @@ -289,8 +296,15 @@ public void readEntriesComplete(List entries, Object ctx) { // This flag is set to true when we skip atleast one local message, // in order to skip remaining local messages. boolean isLocalMessageSkippedOnce = false; + boolean skipRemainingMessages = false; for (int i = 0; i < entries.size(); i++) { Entry entry = entries.get(i); + // Skip the messages since the replicator need to fetch the schema info to replicate the schema to the + // remote cluster. Rewind the cursor first and continue the message read after fetched the schema. + if (skipRemainingMessages) { + entry.release(); + continue; + } int length = entry.getLength(); ByteBuf headersAndPayload = entry.getDataBuffer(); MessageImpl msg; @@ -363,16 +377,34 @@ public void readEntriesComplete(List entries, Object ctx) { headersAndPayload.retain(); - getSchemaInfo(msg).thenAccept(schemaInfo -> { - msg.setSchemaInfoForReplicator(schemaInfo); + CompletableFuture schemaFuture = getSchemaInfo(msg); + if (!schemaFuture.isDone() || schemaFuture.isCompletedExceptionally()) { + entry.release(); + headersAndPayload.release(); + msg.recycle(); + // Mark the replicator is fetching the schema for now and rewind the cursor + // and trigger the next read after complete the schema fetching. + fetchSchemaInProgress = true; + skipRemainingMessages = true; + cursor.cancelPendingReadRequest(); + log.info("[{}][{} -> {}] Pause the data replication due to new detected schema", topicName, + localCluster, remoteCluster); + schemaFuture.whenComplete((__, e) -> { + if (e != null) { + log.warn("[{}][{} -> {}] Failed to get schema from local cluster, will try in the next loop", + topicName, localCluster, remoteCluster, e); + } + log.info("[{}][{} -> {}] Resume the data replication after the schema fetching done", topicName, + localCluster, remoteCluster); + cursor.rewind(); + fetchSchemaInProgress = false; + readMoreEntries(); + }); + } else { + msg.setSchemaInfoForReplicator(schemaFuture.get()); producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg)); - }).exceptionally(ex -> { - log.error("[{}][{} -> {}] Failed to get schema from local cluster", topicName, - localCluster, remoteCluster, ex); - return null; - }); - - atLeastOneMessageSentForReplication = true; + atLeastOneMessageSentForReplication = true; + } } } catch (Exception e) { log.error("[{}][{} -> {}] Unexpected exception: {}", topicName, localCluster, remoteCluster, e.getMessage(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index e78539f985d95..9cb6ec4548797 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -78,6 +78,8 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; @@ -379,8 +381,11 @@ public void testReplication(String namespace) throws Exception { consumer3.receive(1); } - @Test + @Test(invocationCount = 5) public void testReplicationWithSchema() throws Exception { + config1.setBrokerDeduplicationEnabled(true); + config2.setBrokerDeduplicationEnabled(true); + config3.setBrokerDeduplicationEnabled(true); PulsarClient client1 = pulsar1.getClient(); PulsarClient client2 = pulsar2.getClient(); PulsarClient client3 = pulsar3.getClient(); @@ -390,17 +395,29 @@ public void testReplicationWithSchema() throws Exception { final String subName = "my-sub"; @Cleanup - Producer producer = client1.newProducer(Schema.AVRO(Schemas.PersonOne.class)) + Producer producer1 = client1.newProducer(Schema.AVRO(Schemas.PersonOne.class)) + .topic(topic.toString()) + .enableBatching(false) + .create(); + + @Cleanup + Producer producer2 = client1.newProducer(Schema.AVRO(Schemas.PersonThree.class)) .topic(topic.toString()) + .enableBatching(false) .create(); admin1.topics().createSubscription(topic.toString(), subName, MessageId.earliest); admin2.topics().createSubscription(topic.toString(), subName, MessageId.earliest); admin3.topics().createSubscription(topic.toString(), subName, MessageId.earliest); + final int totalMessages = 1000; - for (int i = 0; i < 10; i++) { - producer.send(new Schemas.PersonOne(i)); + for (int i = 0; i < totalMessages / 2; i++) { + producer1.sendAsync(new Schemas.PersonOne(i)); + } + + for (int i = 500; i < totalMessages; i++) { + producer2.sendAsync(new Schemas.PersonThree(i, "name-" + i)); } Awaitility.await().untilAsserted(() -> { @@ -410,29 +427,39 @@ public void testReplicationWithSchema() throws Exception { }); @Cleanup - Consumer consumer1 = client1.newConsumer(Schema.AVRO(Schemas.PersonOne.class)) + Consumer consumer1 = client1.newConsumer(Schema.AUTO_CONSUME()) .topic(topic.toString()) .subscriptionName(subName) .subscribe(); @Cleanup - Consumer consumer2 = client2.newConsumer(Schema.AVRO(Schemas.PersonOne.class)) + Consumer consumer2 = client2.newConsumer(Schema.AUTO_CONSUME()) .topic(topic.toString()) .subscriptionName(subName) .subscribe(); @Cleanup - Consumer consumer3 = client3.newConsumer(Schema.AVRO(Schemas.PersonOne.class)) + Consumer consumer3 = client3.newConsumer(Schema.AUTO_CONSUME()) .topic(topic.toString()) .subscriptionName(subName) .subscribe(); - for (int i = 0; i < 10; i++) { - Message msg1 = consumer1.receive(); - Message msg2 = consumer2.receive(); - Message msg3 = consumer3.receive(); + int lastId = -1; + for (int i = 0; i < totalMessages; i++) { + Message msg1 = consumer1.receive(); + Message msg2 = consumer2.receive(); + Message msg3 = consumer3.receive(); assertTrue(msg1 != null && msg2 != null && msg3 != null); - assertTrue(msg1.getValue().equals(msg2.getValue()) && msg2.getValue().equals(msg3.getValue())); + GenericRecord record1 = msg1.getValue(); + GenericRecord record2 = msg2.getValue(); + GenericRecord record3 = msg3.getValue(); + int id1 = (int) record1.getField("id"); + int id2 = (int) record2.getField("id"); + int id3 = (int) record3.getField("id"); + log.info("Received ids, id1: {}, id2: {}, id3: {}, lastId: {}", id1, id2, id3, lastId); + assertTrue(id1 == id2 && id2 == id3); + assertTrue(id1 > lastId); + lastId = id1; consumer1.acknowledge(msg1); consumer2.acknowledge(msg2); consumer3.acknowledge(msg3); From 22eb42f728aee529be0c71198ce95f6b91a04718 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 8 Sep 2022 13:37:01 +0800 Subject: [PATCH 740/823] [fix][broker] Fix the replicator unnecessary get schema request for bytes schema (#17523) (cherry picked from commit 2ed561436deea9639d177e8e43317c37ea44152d) --- .../pulsar/broker/service/ReplicatorTest.java | 13 ++++++ .../pulsar/client/impl/ProducerImpl.java | 11 ++++- .../pulsar/client/impl/ProducerImplTest.java | 43 +++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 9cb6ec4548797..d9ac04c3fc2dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -464,6 +464,19 @@ public void testReplicationWithSchema() throws Exception { consumer2.acknowledge(msg2); consumer3.acknowledge(msg3); } + + @Cleanup + Producer producerBytes = client1.newProducer() + .topic(topic.toString()) + .enableBatching(false) + .create(); + + byte[] data = "Bytes".getBytes(); + producerBytes.send(data); + + assertEquals(consumer1.receive().getValue().getNativeObject(), data); + assertEquals(consumer2.receive().getValue().getNativeObject(), data); + assertEquals(consumer3.receive().getValue().getNativeObject(), data); } @Test diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 711f8e38dd3bd..9c77d8b4ae6e0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -623,13 +623,22 @@ private void serializeAndSendMessage(MessageImpl msg, ByteBuf payload, } } - private boolean populateMessageSchema(MessageImpl msg, SendCallback callback) { + @VisibleForTesting + boolean populateMessageSchema(MessageImpl msg, SendCallback callback) { MessageMetadata msgMetadataBuilder = msg.getMessageBuilder(); if (msg.getSchemaInternal() == schema) { schemaVersion.ifPresent(v -> msgMetadataBuilder.setSchemaVersion(v)); msg.setSchemaState(MessageImpl.SchemaState.Ready); return true; } + // If the message is from the replicator and without replicated schema + // Which means the message is written with BYTES schema + // So we don't need to replicate schema to the remote cluster + if (msg.hasReplicateFrom() && msg.getSchemaInfoForReplicator() == null) { + msg.setSchemaState(MessageImpl.SchemaState.Ready); + return true; + } + if (!isMultiSchemaEnabled(true)) { PulsarClientException.InvalidMessageException e = new PulsarClientException.InvalidMessageException( format("The producer %s of the topic %s is disabled the `MultiSchema`", producerName, topic) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java new file mode 100644 index 0000000000000..4db3cd0843b92 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import static org.testng.Assert.assertTrue; +import org.apache.pulsar.client.api.Schema; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class ProducerImplTest { + @Test + public void testPopulateMessageSchema() { + MessageImpl msg = mock(MessageImpl.class); + when(msg.hasReplicateFrom()).thenReturn(true); + when(msg.getSchemaInternal()).thenReturn(mock(Schema.class)); + when(msg.getSchemaInfoForReplicator()).thenReturn(null); + ProducerImpl producer = mock(ProducerImpl.class, withSettings() + .defaultAnswer(Mockito.CALLS_REAL_METHODS)); + assertTrue(producer.populateMessageSchema(msg, null)); + verify(msg).setSchemaState(MessageImpl.SchemaState.Ready); + } + +} From 8262f25ddf332afcb5d942c555e317e2cc81150d Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 7 Sep 2022 15:27:32 +0800 Subject: [PATCH 741/823] [improve][broker] Improve cursor.getNumberOfEntries if isUnackedRangesOpenCacheSetEnabled=true (#17465) (cherry picked from commit 09edcceab419ef28a9311ac480c0335c8f9dd87e) --- .../mledger/impl/ManagedCursorImpl.java | 41 +++++++++++-------- .../ConcurrentOpenLongPairRangeSet.java | 24 +++++++++++ .../util/collections/LongPairRangeSet.java | 10 +++++ .../ConcurrentOpenLongPairRangeSetTest.java | 20 +++++++++ 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 5fb4cb356ff09..1b54bb16a3d46 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1369,25 +1369,32 @@ protected long getNumberOfEntries(Range range) { lock.readLock().lock(); try { - individualDeletedMessages.forEach((r) -> { - try { - if (r.isConnected(range)) { - Range commonEntries = r.intersection(range); - long commonCount = ledger.getNumberOfEntries(commonEntries); - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Discounting {} entries for already deleted range {}", ledger.getName(), - name, commonCount, commonEntries); + if (config.isUnackedRangesOpenCacheSetEnabled()) { + int cardinality = individualDeletedMessages.cardinality( + range.lowerEndpoint().ledgerId, range.lowerEndpoint().entryId, + range.upperEndpoint().ledgerId, range.upperEndpoint().entryId); + deletedEntries.addAndGet(cardinality); + } else { + individualDeletedMessages.forEach((r) -> { + try { + if (r.isConnected(range)) { + Range commonEntries = r.intersection(range); + long commonCount = ledger.getNumberOfEntries(commonEntries); + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Discounting {} entries for already deleted range {}", + ledger.getName(), name, commonCount, commonEntries); + } + deletedEntries.addAndGet(commonCount); + } + return true; + } finally { + if (r.lowerEndpoint() instanceof PositionImplRecyclable) { + ((PositionImplRecyclable) r.lowerEndpoint()).recycle(); + ((PositionImplRecyclable) r.upperEndpoint()).recycle(); } - deletedEntries.addAndGet(commonCount); - } - return true; - } finally { - if (r.lowerEndpoint() instanceof PositionImplRecyclable) { - ((PositionImplRecyclable) r.lowerEndpoint()).recycle(); - ((PositionImplRecyclable) r.upperEndpoint()).recycle(); } - } - }, recyclePositionRangeConverter); + }, recyclePositionRangeConverter); + } } finally { lock.readLock().unlock(); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 7ae6c7e96625a..08a63141f4cca 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.lang3.mutable.MutableInt; /** * A Concurrent set comprising zero or more ranges of type {@link LongPair}. This can be alternative of @@ -242,6 +243,29 @@ public Range lastRange() { return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); } + @Override + public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + MutableInt v = new MutableInt(0); + subMap.forEach((key, bitset) -> { + if (key == lowerKey || key == upperKey) { + BitSet temp = (BitSet) bitset.clone(); + // Trim the bitset index which < lowerValue + if (key == lowerKey) { + temp.clear(0, (int) Math.max(0, lowerValue)); + } + // Trim the bitset index which > upperValue + if (key == upperKey) { + temp.clear((int) Math.min(upperValue + 1, temp.length()), temp.length()); + } + v.add(temp.cardinality()); + } else { + v.add(bitset.cardinality()); + } + }); + return v.intValue(); + } + @Override public int size() { if (updatedAfterCachedForSize) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java index 15f32252b90a1..c41a1a1f13fff 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java @@ -127,6 +127,11 @@ public interface LongPairRangeSet> { */ Range lastRange(); + /** + * Return the number bit sets to true from lower (inclusive) to upper (inclusive). + */ + int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue); + /** * Represents a function that accepts two long arguments and produces a result. * @@ -290,6 +295,11 @@ public Range lastRange() { return list.get(list.size() - 1); } + @Override + public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { + throw new UnsupportedOperationException(); + } + @Override public int size() { return set.asRanges().size(); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java index 90c23ae0dd01f..ba29ee0d27f99 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java @@ -460,4 +460,24 @@ private List> getConnectedRange(Set> gRanges) { gRangeConnected.add(lastRange); return gRangeConnected; } + + @Test + public void testCardinality() { + ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + int v = set.cardinality(0, 0, Integer.MAX_VALUE, Integer.MAX_VALUE); + assertEquals(v, 0 ); + set.addOpenClosed(1, 0, 1, 20); + set.addOpenClosed(1, 30, 1, 90); + set.addOpenClosed(2, 0, 3, 30); + v = set.cardinality(1, 0, 1, 100); + assertEquals(v, 80); + v = set.cardinality(1, 11, 1, 100); + assertEquals(v, 70); + v = set.cardinality(1, 0, 1, 90); + assertEquals(v, 80); + v = set.cardinality(1, 0, 1, 80); + assertEquals(v, 70); + v = set.cardinality(1, 0, 3, 30); + assertEquals(v, 80 + 31); + } } From f1be5c32c3cf6da90fd3e885cc4fbb7109eff6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 9 Sep 2022 18:13:51 +0200 Subject: [PATCH 742/823] [ci] Remove post-commit trigger for old release branches (#17570) (cherry picked from commit 0787755df47f54e579227de0de4fba6608f37bcd) --- .github/workflows/ci-build-macos.yaml | 3 --- .github/workflows/ci-cpp-build-centos7.yaml | 7 +------ .github/workflows/ci-cpp-build-windows.yaml | 6 ------ .github/workflows/ci-cpp.yaml | 3 --- .github/workflows/ci-go-functions-style.yaml | 6 ------ .github/workflows/ci-go-functions-test.yaml | 6 ------ .../workflows/ci-integration-backwards-compatibility.yaml | 3 --- .github/workflows/ci-integration-cli.yaml | 3 --- .github/workflows/ci-integration-function.yaml | 3 --- .github/workflows/ci-integration-messaging.yaml | 3 --- .github/workflows/ci-integration-process.yaml | 3 --- .github/workflows/ci-integration-pulsar-io-ora.yaml | 3 --- .github/workflows/ci-integration-pulsar-io.yaml | 3 --- .github/workflows/ci-integration-schema.yaml | 3 --- .github/workflows/ci-integration-sql.yaml | 3 --- .github/workflows/ci-integration-standalone.yaml | 3 --- .github/workflows/ci-integration-thread.yaml | 3 --- .github/workflows/ci-integration-tiered-filesystem.yaml | 3 --- .github/workflows/ci-integration-tiered-jcloud.yaml | 3 --- .github/workflows/ci-integration-transaction.yaml | 3 --- .github/workflows/ci-license.yaml | 3 --- .github/workflows/ci-python-build-3.9-client.yaml | 7 +------ .github/workflows/ci-shade-test.yaml | 3 --- .github/workflows/ci-unit-broker-broker-gp1.yaml | 3 --- .github/workflows/ci-unit-broker-broker-gp2.yaml | 3 --- .github/workflows/ci-unit-broker-client-api.yaml | 3 --- .github/workflows/ci-unit-broker-client-impl.yaml | 3 --- .github/workflows/ci-unit-broker-jdk8.yaml | 3 --- .github/workflows/ci-unit-broker-other.yaml | 3 --- .github/workflows/ci-unit-proxy.yaml | 3 --- .github/workflows/ci-unit.yaml | 3 --- 31 files changed, 2 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml index 30963293d3615..4f4adc4d20604 100644 --- a/.github/workflows/ci-build-macos.yaml +++ b/.github/workflows/ci-build-macos.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-cpp-build-centos7.yaml b/.github/workflows/ci-cpp-build-centos7.yaml index bc480d48162f1..ca5f5632cf29a 100644 --- a/.github/workflows/ci-cpp-build-centos7.yaml +++ b/.github/workflows/ci-cpp-build-centos7.yaml @@ -26,12 +26,7 @@ on: paths: - '.github/workflows/**' - 'pulsar-client-cpp/**' - push: - branches: - - branch-* - paths: - - '.github/workflows/**' - - 'pulsar-client-cpp/**' + jobs: cpp-build-centos7: diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml index 85883ab4c9142..963a717111ba2 100644 --- a/.github/workflows/ci-cpp-build-windows.yaml +++ b/.github/workflows/ci-cpp-build-windows.yaml @@ -26,12 +26,6 @@ on: paths: - '.github/workflows/**' - 'pulsar-client-cpp/**' - push: - branches: - - branch-* - paths: - - '.github/workflows/**' - - 'pulsar-client-cpp/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml index 48a6a3c5115ee..1f30edc42751b 100644 --- a/.github/workflows/ci-cpp.yaml +++ b/.github/workflows/ci-cpp.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions-style.yaml index 2e1aced1fb25e..c4702a0a3fb7c 100644 --- a/.github/workflows/ci-go-functions-style.yaml +++ b/.github/workflows/ci-go-functions-style.yaml @@ -26,12 +26,6 @@ on: paths: - '.github/workflows/**' - 'pulsar-function-go/**' - push: - branches: - - branch-* - paths: - - '.github/workflows/**' - - 'pulsar-function-go/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml index e7cbfc83745e9..1a95ad3750072 100644 --- a/.github/workflows/ci-go-functions-test.yaml +++ b/.github/workflows/ci-go-functions-test.yaml @@ -26,12 +26,6 @@ on: paths: - '.github/workflows/**' - 'pulsar-function-go/**' - push: - branches: - - branch-* - paths: - - '.github/workflows/**' - - 'pulsar-function-go/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml index d6babedb54b3f..f5827d6a3f872 100644 --- a/.github/workflows/ci-integration-backwards-compatibility.yaml +++ b/.github/workflows/ci-integration-backwards-compatibility.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml index 65590dcd6029c..df0698afbe177 100644 --- a/.github/workflows/ci-integration-cli.yaml +++ b/.github/workflows/ci-integration-cli.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-function.yaml b/.github/workflows/ci-integration-function.yaml index 46c38215b0862..fec9941ab39e6 100644 --- a/.github/workflows/ci-integration-function.yaml +++ b/.github/workflows/ci-integration-function.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml index ca1bc3a3adccc..dd099ca5af3e0 100644 --- a/.github/workflows/ci-integration-messaging.yaml +++ b/.github/workflows/ci-integration-messaging.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml index 43567f523004f..1e652d64c5da5 100644 --- a/.github/workflows/ci-integration-process.yaml +++ b/.github/workflows/ci-integration-process.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml index 5fa3bb7150598..bad5aa40d3d57 100644 --- a/.github/workflows/ci-integration-pulsar-io-ora.yaml +++ b/.github/workflows/ci-integration-pulsar-io-ora.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-pulsar-io.yaml b/.github/workflows/ci-integration-pulsar-io.yaml index d7ec7505ccbc1..8ba7d7f5450be 100644 --- a/.github/workflows/ci-integration-pulsar-io.yaml +++ b/.github/workflows/ci-integration-pulsar-io.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml index 8f207bff11315..fdc38383c59d5 100644 --- a/.github/workflows/ci-integration-schema.yaml +++ b/.github/workflows/ci-integration-schema.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml index 8c996b33824ae..6b5122124f929 100644 --- a/.github/workflows/ci-integration-sql.yaml +++ b/.github/workflows/ci-integration-sql.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml index 8902029d21c1d..46ffe3a82fea5 100644 --- a/.github/workflows/ci-integration-standalone.yaml +++ b/.github/workflows/ci-integration-standalone.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml index 5f3ac6b99719a..7e758e7556ba3 100644 --- a/.github/workflows/ci-integration-thread.yaml +++ b/.github/workflows/ci-integration-thread.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml index 03d3a3bec8460..b89f0d81957a6 100644 --- a/.github/workflows/ci-integration-tiered-filesystem.yaml +++ b/.github/workflows/ci-integration-tiered-filesystem.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml index 12ea0b942ec06..19ef68bdcb643 100644 --- a/.github/workflows/ci-integration-tiered-jcloud.yaml +++ b/.github/workflows/ci-integration-tiered-jcloud.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml index 401a1a4e1596d..e020aec7483d1 100644 --- a/.github/workflows/ci-integration-transaction.yaml +++ b/.github/workflows/ci-integration-transaction.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml index 6969054e6344f..3c9756748804c 100644 --- a/.github/workflows/ci-license.yaml +++ b/.github/workflows/ci-license.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-python-build-3.9-client.yaml b/.github/workflows/ci-python-build-3.9-client.yaml index 1b20a327d9e8f..aed5d3568fbd1 100644 --- a/.github/workflows/ci-python-build-3.9-client.yaml +++ b/.github/workflows/ci-python-build-3.9-client.yaml @@ -26,12 +26,7 @@ on: paths: - '.github/workflows/**' - 'pulsar-client-cpp/**' - push: - branches: - - branch-* - paths: - - '.github/workflows/**' - - 'pulsar-client-cpp/**' + jobs: build-wheel: diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml index 36f0e9b42f421..cb66e1638c03f 100644 --- a/.github/workflows/ci-shade-test.yaml +++ b/.github/workflows/ci-shade-test.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml index 44b2180a48b5c..c752c2900997f 100644 --- a/.github/workflows/ci-unit-broker-broker-gp1.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp1.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-broker-broker-gp2.yaml b/.github/workflows/ci-unit-broker-broker-gp2.yaml index 1f85dc3564623..b4b8eabebbc43 100644 --- a/.github/workflows/ci-unit-broker-broker-gp2.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp2.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml index b26dab939757f..319a43b97b985 100644 --- a/.github/workflows/ci-unit-broker-client-api.yaml +++ b/.github/workflows/ci-unit-broker-client-api.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml index 072155a32fdb7..a5ec3b90be1ed 100644 --- a/.github/workflows/ci-unit-broker-client-impl.yaml +++ b/.github/workflows/ci-unit-broker-client-impl.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-broker-jdk8.yaml b/.github/workflows/ci-unit-broker-jdk8.yaml index fef2e1c5e5263..aa31b609ef7d3 100644 --- a/.github/workflows/ci-unit-broker-jdk8.yaml +++ b/.github/workflows/ci-unit-broker-jdk8.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml index fd6eacc975b85..c7f99a7352241 100644 --- a/.github/workflows/ci-unit-broker-other.yaml +++ b/.github/workflows/ci-unit-broker-other.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml index ebbbe4b5749ba..4e32b89d246da 100644 --- a/.github/workflows/ci-unit-proxy.yaml +++ b/.github/workflows/ci-unit-proxy.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml index 05b8140677c7b..01162bdcae3f0 100644 --- a/.github/workflows/ci-unit.yaml +++ b/.github/workflows/ci-unit.yaml @@ -23,9 +23,6 @@ on: branches: - master - branch-* - push: - branches: - - branch-* concurrency: group: ${{ github.workflow }}-${{ github.ref }} From dea4a328fde4da49a3a46cb2402d5b10b6ba7922 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Sat, 10 Sep 2022 21:10:21 +0800 Subject: [PATCH 743/823] [fix][broker] Multiple consumer dispatcher stuck when `unackedMessages` greater than `maxUnackedMessages` (#17483) (cherry picked from commit d7943a5702a48ae1fad6afc72d230f130b5b8150) --- ...PersistentDispatcherMultipleConsumers.java | 4 +- ...tStickyKeyDispatcherMultipleConsumers.java | 6 +- .../impl/KeySharedSubscriptionTest.java | 189 ++++++++++++++++++ 3 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index b5375d664924f..0b8e7e18338ca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -527,7 +527,9 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { // round-robin dispatch batch size for this consumer int availablePermits = c.isWritable() ? c.getAvailablePermits() : 1; if (c.getMaxUnackedMessages() > 0) { - availablePermits = Math.min(availablePermits, c.getMaxUnackedMessages() - c.getUnackedMessages()); + // Avoid negative number + int remainUnAckedMessages = Math.max(c.getMaxUnackedMessages() - c.getUnackedMessages(), 0); + availablePermits = Math.min(availablePermits, remainUnAckedMessages); } if (log.isDebugEnabled() && !c.isWritable()) { log.debug("[{}-{}] consumer is not writable. dispatching only 1 message to {}; " diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 886be363c4120..73eb031d60c87 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -230,8 +230,10 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { int entriesWithSameKeyCount = entriesWithSameKey.size(); int availablePermits = consumer == null ? 0 : Math.max(consumer.getAvailablePermits(), 0); if (consumer != null && consumer.getMaxUnackedMessages() > 0) { - availablePermits = Math.min(availablePermits, - consumer.getMaxUnackedMessages() - consumer.getUnackedMessages()); + int remainUnAckedMessages = + // Avoid negative number + Math.max(consumer.getMaxUnackedMessages() - consumer.getUnackedMessages(), 0); + availablePermits = Math.min(availablePermits, remainUnAckedMessages); } int maxMessagesForC = Math.min(entriesWithSameKeyCount, availablePermits); int messagesForC = getRestrictedMaxEntriesForConsumer(consumer, entriesWithSameKey, maxMessagesForC, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java new file mode 100644 index 0000000000000..36d3bf6c44076 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import lombok.Cleanup; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +@Test(groups = "broker-impl") +public class KeySharedSubscriptionTest extends ProducerConsumerBase { + + @Override + @BeforeMethod + protected void setup() throws Exception { + conf.setMaxUnackedMessagesPerConsumer(10); + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider + public Object[][] subType() { + return new Object[][] { { SubscriptionType.Shared }, { SubscriptionType.Key_Shared } }; + } + + @Test(dataProvider = "subType") + public void testCanRecoverConsumptionWhenLiftMaxUnAckedMessagesRestriction(SubscriptionType subscriptionType) + throws PulsarClientException { + PulsarClient pulsarClient = PulsarClient.builder(). + serviceUrl(lookupUrl.toString()) + .build(); + final int totalMsg = 1000; + String topic = "broker-close-test-" + RandomStringUtils.randomAlphabetic(5); + Map, List> nameToId = Maps.newConcurrentMap(); + Set pubMessages = Sets.newConcurrentHashSet(); + Set recMessages = Sets.newConcurrentHashSet(); + AtomicLong lastActiveTime = new AtomicLong(); + AtomicBoolean canAcknowledgement = new AtomicBoolean(false); + + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub-1") + .subscriptionType(subscriptionType) + .consumerName("con-1") + .messageListener((cons1, msg) -> { + lastActiveTime.set(System.currentTimeMillis()); + nameToId.computeIfAbsent(cons1,(k) -> new ArrayList<>()) + .add(msg.getMessageId()); + recMessages.add(msg.getMessageId()); + if (canAcknowledgement.get()) { + try { + cons1.acknowledge(msg); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }) + .subscribe(); + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub-1") + .subscriptionType(subscriptionType) + .messageListener((cons2, msg) -> { + lastActiveTime.set(System.currentTimeMillis()); + nameToId.computeIfAbsent(cons2,(k) -> new ArrayList<>()) + .add(msg.getMessageId()); + recMessages.add(msg.getMessageId()); + if (canAcknowledgement.get()) { + try { + cons2.acknowledge(msg); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }) + .consumerName("con-2") + .subscribe(); + @Cleanup + Consumer consumer3 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub-1") + .subscriptionType(subscriptionType) + .messageListener((cons3, msg) -> { + lastActiveTime.set(System.currentTimeMillis()); + nameToId.computeIfAbsent(cons3,(k) -> new ArrayList<>()) + .add(msg.getMessageId()); + recMessages.add(msg.getMessageId()); + if (canAcknowledgement.get()) { + try { + cons3.acknowledge(msg); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }) + .consumerName("con-3") + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.MILLISECONDS) + // We chose 9 because the maximum unacked message is 10 + .batchingMaxMessages(9) + .create(); + + for (int i = 0; i < totalMsg; i++) { + producer.sendAsync(UUID.randomUUID().toString() + .getBytes(StandardCharsets.UTF_8)) + .thenAccept(pubMessages::add); + } + + // Wait for all consumers can not read more messages. the consumers are stuck by max unacked messages. + Awaitility.await() + .pollDelay(5, TimeUnit.SECONDS) + .until(() -> + (System.currentTimeMillis() - lastActiveTime.get()) > TimeUnit.SECONDS.toMillis(5)); + + // All consumers can acknowledge messages as they continue to receive messages. + canAcknowledgement.set(true); + + // Acknowledgment of currently received messages to get out of stuck state due to unack message + for (Map.Entry, List> entry : nameToId.entrySet()) { + Consumer consumer = entry.getKey(); + consumer.acknowledge(entry.getValue()); + } + // refresh active time + lastActiveTime.set(System.currentTimeMillis()); + + // Wait for all consumers to continue receiving messages. + Awaitility.await() + .pollDelay(5, TimeUnit.SECONDS) + .until(() -> + (System.currentTimeMillis() - lastActiveTime.get()) > TimeUnit.SECONDS.toMillis(5)); + + //Determine if all messages have been received. + //If the dispatcher is stuck, we can not receive enough messages. + Assert.assertEquals(pubMessages.size(), totalMsg); + Assert.assertEquals(pubMessages.size(), recMessages.size()); + Assert.assertTrue(recMessages.containsAll(pubMessages)); + } +} From 3e632fe5e03d2a3c756ecf6c1ba7b5afa0ec16c6 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 10 Sep 2022 13:48:08 +0800 Subject: [PATCH 744/823] Move into future stage to catch the exception (#17556) (cherry picked from commit 12a6cc46bcde943bfb08fa83a9822b81f20508d9) --- .../SystemTopicBasedTopicPoliciesService.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 1c018141150ff..a5bcaa5728dce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -359,23 +359,26 @@ private void cleanCacheAndCloseReader(NamespaceName namespace, boolean cleanOwne } private void readMorePolicies(SystemTopicClient.Reader reader) { - reader.readNextAsync().whenComplete((msg, ex) -> { - if (ex == null) { - refreshTopicPoliciesCache(msg); - notifyListener(msg); - readMorePolicies(reader); - } else { - Throwable cause = FutureUtil.unwrapCompletionException(ex); - if (cause instanceof PulsarClientException.AlreadyClosedException) { - log.error("Read more topic policies exception, close the read now!", ex); - cleanCacheAndCloseReader( - reader.getSystemTopic().getTopicName().getNamespaceObject(), false); - } else { - log.warn("Read more topic polices exception, read again.", ex); - readMorePolicies(reader); - } - } - }); + reader.readNextAsync() + .thenAccept(msg -> { + refreshTopicPoliciesCache(msg); + notifyListener(msg); + }) + .whenComplete((__, ex) -> { + if (ex == null) { + readMorePolicies(reader); + } else { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof PulsarClientException.AlreadyClosedException) { + log.error("Read more topic policies exception, close the read now!", ex); + cleanCacheAndCloseReader( + reader.getSystemTopic().getTopicName().getNamespaceObject(), false); + } else { + log.warn("Read more topic polices exception, read again.", ex); + readMorePolicies(reader); + } + } + }); } private void refreshTopicPoliciesCache(Message msg) { From db8480b66f0634193e85de01358a1e34648d5595 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 13 Sep 2022 09:06:25 +0800 Subject: [PATCH 745/823] [fix][broker] Topic policy reader can't recover when get any exception (#17562) (cherry picked from commit 5aa1f1101c6fef61163b34558cc98bf362dfa969) --- .../SystemTopicBasedTopicPoliciesService.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index a5bcaa5728dce..38ec5d803c660 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nonnull; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; @@ -230,21 +231,20 @@ public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle name return result; } - private void prepareInitPoliciesCache(NamespaceName namespace, CompletableFuture result) { + private void prepareInitPoliciesCache(@Nonnull NamespaceName namespace, CompletableFuture result) { if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { CompletableFuture> readerCompletableFuture = createSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); - readerCompletableFuture.whenComplete((reader, ex) -> { - if (ex != null) { - log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); - result.completeExceptionally(ex); - cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); - } else { - initPolicesCache(reader, result); - result.thenRun(() -> readMorePolicies(reader)); - } + readerCompletableFuture.thenAccept(reader -> { + initPolicesCache(reader, result); + result.thenRun(() -> readMorePolicies(reader)); + }).exceptionally(ex -> { + log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); + cleanCacheAndCloseReader(namespace, false); + result.completeExceptionally(ex); + return null; }); } } @@ -346,14 +346,18 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp }); } - private void cleanCacheAndCloseReader(NamespaceName namespace, boolean cleanOwnedBundlesCount) { + private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount) { CompletableFuture> readerFuture = readerCaches.remove(namespace); policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); if (cleanOwnedBundlesCount) { ownedBundlesCountPerNamespace.remove(namespace); } if (readerFuture != null && !readerFuture.isCompletedExceptionally()) { - readerFuture.thenAccept(SystemTopicClient.Reader::closeAsync); + readerFuture.thenCompose(SystemTopicClient.Reader::closeAsync) + .exceptionally(ex -> { + log.warn("[{}] Close change_event reader fail.", namespace, ex); + return null; + }); } policyCacheInitMap.remove(namespace); } From 4d628a9638cedb4ec018ba68791d33bad77c271e Mon Sep 17 00:00:00 2001 From: Yuri Mizushima Date: Tue, 13 Sep 2022 11:42:38 +0900 Subject: [PATCH 746/823] [branch-2.9] Rename test file name from `*Test2.java` to `*Test.java` to run all tests correctly (#17048) --- pom.xml | 2 +- .../broker/admin/impl/NamespacesBase.java | 34 ++++++-- ...{AdminApiTest2.java => AdminApi2Test.java} | 82 +++---------------- ...minApiTest2.java => V1_AdminApi2Test.java} | 2 +- 4 files changed, 41 insertions(+), 79 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/{AdminApiTest2.java => AdminApi2Test.java} (96%) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/{V1_AdminApiTest2.java => V1_AdminApi2Test.java} (99%) diff --git a/pom.xml b/pom.xml index 80ae55d147d92..5d5bcb966a4a9 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,7 @@ flexible messaging model and an intuitive client API. 8 8 - + **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 56544a56106f2..a91499ed81b7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -465,7 +465,8 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo } // remove from owned namespace map and ephemeral node from ZK - final List> futures = Lists.newArrayList(); + final List> topicFutures = Lists.newArrayList(); + final List> bundleFutures = Lists.newArrayList(); try { // firstly remove all topics including system topics if (!topics.isEmpty()) { @@ -479,12 +480,12 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo String partitionedTopic = topicName.getPartitionedTopicName(); if (!partitionedTopics.contains(partitionedTopic)) { // Distinguish partitioned topic to avoid duplicate deletion of the same schema - futures.add(pulsar().getAdminClient().topics().deletePartitionedTopicAsync( + topicFutures.add(pulsar().getAdminClient().topics().deletePartitionedTopicAsync( partitionedTopic, true, true)); partitionedTopics.add(partitionedTopic); } } else { - futures.add(pulsar().getAdminClient().topics().deleteAsync( + topicFutures.add(pulsar().getAdminClient().topics().deleteAsync( topic, true, true)); nonPartitionedTopics.add(topic); } @@ -505,14 +506,35 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo + "and non-partitioned-topics:{} in namespace:{}.", partitionedTopics, nonPartitionedTopics, namespaceName); } + + final CompletableFuture topicFutureEx = + FutureUtil.waitForAll(topicFutures).handle((result, exception) -> { + if (exception != null) { + if (exception.getCause() instanceof PulsarAdminException) { + asyncResponse + .resume(new RestException((PulsarAdminException) exception.getCause())); + } else { + log.error("[{}] Failed to remove forcefully owned namespace {}", + clientAppId(), namespaceName, exception); + asyncResponse.resume(new RestException(exception.getCause())); + } + return exception; + } + + return null; + }); + if (topicFutureEx.join() != null) { + return; + } } + // forcefully delete namespace bundles NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() .getBundles(namespaceName); for (NamespaceBundle bundle : bundles.getBundles()) { // check if the bundle is owned by any broker, if not then we do not need to delete the bundle if (pulsar().getNamespaceService().getOwner(bundle).isPresent()) { - futures.add(pulsar().getAdminClient().namespaces() + bundleFutures.add(pulsar().getAdminClient().namespaces() .deleteNamespaceBundleAsync(namespaceName.toString(), bundle.getBundleRange(), true)); } } @@ -522,7 +544,7 @@ protected void internalDeleteNamespaceForcefully(AsyncResponse asyncResponse, bo return; } - FutureUtil.waitForAll(futures).thenCompose(__ -> internalClearZkSources()).handle((result, exception) -> { + FutureUtil.waitForAll(bundleFutures).thenCompose(__ -> internalClearZkSources()).handle((result, exception) -> { if (exception != null) { Throwable cause = FutureUtil.unwrapCompletionException(exception); if (cause instanceof PulsarAdminException.ConflictException) { @@ -1910,7 +1932,7 @@ protected List internalGetAntiAffinityNamespaces(String cluster, String return namespaces.stream().filter(ns -> { Optional policies; try { - policies = getLocalPolicies().getLocalPolicies(namespaceName); + policies = getLocalPolicies().getLocalPolicies(NamespaceName.get(ns)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java similarity index 96% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index baafbc1ba4253..e0fac522f27db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -36,7 +36,6 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -107,7 +106,7 @@ @Slf4j @Test(groups = "broker") -public class AdminApiTest2 extends MockedPulsarServiceBaseTest { +public class AdminApi2Test extends MockedPulsarServiceBaseTest { private MockedPulsarService mockPulsarSetup; @@ -1027,9 +1026,11 @@ public void brokerNamespaceIsolationPoliciesUpdateOnTime() throws Exception { parameters1.put("min_limit", "1"); parameters1.put("usage_threshold", "100"); + final List primaryList = new ArrayList<>(); + primaryList.add(brokerName + ".*"); NamespaceIsolationData nsPolicyData1 = NamespaceIsolationData.builder() .namespaces(Collections.singletonList(ns1Name)) - .primary(Collections.singletonList(brokerName + ".*")) + .primary(primaryList) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) .parameters(parameters1) @@ -1252,7 +1253,7 @@ public void testPreciseBacklog() throws PulsarClientException, PulsarAdminExcept assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); topicStats = admin.topics().getStats(topic, true, true); - assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 43); + assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 40); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 1); consumer.acknowledge(message); @@ -1500,7 +1501,7 @@ public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException topicStats = admin.topics().getPartitionedStats(topic, false, true, true); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 1); - assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 43); + assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 40); } @Test(timeOut = 30000) @@ -1539,7 +1540,7 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti TopicStats topicStats = admin.topics().getPartitionedStats(topic, false, true, true); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); - assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 470); + assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 440); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklogNoDelayed(), 5); for (int i = 0; i < 5; i++) { @@ -1549,7 +1550,7 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti Awaitility.await().untilAsserted(() -> { TopicStats topicStats2 = admin.topics().getPartitionedStats(topic, false, true, true); assertEquals(topicStats2.getSubscriptions().get(subName).getMsgBacklog(), 5); - assertEquals(topicStats2.getSubscriptions().get(subName).getBacklogSize(), 238); + assertEquals(topicStats2.getSubscriptions().get(subName).getBacklogSize(), 223); assertEquals(topicStats2.getSubscriptions().get(subName).getMsgBacklogNoDelayed(), 0); }); @@ -1615,60 +1616,6 @@ public void testForceDeleteNamespace() throws Exception { } } - @Test - public void testDistinguishTopicTypeWhenForceDeleteNamespace() throws Exception { - conf.setForceDeleteNamespaceAllowed(true); - final String ns = "prop-xyz/distinguish-topic-type-ns"; - final String exNs = "prop-xyz/ex-distinguish-topic-type-ns"; - admin.namespaces().createNamespace(ns, 2); - admin.namespaces().createNamespace(exNs, 2); - - final String p1 = "persistent://" + ns + "/p1"; - final String p5 = "persistent://" + ns + "/p5"; - final String np = "persistent://" + ns + "/np"; - - admin.topics().createPartitionedTopic(p1, 1); - admin.topics().createPartitionedTopic(p5, 5); - admin.topics().createNonPartitionedTopic(np); - - final String exNp = "persistent://" + exNs + "/np"; - admin.topics().createNonPartitionedTopic(exNp); - // insert an invalid topic name - pulsar.getLocalMetadataStore().put( - "/managed-ledgers/" + exNs + "/persistent/", "".getBytes(), Optional.empty()).join(); - - List topics = pulsar.getNamespaceService().getFullListOfTopics(NamespaceName.get(ns)).get(); - List exTopics = pulsar.getNamespaceService().getFullListOfTopics(NamespaceName.get(exNs)).get(); - - // ensure that the topic list contains all the topics - List allTopics = new ArrayList<>(Arrays.asList(np, TopicName.get(p1).getPartition(0).toString())); - for (int i = 0; i < 5; i++) { - allTopics.add(TopicName.get(p5).getPartition(i).toString()); - } - Assert.assertEquals(allTopics.stream().filter(t -> !topics.contains(t)).count(), 0); - Assert.assertTrue(exTopics.contains("persistent://" + exNs + "/")); - // partition num = p1 + p5 + np - Assert.assertEquals(topics.size(), 1 + 5 + 1); - Assert.assertEquals(exTopics.size(), 1 + 1); - - admin.namespaces().deleteNamespace(ns, true); - Arrays.asList(p1, p5, np).forEach(t -> { - try { - admin.schemas().getSchemaInfo(t); - } catch (PulsarAdminException e) { - // all the normal topics' schemas have been deleted - Assert.assertEquals(e.getStatusCode(), 404); - } - }); - - try { - admin.namespaces().deleteNamespace(exNs, true); - fail("Should fail due to invalid topic"); - } catch (Exception e) { - //ok - } - } - @Test public void testUpdateClusterWithProxyUrl() throws Exception { ClusterData cluster = ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build(); @@ -1760,11 +1707,7 @@ public void testMaxTopicsPerNamespace() throws Exception { for (int i = 0; i < 5; ++i) { admin.topics().createPartitionedTopic(topic + i, 1); } - admin.topics().createPartitionedTopic("persistent://testTenant/ns1/__change_events", 2); - admin.topics().createPartitionedTopic("persistent://testTenant/ns1/__transaction_buffer_snapshot", 2); - admin.topics().createPartitionedTopic( - "persistent://testTenant/ns1/__transaction_buffer_snapshot-multiTopicsReader" - + "-05c0ded5e9__transaction_pending_ack", 2); + admin.topics().createPartitionedTopic("persistent://testTenant/ns1/__change_events", 6); // check first create system topics, then normal topic, unlimited even setMaxTopicsPerNamespace @@ -1774,11 +1717,7 @@ public void testMaxTopicsPerNamespace() throws Exception { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); admin.tenants().createTenant("testTenant", tenantInfo); admin.namespaces().createNamespace("testTenant/ns1", Sets.newHashSet("test")); - admin.topics().createPartitionedTopic("persistent://testTenant/ns1/__change_events", 2); - admin.topics().createPartitionedTopic("persistent://testTenant/ns1/__transaction_buffer_snapshot", 2); - admin.topics().createPartitionedTopic( - "persistent://testTenant/ns1/__transaction_buffer_snapshot-multiTopicsReader" - + "-05c0ded5e9__transaction_pending_ack", 2); + admin.topics().createPartitionedTopic("persistent://testTenant/ns1/__change_events", 6); for (int i = 0; i < 5; ++i) { admin.topics().createPartitionedTopic(topic + i, 1); } @@ -1949,6 +1888,7 @@ public void testMaxSubPerTopicApi() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopic() throws Exception { + pulsar.getConfiguration().setMaxSubscriptionsPerTopic(0); final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java similarity index 99% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest2.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java index 4f9c8603a9e9a..2eb37c222d614 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest2.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java @@ -73,7 +73,7 @@ import org.testng.annotations.Test; @Test(groups = "broker") -public class V1_AdminApiTest2 extends MockedPulsarServiceBaseTest { +public class V1_AdminApi2Test extends MockedPulsarServiceBaseTest { private MockedPulsarService mockPulsarSetup; From b5ef559d0a79e208b8ede11f4a4372b6d629b08a Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Wed, 31 Aug 2022 10:15:02 +0800 Subject: [PATCH 747/823] [refactor][client c++] Delete PartitionedConsumerImpl, use MultiTopicsConsumerImpl instead (#16969) ### Motivation The code of the C++ client is still relatively old, after #1365, the java client used `MultiTopicConsumerImpl` instead of `PartitionedConsumerImpl`. ### Modifications - Delete PartitionedConsumerImpl, use MultiTopicsConsumerImpl instead. - For MultiTopicConsumerImpl, support seek message and topic partition listener feature. (cherry picked from commit 3a3ae23427bc9c8642b3ec9a081ce01000339875) --- pulsar-client-cpp/.gitignore | 1 + pulsar-client-cpp/include/pulsar/Consumer.h | 1 - pulsar-client-cpp/include/pulsar/Message.h | 1 - pulsar-client-cpp/include/pulsar/MessageId.h | 1 - pulsar-client-cpp/lib/ClientImpl.cc | 6 +- pulsar-client-cpp/lib/ConsumerImpl.h | 1 - .../lib/MultiTopicsConsumerImpl.cc | 151 ++++- .../lib/MultiTopicsConsumerImpl.h | 27 +- .../lib/PartitionedBrokerConsumerStatsImpl.cc | 163 ----- .../lib/PartitionedBrokerConsumerStatsImpl.h | 90 --- .../lib/PartitionedConsumerImpl.cc | 610 ------------------ .../lib/PartitionedConsumerImpl.h | 137 ---- pulsar-client-cpp/tests/BasicEndToEndTest.cc | 2 +- pulsar-client-cpp/tests/ConsumerStatsTest.cc | 7 +- pulsar-client-cpp/tests/ConsumerTest.cc | 10 +- pulsar-client-cpp/tests/PulsarFriend.h | 5 - 16 files changed, 164 insertions(+), 1049 deletions(-) delete mode 100644 pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.cc delete mode 100644 pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.h delete mode 100644 pulsar-client-cpp/lib/PartitionedConsumerImpl.cc delete mode 100644 pulsar-client-cpp/lib/PartitionedConsumerImpl.h diff --git a/pulsar-client-cpp/.gitignore b/pulsar-client-cpp/.gitignore index f2a623dfad71a..cc4c9043e1e96 100644 --- a/pulsar-client-cpp/.gitignore +++ b/pulsar-client-cpp/.gitignore @@ -70,6 +70,7 @@ apidocs/ generated/ # CMAKE +.cmake Makefile cmake_install.cmake CMakeFiles diff --git a/pulsar-client-cpp/include/pulsar/Consumer.h b/pulsar-client-cpp/include/pulsar/Consumer.h index e82d2c07fbcdf..0db8416ec6e4e 100644 --- a/pulsar-client-cpp/include/pulsar/Consumer.h +++ b/pulsar-client-cpp/include/pulsar/Consumer.h @@ -396,7 +396,6 @@ class PULSAR_PUBLIC Consumer { friend class PulsarFriend; friend class PulsarWrapper; - friend class PartitionedConsumerImpl; friend class MultiTopicsConsumerImpl; friend class ConsumerImpl; friend class ClientImpl; diff --git a/pulsar-client-cpp/include/pulsar/Message.h b/pulsar-client-cpp/include/pulsar/Message.h index 9cea48f26580d..935236bd5bb5b 100644 --- a/pulsar-client-cpp/include/pulsar/Message.h +++ b/pulsar-client-cpp/include/pulsar/Message.h @@ -179,7 +179,6 @@ class PULSAR_PUBLIC Message { Message(const MessageId& messageId, proto::MessageMetadata& metadata, SharedBuffer& payload, proto::SingleMessageMetadata& singleMetadata, const std::string& topicName); friend class PartitionedProducerImpl; - friend class PartitionedConsumerImpl; friend class MultiTopicsConsumerImpl; friend class MessageBuilder; friend class ConsumerImpl; diff --git a/pulsar-client-cpp/include/pulsar/MessageId.h b/pulsar-client-cpp/include/pulsar/MessageId.h index de64d1d822413..06be790c1ea4b 100644 --- a/pulsar-client-cpp/include/pulsar/MessageId.h +++ b/pulsar-client-cpp/include/pulsar/MessageId.h @@ -94,7 +94,6 @@ class PULSAR_PUBLIC MessageId { friend class MessageImpl; friend class Commands; friend class PartitionedProducerImpl; - friend class PartitionedConsumerImpl; friend class MultiTopicsConsumerImpl; friend class UnAckedMessageTrackerEnabled; friend class BatchAcknowledgementTracker; diff --git a/pulsar-client-cpp/lib/ClientImpl.cc b/pulsar-client-cpp/lib/ClientImpl.cc index d419ffe7dc596..5ae23cb411435 100644 --- a/pulsar-client-cpp/lib/ClientImpl.cc +++ b/pulsar-client-cpp/lib/ClientImpl.cc @@ -23,7 +23,6 @@ #include "ProducerImpl.h" #include "ReaderImpl.h" #include "PartitionedProducerImpl.h" -#include "PartitionedConsumerImpl.h" #include "MultiTopicsConsumerImpl.h" #include "PatternMultiTopicsConsumerImpl.h" #include "TimeUtils.h" @@ -377,8 +376,9 @@ void ClientImpl::handleSubscribe(const Result result, const LookupDataResultPtr callback(ResultInvalidConfiguration, Consumer()); return; } - consumer = std::make_shared( - shared_from_this(), subscriptionName, topicName, partitionMetadata->getPartitions(), conf); + consumer = std::make_shared(shared_from_this(), topicName, + partitionMetadata->getPartitions(), + subscriptionName, conf, lookupServicePtr_); } else { auto consumerImpl = std::make_shared(shared_from_this(), topicName->toString(), subscriptionName, conf); diff --git a/pulsar-client-cpp/lib/ConsumerImpl.h b/pulsar-client-cpp/lib/ConsumerImpl.h index 80d96d039156d..4db868c9f853a 100644 --- a/pulsar-client-cpp/lib/ConsumerImpl.h +++ b/pulsar-client-cpp/lib/ConsumerImpl.h @@ -224,7 +224,6 @@ class ConsumerImpl : public ConsumerImplBase, // these two declared friend to access setNegativeAcknowledgeEnabledForTesting friend class MultiTopicsConsumerImpl; - friend class PartitionedConsumerImpl; FRIEND_TEST(ConsumerTest, testPartitionedConsumerUnAckedMessageRedelivery); FRIEND_TEST(ConsumerTest, testMultiTopicsConsumerUnAckedMessageRedelivery); diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc index 5ed66400983c6..5fe9446b18668 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc @@ -17,6 +17,7 @@ * under the License. */ #include "MultiTopicsConsumerImpl.h" +#include "MultiResultCallback.h" DECLARE_LOG_OBJECT() @@ -25,7 +26,7 @@ using namespace pulsar; MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std::vector& topics, const std::string& subscriptionName, TopicNamePtr topicName, const ConsumerConfiguration& conf, - const LookupServicePtr lookupServicePtr) + LookupServicePtr lookupServicePtr) : client_(client), subscriptionName_(subscriptionName), topic_(topicName ? topicName->toString() : "EmptyTopics"), @@ -52,6 +53,12 @@ MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std } else { unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled()); } + auto partitionsUpdateInterval = static_cast(client_->conf().getPartitionsUpdateInterval()); + if (partitionsUpdateInterval > 0) { + partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); + partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); + lookupServicePtr_ = client_->getLookup(); + } } void MultiTopicsConsumerImpl::start() { @@ -125,25 +132,32 @@ Future MultiTopicsConsumerImpl::subscribeOneTopicAsync(const s } // subscribe for each partition, when all partitions completed, complete promise - lookupServicePtr_->getPartitionMetadataAsync(topicName).addListener(std::bind( - &MultiTopicsConsumerImpl::subscribeTopicPartitions, shared_from_this(), std::placeholders::_1, - std::placeholders::_2, topicName, subscriptionName_, conf_, topicPromise)); + Lock lock(mutex_); + auto entry = topicsPartitions_.find(topic); + if (entry == topicsPartitions_.end()) { + lock.unlock(); + lookupServicePtr_->getPartitionMetadataAsync(topicName).addListener( + [this, topicName, topicPromise](Result result, const LookupDataResultPtr& lookupDataResult) { + if (result != ResultOk) { + LOG_ERROR("Error Checking/Getting Partition Metadata while MultiTopics Subscribing- " + << consumerStr_ << " result: " << result) + topicPromise->setFailed(result); + return; + } + subscribeTopicPartitions(lookupDataResult->getPartitions(), topicName, subscriptionName_, + topicPromise); + }); + } else { + auto numPartitions = entry->second; + lock.unlock(); + subscribeTopicPartitions(numPartitions, topicName, subscriptionName_, topicPromise); + } return topicPromise->getFuture(); } -void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, - const LookupDataResultPtr partitionMetadata, - TopicNamePtr topicName, +void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicNamePtr topicName, const std::string& consumerName, - ConsumerConfiguration conf, ConsumerSubResultPromisePtr topicSubResultPromise) { - if (result != ResultOk) { - LOG_ERROR("Error Checking/Getting Partition Metadata while MultiTopics Subscribing- " - << consumerStr_ << " result: " << result) - topicSubResultPromise->setFailed(result); - return; - } - std::shared_ptr consumer; ConsumerConfiguration config = conf_.clone(); ExecutorServicePtr internalListenerExecutor = client_->getPartitionListenerExecutorProvider()->get(); @@ -151,7 +165,6 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, shared_from_this(), std::placeholders::_1, std::placeholders::_2)); - int numPartitions = partitionMetadata->getPartitions(); int partitions = numPartitions == 0 ? 1 : numPartitions; // Apply total limit of receiver queue size across partitions @@ -160,7 +173,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, (int)(conf_.getMaxTotalReceiverQueueSizeAcrossPartitions() / partitions))); Lock lock(mutex_); - topicsPartitions_.insert(std::make_pair(topicName->toString(), partitions)); + topicsPartitions_[topicName->toString()] = partitions; lock.unlock(); numberTopicPartitions_->fetch_add(partitions); @@ -214,10 +227,13 @@ void MultiTopicsConsumerImpl::handleSingleConsumerCreated( return; } - LOG_DEBUG("Successfully Subscribed to a single partition of topic in TopicsConsumer. " - << "Partitions need to create - " << previous - 1); + LOG_INFO("Successfully Subscribed to a single partition of topic in TopicsConsumer. " + << "Partitions need to create : " << previous - 1); if (partitionsNeedCreate->load() == 0) { + if (partitionsUpdateTimer_) { + runPartitionUpdateTask(); + } topicSubResultPromise->setValue(Consumer(shared_from_this())); } } @@ -274,13 +290,17 @@ void MultiTopicsConsumerImpl::handleUnsubscribedAsync(Result result, } void MultiTopicsConsumerImpl::unsubscribeOneTopicAsync(const std::string& topic, ResultCallback callback) { + Lock lock(mutex_); std::map::iterator it = topicsPartitions_.find(topic); if (it == topicsPartitions_.end()) { + lock.unlock(); LOG_ERROR("TopicsConsumer does not subscribe topic : " << topic << " subscription - " << subscriptionName_); callback(ResultTopicNotFound); return; } + int numberPartitions = it->second; + lock.unlock(); const auto state = state_.load(); if (state == Closing || state == Closed) { @@ -295,7 +315,6 @@ void MultiTopicsConsumerImpl::unsubscribeOneTopicAsync(const std::string& topic, LOG_ERROR("TopicName invalid: " << topic); callback(ResultUnknownError); } - int numberPartitions = it->second; std::shared_ptr> consumerUnsubed = std::make_shared>(0); for (int i = 0; i < numberPartitions; i++) { @@ -677,7 +696,15 @@ void MultiTopicsConsumerImpl::seekAsync(const MessageId& msgId, ResultCallback c } void MultiTopicsConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { - callback(ResultOperationNotSupported); + if (state_ != Ready) { + callback(ResultAlreadyClosed); + return; + } + + MultiResultCallback multiResultCallback(callback, consumers_.size()); + consumers_.forEachValue([×tamp, &multiResultCallback](ConsumerImplPtr consumer) { + consumer->seekAsync(timestamp, multiResultCallback); + }); } void MultiTopicsConsumerImpl::setNegativeAcknowledgeEnabledForTesting(bool enabled) { @@ -705,3 +732,85 @@ uint64_t MultiTopicsConsumerImpl::getNumberOfConnectedConsumer() { }); return numberOfConnectedConsumer; } +void MultiTopicsConsumerImpl::runPartitionUpdateTask() { + partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); + auto self = shared_from_this(); + partitionsUpdateTimer_->async_wait([self](const boost::system::error_code& ec) { + // If two requests call runPartitionUpdateTask at the same time, the timer will fail, and it + // cannot continue at this time, and the request needs to be ignored. + if (!ec) { + self->topicPartitionUpdate(); + } + }); +} +void MultiTopicsConsumerImpl::topicPartitionUpdate() { + using namespace std::placeholders; + Lock lock(mutex_); + auto topicsPartitions = topicsPartitions_; + lock.unlock(); + for (const auto& item : topicsPartitions) { + auto topicName = TopicName::get(item.first); + auto currentNumPartitions = item.second; + lookupServicePtr_->getPartitionMetadataAsync(topicName).addListener( + std::bind(&MultiTopicsConsumerImpl::handleGetPartitions, shared_from_this(), topicName, + std::placeholders::_1, std::placeholders::_2, currentNumPartitions)); + } +} +void MultiTopicsConsumerImpl::handleGetPartitions(TopicNamePtr topicName, Result result, + const LookupDataResultPtr& lookupDataResult, + int currentNumPartitions) { + if (state_ != Ready) { + return; + } + if (!result) { + const auto newNumPartitions = static_cast(lookupDataResult->getPartitions()); + if (newNumPartitions > currentNumPartitions) { + LOG_INFO("new partition count: " << newNumPartitions + << " current partition count: " << currentNumPartitions); + auto partitionsNeedCreate = + std::make_shared>(newNumPartitions - currentNumPartitions); + ConsumerSubResultPromisePtr topicPromise = std::make_shared>(); + Lock lock(mutex_); + topicsPartitions_[topicName->toString()] = newNumPartitions; + lock.unlock(); + numberTopicPartitions_->fetch_add(newNumPartitions - currentNumPartitions); + for (unsigned int i = currentNumPartitions; i < newNumPartitions; i++) { + subscribeSingleNewConsumer(newNumPartitions, topicName, i, topicPromise, + partitionsNeedCreate); + } + // `runPartitionUpdateTask()` will be called in `handleSingleConsumerCreated()` + return; + } + } else { + LOG_WARN("Failed to getPartitionMetadata: " << strResult(result)); + } + runPartitionUpdateTask(); +} + +void MultiTopicsConsumerImpl::subscribeSingleNewConsumer( + int numPartitions, TopicNamePtr topicName, int partitionIndex, + ConsumerSubResultPromisePtr topicSubResultPromise, + std::shared_ptr> partitionsNeedCreate) { + ConsumerConfiguration config = conf_.clone(); + ExecutorServicePtr internalListenerExecutor = client_->getPartitionListenerExecutorProvider()->get(); + config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, shared_from_this(), + std::placeholders::_1, std::placeholders::_2)); + + // Apply total limit of receiver queue size across partitions + config.setReceiverQueueSize( + std::min(conf_.getReceiverQueueSize(), + (int)(conf_.getMaxTotalReceiverQueueSizeAcrossPartitions() / numPartitions))); + + std::string topicPartitionName = topicName->getTopicPartitionName(partitionIndex); + + auto consumer = std::make_shared(client_, topicPartitionName, subscriptionName_, config, + internalListenerExecutor, true, Partitioned); + consumer->getConsumerCreatedFuture().addListener( + std::bind(&MultiTopicsConsumerImpl::handleSingleConsumerCreated, shared_from_this(), + std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); + consumer->setPartitionIndex(partitionIndex); + consumer->start(); + consumers_.emplace(topicPartitionName, consumer); + LOG_INFO("Add Creating Consumer for - " << topicPartitionName << " - " << consumerStr_ + << " consumerSize: " << consumers_.size()); +} diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h index 0f111110c44a1..95c24f68c5b78 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h @@ -51,7 +51,14 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, }; MultiTopicsConsumerImpl(ClientImplPtr client, const std::vector& topics, const std::string& subscriptionName, TopicNamePtr topicName, - const ConsumerConfiguration& conf, const LookupServicePtr lookupServicePtr_); + const ConsumerConfiguration& conf, LookupServicePtr lookupServicePtr_); + MultiTopicsConsumerImpl(ClientImplPtr client, TopicNamePtr topicName, int numPartitions, + const std::string& subscriptionName, const ConsumerConfiguration& conf, + LookupServicePtr lookupServicePtr) + : MultiTopicsConsumerImpl(client, {topicName->toString()}, subscriptionName, topicName, conf, + lookupServicePtr) { + topicsPartitions_[topicName->toString()] = numPartitions; + } ~MultiTopicsConsumerImpl(); // overrided methods from ConsumerImplBase Future getConsumerCreatedFuture() override; @@ -101,14 +108,16 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, std::mutex pendingReceiveMutex_; std::atomic state_{Pending}; BlockingQueue messages_; - ExecutorServicePtr listenerExecutor_; + const ExecutorServicePtr listenerExecutor_; MessageListener messageListener_; + DeadlineTimerPtr partitionsUpdateTimer_; + boost::posix_time::time_duration partitionsUpdateInterval_; LookupServicePtr lookupServicePtr_; std::shared_ptr> numberTopicPartitions_; std::atomic failedResult{ResultOk}; Promise multiTopicsConsumerCreatedPromise_; UnAckedMessageTrackerPtr unAckedMessageTrackerPtr_; - const std::vector& topics_; + const std::vector topics_; std::queue pendingReceives_; /* methods */ @@ -122,9 +131,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, void handleOneTopicSubscribed(Result result, Consumer consumer, const std::string& topic, std::shared_ptr> topicsNeedCreate); - void subscribeTopicPartitions(const Result result, const LookupDataResultPtr partitionMetadata, - TopicNamePtr topicName, const std::string& consumerName, - ConsumerConfiguration conf, + void subscribeTopicPartitions(int numPartitions, TopicNamePtr topicName, const std::string& consumerName, ConsumerSubResultPromisePtr topicSubResultPromise); void handleSingleConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, std::shared_ptr> partitionsNeedCreate, @@ -134,11 +141,19 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, void handleOneTopicUnsubscribedAsync(Result result, std::shared_ptr> consumerUnsubed, int numberPartitions, TopicNamePtr topicNamePtr, std::string& topicPartitionName, ResultCallback callback); + void runPartitionUpdateTask(); + void topicPartitionUpdate(); + void handleGetPartitions(TopicNamePtr topicName, Result result, + const LookupDataResultPtr& lookupDataResult, int currentNumPartitions); + void subscribeSingleNewConsumer(int numPartitions, TopicNamePtr topicName, int partitionIndex, + ConsumerSubResultPromisePtr topicSubResultPromise, + std::shared_ptr> partitionsNeedCreate); private: void setNegativeAcknowledgeEnabledForTesting(bool enabled) override; FRIEND_TEST(ConsumerTest, testMultiTopicsConsumerUnAckedMessageRedelivery); + FRIEND_TEST(ConsumerTest, testPartitionedConsumerUnAckedMessageRedelivery); }; typedef std::shared_ptr MultiTopicsConsumerImplPtr; diff --git a/pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.cc b/pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.cc deleted file mode 100644 index 9d5965b24bd92..0000000000000 --- a/pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.cc +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include -#include -#include -#include - -namespace pulsar { - -const std::string PartitionedBrokerConsumerStatsImpl::DELIMITER = ";"; - -PartitionedBrokerConsumerStatsImpl::PartitionedBrokerConsumerStatsImpl(size_t size) { - statsList_.resize(size); -} - -bool PartitionedBrokerConsumerStatsImpl::isValid() const { - bool isValid = true; - for (int i = 0; i < statsList_.size(); i++) { - isValid &= statsList_[i].isValid(); - } - return isValid; -} - -std::ostream& operator<<(std::ostream& os, const PartitionedBrokerConsumerStatsImpl& obj) { - os << "\nPartitionedBrokerConsumerStatsImpl [" - << "validTill_ = " << obj.isValid() << ", msgRateOut_ = " << obj.getMsgRateOut() - << ", msgThroughputOut_ = " << obj.getMsgThroughputOut() - << ", msgRateRedeliver_ = " << obj.getMsgRateRedeliver() - << ", consumerName_ = " << obj.getConsumerName() - << ", availablePermits_ = " << obj.getAvailablePermits() - << ", unackedMessages_ = " << obj.getUnackedMessages() - << ", blockedConsumerOnUnackedMsgs_ = " << obj.isBlockedConsumerOnUnackedMsgs() - << ", address_ = " << obj.getAddress() << ", connectedSince_ = " << obj.getConnectedSince() - << ", type_ = " << obj.getType() << ", msgRateExpired_ = " << obj.getMsgRateExpired() - << ", msgBacklog_ = " << obj.getMsgBacklog() << "]"; - return os; -} - -double PartitionedBrokerConsumerStatsImpl::getMsgRateOut() const { - double sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getMsgRateOut(); - } - return sum; -} - -double PartitionedBrokerConsumerStatsImpl::getMsgThroughputOut() const { - double sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getMsgThroughputOut(); - } - return sum; -} - -double PartitionedBrokerConsumerStatsImpl::getMsgRateRedeliver() const { - double sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getMsgRateRedeliver(); - } - return sum; -} - -const std::string PartitionedBrokerConsumerStatsImpl::getConsumerName() const { - std::string str; - for (int i = 0; i < statsList_.size(); i++) { - str += statsList_[i].getConsumerName() + DELIMITER; - } - return str; -} - -uint64_t PartitionedBrokerConsumerStatsImpl::getAvailablePermits() const { - uint64_t sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getAvailablePermits(); - } - return sum; -} - -uint64_t PartitionedBrokerConsumerStatsImpl::getUnackedMessages() const { - uint64_t sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getUnackedMessages(); - } - return sum; -} - -bool PartitionedBrokerConsumerStatsImpl::isBlockedConsumerOnUnackedMsgs() const { - if (statsList_.size() == 0) { - return false; - } - - bool isValid = true; - for (int i = 0; i < statsList_.size(); i++) { - isValid &= statsList_[i].isValid(); - } - return isValid; -} - -const std::string PartitionedBrokerConsumerStatsImpl::getAddress() const { - std::string str; - for (int i = 0; i < statsList_.size(); i++) { - str += statsList_[i].getAddress() + DELIMITER; - } - return str; -} - -const std::string PartitionedBrokerConsumerStatsImpl::getConnectedSince() const { - std::string str; - for (int i = 0; i < statsList_.size(); i++) { - str += statsList_[i].getConnectedSince() + DELIMITER; - } - return str; -} - -const ConsumerType PartitionedBrokerConsumerStatsImpl::getType() const { - if (!statsList_.size()) { - return ConsumerExclusive; - } - return statsList_[0].getType(); -} - -double PartitionedBrokerConsumerStatsImpl::getMsgRateExpired() const { - double sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getMsgRateExpired(); - } - return sum; -} - -uint64_t PartitionedBrokerConsumerStatsImpl::getMsgBacklog() const { - uint64_t sum = 0; - for (int i = 0; i < statsList_.size(); i++) { - sum += statsList_[i].getMsgBacklog(); - } - return sum; -} - -BrokerConsumerStats PartitionedBrokerConsumerStatsImpl::getBrokerConsumerStats(int index) { - return statsList_[index]; -} - -void PartitionedBrokerConsumerStatsImpl::add(BrokerConsumerStats stats, int index) { - statsList_[index] = stats; -} - -void PartitionedBrokerConsumerStatsImpl::clear() { statsList_.clear(); } -} // namespace pulsar diff --git a/pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.h b/pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.h deleted file mode 100644 index 683f5245dbb2c..0000000000000 --- a/pulsar-client-cpp/lib/PartitionedBrokerConsumerStatsImpl.h +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef PULSAR_CPP_PARTITIONEDBROKERCONSUMERSTATSIMPL_H -#define PULSAR_CPP_PARTITIONEDBROKERCONSUMERSTATSIMPL_H - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pulsar { -class PULSAR_PUBLIC PartitionedBrokerConsumerStatsImpl : public BrokerConsumerStatsImplBase { - private: - std::vector statsList_; - static const std::string DELIMITER; - - public: - PartitionedBrokerConsumerStatsImpl(size_t size); - - /** Returns true if the Stats are still valid **/ - virtual bool isValid() const; - - /** Returns the rate of messages delivered to the consumer. msg/s */ - virtual double getMsgRateOut() const; - - /** Returns the throughput delivered to the consumer. bytes/s */ - virtual double getMsgThroughputOut() const; - - /** Returns the rate of messages redelivered by this consumer. msg/s */ - virtual double getMsgRateRedeliver() const; - - /** Returns the Name of the consumer */ - virtual const std::string getConsumerName() const; - - /** Returns the Number of available message permits for the consumer */ - virtual uint64_t getAvailablePermits() const; - - /** Returns the Number of unacknowledged messages for the consumer */ - virtual uint64_t getUnackedMessages() const; - - /** Returns true if the consumer is blocked due to unacked messages. */ - virtual bool isBlockedConsumerOnUnackedMsgs() const; - - /** Returns the Address of this consumer */ - virtual const std::string getAddress() const; - - /** Returns the Timestamp of connection */ - virtual const std::string getConnectedSince() const; - - /** Returns Whether this subscription is Exclusive or Shared or Failover */ - virtual const ConsumerType getType() const; - - /** Returns the rate of messages expired on this subscription. msg/s */ - virtual double getMsgRateExpired() const; - - /** Returns the Number of messages in the subscription backlog */ - virtual uint64_t getMsgBacklog() const; - - /** Returns the BrokerConsumerStatsImpl at of ith partition */ - BrokerConsumerStats getBrokerConsumerStats(int index); - - void add(BrokerConsumerStats stats, int index); - - void clear(); - - friend std::ostream &operator<<(std::ostream &os, const PartitionedBrokerConsumerStatsImpl &obj); -}; -typedef std::shared_ptr PartitionedBrokerConsumerStatsPtr; -} // namespace pulsar -#endif // PULSAR_CPP_BROKERCONSUMERSTATSIMPL_H diff --git a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc deleted file mode 100644 index 0dc9135be59bc..0000000000000 --- a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc +++ /dev/null @@ -1,610 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include "PartitionedConsumerImpl.h" -#include "MultiResultCallback.h" - -DECLARE_LOG_OBJECT() - -namespace pulsar { - -PartitionedConsumerImpl::PartitionedConsumerImpl(ClientImplPtr client, const std::string& subscriptionName, - const TopicNamePtr topicName, - const unsigned int numPartitions, - const ConsumerConfiguration& conf) - : client_(client), - subscriptionName_(subscriptionName), - topicName_(topicName), - numPartitions_(numPartitions), - conf_(conf), - messages_(1000), - listenerExecutor_(client->getListenerExecutorProvider()->get()), - messageListener_(conf.getMessageListener()), - topic_(topicName->toString()) { - std::stringstream consumerStrStream; - consumerStrStream << "[Partitioned Consumer: " << topic_ << "," << subscriptionName << "," - << numPartitions << "]"; - if (conf.getUnAckedMessagesTimeoutMs() != 0) { - if (conf.getTickDurationInMs() > 0) { - unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerEnabled( - conf.getUnAckedMessagesTimeoutMs(), conf.getTickDurationInMs(), client, *this)); - } else { - unAckedMessageTrackerPtr_.reset( - new UnAckedMessageTrackerEnabled(conf.getUnAckedMessagesTimeoutMs(), client, *this)); - } - } else { - unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled()); - } - auto partitionsUpdateInterval = static_cast(client_->conf().getPartitionsUpdateInterval()); - if (partitionsUpdateInterval > 0) { - partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); - partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); - lookupServicePtr_ = client_->getLookup(); - } -} - -PartitionedConsumerImpl::~PartitionedConsumerImpl() {} - -Future PartitionedConsumerImpl::getConsumerCreatedFuture() { - return partitionedConsumerCreatedPromise_.getFuture(); -} -const std::string& PartitionedConsumerImpl::getSubscriptionName() const { return subscriptionName_; } - -const std::string& PartitionedConsumerImpl::getTopic() const { return topic_; } - -Result PartitionedConsumerImpl::receive(Message& msg) { - if (state_ != Ready) { - return ResultAlreadyClosed; - } - // See comments in `receive(Message&, int)` - - if (messageListener_) { - LOG_ERROR("Can not receive when a listener has been set"); - return ResultInvalidConfiguration; - } - - messages_.pop(msg); - unAckedMessageTrackerPtr_->add(msg.getMessageId()); - return ResultOk; -} - -Result PartitionedConsumerImpl::receive(Message& msg, int timeout) { - if (state_ != Ready) { - return ResultAlreadyClosed; - } - - if (messageListener_) { - LOG_ERROR("Can not receive when a listener has been set"); - return ResultInvalidConfiguration; - } - - if (messages_.pop(msg, std::chrono::milliseconds(timeout))) { - unAckedMessageTrackerPtr_->add(msg.getMessageId()); - return ResultOk; - } else { - return ResultTimeout; - } -} - -void PartitionedConsumerImpl::receiveAsync(ReceiveCallback& callback) { - Message msg; - - // fail the callback if consumer is closing or closed - if (state_ != Ready) { - callback(ResultAlreadyClosed, msg); - return; - } - - Lock lock(pendingReceiveMutex_); - if (messages_.pop(msg, std::chrono::milliseconds(0))) { - lock.unlock(); - unAckedMessageTrackerPtr_->add(msg.getMessageId()); - callback(ResultOk, msg); - } else { - pendingReceives_.push(callback); - } -} - -void PartitionedConsumerImpl::unsubscribeAsync(ResultCallback callback) { - LOG_INFO("[" << topicName_->toString() << "," << subscriptionName_ << "] Unsubscribing"); - // change state to Closing, so that no Ready state operation is permitted during unsubscribe - state_ = Closing; - // do not accept un subscribe until we have subscribe to all of the partitions of a topic - // it's a logical single topic so it should behave like a single topic, even if it's sharded - unsigned int index = 0; - for (ConsumerList::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - LOG_DEBUG("Unsubcribing Consumer - " << index << " for Subscription - " << subscriptionName_ - << " for Topic - " << topicName_->toString()); - (*consumer)->unsubscribeAsync(std::bind(&PartitionedConsumerImpl::handleUnsubscribeAsync, - shared_from_this(), std::placeholders::_1, index++, - callback)); - } -} - -void PartitionedConsumerImpl::handleUnsubscribeAsync(Result result, unsigned int consumerIndex, - ResultCallback callback) { - if (state_ == Failed) { - // we have already informed the client that unsubcribe has failed so, ignore this callbacks - // or do we still go ahead and check how many could we close successfully? - LOG_DEBUG("handleUnsubscribeAsync callback received in Failed State for consumerIndex - " - << consumerIndex << "with Result - " << result << " for Subscription - " - << subscriptionName_ << " for Topic - " << topicName_->toString()); - return; - } - if (result != ResultOk) { - state_ = Failed; - LOG_ERROR("Error Closing one of the parition consumers, consumerIndex - " << consumerIndex); - callback(ResultUnknownError); - return; - } - const auto numPartitions = getNumPartitionsWithLock(); - assert(unsubscribedSoFar_ <= numPartitions); - assert(consumerIndex <= numPartitions); - // this means we have successfully closed this partition consumer and no unsubscribe has failed so far - LOG_INFO("Successfully Unsubscribed Consumer - " << consumerIndex << " for Subscription - " - << subscriptionName_ << " for Topic - " - << topicName_->toString()); - unsubscribedSoFar_++; - if (unsubscribedSoFar_ == numPartitions) { - LOG_DEBUG("Unsubscribed all of the partition consumer for subscription - " << subscriptionName_); - state_ = Closed; - callback(ResultOk); - return; - } -} - -void PartitionedConsumerImpl::acknowledgeAsync(const MessageId& msgId, ResultCallback callback) { - int32_t partition = msgId.partition(); -#ifndef NDEBUG - Lock consumersLock(consumersMutex_); - assert(partition < getNumPartitions() && partition >= 0 && consumers_.size() > partition); - consumersLock.unlock(); -#endif - unAckedMessageTrackerPtr_->remove(msgId); - consumers_[partition]->acknowledgeAsync(msgId, callback); -} - -void PartitionedConsumerImpl::acknowledgeCumulativeAsync(const MessageId& msgId, ResultCallback callback) { - callback(ResultOperationNotSupported); -} - -void PartitionedConsumerImpl::negativeAcknowledge(const MessageId& msgId) { - int32_t partition = msgId.partition(); - unAckedMessageTrackerPtr_->remove(msgId); - consumers_[partition]->negativeAcknowledge(msgId); -} - -unsigned int PartitionedConsumerImpl::getNumPartitions() const { return numPartitions_; } - -unsigned int PartitionedConsumerImpl::getNumPartitionsWithLock() const { - Lock consumersLock(consumersMutex_); - return getNumPartitions(); -} - -ConsumerConfiguration PartitionedConsumerImpl::getSinglePartitionConsumerConfig() const { - using namespace std::placeholders; - - ConsumerConfiguration config = conf_.clone(); - // all the partitioned-consumer belonging to one partitioned topic should have same name - config.setConsumerName(conf_.getConsumerName()); - config.setConsumerType(conf_.getConsumerType()); - config.setBrokerConsumerStatsCacheTimeInMs(conf_.getBrokerConsumerStatsCacheTimeInMs()); - - const auto shared_this = const_cast(this)->shared_from_this(); - config.setMessageListener(std::bind(&PartitionedConsumerImpl::messageReceived, shared_this, - std::placeholders::_1, std::placeholders::_2)); - - // Apply total limit of receiver queue size across partitions - // NOTE: if it's called by handleGetPartitions(), the queue size of new internal consumers may be smaller - // than previous created internal consumers. - config.setReceiverQueueSize( - std::min(conf_.getReceiverQueueSize(), - (int)(conf_.getMaxTotalReceiverQueueSizeAcrossPartitions() / getNumPartitions()))); - - return config; -} - -ConsumerImplPtr PartitionedConsumerImpl::newInternalConsumer(unsigned int partition, - const ConsumerConfiguration& config) const { - using namespace std::placeholders; - - std::string topicPartitionName = topicName_->getTopicPartitionName(partition); - auto consumer = std::make_shared(client_, topicPartitionName, subscriptionName_, config, - internalListenerExecutor_, true, Partitioned); - - const auto shared_this = const_cast(this)->shared_from_this(); - consumer->getConsumerCreatedFuture().addListener( - std::bind(&PartitionedConsumerImpl::handleSinglePartitionConsumerCreated, shared_this, - std::placeholders::_1, std::placeholders::_2, partition)); - consumer->setPartitionIndex(partition); - - LOG_DEBUG("Creating Consumer for single Partition - " << topicPartitionName << "SubName - " - << subscriptionName_); - return consumer; -} - -void PartitionedConsumerImpl::start() { - internalListenerExecutor_ = client_->getPartitionListenerExecutorProvider()->get(); - const auto config = getSinglePartitionConsumerConfig(); - - // create consumer on each partition - // Here we don't need `consumersMutex` to protect `consumers_`, because `consumers_` can only be increased - // when `state_` is Ready - for (unsigned int i = 0; i < getNumPartitions(); i++) { - consumers_.push_back(newInternalConsumer(i, config)); - } - for (ConsumerList::const_iterator consumer = consumers_.begin(); consumer != consumers_.end(); - consumer++) { - (*consumer)->start(); - } -} - -void PartitionedConsumerImpl::handleSinglePartitionConsumerCreated( - Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, unsigned int partitionIndex) { - ResultCallback nullCallbackForCleanup = NULL; - if (state_ == Failed) { - // one of the consumer creation failed, and we are cleaning up - return; - } - const auto numPartitions = getNumPartitionsWithLock(); - assert(numConsumersCreated_ < numPartitions); - - if (result != ResultOk) { - state_ = Failed; - partitionedConsumerCreatedPromise_.setFailed(result); - // unsubscribed all of the successfully subscribed partitioned consumers - closeAsync(nullCallbackForCleanup); - LOG_ERROR("Unable to create Consumer for partition - " << partitionIndex << " Error - " << result); - return; - } - - assert(partitionIndex < numPartitions && partitionIndex >= 0); - Lock lock(mutex_); - numConsumersCreated_++; - lock.unlock(); - if (numConsumersCreated_ == numPartitions) { - LOG_INFO("Successfully Subscribed to Partitioned Topic - " << topicName_->toString() << " with - " - << numPartitions << " Partitions."); - state_ = Ready; - if (partitionsUpdateTimer_) { - runPartitionUpdateTask(); - } - receiveMessages(); - partitionedConsumerCreatedPromise_.setValue(shared_from_this()); - return; - } -} - -void PartitionedConsumerImpl::handleSinglePartitionConsumerClose(Result result, unsigned int partitionIndex, - CloseCallback callback) { - if (state_ == Failed) { - // we should have already notified the client by callback - return; - } - if (result != ResultOk) { - state_ = Failed; - LOG_ERROR("Closing the consumer failed for partition - " << partitionIndex); - partitionedConsumerCreatedPromise_.setFailed(result); - if (callback) { - callback(result); - } - return; - } - assert(partitionIndex < getNumPartitionsWithLock() && partitionIndex >= 0); - Lock lock(mutex_); - if (numConsumersCreated_ > 0) { - numConsumersCreated_--; - } - lock.unlock(); - // closed all successfully - if (!numConsumersCreated_) { - state_ = Closed; - // set the producerCreatedPromise to failure - partitionedConsumerCreatedPromise_.setFailed(ResultUnknownError); - if (callback) { - callback(result); - } - return; - } -} -void PartitionedConsumerImpl::closeAsync(ResultCallback callback) { - Lock lock(consumersMutex_); - if (consumers_.empty()) { - notifyResult(callback); - return; - } - state_ = Closed; - unsigned int consumerAlreadyClosed = 0; - // close successfully subscribed consumers - // Here we don't need `consumersMutex` to protect `consumers_`, because `consumers_` can only be increased - // when `state_` is Ready - for (auto& consumer : consumers_) { - if (!consumer->isClosed()) { - auto self = shared_from_this(); - const auto partition = consumer->getPartitionIndex(); - consumer->closeAsync([this, self, partition, callback](Result result) { - handleSinglePartitionConsumerClose(result, partition, callback); - }); - } else { - if (++consumerAlreadyClosed == consumers_.size()) { - // everything is closed already. so we are good. - notifyResult(callback); - return; - } - } - } - - // fail pending recieve - failPendingReceiveCallback(); -} - -void PartitionedConsumerImpl::notifyResult(CloseCallback closeCallback) { - if (closeCallback) { - // this means client invoked the closeAsync with a valid callback - state_ = Closed; - closeCallback(ResultOk); - } else { - // consumer create failed, closeAsync called to cleanup the successfully created producers - state_ = Failed; - partitionedConsumerCreatedPromise_.setFailed(ResultUnknownError); - } -} - -void PartitionedConsumerImpl::shutdown() {} - -bool PartitionedConsumerImpl::isClosed() { return state_ == Closed; } - -bool PartitionedConsumerImpl::isOpen() { return state_ == Ready; } - -void PartitionedConsumerImpl::messageReceived(Consumer consumer, const Message& msg) { - LOG_DEBUG("Received Message from one of the partition - " << msg.impl_->messageId.partition()); - const std::string& topicPartitionName = consumer.getTopic(); - msg.impl_->setTopicName(topicPartitionName); - // messages_ is a blocking queue: if queue is already full then no need of lock as receiveAsync already - // gets available-msg and no need to put request in pendingReceives_ - Lock lock(pendingReceiveMutex_); - if (!pendingReceives_.empty()) { - ReceiveCallback callback = pendingReceives_.front(); - pendingReceives_.pop(); - lock.unlock(); - unAckedMessageTrackerPtr_->add(msg.getMessageId()); - listenerExecutor_->postWork(std::bind(callback, ResultOk, msg)); - } else { - if (messages_.full()) { - lock.unlock(); - } - messages_.push(msg); - if (messageListener_) { - unAckedMessageTrackerPtr_->add(msg.getMessageId()); - listenerExecutor_->postWork( - std::bind(&PartitionedConsumerImpl::internalListener, shared_from_this(), consumer)); - } - } -} - -void PartitionedConsumerImpl::failPendingReceiveCallback() { - Message msg; - Lock lock(pendingReceiveMutex_); - while (!pendingReceives_.empty()) { - ReceiveCallback callback = pendingReceives_.front(); - pendingReceives_.pop(); - listenerExecutor_->postWork(std::bind(callback, ResultAlreadyClosed, msg)); - } - lock.unlock(); -} - -void PartitionedConsumerImpl::internalListener(Consumer consumer) { - Message m; - messages_.pop(m); - try { - messageListener_(Consumer(shared_from_this()), m); - } catch (const std::exception& e) { - LOG_ERROR("Exception thrown from listener of Partitioned Consumer" << e.what()); - } -} - -void PartitionedConsumerImpl::receiveMessages() { - for (ConsumerList::const_iterator i = consumers_.begin(); i != consumers_.end(); i++) { - ConsumerImplPtr consumer = *i; - consumer->sendFlowPermitsToBroker(consumer->getCnx().lock(), conf_.getReceiverQueueSize()); - LOG_DEBUG("Sending FLOW command for consumer - " << consumer->getConsumerId()); - } -} - -Result PartitionedConsumerImpl::pauseMessageListener() { - if (!messageListener_) { - return ResultInvalidConfiguration; - } - for (ConsumerList::const_iterator i = consumers_.begin(); i != consumers_.end(); i++) { - (*i)->pauseMessageListener(); - } - return ResultOk; -} - -Result PartitionedConsumerImpl::resumeMessageListener() { - if (!messageListener_) { - return ResultInvalidConfiguration; - } - for (ConsumerList::const_iterator i = consumers_.begin(); i != consumers_.end(); i++) { - (*i)->resumeMessageListener(); - } - return ResultOk; -} - -void PartitionedConsumerImpl::redeliverUnacknowledgedMessages() { - LOG_DEBUG("Sending RedeliverUnacknowledgedMessages command for partitioned consumer."); - for (ConsumerList::const_iterator i = consumers_.begin(); i != consumers_.end(); i++) { - (*i)->redeliverUnacknowledgedMessages(); - } - unAckedMessageTrackerPtr_->clear(); -} - -void PartitionedConsumerImpl::redeliverUnacknowledgedMessages(const std::set& messageIds) { - if (messageIds.empty()) { - return; - } - if (conf_.getConsumerType() != ConsumerShared && conf_.getConsumerType() != ConsumerKeyShared) { - redeliverUnacknowledgedMessages(); - return; - } - LOG_DEBUG("Sending RedeliverUnacknowledgedMessages command for partitioned consumer."); - for (ConsumerList::const_iterator i = consumers_.begin(); i != consumers_.end(); i++) { - (*i)->redeliverUnacknowledgedMessages(messageIds); - } -} - -const std::string& PartitionedConsumerImpl::getName() const { return partitionStr_; } - -int PartitionedConsumerImpl::getNumOfPrefetchedMessages() const { return messages_.size(); } - -void PartitionedConsumerImpl::getBrokerConsumerStatsAsync(BrokerConsumerStatsCallback callback) { - if (state_ != Ready) { - callback(ResultConsumerNotInitialized, BrokerConsumerStats()); - return; - } - const auto numPartitions = getNumPartitionsWithLock(); - PartitionedBrokerConsumerStatsPtr statsPtr = - std::make_shared(numPartitions); - LatchPtr latchPtr = std::make_shared(numPartitions); - ConsumerList consumerList = consumers_; - for (int i = 0; i < consumerList.size(); i++) { - consumerList[i]->getBrokerConsumerStatsAsync( - std::bind(&PartitionedConsumerImpl::handleGetConsumerStats, shared_from_this(), - std::placeholders::_1, std::placeholders::_2, latchPtr, statsPtr, i, callback)); - } -} - -void PartitionedConsumerImpl::handleGetConsumerStats(Result res, BrokerConsumerStats brokerConsumerStats, - LatchPtr latchPtr, - PartitionedBrokerConsumerStatsPtr statsPtr, size_t index, - BrokerConsumerStatsCallback callback) { - Lock lock(mutex_); - if (res == ResultOk) { - latchPtr->countdown(); - statsPtr->add(brokerConsumerStats, index); - } else { - lock.unlock(); - callback(res, BrokerConsumerStats()); - return; - } - if (latchPtr->getCount() == 0) { - lock.unlock(); - callback(ResultOk, BrokerConsumerStats(statsPtr)); - } -} - -void PartitionedConsumerImpl::seekAsync(const MessageId& msgId, ResultCallback callback) { - callback(ResultOperationNotSupported); -} - -void PartitionedConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { - if (state_ != Ready) { - callback(ResultAlreadyClosed); - return; - } - - // consumers_ could only be modified when state_ is Ready, so we needn't lock consumersMutex_ here - ConsumerList consumerList = consumers_; - - MultiResultCallback multiResultCallback(callback, consumers_.size()); - for (ConsumerList::const_iterator i = consumerList.begin(); i != consumerList.end(); i++) { - (*i)->seekAsync(timestamp, multiResultCallback); - } -} - -void PartitionedConsumerImpl::runPartitionUpdateTask() { - partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); - partitionsUpdateTimer_->async_wait( - std::bind(&PartitionedConsumerImpl::getPartitionMetadata, shared_from_this())); -} - -void PartitionedConsumerImpl::getPartitionMetadata() { - using namespace std::placeholders; - lookupServicePtr_->getPartitionMetadataAsync(topicName_) - .addListener(std::bind(&PartitionedConsumerImpl::handleGetPartitions, shared_from_this(), - std::placeholders::_1, std::placeholders::_2)); -} - -void PartitionedConsumerImpl::handleGetPartitions(Result result, - const LookupDataResultPtr& lookupDataResult) { - if (state_ != Ready) { - return; - } - - if (!result) { - const auto newNumPartitions = static_cast(lookupDataResult->getPartitions()); - Lock consumersLock(consumersMutex_); - const auto currentNumPartitions = getNumPartitions(); - assert(currentNumPartitions == consumers_.size()); - if (newNumPartitions > currentNumPartitions) { - LOG_INFO("new partition count: " << newNumPartitions); - numPartitions_ = newNumPartitions; - const auto config = getSinglePartitionConsumerConfig(); - for (unsigned int i = currentNumPartitions; i < newNumPartitions; i++) { - auto consumer = newInternalConsumer(i, config); - consumer->start(); - consumers_.push_back(consumer); - } - // `runPartitionUpdateTask()` will be called in `handleSinglePartitionConsumerCreated()` - return; - } - } else { - LOG_WARN("Failed to getPartitionMetadata: " << strResult(result)); - } - - runPartitionUpdateTask(); -} - -void PartitionedConsumerImpl::setNegativeAcknowledgeEnabledForTesting(bool enabled) { - Lock lock(mutex_); - for (auto&& c : consumers_) { - c->setNegativeAcknowledgeEnabledForTesting(enabled); - } -} - -bool PartitionedConsumerImpl::isConnected() const { - if (state_ != Ready) { - return false; - } - - Lock consumersLock(consumersMutex_); - const auto consumers = consumers_; - consumersLock.unlock(); - for (const auto& consumer : consumers_) { - if (!consumer->isConnected()) { - return false; - } - } - return true; -} - -uint64_t PartitionedConsumerImpl::getNumberOfConnectedConsumer() { - uint64_t numberOfConnectedConsumer = 0; - Lock consumersLock(consumersMutex_); - const auto consumers = consumers_; - consumersLock.unlock(); - for (const auto& consumer : consumers) { - if (consumer->isConnected()) { - numberOfConnectedConsumer++; - } - } - return numberOfConnectedConsumer; -} - -} // namespace pulsar diff --git a/pulsar-client-cpp/lib/PartitionedConsumerImpl.h b/pulsar-client-cpp/lib/PartitionedConsumerImpl.h deleted file mode 100644 index 8f4faf09954f2..0000000000000 --- a/pulsar-client-cpp/lib/PartitionedConsumerImpl.h +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef PULSAR_PARTITIONED_CONSUMER_HEADER -#define PULSAR_PARTITIONED_CONSUMER_HEADER -#include "lib/TestUtil.h" -#include "ConsumerImpl.h" -#include "ClientImpl.h" -#include -#include - -#include -#include "ConsumerImplBase.h" -#include "lib/UnAckedMessageTrackerDisabled.h" -#include -#include -#include - -namespace pulsar { -class PartitionedConsumerImpl; -class PartitionedConsumerImpl : public ConsumerImplBase, - public std::enable_shared_from_this { - public: - enum PartitionedConsumerState - { - Pending, - Ready, - Closing, - Closed, - Failed - }; - PartitionedConsumerImpl(ClientImplPtr client, const std::string& subscriptionName, - const TopicNamePtr topicName, const unsigned int numPartitions, - const ConsumerConfiguration& conf); - ~PartitionedConsumerImpl(); - // overrided methods from ConsumerImplBase - Future getConsumerCreatedFuture() override; - const std::string& getSubscriptionName() const override; - const std::string& getTopic() const override; - Result receive(Message& msg) override; - Result receive(Message& msg, int timeout) override; - void receiveAsync(ReceiveCallback& callback) override; - void unsubscribeAsync(ResultCallback callback) override; - void acknowledgeAsync(const MessageId& msgId, ResultCallback callback) override; - void acknowledgeCumulativeAsync(const MessageId& msgId, ResultCallback callback) override; - void closeAsync(ResultCallback callback) override; - void start() override; - void shutdown() override; - bool isClosed() override; - bool isOpen() override; - Result pauseMessageListener() override; - Result resumeMessageListener() override; - void redeliverUnacknowledgedMessages() override; - void redeliverUnacknowledgedMessages(const std::set& messageIds) override; - const std::string& getName() const override; - int getNumOfPrefetchedMessages() const override; - void getBrokerConsumerStatsAsync(BrokerConsumerStatsCallback callback) override; - void seekAsync(const MessageId& msgId, ResultCallback callback) override; - void seekAsync(uint64_t timestamp, ResultCallback callback) override; - void negativeAcknowledge(const MessageId& msgId) override; - bool isConnected() const override; - uint64_t getNumberOfConnectedConsumer() override; - - void handleGetConsumerStats(Result, BrokerConsumerStats, LatchPtr, PartitionedBrokerConsumerStatsPtr, - size_t, BrokerConsumerStatsCallback); - - private: - const ClientImplPtr client_; - const std::string subscriptionName_; - const TopicNamePtr topicName_; - unsigned int numPartitions_; - unsigned int numConsumersCreated_ = 0; - const ConsumerConfiguration conf_; - typedef std::vector ConsumerList; - ConsumerList consumers_; - // consumersMutex_ is used to share consumers_ and numPartitions_ - mutable std::mutex consumersMutex_; - mutable std::mutex mutex_; - std::mutex pendingReceiveMutex_; - std::atomic state_{Pending}; - unsigned int unsubscribedSoFar_ = 0; - BlockingQueue messages_; - ExecutorServicePtr listenerExecutor_; - MessageListener messageListener_; - const std::string topic_; - const std::string name_; - const std::string partitionStr_; - ExecutorServicePtr internalListenerExecutor_; - DeadlineTimerPtr partitionsUpdateTimer_; - boost::posix_time::time_duration partitionsUpdateInterval_; - LookupServicePtr lookupServicePtr_; - - unsigned int getNumPartitions() const; - unsigned int getNumPartitionsWithLock() const; - ConsumerConfiguration getSinglePartitionConsumerConfig() const; - ConsumerImplPtr newInternalConsumer(unsigned int partition, const ConsumerConfiguration& config) const; - void handleUnsubscribeAsync(Result result, unsigned int consumerIndex, ResultCallback callback); - void handleSinglePartitionConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, - unsigned int partitionIndex); - void handleSinglePartitionConsumerClose(Result result, unsigned int partitionIndex, - CloseCallback callback); - void notifyResult(CloseCallback closeCallback); - void messageReceived(Consumer consumer, const Message& msg); - void internalListener(Consumer consumer); - void receiveMessages(); - void failPendingReceiveCallback(); - void setNegativeAcknowledgeEnabledForTesting(bool enabled) override; - Promise partitionedConsumerCreatedPromise_; - UnAckedMessageTrackerPtr unAckedMessageTrackerPtr_; - std::queue pendingReceives_; - void runPartitionUpdateTask(); - void getPartitionMetadata(); - void handleGetPartitions(const Result result, const LookupDataResultPtr& lookupDataResult); - - friend class PulsarFriend; - - FRIEND_TEST(ConsumerTest, testPartitionedConsumerUnAckedMessageRedelivery); -}; -typedef std::weak_ptr PartitionedConsumerImplWeakPtr; -typedef std::shared_ptr PartitionedConsumerImplPtr; -} // namespace pulsar -#endif // PULSAR_PARTITIONED_CONSUMER_HEADER diff --git a/pulsar-client-cpp/tests/BasicEndToEndTest.cc b/pulsar-client-cpp/tests/BasicEndToEndTest.cc index be6577523ed7d..b8cfcdb0fd0b5 100644 --- a/pulsar-client-cpp/tests/BasicEndToEndTest.cc +++ b/pulsar-client-cpp/tests/BasicEndToEndTest.cc @@ -1708,7 +1708,7 @@ TEST(BasicEndToEndTest, testSeekOnPartitionedTopic) { ASSERT_EQ(expected.str(), msgReceived.getDataAsString()); ASSERT_EQ(ResultOk, consumer.acknowledge(msgReceived)); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultOk, consumer.close()); + ASSERT_EQ(ResultAlreadyClosed, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } diff --git a/pulsar-client-cpp/tests/ConsumerStatsTest.cc b/pulsar-client-cpp/tests/ConsumerStatsTest.cc index ada86760a2a02..c398a532e68cc 100644 --- a/pulsar-client-cpp/tests/ConsumerStatsTest.cc +++ b/pulsar-client-cpp/tests/ConsumerStatsTest.cc @@ -20,15 +20,12 @@ #include #include #include -#include "CustomRoutingPolicy.h" #include "lib/Future.h" #include "lib/Utils.h" #include "PulsarFriend.h" #include "ConsumerTest.h" #include "HttpHelper.h" #include -#include -#include #include #include @@ -42,8 +39,8 @@ static std::string adminUrl = "http://localhost:8080/"; void partitionedCallbackFunction(Result result, BrokerConsumerStats brokerConsumerStats, long expectedBacklog, Latch& latch, int index, bool accurate) { ASSERT_EQ(result, ResultOk); - PartitionedBrokerConsumerStatsImpl* statsPtr = - (PartitionedBrokerConsumerStatsImpl*)(brokerConsumerStats.getImpl().get()); + MultiTopicsBrokerConsumerStatsImpl* statsPtr = + (MultiTopicsBrokerConsumerStatsImpl*)(brokerConsumerStats.getImpl().get()); LOG_DEBUG(statsPtr); if (accurate) { ASSERT_EQ(expectedBacklog, statsPtr->getBrokerConsumerStats(index).getMsgBacklog()); diff --git a/pulsar-client-cpp/tests/ConsumerTest.cc b/pulsar-client-cpp/tests/ConsumerTest.cc index b61c15a886630..b1fc11cec8fe3 100644 --- a/pulsar-client-cpp/tests/ConsumerTest.cc +++ b/pulsar-client-cpp/tests/ConsumerTest.cc @@ -30,7 +30,6 @@ #include "lib/Future.h" #include "lib/Utils.h" #include "lib/LogUtils.h" -#include "lib/PartitionedConsumerImpl.h" #include "lib/MultiTopicsConsumerImpl.h" #include "HttpHelper.h" @@ -406,8 +405,9 @@ TEST(ConsumerTest, testPartitionedConsumerUnAckedMessageRedelivery) { consumerConfig.setUnAckedMessagesTimeoutMs(unAckedMessagesTimeoutMs); consumerConfig.setTickDurationInMs(tickDurationInMs); ASSERT_EQ(ResultOk, client.subscribe(partitionedTopic, subName, consumerConfig, consumer)); - PartitionedConsumerImplPtr partitionedConsumerImplPtr = - PulsarFriend::getPartitionedConsumerImplPtr(consumer); + + MultiTopicsConsumerImplPtr partitionedConsumerImplPtr = + PulsarFriend::getMultiTopicsConsumerImplPtr(consumer); ASSERT_EQ(numPartitions, partitionedConsumerImplPtr->consumers_.size()); // send messages @@ -442,8 +442,10 @@ TEST(ConsumerTest, testPartitionedConsumerUnAckedMessageRedelivery) { ASSERT_EQ(numOfMessages, partitionedTracker->size()); ASSERT_FALSE(partitionedTracker->isEmpty()); for (auto i = 0; i < numPartitions; i++) { + auto topicName = + "persistent://public/default/" + partitionedTopic + "-partition-" + std::to_string(i); ASSERT_EQ(numOfMessages / numPartitions, messageIds[i].size()); - auto subConsumerPtr = partitionedConsumerImplPtr->consumers_[i]; + auto subConsumerPtr = partitionedConsumerImplPtr->consumers_.find(topicName).value(); auto tracker = static_cast(subConsumerPtr->unAckedMessageTrackerPtr_.get()); ASSERT_EQ(0, tracker->size()); diff --git a/pulsar-client-cpp/tests/PulsarFriend.h b/pulsar-client-cpp/tests/PulsarFriend.h index 2d9b558e0fa16..9325d3877b566 100644 --- a/pulsar-client-cpp/tests/PulsarFriend.h +++ b/pulsar-client-cpp/tests/PulsarFriend.h @@ -23,7 +23,6 @@ #include "lib/ProducerImpl.h" #include "lib/PartitionedProducerImpl.h" #include "lib/ConsumerImpl.h" -#include "lib/PartitionedConsumerImpl.h" #include "lib/MultiTopicsConsumerImpl.h" #include "lib/ReaderImpl.h" @@ -86,10 +85,6 @@ class PulsarFriend { static ReaderImplWeakPtr getReaderImplWeakPtr(Reader reader) { return reader.impl_; } - static std::shared_ptr getPartitionedConsumerImplPtr(Consumer consumer) { - return std::static_pointer_cast(consumer.impl_); - } - static std::shared_ptr getMultiTopicsConsumerImplPtr(Consumer consumer) { return std::static_pointer_cast(consumer.impl_); } From 62dca4081b46d25a860e9ff23eaae338bd86d3f5 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 6 Sep 2022 10:39:35 +0800 Subject: [PATCH 748/823] [fix][cpp] Fix potential segfault when resending messages (#17395) Fixes #17392 ### Motivation All timers in `ProducerImpl` are `std::shared_ptr` objects that can be reset with `nullptr` in `ProducerImpl::cancelTimers`. It could lead to null pointer access in some cases. See https://github.com/apache/pulsar/issues/17392#issuecomment-1233929427 for the analysis. Generally it's not necessary to hold a nullable pointer to the timer. However, to resolve the cyclic reference issue, #5246 reset the shared pointer to reduce the reference count manually. It's not a good solution because we have to perform null check for timers everywhere. The null check still has some race condition issue like: Thread 1: ```c++ if (timer) { // [1] timer is not nullptr timer->async_wait(/* ... */); // [3] timer is null now, see [2] below } ``` Thread 2: ```c++ timer.reset(); // [2] ``` The best solution is to capture `weak_ptr` in timer's callback and call `lock()` to check if the referenced object is still valid. ### Modifications - Change the type of `sendTimer_` and `batchTimer_` to `deadline_timer`, not a `shared_ptr`. - Use `PeriodicTask` instead of the `deadline_timer` for token refresh. - Migrate `weak_from_this()` method from C++17 and capture `weak_from_this()` instead of `shared_from_this()` in callbacks. ### Verifying this change Run the `testResendViaSendCallback` for many times and we can see it won't fail after this patch. ```bash ./tests/main --gtest_filter='BasicEndToEndTest.testResendViaSendCallback' --gtest_repeat=30 ``` (cherry picked from commit 7d6f394b3c8ed693dc4ce1e138e9928930118fdb) --- pulsar-client-cpp/lib/PeriodicTask.cc | 9 ++- pulsar-client-cpp/lib/ProducerImpl.cc | 111 +++++++++++--------------- pulsar-client-cpp/lib/ProducerImpl.h | 13 +-- 3 files changed, 62 insertions(+), 71 deletions(-) diff --git a/pulsar-client-cpp/lib/PeriodicTask.cc b/pulsar-client-cpp/lib/PeriodicTask.cc index 533d38b5efa94..4e91ef5f7e150 100644 --- a/pulsar-client-cpp/lib/PeriodicTask.cc +++ b/pulsar-client-cpp/lib/PeriodicTask.cc @@ -27,9 +27,14 @@ void PeriodicTask::start() { } state_ = Ready; if (periodMs_ >= 0) { - auto self = shared_from_this(); + std::weak_ptr weakSelf{shared_from_this()}; timer_.expires_from_now(boost::posix_time::millisec(periodMs_)); - timer_.async_wait([this, self](const ErrorCode& ec) { handleTimeout(ec); }); + timer_.async_wait([weakSelf](const ErrorCode& ec) { + auto self = weakSelf.lock(); + if (self) { + self->handleTimeout(ec); + } + }); } } diff --git a/pulsar-client-cpp/lib/ProducerImpl.cc b/pulsar-client-cpp/lib/ProducerImpl.cc index a446c2b4b255c..b61897bbf0189 100644 --- a/pulsar-client-cpp/lib/ProducerImpl.cc +++ b/pulsar-client-cpp/lib/ProducerImpl.cc @@ -56,7 +56,9 @@ ProducerImpl::ProducerImpl(ClientImplPtr client, const std::string& topic, const producerStr_("[" + topic_ + ", " + producerName_ + "] "), producerId_(client->newProducerId()), msgSequenceGenerator_(0), - dataKeyGenIntervalSec_(4 * 60 * 60), + batchTimer_(executor_->getIOService()), + sendTimer_(executor_->getIOService()), + dataKeyRefreshTask_(executor_->getIOService(), 4 * 60 * 60 * 1000), memoryLimitController_(client->getMemoryLimitController()) { LOG_DEBUG("ProducerName - " << producerName_ << " Created producer on topic " << topic_ << " id: " << producerId_); @@ -101,7 +103,6 @@ ProducerImpl::ProducerImpl(ClientImplPtr client, const std::string& topic, const LOG_ERROR("Unknown batching type: " << conf_.getBatchingType()); return; } - batchTimer_ = executor_->createDeadlineTimer(); } } @@ -122,19 +123,6 @@ int64_t ProducerImpl::getLastSequenceId() const { return lastSequenceIdPublished const std::string& ProducerImpl::getSchemaVersion() const { return schemaVersion_; } -void ProducerImpl::refreshEncryptionKey(const boost::system::error_code& ec) { - if (ec) { - LOG_DEBUG("Ignoring timer cancelled event, code[" << ec << "]"); - return; - } - - msgCrypto_->addPublicKeyCipher(conf_.getEncryptionKeys(), conf_.getCryptoKeyReader()); - - dataKeyGenTImer_->expires_from_now(boost::posix_time::seconds(dataKeyGenIntervalSec_)); - dataKeyGenTImer_->async_wait( - std::bind(&pulsar::ProducerImpl::refreshEncryptionKey, shared_from_this(), std::placeholders::_1)); -} - void ProducerImpl::connectionOpened(const ClientConnectionPtr& cnx) { if (state_ == Closed) { LOG_DEBUG(getName() << "connectionOpened : Producer is already closed"); @@ -199,11 +187,19 @@ void ProducerImpl::handleCreateProducer(const ClientConnectionPtr& cnx, Result r backoff_.reset(); lock.unlock(); - if (!dataKeyGenTImer_ && conf_.isEncryptionEnabled()) { - dataKeyGenTImer_ = executor_->createDeadlineTimer(); - dataKeyGenTImer_->expires_from_now(boost::posix_time::seconds(dataKeyGenIntervalSec_)); - dataKeyGenTImer_->async_wait(std::bind(&pulsar::ProducerImpl::refreshEncryptionKey, - shared_from_this(), std::placeholders::_1)); + if (conf_.isEncryptionEnabled()) { + auto weakSelf = weak_from_this(); + dataKeyRefreshTask_.setCallback([this, weakSelf](const PeriodicTask::ErrorCode& ec) { + auto self = weakSelf.lock(); + if (!self) { + return; + } + if (ec) { + LOG_ERROR("DataKeyRefresh timer failed: " << ec.message()); + return; + } + msgCrypto_->addPublicKeyCipher(conf_.getEncryptionKeys(), conf_.getCryptoKeyReader()); + }); } // if the producer is lazy the send timeout timer is already running @@ -437,10 +433,29 @@ void ProducerImpl::sendAsync(const Message& msg, SendCallback callback) { bool isFirstMessage = batchMessageContainer_->isFirstMessageToAdd(msg); bool isFull = batchMessageContainer_->add(msg, cb); if (isFirstMessage) { - batchTimer_->expires_from_now( + batchTimer_.expires_from_now( boost::posix_time::milliseconds(conf_.getBatchingMaxPublishDelayMs())); - batchTimer_->async_wait(std::bind(&ProducerImpl::batchMessageTimeoutHandler, shared_from_this(), - std::placeholders::_1)); + auto weakSelf = weak_from_this(); + batchTimer_.async_wait([this, weakSelf](const boost::system::error_code& ec) { + auto self = weakSelf.lock(); + if (!self) { + return; + } + if (ec) { + LOG_DEBUG(getName() << " Ignoring timer cancelled event, code[" << ec << "]"); + return; + } + LOG_DEBUG(getName() << " - Batch Message Timer expired"); + + // ignore if the producer is already closing/closed + const auto state = state_.load(); + if (state == Pending || state == Ready) { + Lock lock(mutex_); + auto failures = batchMessageAndSend(); + lock.unlock(); + failures.complete(); + } + }); } if (isFull) { @@ -499,7 +514,7 @@ void ProducerImpl::releaseSemaphoreForSendOp(const OpSendMsg& op) { PendingFailures ProducerImpl::batchMessageAndSend(const FlushCallback& flushCallback) { PendingFailures failures; LOG_DEBUG("batchMessageAndSend " << *batchMessageContainer_); - batchTimer_->cancel(); + batchTimer_.cancel(); batchMessageContainer_->processAndClear( [this, &failures](Result result, const OpSendMsg& opSendMsg) { @@ -536,23 +551,6 @@ void ProducerImpl::sendMessage(const OpSendMsg& op) { } } -void ProducerImpl::batchMessageTimeoutHandler(const boost::system::error_code& ec) { - if (ec) { - LOG_DEBUG(getName() << " Ignoring timer cancelled event, code[" << ec << "]"); - return; - } - LOG_DEBUG(getName() << " - Batch Message Timer expired"); - - // ignore if the producer is already closing/closed - const auto state = state_.load(); - if (state == Pending || state == Ready) { - Lock lock(mutex_); - auto failures = batchMessageAndSend(); - lock.unlock(); - failures.complete(); - } -} - void ProducerImpl::printStats() { if (batchMessageContainer_) { LOG_INFO("Producer - " << producerStr_ << ", [batchMessageContainer = " << *batchMessageContainer_ @@ -809,20 +807,9 @@ void ProducerImpl::shutdown() { } void ProducerImpl::cancelTimers() { - if (dataKeyGenTImer_) { - dataKeyGenTImer_->cancel(); - dataKeyGenTImer_.reset(); - } - - if (batchTimer_) { - batchTimer_->cancel(); - batchTimer_.reset(); - } - - if (sendTimer_) { - sendTimer_->cancel(); - sendTimer_.reset(); - } + dataKeyRefreshTask_.stop(); + batchTimer_.cancel(); + sendTimer_.cancel(); } bool ProducerImplCmp::operator()(const ProducerImplPtr& a, const ProducerImplPtr& b) const { @@ -837,20 +824,16 @@ uint64_t ProducerImpl::getNumberOfConnectedProducer() { return isConnected() ? 1 bool ProducerImpl::isStarted() const { return state_ != NotStarted; } void ProducerImpl::startSendTimeoutTimer() { - // Initialize the sendTimer only once per producer and only when producer timeout is - // configured. Set the timeout as configured value and asynchronously wait for the - // timeout to happen. - if (!sendTimer_ && conf_.getSendTimeout() > 0) { - sendTimer_ = executor_->createDeadlineTimer(); + if (conf_.getSendTimeout() > 0) { asyncWaitSendTimeout(milliseconds(conf_.getSendTimeout())); } } void ProducerImpl::asyncWaitSendTimeout(DurationType expiryTime) { - sendTimer_->expires_from_now(expiryTime); + sendTimer_.expires_from_now(expiryTime); - ProducerImplBaseWeakPtr weakSelf = shared_from_this(); - sendTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + auto weakSelf = weak_from_this(); + sendTimer_.async_wait([weakSelf](const boost::system::error_code& err) { auto self = weakSelf.lock(); if (self) { std::static_pointer_cast(self)->handleSendTimeout(err); @@ -858,5 +841,7 @@ void ProducerImpl::asyncWaitSendTimeout(DurationType expiryTime) { }); } +ProducerImplWeakPtr ProducerImpl::weak_from_this() noexcept { return shared_from_this(); } + } // namespace pulsar /* namespace pulsar */ diff --git a/pulsar-client-cpp/lib/ProducerImpl.h b/pulsar-client-cpp/lib/ProducerImpl.h index a9eb12b7e9409..3db10533a1286 100644 --- a/pulsar-client-cpp/lib/ProducerImpl.h +++ b/pulsar-client-cpp/lib/ProducerImpl.h @@ -35,6 +35,7 @@ #include "BatchMessageContainerBase.h" #include "PendingFailures.h" #include "Semaphore.h" +#include "PeriodicTask.h" using namespace pulsar; @@ -83,6 +84,9 @@ class ProducerImpl : public HandlerBase, int32_t partition() const noexcept { return partition_; } + // NOTE: this method is introduced into `enable_shared_from_this` since C++17 + ProducerImplWeakPtr weak_from_this() noexcept; + protected: ProducerStatsBasePtr producerStatsBasePtr_; @@ -92,8 +96,6 @@ class ProducerImpl : public HandlerBase, void sendMessage(const OpSendMsg& opSendMsg); - void batchMessageTimeoutHandler(const boost::system::error_code& ec); - void startSendTimeoutTimer(); friend class PulsarFriend; @@ -147,13 +149,13 @@ class ProducerImpl : public HandlerBase, proto::BaseCommand cmd_; std::unique_ptr batchMessageContainer_; - DeadlineTimerPtr batchTimer_; + boost::asio::deadline_timer batchTimer_; PendingFailures batchMessageAndSend(const FlushCallback& flushCallback = nullptr); volatile int64_t lastSequenceIdPublished_; std::string schemaVersion_; - DeadlineTimerPtr sendTimer_; + boost::asio::deadline_timer sendTimer_; void handleSendTimeout(const boost::system::error_code& err); using DurationType = typename boost::asio::deadline_timer::duration_type; void asyncWaitSendTimeout(DurationType expiryTime); @@ -167,8 +169,7 @@ class ProducerImpl : public HandlerBase, void failPendingMessages(Result result, bool withLock); MessageCryptoPtr msgCrypto_; - DeadlineTimerPtr dataKeyGenTImer_; - uint32_t dataKeyGenIntervalSec_; + PeriodicTask dataKeyRefreshTask_; MemoryLimitController& memoryLimitController_; }; From a6842aa3862e6335d157a480a9c1497d9d44de19 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 14 Sep 2022 17:55:10 +0800 Subject: [PATCH 749/823] [fix][admin] Add SNI header when tlsHostnameVerification is not enabled (#17543) (cherry picked from commit 99b52ebfcbcd97793c49e4b64596108b480f55b3) --- .../internal/http/AsyncHttpConnector.java | 5 +++ .../apache/pulsar/client/impl/HttpClient.java | 5 +++ .../client/util/WithSNISslEngineFactory.java | 42 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java index 2b08bfc0048de..b8e256268ea78 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java @@ -51,6 +51,7 @@ import org.apache.pulsar.client.api.KeyStoreParams; import org.apache.pulsar.client.impl.PulsarServiceNameResolver; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.util.WithSNISslEngineFactory; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.SecurityUtility; import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; @@ -167,6 +168,10 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, conf.getTlsProtocols()); } confBuilder.setSslContext(sslCtx); + if (!conf.isTlsHostnameVerificationEnable()) { + confBuilder.setSslEngineFactory(new WithSNISslEngineFactory(serviceNameResolver + .resolveHostUri().getHost())); + } } } confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index 323f3bcad5e70..12830ed5a6b78 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -43,6 +43,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.util.WithSNISslEngineFactory; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.SecurityUtility; import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; @@ -134,6 +135,10 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, conf.getTlsTrustCertsFilePath(), conf.getTlsCiphers(), conf.getTlsProtocols()); } confBuilder.setSslContext(sslCtx); + if (!conf.isTlsHostnameVerificationEnable()) { + confBuilder.setSslEngineFactory(new WithSNISslEngineFactory(serviceNameResolver + .resolveHostUri().getHost())); + } } confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java new file mode 100644 index 0000000000000..965a7f2aec328 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.util; + +import java.util.Collections; +import javax.net.ssl.SNIHostName; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLParameters; +import org.asynchttpclient.AsyncHttpClientConfig; +import org.asynchttpclient.netty.ssl.DefaultSslEngineFactory; + +public class WithSNISslEngineFactory extends DefaultSslEngineFactory { + private final String host; + + public WithSNISslEngineFactory(String host) { + this.host = host; + } + + @Override + protected void configureSslEngine(SSLEngine sslEngine, AsyncHttpClientConfig config) { + super.configureSslEngine(sslEngine, config); + SSLParameters params = sslEngine.getSSLParameters(); + params.setServerNames(Collections.singletonList(new SNIHostName(host))); + sslEngine.setSSLParameters(params); + } +} From db6393aee84323ec42dbed6b6909935fbae5a81a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 14 Sep 2022 20:33:10 -0700 Subject: [PATCH 750/823] [C++] Reset `havePendingPingRequest` flag for any data received from broker (#17658) --- pulsar-client-cpp/lib/ClientConnection.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 95d1bf83adcb5..69f61ca0de415 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -810,6 +810,10 @@ void ClientConnection::handleIncomingCommand() { } case Ready: { + // Since we are receiving data from the connection, we are assuming that for now the connection is + // still working well. + havePendingPingRequest_ = false; + // Handle normal commands switch (incomingCmd_.type()) { case BaseCommand::SEND_RECEIPT: { @@ -1165,7 +1169,6 @@ void ClientConnection::handleIncomingCommand() { case BaseCommand::PONG: { LOG_DEBUG(cnxString_ << "Received response to ping message"); - havePendingPingRequest_ = false; break; } From fb3667328e4988dd4321bc557e821cdeb2026cff Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 16 Sep 2022 11:34:53 +0800 Subject: [PATCH 751/823] Fix wrong consumers size: execute `callback` before executing `readerCreatedCallback_`. (#17325) (#17629) Fixes #14848 ### Motivation We should execute `callback` before executing `readerCreatedCallback_`, otherwise, we may get the wrong consumers size. More see: https://github.com/apache/pulsar/blob/e23d312c04da1d82d35f9e2faf8a446f8e8a4eeb/pulsar-client-cpp/lib/ReaderImpl.cc#L84-L92 https://github.com/apache/pulsar/blob/c48a3243287c7d775459b6437d9f4b24ed44cf4c/pulsar-client-cpp/lib/ClientImpl.cc#L250-L254 ### Modifications execute `callback` before executing `readerCreatedCallback_` (cherry picked from commit 3bc50a4277a102c1683041d7311ebd2266f32d9c) --- pulsar-client-cpp/lib/ReaderImpl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-cpp/lib/ReaderImpl.cc b/pulsar-client-cpp/lib/ReaderImpl.cc index c660c01ab2a16..5f78068228ff4 100644 --- a/pulsar-client-cpp/lib/ReaderImpl.cc +++ b/pulsar-client-cpp/lib/ReaderImpl.cc @@ -84,8 +84,8 @@ void ReaderImpl::start(const MessageId& startMessageId, consumer_->getConsumerCreatedFuture().addListener( [this, self, callback](Result result, const ConsumerImplBaseWeakPtr& weakConsumerPtr) { if (result == ResultOk) { - readerCreatedCallback_(result, Reader(self)); callback(weakConsumerPtr); + readerCreatedCallback_(result, Reader(self)); } else { readerCreatedCallback_(result, {}); } From 4dc61755d0e8cce3e55b9afa6ec33b0241aef5f4 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 17 Sep 2022 12:22:03 +0800 Subject: [PATCH 752/823] [branch-2.9][fix][broker]Persist cursor info error when cursor close (#17604) --- .../mledger/impl/ManagedCursorImpl.java | 18 ++++++++-- .../mledger/impl/ManagedCursorTest.java | 35 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 1b54bb16a3d46..8deb70c6a19e5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -2455,8 +2455,22 @@ public void asyncClose(final AsyncCallbacks.CloseCallback callback, final Object callback.closeComplete(ctx); return; } - persistPositionWhenClosing(lastMarkDeleteEntry.newPosition, lastMarkDeleteEntry.properties, callback, ctx); - STATE_UPDATER.set(this, State.Closed); + persistPositionWhenClosing(lastMarkDeleteEntry.newPosition, lastMarkDeleteEntry.properties, + new AsyncCallbacks.CloseCallback(){ + + @Override + public void closeComplete(Object ctx) { + STATE_UPDATER.set(ManagedCursorImpl.this, State.Closed); + callback.closeComplete(ctx); + } + + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + log.warn("[{}] [{}] persistent position failure when closing, the state will remain in" + + " state-closing and will no longer work", ledger.getName(), name); + callback.closeFailed(exception, ctx); + } + }, ctx); } /** diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index db351aa2a74f6..5f1ed11836b54 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -69,6 +69,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -102,6 +103,7 @@ import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.reflect.Whitebox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -129,6 +131,39 @@ public static Object[][] useOpenRangeSet() { } + @Test + public void testCloseCursor() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxUnackedRangesToPersistInZk(0); + config.setThrottleMarkDelete(0); + ManagedLedger ledger = factory.open("my_test_ledger", config); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); + // Write some data. + ledger.addEntry(new byte[]{1}); + ledger.addEntry(new byte[]{2}); + ledger.addEntry(new byte[]{3}); + ledger.addEntry(new byte[]{4}); + ledger.addEntry(new byte[]{5}); + // Persistent cursor info to ledger. + c1.delete(PositionImpl.get(c1.getReadPosition().getLedgerId(), c1.getReadPosition().getEntryId())); + Awaitility.await().until(() ->c1.getStats().getPersistLedgerSucceed() > 0); + // Make cursor ledger can not work. + closeCursorLedger(c1); + c1.delete(PositionImpl.get(c1.getReadPosition().getLedgerId(), c1.getReadPosition().getEntryId() + 2)); + ledger.close(); + } + + private static void closeCursorLedger(ManagedCursorImpl managedCursor) { + Awaitility.await().until(() -> { + LedgerHandle ledgerHandle = Whitebox.getInternalState(managedCursor, "cursorLedger"); + if (ledgerHandle == null) { + return false; + } + ledgerHandle.close(); + return true; + }); + } + @Test(timeOut = 20000) void readFromEmptyLedger() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); From ab78a7e0c912d014aa7f9ed3e396b63f9e6ba9b1 Mon Sep 17 00:00:00 2001 From: Guangning E Date: Wed, 14 Sep 2022 09:47:28 +0800 Subject: [PATCH 753/823] Update proxy lookup throw exception type (#17600) (cherry picked from commit 52a380f7e930e633dcefcbdb29ee1460e867dcfe) --- .../pulsar/proxy/server/LookupProxyHandler.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java index d9d208085fd2a..cbebfa34fa1f4 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java @@ -151,7 +151,7 @@ private void performLookup(long clientRequestId, String topic, String brokerServ if (t != null) { log.warn("[{}] Failed to lookup topic {}: {}", clientAddress, topic, t.getMessage()); proxyConnection.ctx().writeAndFlush( - Commands.newLookupErrorResponse(ServerError.ServiceNotReady, t.getMessage(), clientRequestId)); + Commands.newLookupErrorResponse(getServerError(t), t.getMessage(), clientRequestId)); } else { String brokerUrl = connectWithTLS ? r.brokerUrlTls : r.brokerUrl; if (r.redirect) { @@ -179,7 +179,7 @@ private void performLookup(long clientRequestId, String topic, String brokerServ }).exceptionally(ex -> { // Failed to connect to backend broker proxyConnection.ctx().writeAndFlush( - Commands.newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), clientRequestId)); + Commands.newLookupErrorResponse(getServerError(ex), ex.getMessage(), clientRequestId)); return null; }); } @@ -249,7 +249,7 @@ private void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata par }); }).exceptionally(ex -> { // Failed to connect to backend broker - proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.ServiceNotReady, + proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(getServerError(ex), ex.getMessage(), clientRequestId)); return null; }); @@ -322,7 +322,7 @@ private void performGetTopicsOfNamespace(long clientRequestId, if (t != null) { log.warn("[{}] Failed to get TopicsOfNamespace {}: {}", clientAddress, namespaceName, t.getMessage()); proxyConnection.ctx().writeAndFlush( - Commands.newError(clientRequestId, ServerError.ServiceNotReady, t.getMessage())); + Commands.newError(clientRequestId, getServerError(t), t.getMessage())); } else { proxyConnection.ctx().writeAndFlush( Commands.newGetTopicsOfNamespaceResponse(r, clientRequestId)); @@ -333,7 +333,7 @@ private void performGetTopicsOfNamespace(long clientRequestId, }).exceptionally(ex -> { // Failed to connect to backend broker proxyConnection.ctx().writeAndFlush( - Commands.newError(clientRequestId, ServerError.ServiceNotReady, ex.getMessage())); + Commands.newError(clientRequestId, getServerError(ex), ex.getMessage())); return null; }); } @@ -376,7 +376,7 @@ public void handleGetSchema(CommandGetSchema commandGetSchema) { if (t != null) { log.warn("[{}] Failed to get schema {}: {}", clientAddress, topic, t); proxyConnection.ctx().writeAndFlush( - Commands.newError(clientRequestId, ServerError.ServiceNotReady, t.getMessage())); + Commands.newError(clientRequestId, getServerError(t), t.getMessage())); } else { proxyConnection.ctx().writeAndFlush( Commands.newGetSchemaResponse(clientRequestId, r)); @@ -387,7 +387,7 @@ public void handleGetSchema(CommandGetSchema commandGetSchema) { }).exceptionally(ex -> { // Failed to connect to backend broker proxyConnection.ctx().writeAndFlush( - Commands.newError(clientRequestId, ServerError.ServiceNotReady, ex.getMessage())); + Commands.newError(clientRequestId, getServerError(ex), ex.getMessage())); return null; }); From d1d1273e2c4a1cdb5484345f778f97bb6e6312e9 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Sun, 18 Sep 2022 06:27:59 -0700 Subject: [PATCH 754/823] [fix][metadata] Set revalidateAfterReconnection true for certain failures (#17664) (cherry picked from commit c40c7ee9fb35eafea9c9923dfcf62706ea5d36bf) --- .../pulsar/metadata/coordination/impl/ResourceLockImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java index ec9d3c42904b4..afc7feeaa2530 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java @@ -197,6 +197,7 @@ synchronized void lockWasInvalidated() { // We failed to revalidate the lock due to connectivity issue // Continue assuming we hold the lock, until we can revalidate it, either // on Reconnected or SessionReestablished events. + revalidateAfterReconnection = true; log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path, ex.getCause().getMessage()); } From 15da3ed5d4622aeb5317698f6843f48c3eb309b1 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 20 Sep 2022 15:41:15 +0800 Subject: [PATCH 755/823] [fix][metadata] Cleanup state when lock revalidation gets `LockBusyException` (#17700) ### Motivation In the production environment, we found two brokers holding the same valid locks. and one has an exceptional revalidate future with `lockBusyException`. after reading the code, there may forget the reset the cache and complete expire exception when getting lockBusyException. (cherry picked from commit 955ae340293185ba17b9d5baf1a0f5bd7e547186) --- .../coordination/impl/LockManagerImpl.java | 2 +- .../coordination/impl/ResourceLockImpl.java | 48 ++++++++------- .../pulsar/metadata/LockManagerTest.java | 59 +++++++++++++++++++ 3 files changed, 86 insertions(+), 23 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java index f45c0c7dd7d17..046b174d35218 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java @@ -125,7 +125,7 @@ private void handleSessionEvent(SessionEvent se) { if (se == SessionEvent.SessionReestablished) { log.info("Metadata store session has been re-established. Revalidating all the existing locks."); for (ResourceLockImpl lock : locks.values()) { - futures.add(lock.revalidate(lock.getValue())); + futures.add(lock.revalidate(lock.getValue(), true)); } } else if (se == SessionEvent.Reconnected) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java index afc7feeaa2530..05758ef114648 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException; @@ -128,7 +129,7 @@ synchronized CompletableFuture acquire(T newValue) { .thenRun(() -> result.complete(null)) .exceptionally(ex -> { if (ex.getCause() instanceof LockBusyException) { - revalidate(newValue) + revalidate(newValue, false) .thenAccept(__ -> result.complete(null)) .exceptionally(ex1 -> { result.completeExceptionally(ex1); @@ -185,38 +186,21 @@ synchronized void lockWasInvalidated() { } log.info("Lock on resource {} was invalidated", path); - revalidate(value) - .thenRun(() -> log.info("Successfully revalidated the lock on {}", path)) - .exceptionally(ex -> { - synchronized (ResourceLockImpl.this) { - if (ex.getCause() instanceof BadVersionException) { - log.warn("Failed to revalidate the lock at {}. Marked as expired", path); - state = State.Released; - expiredFuture.complete(null); - } else { - // We failed to revalidate the lock due to connectivity issue - // Continue assuming we hold the lock, until we can revalidate it, either - // on Reconnected or SessionReestablished events. - revalidateAfterReconnection = true; - log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path, - ex.getCause().getMessage()); - } - } - return null; - }); + revalidate(value, true) + .thenRun(() -> log.info("Successfully revalidated the lock on {}", path)); } synchronized CompletableFuture revalidateIfNeededAfterReconnection() { if (revalidateAfterReconnection) { revalidateAfterReconnection = false; log.warn("Revalidate lock at {} after reconnection", path); - return revalidate(value); + return revalidate(value, true); } else { return CompletableFuture.completedFuture(null); } } - synchronized CompletableFuture revalidate(T newValue) { + synchronized CompletableFuture revalidate(T newValue, boolean revalidateAfterReconnection) { if (revalidateFuture == null || revalidateFuture.isDone()) { revalidateFuture = doRevalidate(newValue); } else { @@ -234,6 +218,26 @@ synchronized CompletableFuture revalidate(T newValue) { }); revalidateFuture = newFuture; } + revalidateFuture.exceptionally(ex -> { + synchronized (ResourceLockImpl.this) { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (!revalidateAfterReconnection || realCause instanceof BadVersionException + || realCause instanceof LockBusyException) { + log.warn("Failed to revalidate the lock at {}. Marked as expired. {}", + path, realCause.getMessage()); + state = State.Released; + expiredFuture.complete(null); + } else { + // We failed to revalidate the lock due to connectivity issue + // Continue assuming we hold the lock, until we can revalidate it, either + // on Reconnected or SessionReestablished events. + ResourceLockImpl.this.revalidateAfterReconnection = true; + log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path, + realCause.getMessage()); + } + } + return null; + }); return revalidateFuture; } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java index da080baacab58..ef94f2f3fe22a 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java @@ -20,6 +20,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -29,6 +31,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import lombok.Cleanup; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -286,4 +289,60 @@ public void revalidateLockOnDifferentSession(String provider, Supplier u assertEquals(new String(store1.get(path2).join().get().getValue()), "\"value-1\""); }); } + + @Test(dataProvider = "impl") + public void testCleanUpStateWhenRevalidationGotLockBusy(String provider, Supplier urlSupplier) + throws Exception { + + if (provider.equals("Memory") || provider.equals("RocksDB")) { + // Local memory provider doesn't really have the concept of multiple sessions + return; + } + + @Cleanup + MetadataStoreExtended store1 = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + @Cleanup + MetadataStoreExtended store2 = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + + @Cleanup + CoordinationService cs1 = new CoordinationServiceImpl(store1); + @Cleanup + LockManager lm1 = cs1.getLockManager(String.class); + + @Cleanup + CoordinationService cs2 = new CoordinationServiceImpl(store2); + @Cleanup + LockManager lm2 = cs2.getLockManager(String.class); + + String path1 = newKey(); + + ResourceLock lock1 = lm1.acquireLock(path1, "value-1").join(); + AtomicReference> lock2 = new AtomicReference<>(); + // lock 2 will steal the distributed lock first. + Awaitility.await().until(()-> { + // Ensure steal the lock success. + try { + lock2.set(lm2.acquireLock(path1, "value-1").join()); + return true; + } catch (Exception ex) { + return false; + } + }); + + // Since we can steal the lock repeatedly, we don't know which one will get it. + // But we can verify the final state. + Awaitility.await().untilAsserted(() -> { + if (lock1.getLockExpiredFuture().isDone()) { + assertTrue(lm1.listLocks(path1).join().isEmpty()); + assertFalse(lock2.get().getLockExpiredFuture().isDone()); + } else if (lock2.get().getLockExpiredFuture().isDone()) { + assertTrue(lm2.listLocks(path1).join().isEmpty()); + assertFalse(lock1.getLockExpiredFuture().isDone()); + } else { + fail("unexpected behaviour"); + } + }); + } } From ce2c15782b21ae82aacc1be987d037bce9d276f0 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 20 Sep 2022 16:25:10 +0800 Subject: [PATCH 756/823] =?UTF-8?q?[branch-2.9][cherry-pick]=20Fix=20produ?= =?UTF-8?q?cer/consume=20permission=20can=E2=80=99t=20get=20schema.=20(#17?= =?UTF-8?q?730)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../admin/impl/SchemasResourceBase.java | 22 ++- .../admin/AdminApiSchemaWithAuthTest.java | 145 ++++++++++++++++++ 2 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 5b119ec881db4..304b311cbea54 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -38,6 +38,7 @@ import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.protocol.schema.DeleteSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetAllVersionsSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetSchemaResponse; @@ -83,7 +84,7 @@ private String getSchemaId() { } public void getSchema(boolean authoritative, AsyncResponse response) { - validateDestinationAndAdminOperation(authoritative); + validateOwnershipAndOperation(authoritative, TopicOperation.GET_METADATA); String schemaId = getSchemaId(); pulsar().getSchemaRegistryService().getSchema(schemaId).handle((schema, error) -> { handleGetSchemaResponse(response, schema, error); @@ -92,7 +93,7 @@ public void getSchema(boolean authoritative, AsyncResponse response) { } public void getSchema(boolean authoritative, String version, AsyncResponse response) { - validateDestinationAndAdminOperation(authoritative); + validateOwnershipAndOperation(authoritative, TopicOperation.GET_METADATA); String schemaId = getSchemaId(); ByteBuffer bbVersion = ByteBuffer.allocate(Long.BYTES); bbVersion.putLong(Long.parseLong(version)); @@ -104,7 +105,7 @@ public void getSchema(boolean authoritative, String version, AsyncResponse respo } public void getAllSchemas(boolean authoritative, AsyncResponse response) { - validateDestinationAndAdminOperation(authoritative); + validateOwnershipAndOperation(authoritative, TopicOperation.GET_METADATA); String schemaId = getSchemaId(); pulsar().getSchemaRegistryService().trimDeletedSchemaAndGetList(schemaId).handle((schema, error) -> { @@ -208,7 +209,7 @@ public void testCompatibility(PostSchemaPayload payload, boolean authoritative, public void getVersionBySchema( PostSchemaPayload payload, boolean authoritative, AsyncResponse response) { - validateDestinationAndAdminOperation(authoritative); + validateOwnershipAndOperation(authoritative, TopicOperation.GET_METADATA); String schemaId = getSchemaId(); @@ -302,5 +303,18 @@ private void validateDestinationAndAdminOperation(boolean authoritative) { } } + private void validateOwnershipAndOperation(boolean authoritative, TopicOperation operation) { + try { + validateTopicOwnership(topicName, authoritative); + validateTopicOperation(topicName, operation); + } catch (RestException e) { + if (e.getResponse().getStatus() == Response.Status.UNAUTHORIZED.getStatusCode()) { + throw new RestException(Response.Status.UNAUTHORIZED, e.getMessage()); + } else { + throw e; + } + } + } + private static final Logger log = LoggerFactory.getLogger(SchemasResourceBase.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java new file mode 100644 index 0000000000000..6e8fa4c80275e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.assertThrows; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import java.util.Base64; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.crypto.SecretKey; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit tests for schema admin api. + */ +@Slf4j +@Test(groups = "broker-admin") +public class AdminApiSchemaWithAuthTest extends MockedPulsarServiceBaseTest { + + private static final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + private static final String ADMIN_TOKEN = Jwts.builder().setSubject("admin").signWith(SECRET_KEY).compact(); + private static final String CONSUME_TOKEN = Jwts.builder().setSubject("consumer").signWith(SECRET_KEY).compact(); + + @BeforeMethod + @Override + public void setup() throws Exception { + conf.setAuthorizationEnabled(true); + conf.setAuthenticationEnabled(true); + conf.getProperties().setProperty("tokenSecretKey", "data:;base64," + + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + Set superUserRoles = new HashSet<>(); + superUserRoles.add("admin"); + conf.setSuperUserRoles(superUserRoles); + conf.setAuthenticationProviders(providers); + conf.setSystemTopicEnabled(false); + conf.setTopicLevelPoliciesEnabled(false); + super.internalSetup(); + + PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null + ? brokerUrl.toString() : brokerUrlTls.toString()) + .authentication(AuthenticationToken.class.getName(), + ADMIN_TOKEN); + admin = Mockito.spy(pulsarAdminBuilder.build()); + + // Setup namespaces + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + TenantInfoImpl tenantInfo = new TenantInfoImpl(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test")); + admin.tenants().createTenant("schematest", tenantInfo); + admin.namespaces().createNamespace("schematest/test", Sets.newHashSet("test")); + } + + @AfterMethod(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testGetCreateDeleteSchema() throws Exception { + String topicName = "persistent://schematest/test/testCreateSchema"; + PulsarAdmin adminWithoutPermission = PulsarAdmin.builder() + .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) + .build(); + PulsarAdmin adminWithAdminPermission = PulsarAdmin.builder() + .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) + .authentication(AuthenticationToken.class.getName(), ADMIN_TOKEN) + .build(); + PulsarAdmin adminWithConsumePermission = PulsarAdmin.builder() + .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) + .authentication(AuthenticationToken.class.getName(), CONSUME_TOKEN) + .build(); + admin.topics().grantPermission(topicName, "consumer", EnumSet.of(AuthAction.consume)); + admin.topics().grantPermission(topicName, "producer", EnumSet.of(AuthAction.produce)); + + SchemaInfo si = Schema.BOOL.getSchemaInfo(); + assertThrows(PulsarAdminException.class, () -> adminWithoutPermission.schemas().createSchema(topicName, si)); + adminWithAdminPermission.schemas().createSchema(topicName, si); + + assertThrows(PulsarAdminException.class, () -> adminWithoutPermission.schemas().getSchemaInfo(topicName)); + SchemaInfo readSi = adminWithConsumePermission.schemas().getSchemaInfo(topicName); + ((SchemaInfoImpl) readSi).setTimestamp(0); + assertEquals(readSi, si); + + assertThrows(PulsarAdminException.class, () -> adminWithoutPermission.schemas().getSchemaInfo(topicName, 0)); + readSi = adminWithConsumePermission.schemas().getSchemaInfo(topicName, 0); + ((SchemaInfoImpl) readSi).setTimestamp(0); + assertEquals(readSi, si); + List allSchemas = adminWithConsumePermission.schemas().getAllSchemas(topicName); + assertEquals(allSchemas.size(), 1); + + SchemaInfo schemaInfo2 = Schema.BOOL.getSchemaInfo(); + assertThrows(PulsarAdminException.class, () -> + adminWithoutPermission.schemas().testCompatibility(topicName, schemaInfo2)); + assertTrue(adminWithAdminPermission.schemas().testCompatibility(topicName, schemaInfo2).isCompatibility()); + + assertThrows(PulsarAdminException.class, () -> + adminWithoutPermission.schemas().getVersionBySchema(topicName, si)); + Long versionBySchema = adminWithConsumePermission.schemas().getVersionBySchema(topicName, si); + assertEquals(versionBySchema, Long.valueOf(0L)); + + assertThrows(PulsarAdminException.class, () -> adminWithoutPermission.schemas().deleteSchema(topicName)); + adminWithAdminPermission.schemas().deleteSchema(topicName); + } +} From bc69cfbe9d3cf06fc6e407c284eed45f095b41be Mon Sep 17 00:00:00 2001 From: Mark Silcox <63227862+marksilcox@users.noreply.github.com> Date: Tue, 20 Sep 2022 09:38:06 +0100 Subject: [PATCH 757/823] [fix][broker][functions-worker] Ensure prometheus metrics are grouped by type (#8407, #13865) (#17618) --- .../prometheus/AggregatedNamespaceStats.java | 2 +- .../prometheus/NamespaceStatsAggregator.java | 354 ++++++----- .../prometheus/PrometheusMetricStreams.java | 75 +++ .../PrometheusMetricsGenerator.java | 60 +- .../broker/stats/prometheus/TopicStats.java | 598 ++++++++---------- .../prometheus/TransactionAggregator.java | 321 ++++------ .../metrics/PrometheusTextFormatUtil.java | 32 - .../broker/stats/PrometheusMetricsTest.java | 58 ++ .../PrometheusMetricStreamsTest.java | 85 +++ .../common/util/SimpleTextOutputStream.java | 13 +- .../instance/stats/PrometheusTextFormat.java | 5 + .../functions/worker/WorkerStatsManager.java | 5 + 12 files changed, 878 insertions(+), 730 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 5610dbab218e0..1980af91b7b54 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -96,7 +96,7 @@ void updateStats(TopicStats stats) { stats.replicationStats.forEach((n, as) -> { AggregatedReplicationStats replStats = - replicationStats.computeIfAbsent(n, k -> new AggregatedReplicationStats()); + replicationStats.computeIfAbsent(n, k -> new AggregatedReplicationStats()); replStats.msgRateIn += as.msgRateIn; replStats.msgRateOut += as.msgRateOut; replStats.msgThroughputIn += as.msgThroughputIn; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 16e438e2a2eb3..29915f071c099 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -19,8 +19,11 @@ package org.apache.pulsar.broker.stats.prometheus; import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -32,7 +35,6 @@ import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; -import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.CompactorMXBean; @@ -40,72 +42,75 @@ @Slf4j public class NamespaceStatsAggregator { - private static FastThreadLocal localNamespaceStats = + private static final FastThreadLocal localNamespaceStats = new FastThreadLocal() { @Override - protected AggregatedNamespaceStats initialValue() throws Exception { + protected AggregatedNamespaceStats initialValue() { return new AggregatedNamespaceStats(); } }; - private static FastThreadLocal localTopicStats = new FastThreadLocal() { + private static final FastThreadLocal localTopicStats = new FastThreadLocal() { @Override - protected TopicStats initialValue() throws Exception { + protected TopicStats initialValue() { return new TopicStats(); } }; public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, SimpleTextOutputStream stream) { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + PrometheusMetricStreams stream) { String cluster = pulsar.getConfiguration().getClusterName(); AggregatedNamespaceStats namespaceStats = localNamespaceStats.get(); - TopicStats.resetTypes(); TopicStats topicStats = localTopicStats.get(); + Optional compactorMXBean = getCompactorMXBean(pulsar); + LongAdder topicsCount = new LongAdder(); + Map localNamespaceTopicCount = new HashMap<>(); printDefaultBrokerStats(stream, cluster); - Optional compactorMXBean = getCompactorMXBean(pulsar); - LongAdder topicsCount = new LongAdder(); pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { namespaceStats.reset(); topicsCount.reset(); - bundlesMap.forEach((bundle, topicsMap) -> { - topicsMap.forEach((name, topic) -> { - getTopicStats(topic, topicStats, includeConsumerMetrics, includeProducerMetrics, - pulsar.getConfiguration().isExposePreciseBacklogInPrometheus(), - pulsar.getConfiguration().isExposeSubscriptionBacklogSizeInPrometheus(), - compactorMXBean - ); - - if (includeTopicMetrics) { - topicsCount.add(1); - TopicStats.printTopicStats(stream, cluster, namespace, name, topicStats, compactorMXBean, - splitTopicAndPartitionIndexLabel); - } else { - namespaceStats.updateStats(topicStats); - } - }); - }); + bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { + getTopicStats(topic, topicStats, includeConsumerMetrics, includeProducerMetrics, + pulsar.getConfiguration().isExposePreciseBacklogInPrometheus(), + pulsar.getConfiguration().isExposeSubscriptionBacklogSizeInPrometheus(), + compactorMXBean + ); + + if (includeTopicMetrics) { + topicsCount.add(1); + TopicStats.printTopicStats(stream, topicStats, compactorMXBean, cluster, namespace, name, + splitTopicAndPartitionIndexLabel); + } else { + namespaceStats.updateStats(topicStats); + } + })); if (!includeTopicMetrics) { - // Only include namespace level stats if we don't have the per-topic, otherwise we're going to report - // the same data twice, and it will make the aggregation difficult - printNamespaceStats(stream, cluster, namespace, namespaceStats); + // Only include namespace level stats if we don't have the per-topic, otherwise we're going to + // report the same data twice, and it will make the aggregation difficult + printNamespaceStats(stream, namespaceStats, cluster, namespace); } else { - printTopicsCountStats(stream, cluster, namespace, topicsCount); + localNamespaceTopicCount.put(namespace, topicsCount.sum()); } }); + + if (includeTopicMetrics) { + printTopicsCountStats(stream, localNamespaceTopicCount, cluster); + } } private static Optional getCompactorMXBean(PulsarService pulsar) { Compactor compactor = pulsar.getNullableCompactor(); - return Optional.ofNullable(compactor).map(c -> c.getStats()); + return Optional.ofNullable(compactor).map(Compactor::getStats); } private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean getPreciseBacklog, boolean subscriptionBacklogSize, - Optional compactorMXBean) { + boolean includeProducerMetrics, boolean getPreciseBacklog, + boolean subscriptionBacklogSize, Optional compactorMXBean) { stats.reset(); if (topic instanceof PersistentTopic) { @@ -267,161 +272,176 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include }); } - private static void printDefaultBrokerStats(SimpleTextOutputStream stream, String cluster) { + private static void printDefaultBrokerStats(PrometheusMetricStreams stream, String cluster) { // Print metrics with 0 values. This is necessary to have the available brokers being // reported in the brokers dashboard even if they don't have any topic or traffic - metric(stream, cluster, "pulsar_topics_count", 0); - metric(stream, cluster, "pulsar_subscriptions_count", 0); - metric(stream, cluster, "pulsar_producers_count", 0); - metric(stream, cluster, "pulsar_consumers_count", 0); - metric(stream, cluster, "pulsar_rate_in", 0); - metric(stream, cluster, "pulsar_rate_out", 0); - metric(stream, cluster, "pulsar_throughput_in", 0); - metric(stream, cluster, "pulsar_throughput_out", 0); - metric(stream, cluster, "pulsar_storage_size", 0); - metric(stream, cluster, "pulsar_storage_logical_size", 0); - metric(stream, cluster, "pulsar_storage_write_rate", 0); - metric(stream, cluster, "pulsar_storage_read_rate", 0); - metric(stream, cluster, "pulsar_msg_backlog", 0); + writeMetric(stream, "pulsar_topics_count", 0, cluster); + writeMetric(stream, "pulsar_subscriptions_count", 0, cluster); + writeMetric(stream, "pulsar_producers_count", 0, cluster); + writeMetric(stream, "pulsar_consumers_count", 0, cluster); + writeMetric(stream, "pulsar_rate_in", 0, cluster); + writeMetric(stream, "pulsar_rate_out", 0, cluster); + writeMetric(stream, "pulsar_throughput_in", 0, cluster); + writeMetric(stream, "pulsar_throughput_out", 0, cluster); + writeMetric(stream, "pulsar_storage_size", 0, cluster); + writeMetric(stream, "pulsar_storage_logical_size", 0, cluster); + writeMetric(stream, "pulsar_storage_write_rate", 0, cluster); + writeMetric(stream, "pulsar_storage_read_rate", 0, cluster); + writeMetric(stream, "pulsar_msg_backlog", 0, cluster); } - private static void printTopicsCountStats(SimpleTextOutputStream stream, String cluster, String namespace, - LongAdder topicsCount) { - metric(stream, cluster, namespace, "pulsar_topics_count", topicsCount.sum()); + private static void printTopicsCountStats(PrometheusMetricStreams stream, Map namespaceTopicsCount, + String cluster) { + namespaceTopicsCount.forEach( + (ns, topicCount) -> writeMetric(stream, "pulsar_topics_count", topicCount, cluster, ns) + ); } - private static void printNamespaceStats(SimpleTextOutputStream stream, String cluster, String namespace, - AggregatedNamespaceStats stats) { - metric(stream, cluster, namespace, "pulsar_topics_count", stats.topicsCount); - metric(stream, cluster, namespace, "pulsar_subscriptions_count", stats.subscriptionsCount); - metric(stream, cluster, namespace, "pulsar_producers_count", stats.producersCount); - metric(stream, cluster, namespace, "pulsar_consumers_count", stats.consumersCount); - - metric(stream, cluster, namespace, "pulsar_rate_in", stats.rateIn); - metric(stream, cluster, namespace, "pulsar_rate_out", stats.rateOut); - metric(stream, cluster, namespace, "pulsar_throughput_in", stats.throughputIn); - metric(stream, cluster, namespace, "pulsar_throughput_out", stats.throughputOut); - metric(stream, cluster, namespace, "pulsar_consumer_msg_ack_rate", stats.messageAckRate); - - metric(stream, cluster, namespace, "pulsar_in_bytes_total", stats.bytesInCounter); - metric(stream, cluster, namespace, "pulsar_in_messages_total", stats.msgInCounter); - metric(stream, cluster, namespace, "pulsar_out_bytes_total", stats.bytesOutCounter); - metric(stream, cluster, namespace, "pulsar_out_messages_total", stats.msgOutCounter); - - metric(stream, cluster, namespace, "pulsar_storage_size", stats.managedLedgerStats.storageSize); - metric(stream, cluster, namespace, "pulsar_storage_logical_size", stats.managedLedgerStats.storageLogicalSize); - metric(stream, cluster, namespace, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize); - metric(stream, cluster, namespace, "pulsar_storage_offloaded_size", - stats.managedLedgerStats.offloadedStorageUsed); - - metric(stream, cluster, namespace, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate); - metric(stream, cluster, namespace, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate); - - metric(stream, cluster, namespace, "pulsar_subscription_delayed", stats.msgDelayed); - - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_msg_backlog", "local", stats.msgBacklog); + private static void printNamespaceStats(PrometheusMetricStreams stream, AggregatedNamespaceStats stats, + String cluster, String namespace) { + writeMetric(stream, "pulsar_topics_count", stats.topicsCount, cluster, namespace); + writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, cluster, + namespace); + writeMetric(stream, "pulsar_producers_count", stats.producersCount, cluster, namespace); + writeMetric(stream, "pulsar_consumers_count", stats.consumersCount, cluster, namespace); + + writeMetric(stream, "pulsar_rate_in", stats.rateIn, cluster, namespace); + writeMetric(stream, "pulsar_rate_out", stats.rateOut, cluster, namespace); + writeMetric(stream, "pulsar_throughput_in", stats.throughputIn, cluster, namespace); + writeMetric(stream, "pulsar_throughput_out", stats.throughputOut, cluster, namespace); + writeMetric(stream, "pulsar_consumer_msg_ack_rate", stats.messageAckRate, cluster, namespace); + + writeMetric(stream, "pulsar_in_bytes_total", stats.bytesInCounter, cluster, namespace); + writeMetric(stream, "pulsar_in_messages_total", stats.msgInCounter, cluster, namespace); + writeMetric(stream, "pulsar_out_bytes_total", stats.bytesOutCounter, cluster, namespace); + writeMetric(stream, "pulsar_out_messages_total", stats.msgOutCounter, cluster, namespace); + + writeMetric(stream, "pulsar_storage_size", stats.managedLedgerStats.storageSize, cluster, + namespace); + writeMetric(stream, "pulsar_storage_logical_size", + stats.managedLedgerStats.storageLogicalSize, cluster, namespace); + writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, cluster, + namespace); + writeMetric(stream, "pulsar_storage_offloaded_size", + stats.managedLedgerStats.offloadedStorageUsed, cluster, namespace); + + writeMetric(stream, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate, + cluster, namespace); + writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, + cluster, namespace); + + writeMetric(stream, "pulsar_subscription_delayed", stats.msgDelayed, cluster, namespace); + + writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace); stats.managedLedgerStats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_1", latencyBuckets[1]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_5", latencyBuckets[2]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_10", latencyBuckets[3]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_20", latencyBuckets[4]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_50", latencyBuckets[5]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_100", latencyBuckets[6]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_200", latencyBuckets[7]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_1000", latencyBuckets[8]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_overflow", latencyBuckets[9]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_count", - stats.managedLedgerStats.storageWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_sum", - stats.managedLedgerStats.storageWriteLatencyBuckets.getSum()); + writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_1", latencyBuckets[1], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_5", latencyBuckets[2], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_10", latencyBuckets[3], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_20", latencyBuckets[4], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_50", latencyBuckets[5], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_100", latencyBuckets[6], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_200", latencyBuckets[7], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_overflow", latencyBuckets[9], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_count", + stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_sum", + stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), cluster, namespace); stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.refresh(); - long[] ledgerWritelatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWritelatencyBuckets[0]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_1", ledgerWritelatencyBuckets[1]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_5", ledgerWritelatencyBuckets[2]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_10", ledgerWritelatencyBuckets[3]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_20", ledgerWritelatencyBuckets[4]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_50", ledgerWritelatencyBuckets[5]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_100", ledgerWritelatencyBuckets[6]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_200", ledgerWritelatencyBuckets[7]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_1000", ledgerWritelatencyBuckets[8]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_overflow", - ledgerWritelatencyBuckets[9]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_count", - stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_sum", - stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum()); + long[] ledgerWriteLatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWriteLatencyBuckets[0], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", ledgerWriteLatencyBuckets[1], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", ledgerWriteLatencyBuckets[2], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", ledgerWriteLatencyBuckets[3], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", ledgerWriteLatencyBuckets[4], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", ledgerWriteLatencyBuckets[5], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", ledgerWriteLatencyBuckets[6], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", ledgerWriteLatencyBuckets[7], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", ledgerWriteLatencyBuckets[8], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", ledgerWriteLatencyBuckets[9], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_count", + stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount(), cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", + stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum(), cluster, namespace); stats.managedLedgerStats.entrySizeBuckets.refresh(); long[] entrySizeBuckets = stats.managedLedgerStats.entrySizeBuckets.getBuckets(); - metric(stream, cluster, namespace, "pulsar_entry_size_le_128", entrySizeBuckets[0]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_512", entrySizeBuckets[1]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_overflow", entrySizeBuckets[8]); - metric(stream, cluster, namespace, "pulsar_entry_size_count", - stats.managedLedgerStats.entrySizeBuckets.getCount()); - metric(stream, cluster, namespace, "pulsar_entry_size_sum", - stats.managedLedgerStats.entrySizeBuckets.getSum()); - - if (!stats.replicationStats.isEmpty()) { - stats.replicationStats.forEach((remoteCluster, replStats) -> { - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_in", remoteCluster, - replStats.msgRateIn); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_out", remoteCluster, - replStats.msgRateOut); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_throughput_in", remoteCluster, - replStats.msgThroughputIn); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_throughput_out", remoteCluster, - replStats.msgThroughputOut); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_backlog", remoteCluster, - replStats.replicationBacklog); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_connected_count", remoteCluster, - replStats.connectedCount); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_expired", remoteCluster, - replStats.msgRateExpired); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_delay_in_seconds", - remoteCluster, replStats.replicationDelayInSeconds); - }); - } + writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_count", stats.managedLedgerStats.entrySizeBuckets.getCount(), + cluster, namespace); + writeMetric(stream, "pulsar_entry_size_sum", stats.managedLedgerStats.entrySizeBuckets.getSum(), + cluster, namespace); + + writeReplicationStat(stream, "pulsar_replication_rate_in", stats, + replStats -> replStats.msgRateIn, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_rate_out", stats, + replStats -> replStats.msgRateOut, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_throughput_in", stats, + replStats -> replStats.msgThroughputIn, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_throughput_out", stats, + replStats -> replStats.msgThroughputOut, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_backlog", stats, + replStats -> replStats.replicationBacklog, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_connected_count", stats, + replStats -> replStats.connectedCount, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_rate_expired", stats, + replStats -> replStats.msgRateExpired, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_delay_in_seconds", stats, + replStats -> replStats.replicationDelayInSeconds, cluster, namespace); } - private static void metric(SimpleTextOutputStream stream, String cluster, String name, - long value) { - TopicStats.metricType(stream, name); - stream.write(name) - .write("{cluster=\"").write(cluster).write("\"} ") - .write(value).write(' ').write(System.currentTimeMillis()) - .write('\n'); + private static void writePulsarMsgBacklog(PrometheusMetricStreams stream, Number value, + String cluster, String namespace) { + stream.writeSample("pulsar_msg_backlog", value, "cluster", cluster, "namespace", namespace, + "remote_cluster", + "local"); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String name, - long value) { - TopicStats.metricType(stream, name); - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster) { + stream.writeSample(metricName, value, "cluster", cluster); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String name, - double value) { - TopicStats.metricType(stream, name); - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace) { + stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace); } - private static void metricWithRemoteCluster(SimpleTextOutputStream stream, String cluster, String namespace, - String name, String remoteCluster, double value) { - TopicStats.metricType(stream, name); - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace); - stream.write("\",remote_cluster=\"").write(remoteCluster).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName, + AggregatedNamespaceStats namespaceStats, + Function sampleValueFunction, + String cluster, String namespace) { + if (!namespaceStats.replicationStats.isEmpty()) { + namespaceStats.replicationStats.forEach((remoteCluster, replStats) -> + stream.writeSample(metricName, sampleValueFunction.apply(replStats), + "cluster", cluster, + "namespace", namespace, + "remote_cluster", remoteCluster) + ); + } } + + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java new file mode 100644 index 0000000000000..6b6b972c175f0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import java.util.HashMap; +import java.util.Map; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.util.SimpleTextOutputStream; + +/** + * Helper class to ensure that metrics of the same name are grouped together under the same TYPE header when written. + * Those are the requirements of the + * Prometheus Exposition Format. + */ +public class PrometheusMetricStreams { + private final Map metricStreamMap = new HashMap<>(); + + /** + * Write the given metric and sample value to the stream. Will write #TYPE header if metric not seen before. + * @param metricName name of the metric. + * @param value value of the sample + * @param labelsAndValuesArray varargs of label and label value + */ + void writeSample(String metricName, Number value, String... labelsAndValuesArray) { + SimpleTextOutputStream stream = initGaugeType(metricName); + stream.write(metricName).write('{'); + for (int i = 0; i < labelsAndValuesArray.length; i += 2) { + stream.write(labelsAndValuesArray[i]).write("=\"").write(labelsAndValuesArray[i + 1]).write('\"'); + if (i + 2 != labelsAndValuesArray.length) { + stream.write(','); + } + } + stream.write("} ").write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + } + + /** + * Flush all the stored metrics to the supplied stream. + * @param stream the stream to write to. + */ + void flushAllToStream(SimpleTextOutputStream stream) { + metricStreamMap.values().forEach(s -> stream.write(s.getBuffer())); + } + + /** + * Release all the streams to clean up resources. + */ + void releaseAll() { + metricStreamMap.values().forEach(s -> s.getBuffer().release()); + metricStreamMap.clear(); + } + + private SimpleTextOutputStream initGaugeType(String metricName) { + return metricStreamMap.computeIfAbsent(metricName, s -> { + SimpleTextOutputStream stream = new SimpleTextOutputStream(PulsarByteBufAllocator.DEFAULT.directBuffer()); + stream.write("# TYPE ").write(metricName).write(" gauge\n"); + return stream; + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index cd6afd1535dec..a993d1edf3a3d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -52,7 +52,8 @@ /** * Generate metrics aggregated at the namespace level and optionally at a topic level and formats them out * in a text format suitable to be consumed by Prometheus. - * Format specification can be found at {@link https://prometheus.io/docs/instrumenting/exposition_formats/} + * Format specification can be found at Exposition Formats */ public class PrometheusMetricsGenerator { @@ -86,38 +87,44 @@ public double get() { } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, OutputStream out) throws IOException { + boolean includeProducerMetrics, OutputStream out) throws IOException { generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, false, out, null); } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - OutputStream out) throws IOException { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + OutputStream out) throws IOException { generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, splitTopicAndPartitionIndexLabel, out, null); } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, OutputStream out, - List metricsProviders) - throws IOException { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + OutputStream out, + List metricsProviders) + throws IOException { ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer(); + boolean exceptionHappens = false; + //Used in namespace/topic and transaction aggregators as share metric names + PrometheusMetricStreams metricStreams = new PrometheusMetricStreams(); try { SimpleTextOutputStream stream = new SimpleTextOutputStream(buf); generateSystemMetrics(stream, pulsar.getConfiguration().getClusterName()); NamespaceStatsAggregator.generate(pulsar, includeTopicMetrics, includeConsumerMetrics, - includeProducerMetrics, splitTopicAndPartitionIndexLabel, stream); + includeProducerMetrics, splitTopicAndPartitionIndexLabel, metricStreams); if (pulsar.getWorkerServiceOpt().isPresent()) { pulsar.getWorkerService().generateFunctionsStats(stream); } if (pulsar.getConfiguration().isTransactionCoordinatorEnabled()) { - TransactionAggregator.generate(pulsar, stream, includeTopicMetrics); + TransactionAggregator.generate(pulsar, metricStreams, includeTopicMetrics); } + metricStreams.flushAllToStream(stream); + generateBrokerBasicMetrics(pulsar, stream); generateManagedLedgerBookieClientMetrics(pulsar, stream); @@ -129,7 +136,12 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b } out.write(buf.array(), buf.arrayOffset(), buf.readableBytes()); } finally { - buf.release(); + //release all the metrics buffers + metricStreams.releaseAll(); + //if exception happens, release buffer + if (exceptionHappens) { + buf.release(); + } } } @@ -142,17 +154,17 @@ private static void generateBrokerBasicMetrics(PulsarService pulsar, SimpleTextO if (pulsar.getConfiguration().isExposeManagedLedgerMetricsInPrometheus()) { // generate managedLedger metrics parseMetricsToPrometheusMetrics(new ManagedLedgerMetrics(pulsar).generate(), - clusterName, Collector.Type.GAUGE, stream); + clusterName, Collector.Type.GAUGE, stream); } if (pulsar.getConfiguration().isExposeManagedCursorMetricsInPrometheus()) { // generate managedCursor metrics parseMetricsToPrometheusMetrics(new ManagedCursorMetrics(pulsar).generate(), - clusterName, Collector.Type.GAUGE, stream); + clusterName, Collector.Type.GAUGE, stream); } parseMetricsToPrometheusMetrics(Collections.singletonList(pulsar.getBrokerService() - .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), + .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), clusterName, Collector.Type.GAUGE, stream); // generate loadBalance metrics @@ -267,17 +279,17 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String static String getTypeStr(Collector.Type type) { switch (type) { - case COUNTER: - return "counter"; - case GAUGE: - return "gauge"; - case SUMMARY : - return "summary"; - case HISTOGRAM: - return "histogram"; - case UNTYPED: - default: - return "untyped"; + case COUNTER: + return "counter"; + case GAUGE: + return "gauge"; + case SUMMARY: + return "summary"; + case HISTOGRAM: + return "histogram"; + case UNTYPED: + default: + return "untyped"; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index e6e5883847df2..e91521aff5511 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -23,12 +23,12 @@ import java.util.Map; import java.util.Optional; import org.apache.bookkeeper.mledger.util.StatsBuckets; -import org.apache.pulsar.common.util.SimpleTextOutputStream; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.compaction.CompactionRecord; import org.apache.pulsar.compaction.CompactorMXBean; class TopicStats { - int subscriptionsCount; int producersCount; int consumersCount; @@ -43,7 +43,6 @@ class TopicStats { double averageMsgSize; public long msgBacklog; - long publishRateLimitedTimes; long backlogQuotaLimit; @@ -55,9 +54,6 @@ class TopicStats { Map subscriptionStats = new HashMap<>(); Map producerStats = new HashMap<>(); - // Used for tracking duplicate TYPE definitions - static Map metricWithTypeDefinition = new HashMap<>(); - // For compaction long compactionRemovedEventCount; long compactionSucceedCount; @@ -103,378 +99,340 @@ public void reset() { compactionLatencyBuckets.reset(); } - static void resetTypes() { - metricWithTypeDefinition.clear(); - } - - static void printTopicStats(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - TopicStats stats, Optional compactorMXBean, - boolean splitTopicAndPartitionIndexLabel) { - metric(stream, cluster, namespace, topic, "pulsar_subscriptions_count", stats.subscriptionsCount, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_producers_count", stats.producersCount, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_consumers_count", stats.consumersCount, - splitTopicAndPartitionIndexLabel); - - metric(stream, cluster, namespace, topic, "pulsar_rate_in", stats.rateIn, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_rate_out", stats.rateOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_throughput_in", stats.throughputIn, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_throughput_out", stats.throughputOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_average_msg_size", stats.averageMsgSize, - splitTopicAndPartitionIndexLabel); - - metric(stream, cluster, namespace, topic, "pulsar_storage_size", stats.managedLedgerStats.storageSize, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_logical_size", - stats.managedLedgerStats.storageLogicalSize, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_msg_backlog", stats.msgBacklog, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_rate", - stats.managedLedgerStats.storageWriteRate, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_size", - stats.managedLedgerStats.backlogSize, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_offloaded_size", stats.managedLedgerStats - .offloadedStorageUsed, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit_time", - stats.backlogQuotaLimitTime, splitTopicAndPartitionIndexLabel); + public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats, + Optional compactorMXBean, String cluster, String namespace, + String topic, boolean splitTopicAndPartitionIndexLabel) { + writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_producers_count", stats.producersCount, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_consumers_count", stats.consumersCount, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + writeMetric(stream, "pulsar_rate_in", stats.rateIn, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_rate_out", stats.rateOut, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_throughput_in", stats.throughputIn, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_throughput_out", stats.throughputOut, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_average_msg_size", stats.averageMsgSize, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + writeMetric(stream, "pulsar_storage_size", stats.managedLedgerStats.storageSize, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_logical_size", + stats.managedLedgerStats.storageLogicalSize, cluster, namespace, topic, + splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_msg_backlog", stats.msgBacklog, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_offloaded_size", stats.managedLedgerStats + .offloadedStorageUsed, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_quota_limit_time", stats.backlogQuotaLimitTime, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_1", latencyBuckets[1], + writeMetric(stream, "pulsar_storage_write_latency_le_0_5", + latencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_1", + latencyBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_5", + latencyBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_10", + latencyBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_20", + latencyBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_50", + latencyBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_100", + latencyBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_200", + latencyBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_1000", + latencyBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_overflow", + latencyBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_count", + stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_sum", + stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_5", latencyBuckets[2], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_10", latencyBuckets[3], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_20", latencyBuckets[4], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_50", latencyBuckets[5], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_100", latencyBuckets[6], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_200", latencyBuckets[7], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_overflow", latencyBuckets[9], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_count", - stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_sum", - stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), splitTopicAndPartitionIndexLabel); long[] ledgerWriteLatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_0_5", - ledgerWriteLatencyBuckets[0], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_1", - ledgerWriteLatencyBuckets[1], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_5", - ledgerWriteLatencyBuckets[2], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_10", - ledgerWriteLatencyBuckets[3], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_20", - ledgerWriteLatencyBuckets[4], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_50", - ledgerWriteLatencyBuckets[5], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_100", - ledgerWriteLatencyBuckets[6], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_200", - ledgerWriteLatencyBuckets[7], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_1000", - ledgerWriteLatencyBuckets[8], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_overflow", - ledgerWriteLatencyBuckets[9], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_count", + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", + ledgerWriteLatencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", + ledgerWriteLatencyBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", + ledgerWriteLatencyBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", + ledgerWriteLatencyBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", + ledgerWriteLatencyBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", + ledgerWriteLatencyBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", + ledgerWriteLatencyBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", + ledgerWriteLatencyBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", + ledgerWriteLatencyBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", + ledgerWriteLatencyBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_count", stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount(), - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_sum", + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum(), - splitTopicAndPartitionIndexLabel); + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); long[] entrySizeBuckets = stats.managedLedgerStats.entrySizeBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_128", entrySizeBuckets[0], + writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_512", entrySizeBuckets[1], + writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], + writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], + writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], + writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], + writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], + writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], + writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], + writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_count", - stats.managedLedgerStats.entrySizeBuckets.getCount(), splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_sum", - stats.managedLedgerStats.entrySizeBuckets.getSum(), splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_entry_size_count", stats.managedLedgerStats.entrySizeBuckets.getCount(), + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_entry_size_sum", stats.managedLedgerStats.entrySizeBuckets.getSum(), + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); stats.producerStats.forEach((p, producerStats) -> { - metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_rate_in", - producerStats.msgRateIn, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_throughput_in", - producerStats.msgThroughputIn, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_average_Size", - producerStats.averageMsgSize, splitTopicAndPartitionIndexLabel); + writeProducerMetric(stream, "pulsar_producer_msg_rate_in", producerStats.msgRateIn, + cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); + writeProducerMetric(stream, "pulsar_producer_msg_throughput_in", producerStats.msgThroughputIn, + cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); + writeProducerMetric(stream, "pulsar_producer_msg_average_Size", producerStats.averageMsgSize, + cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); }); - stats.subscriptionStats.forEach((n, subsStats) -> { - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_back_log", - subsStats.msgBacklog, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_back_log_no_delayed", - subsStats.msgBacklogNoDelayed, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_delayed", - subsStats.msgDelayed, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_redeliver", - subsStats.msgRateRedeliver, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_unacked_messages", - subsStats.unackedMessages, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_blocked_on_unacked_messages", - subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_out", - subsStats.msgRateOut, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_ack_rate", - subsStats.messageAckRate, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_throughput_out", - subsStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_out_bytes_total", - subsStats.bytesOutCounter, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_out_messages_total", - subsStats.msgOutCounter, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_expire_timestamp", - subsStats.lastExpireTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_acked_timestamp", - subsStats.lastAckedTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_consumed_flow_timestamp", - subsStats.lastConsumedFlowTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_consumed_timestamp", - subsStats.lastConsumedTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_mark_delete_advanced_timestamp", - subsStats.lastMarkDeleteAdvancedTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_expired", - subsStats.msgRateExpired, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_total_msg_expired", - subsStats.totalMsgExpired, splitTopicAndPartitionIndexLabel); + stats.subscriptionStats.forEach((sub, subsStats) -> { + writeSubscriptionMetric(stream, "pulsar_subscription_back_log", subsStats.msgBacklog, + cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_back_log_no_delayed", + subsStats.msgBacklogNoDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_delayed", + subsStats.msgDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_redeliver", + subsStats.msgRateRedeliver, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_unacked_messages", + subsStats.unackedMessages, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_blocked_on_unacked_messages", + subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, cluster, namespace, topic, sub, + splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_out", + subsStats.msgRateOut, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_ack_rate", + subsStats.messageAckRate, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_throughput_out", + subsStats.msgThroughputOut, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_out_bytes_total", + subsStats.bytesOutCounter, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_out_messages_total", + subsStats.msgOutCounter, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_expire_timestamp", + subsStats.lastExpireTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_acked_timestamp", + subsStats.lastAckedTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_consumed_flow_timestamp", + subsStats.lastConsumedFlowTimestamp, cluster, namespace, topic, sub, + splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_consumed_timestamp", + subsStats.lastConsumedTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_mark_delete_advanced_timestamp", + subsStats.lastMarkDeleteAdvancedTimestamp, cluster, namespace, topic, sub, + splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_expired", + subsStats.msgRateExpired, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_total_msg_expired", + subsStats.totalMsgExpired, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + subsStats.consumerStat.forEach((c, consumerStats) -> { - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_unacked_messages", consumerStats.unackedMessages, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_blocked_on_unacked_messages", + writeConsumerMetric(stream, "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_unacked_messages", consumerStats.unackedMessages, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_blocked_on_unacked_messages", consumerStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_available_permits", consumerStats.availablePermits, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_out_bytes_total", consumerStats.bytesOutCounter, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_out_messages_total", consumerStats.msgOutCounter, - splitTopicAndPartitionIndexLabel); + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + + writeConsumerMetric(stream, "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + + writeConsumerMetric(stream, "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_available_permits", consumerStats.availablePermits, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_out_bytes_total", consumerStats.bytesOutCounter, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_out_messages_total", consumerStats.msgOutCounter, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); }); }); if (!stats.replicationStats.isEmpty()) { stats.replicationStats.forEach((remoteCluster, replStats) -> { - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_in", remoteCluster, - replStats.msgRateIn, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_out", remoteCluster, - replStats.msgRateOut, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_throughput_in", - remoteCluster, replStats.msgThroughputIn, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_throughput_out", - remoteCluster, replStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_backlog", remoteCluster, - replStats.replicationBacklog, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_connected_count", - remoteCluster, replStats.connectedCount, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_expired", - remoteCluster, replStats.msgRateExpired, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_delay_in_seconds", - remoteCluster, replStats.replicationDelayInSeconds, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_rate_in", replStats.msgRateIn, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_rate_out", replStats.msgRateOut, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_throughput_in", replStats.msgThroughputIn, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_throughput_out", replStats.msgThroughputOut, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_backlog", replStats.replicationBacklog, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_connected_count", replStats.connectedCount, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_rate_expired", replStats.msgRateExpired, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_delay_in_seconds", replStats.replicationDelayInSeconds, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); }); } - metric(stream, cluster, namespace, topic, "pulsar_in_bytes_total", stats.bytesInCounter, + writeMetric(stream, "pulsar_in_bytes_total", stats.bytesInCounter, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_in_messages_total", stats.msgInCounter, + writeMetric(stream, "pulsar_in_messages_total", stats.msgInCounter, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); // Compaction boolean hasCompaction = compactorMXBean.flatMap(mxBean -> mxBean.getCompactionRecordForTopic(topic)) - .map(__ -> true).orElse(false); + .isPresent(); if (hasCompaction) { - metric(stream, cluster, namespace, topic, "pulsar_compaction_removed_event_count", - stats.compactionRemovedEventCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_succeed_count", - stats.compactionSucceedCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_failed_count", - stats.compactionFailedCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_duration_time_in_mills", - stats.compactionDurationTimeInMills, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_read_throughput", - stats.compactionReadThroughput, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_write_throughput", - stats.compactionWriteThroughput, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_compacted_entries_count", - stats.compactionCompactedEntriesCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_compacted_entries_size", - stats.compactionCompactedEntriesSize, splitTopicAndPartitionIndexLabel); - long[] compactionLatencyBuckets = stats.compactionLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_0_5", - compactionLatencyBuckets[0], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_1", - compactionLatencyBuckets[1], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_5", - compactionLatencyBuckets[2], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_10", - compactionLatencyBuckets[3], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_20", - compactionLatencyBuckets[4], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_50", - compactionLatencyBuckets[5], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_100", - compactionLatencyBuckets[6], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_200", - compactionLatencyBuckets[7], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_1000", - compactionLatencyBuckets[8], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_overflow", - compactionLatencyBuckets[9], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_sum", - stats.compactionLatencyBuckets.getSum(), splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_count", - stats.compactionLatencyBuckets.getCount(), splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_removed_event_count", + stats.compactionRemovedEventCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_succeed_count", + stats.compactionSucceedCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_failed_count", + stats.compactionFailedCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_duration_time_in_mills", + stats.compactionDurationTimeInMills, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_read_throughput", + stats.compactionReadThroughput, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_write_throughput", + stats.compactionWriteThroughput, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_compacted_entries_count", + stats.compactionCompactedEntriesCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_compacted_entries_size", + stats.compactionCompactedEntriesSize, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + long[] compactionBuckets = stats.compactionLatencyBuckets.getBuckets(); + writeMetric(stream, "pulsar_compaction_latency_le_0_5", + compactionBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_1", + compactionBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_5", + compactionBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_10", + compactionBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_20", + compactionBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_50", + compactionBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_100", + compactionBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_200", + compactionBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_1000", + compactionBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_overflow", + compactionBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_sum", + stats.compactionLatencyBuckets.getSum(), cluster, namespace, topic, + splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_count", + stats.compactionLatencyBuckets.getCount(), cluster, namespace, topic, + splitTopicAndPartitionIndexLabel); } } - static void metricType(SimpleTextOutputStream stream, String name) { - - if (!metricWithTypeDefinition.containsKey(name)) { - metricWithTypeDefinition.put(name, "gauge"); - stream.write("# TYPE ").write(name).write(" gauge\n"); - } - - } - - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String name, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String name, long value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, String remoteCluster, + boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "remote_cluster", remoteCluster); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String producerName, long produceId, String name, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",producer_name=\"").write(producerName) - .write("\",producer_id=\"").write(produceId).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeProducerMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, String producer, + long producerId, boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "producer_name", producer, "producer_id", String.valueOf(producerId)); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String name, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value); - appendEndings(stream); - } - - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String consumerName, long consumerId, String name, long value, - boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription) - .write("\",consumer_name=\"").write(consumerName).write("\",consumer_id=\"").write(consumerId) - .write("\"} "); - stream.write(value); - appendEndings(stream); - } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String consumerName, long consumerId, String name, double value, - boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription) - .write("\",consumer_name=\"").write(consumerName).write("\",consumer_id=\"") - .write(consumerId).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeSubscriptionMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, String subscription, + boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "subscription", subscription); } - private static void metricWithRemoteCluster(SimpleTextOutputStream stream, String cluster, String namespace, - String topic, String name, String remoteCluster, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",remote_cluster=\"").write(remoteCluster).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeConsumerMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, String subscription, + Consumer consumer, boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "subscription", subscription, "consumer_name", consumer.consumerName(), + "consumer_id", String.valueOf(consumer.consumerId())); } - private static SimpleTextOutputStream appendRequiredLabels(SimpleTextOutputStream stream, String cluster, - String namespace, String topic, String name, boolean splitTopicAndPartitionIndexLabel) { - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace); + static void writeTopicMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, boolean splitTopicAndPartitionIndexLabel, + String... extraLabelsAndValues) { + String[] labelsAndValues = new String[splitTopicAndPartitionIndexLabel ? 8 : 6]; + labelsAndValues[0] = "cluster"; + labelsAndValues[1] = cluster; + labelsAndValues[2] = "namespace"; + labelsAndValues[3] = namespace; + labelsAndValues[4] = "topic"; if (splitTopicAndPartitionIndexLabel) { int index = topic.indexOf(PARTITIONED_TOPIC_SUFFIX); if (index > 0) { - stream.write("\",topic=\"").write(topic.substring(0, index)).write("\",partition=\"") - .write(topic.substring(index + PARTITIONED_TOPIC_SUFFIX.length())); + labelsAndValues[5] = topic.substring(0, index); + labelsAndValues[6] = "partition"; + labelsAndValues[7] = topic.substring(index + PARTITIONED_TOPIC_SUFFIX.length()); } else { - stream.write("\",topic=\"").write(topic).write("\",partition=\"").write("-1"); + labelsAndValues[5] = topic; + labelsAndValues[6] = "partition"; + labelsAndValues[7] = "-1"; } } else { - stream.write("\",topic=\"").write(topic); + labelsAndValues[5] = topic; } - return stream; - } - - private static void appendEndings(SimpleTextOutputStream stream) { - stream.write(' ').write(System.currentTimeMillis()).write('\n'); + String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues); + stream.writeSample(metricName, value, labels); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java index e6ac1535f43c7..8c58b516333f5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java @@ -20,8 +20,6 @@ import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import io.netty.util.concurrent.FastThreadLocal; -import java.util.HashMap; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; @@ -30,7 +28,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.impl.TransactionMetadataStoreStats; @@ -38,21 +35,10 @@ @Slf4j public class TransactionAggregator { - /** - * Used for tracking duplicate TYPE definitions. - */ - private static final FastThreadLocal> threadLocalMetricWithTypeDefinition = - new FastThreadLocal() { - @Override - protected Map initialValue() { - return new HashMap<>(); - } - }; - private static final FastThreadLocal localTransactionCoordinatorStats = new FastThreadLocal() { @Override - protected AggregatedTransactionCoordinatorStats initialValue() throws Exception { + protected AggregatedTransactionCoordinatorStats initialValue() { return new AggregatedTransactionCoordinatorStats(); } }; @@ -60,21 +46,18 @@ protected AggregatedTransactionCoordinatorStats initialValue() throws Exception private static final FastThreadLocal localManageLedgerStats = new FastThreadLocal() { @Override - protected ManagedLedgerStats initialValue() throws Exception { + protected ManagedLedgerStats initialValue() { return new ManagedLedgerStats(); } }; - public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, boolean includeTopicMetrics) { + public static void generate(PulsarService pulsar, PrometheusMetricStreams stream, boolean includeTopicMetrics) { String cluster = pulsar.getConfiguration().getClusterName(); - Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); - metricWithTypeDefinition.clear(); if (includeTopicMetrics) { - pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { - bundlesMap.forEach((bundle, topicsMap) -> { - topicsMap.forEach((name, topic) -> { + pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> + bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { if (topic instanceof PersistentTopic) { topic.getSubscriptions().values().forEach(subscription -> { try { @@ -82,9 +65,8 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, if (!checkTopicIsEventsNames(TopicName.get(subscription.getTopic().getName())) && subscription instanceof PersistentSubscription && ((PersistentSubscription) subscription).checkIfPendingAckStoreInit()) { - ManagedLedger managedLedger = - ((PersistentSubscription) subscription) - .getPendingAckManageLedger().get(); + ManagedLedger managedLedger = ((PersistentSubscription) subscription) + .getPendingAckManageLedger().get(); generateManageLedgerStats(managedLedger, stream, cluster, namespace, name, subscription.getName()); } @@ -93,9 +75,7 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, } }); } - }); - }); - }); + }))); } AggregatedTransactionCoordinatorStats transactionCoordinatorStats = localTransactionCoordinatorStats.get(); @@ -124,18 +104,18 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, localManageLedgerStats.get().reset(); if (transactionMetadataStore instanceof MLTransactionMetadataStore) { - ManagedLedger managedLedger = - ((MLTransactionMetadataStore) transactionMetadataStore).getManagedLedger(); + ManagedLedger managedLedger = + ((MLTransactionMetadataStore) transactionMetadataStore).getManagedLedger(); generateManageLedgerStats(managedLedger, stream, cluster, NamespaceName.SYSTEM_NAMESPACE.toString(), MLTransactionLogImpl.TRANSACTION_LOG_PREFIX + transactionCoordinatorID.getId(), MLTransactionLogImpl.TRANSACTION_SUBSCRIPTION_NAME); } - }); + }); } - private static void generateManageLedgerStats(ManagedLedger managedLedger, SimpleTextOutputStream stream, + private static void generateManageLedgerStats(ManagedLedger managedLedger, PrometheusMetricStreams stream, String cluster, String namespace, String topic, String subscription) { ManagedLedgerStats managedLedgerStats = localManageLedgerStats.get(); ManagedLedgerMBeanImpl mlStats = (ManagedLedgerMBeanImpl) managedLedger.getStats(); @@ -157,174 +137,149 @@ private static void generateManageLedgerStats(ManagedLedger managedLedger, Simpl managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate(); managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate(); - printManageLedgerStats(stream, cluster, namespace, topic, - subscription, managedLedgerStats); - } - - private static void metricType(SimpleTextOutputStream stream, String name) { - Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); - if (!metricWithTypeDefinition.containsKey(name)) { - metricWithTypeDefinition.put(name, "gauge"); - stream.write("# TYPE ").write(name).write(" gauge\n"); - } - - } - - private static void metric(SimpleTextOutputStream stream, String cluster, String name, - double value, long coordinatorId) { - metricType(stream, name); - stream.write(name) - .write("{cluster=\"").write(cluster) - .write("\",coordinator_id=\"").write(coordinatorId).write("\"} ") - .write(value).write(' ').write(System.currentTimeMillis()) - .write('\n'); + printManageLedgerStats(stream, cluster, namespace, topic, subscription, managedLedgerStats); } - private static void metrics(SimpleTextOutputStream stream, String cluster, String namespace, - String topic, String subscription, String name, long value) { - stream.write(name).write("{cluster=\"").write(cluster).write("\", namespace=\"").write(namespace) - .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); - } - - private static void metrics(SimpleTextOutputStream stream, String cluster, String namespace, - String topic, String subscription, String name, double value) { - stream.write(name).write("{cluster=\"").write(cluster).write("\", namespace=\"").write(namespace) - .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); - } - - private static void printManageLedgerStats(SimpleTextOutputStream stream, String cluster, String namespace, + private static void printManageLedgerStats(PrometheusMetricStreams stream, String cluster, String namespace, String topic, String subscription, ManagedLedgerStats stats) { - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_size", stats.storageSize); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_logical_size", stats.storageLogicalSize); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_backlog_size", stats.backlogSize); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_offloaded_size", stats.offloadedStorageUsed); + writeMetric(stream, "pulsar_storage_size", stats.storageSize, cluster, namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_logical_size", stats.storageLogicalSize, cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_backlog_size", stats.backlogSize, cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_offloaded_size", stats.offloadedStorageUsed, cluster, namespace, topic, + subscription); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_rate", stats.storageWriteRate); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_read_rate", stats.storageReadRate); + writeMetric(stream, "pulsar_storage_write_rate", stats.storageWriteRate, cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_read_rate", stats.storageReadRate, cluster, namespace, topic, + subscription); stats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.storageWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_0_5", latencyBuckets[0]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_1", latencyBuckets[1]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_5", latencyBuckets[2]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_10", latencyBuckets[3]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_20", latencyBuckets[4]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_50", latencyBuckets[5]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_100", latencyBuckets[6]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_200", latencyBuckets[7]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_1000", latencyBuckets[8]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_overflow", latencyBuckets[9]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_write_latency_count", - stats.storageWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_write_latency_sum", - stats.storageWriteLatencyBuckets.getSum()); + writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_1", latencyBuckets[1], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_5", latencyBuckets[2], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_10", latencyBuckets[3], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_20", latencyBuckets[4], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_50", latencyBuckets[5], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_100", latencyBuckets[6], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_200", latencyBuckets[7], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_overflow", latencyBuckets[9], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_count", stats.storageWriteLatencyBuckets.getCount(), + cluster, namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_write_latency_sum", stats.storageWriteLatencyBuckets.getSum(), cluster, + namespace, topic, subscription); stats.storageLedgerWriteLatencyBuckets.refresh(); - long[] ledgerWritelatencyBuckets = stats.storageLedgerWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_0_5", ledgerWritelatencyBuckets[0]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_1", ledgerWritelatencyBuckets[1]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_5", ledgerWritelatencyBuckets[2]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_10", ledgerWritelatencyBuckets[3]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_20", ledgerWritelatencyBuckets[4]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_50", ledgerWritelatencyBuckets[5]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_100", ledgerWritelatencyBuckets[6]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_200", ledgerWritelatencyBuckets[7]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_1000", ledgerWritelatencyBuckets[8]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_overflow", - ledgerWritelatencyBuckets[9]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_count", - stats.storageLedgerWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_sum", - stats.storageLedgerWriteLatencyBuckets.getSum()); + long[] ledgerWriteLatencyBuckets = stats.storageLedgerWriteLatencyBuckets.getBuckets(); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWriteLatencyBuckets[0], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", ledgerWriteLatencyBuckets[1], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", ledgerWriteLatencyBuckets[2], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", ledgerWriteLatencyBuckets[3], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", ledgerWriteLatencyBuckets[4], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", ledgerWriteLatencyBuckets[5], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", ledgerWriteLatencyBuckets[6], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", ledgerWriteLatencyBuckets[7], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", ledgerWriteLatencyBuckets[8], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", ledgerWriteLatencyBuckets[9], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_count", + stats.storageLedgerWriteLatencyBuckets.getCount(), cluster, namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", + stats.storageLedgerWriteLatencyBuckets.getSum(), cluster, namespace, topic, subscription); stats.entrySizeBuckets.refresh(); long[] entrySizeBuckets = stats.entrySizeBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_128", entrySizeBuckets[0]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_512", entrySizeBuckets[1]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_le_100_kb", entrySizeBuckets[6]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_le_overflow", entrySizeBuckets[8]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_count", stats.entrySizeBuckets.getCount()); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_sum", stats.entrySizeBuckets.getSum()); - } - - private static void metric(SimpleTextOutputStream stream, String cluster, - String namespace, String topic, String subscription, - String name, long value) { - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace) - .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_count", stats.entrySizeBuckets.getCount(), cluster, namespace, + topic, subscription); + writeMetric(stream, "pulsar_entry_size_sum", stats.entrySizeBuckets.getSum(), cluster, namespace, topic, + subscription); } - static void printTransactionCoordinatorStats(SimpleTextOutputStream stream, String cluster, + static void printTransactionCoordinatorStats(PrometheusMetricStreams stream, String cluster, AggregatedTransactionCoordinatorStats stats, long coordinatorId) { - metric(stream, cluster, "pulsar_txn_active_count", - stats.actives, coordinatorId); - metric(stream, cluster, "pulsar_txn_committed_count", - stats.committedCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_aborted_count", - stats.abortedCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_created_count", - stats.createdCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_timeout_count", - stats.timeoutCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_append_log_count", - stats.appendLogCount, coordinatorId); + writeMetric(stream, "pulsar_txn_active_count", stats.actives, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_committed_count", stats.committedCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_aborted_count", stats.abortedCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_created_count", stats.createdCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_timeout_count", stats.timeoutCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_append_log_count", stats.appendLogCount, cluster, + coordinatorId); long[] latencyBuckets = stats.executionLatency; - metric(stream, cluster, "pulsar_txn_execution_latency_le_10", latencyBuckets[0], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_20", latencyBuckets[1], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_50", latencyBuckets[2], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_100", latencyBuckets[3], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_500", latencyBuckets[4], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_1000", latencyBuckets[5], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_5000", latencyBuckets[6], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_15000", latencyBuckets[7], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_30000", latencyBuckets[8], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_60000", latencyBuckets[9], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_300000", - latencyBuckets[10], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_1500000", - latencyBuckets[11], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_3000000", - latencyBuckets[12], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_overflow", - latencyBuckets[13], coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_10", latencyBuckets[0], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_20", latencyBuckets[1], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_50", latencyBuckets[2], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_100", latencyBuckets[3], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_500", latencyBuckets[4], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_1000", latencyBuckets[5], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_5000", latencyBuckets[6], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_15000", latencyBuckets[7], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_30000", latencyBuckets[8], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_60000", latencyBuckets[9], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_300000", latencyBuckets[10], cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_1500000", latencyBuckets[11], cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_3000000", latencyBuckets[12], cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_overflow", latencyBuckets[13], cluster, + coordinatorId); + } + + private static void writeMetric(PrometheusMetricStreams stream, String metricName, double value, String cluster, + long coordinatorId) { + stream.writeSample(metricName, value, "cluster", cluster, "coordinator_id", String.valueOf(coordinatorId)); + } + + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, String subscription) { + stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace, "topic", topic, + "subscription", subscription); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java index 7550096c2b584..8f704b11e764c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java @@ -18,13 +18,8 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; -import io.prometheus.client.Collector; -import io.prometheus.client.Collector.MetricFamilySamples; -import io.prometheus.client.Collector.MetricFamilySamples.Sample; -import io.prometheus.client.CollectorRegistry; import java.io.IOException; import java.io.Writer; -import java.util.Enumeration; import org.apache.bookkeeper.stats.Counter; /** @@ -140,31 +135,4 @@ private static void writeSum(Writer w, DataSketchesOpStatsLogger opStat, String .append(success.toString()).append("\"} ") .append(Double.toString(opStat.getSum(success))).append('\n'); } - - public static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) - throws IOException { - Enumeration metricFamilySamples = registry.metricFamilySamples(); - while (metricFamilySamples.hasMoreElements()) { - MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); - - for (int i = 0; i < metricFamily.samples.size(); i++) { - Sample sample = metricFamily.samples.get(i); - w.write(sample.name); - w.write('{'); - for (int j = 0; j < sample.labelNames.size(); j++) { - if (j != 0) { - w.write(", "); - } - w.write(sample.labelNames.get(j)); - w.write("=\""); - w.write(sample.labelValues.get(j)); - w.write('"'); - } - - w.write("} "); - w.write(Collector.doubleToGoString(sample.value)); - w.write('\n'); - } - } - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index f28412ea75160..be10c8dc65e94 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -50,6 +50,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.crypto.SecretKey; @@ -1404,6 +1405,63 @@ private void compareCompactionStateCount(List cm, double count) { assertEquals(cm.get(0).value, count); } + @Test + public void testMetricsGroupedByTypeDefinitions() throws Exception { + Producer p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create(); + Producer p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create(); + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + p1.send(message.getBytes()); + p2.send(message.getBytes()); + } + + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + String metricsStr = statsOut.toString(); + + Pattern typePattern = Pattern.compile("^#\\s+TYPE\\s+(\\w+)\\s+(\\w+)"); + Pattern metricNamePattern = Pattern.compile("^(\\w+)\\{.+"); + + AtomicReference currentMetric = new AtomicReference<>(); + Splitter.on("\n").split(metricsStr).forEach(line -> { + if (line.isEmpty()) { + return; + } + if (line.startsWith("#")) { + // Get the current type definition + Matcher typeMatcher = typePattern.matcher(line); + checkArgument(typeMatcher.matches()); + String metricName = typeMatcher.group(1); + currentMetric.set(metricName); + } else { + Matcher metricMatcher = metricNamePattern.matcher(line); + checkArgument(metricMatcher.matches()); + String metricName = metricMatcher.group(1); + + if (metricName.endsWith("_bucket")) { + metricName = metricName.substring(0, metricName.indexOf("_bucket")); + } else if (metricName.endsWith("_count") && !currentMetric.get().endsWith("_count")) { + metricName = metricName.substring(0, metricName.indexOf("_count")); + } else if (metricName.endsWith("_sum") && !currentMetric.get().endsWith("_sum")) { + metricName = metricName.substring(0, metricName.indexOf("_sum")); + } else if (metricName.endsWith("_total") && !currentMetric.get().endsWith("_total")) { + metricName = metricName.substring(0, metricName.indexOf("_total")); + } else if (metricName.endsWith("_created") && !currentMetric.get().endsWith("_created")) { + metricName = metricName.substring(0, metricName.indexOf("_created")); + } + + if (!metricName.equals(currentMetric.get())) { + System.out.println(metricsStr); + fail("Metric not grouped under its type definition: " + line); + } + + } + }); + + p1.close(); + p2.close(); + } + /** * Hacky parsing of Prometheus text format. Should be good enough for unit tests */ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java new file mode 100644 index 0000000000000..15c29a0dc66bb --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import org.apache.pulsar.common.util.SimpleTextOutputStream; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class PrometheusMetricStreamsTest { + + private PrometheusMetricStreams underTest; + + @BeforeMethod(alwaysRun = true) + protected void setup() throws Exception { + underTest = new PrometheusMetricStreams(); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + underTest.releaseAll(); + } + + @Test + public void canWriteSampleWithoutLabels() { + underTest.writeSample("my-metric", 123); + + String actual = writeToString(); + + assertTrue(actual.startsWith("# TYPE my-metric gauge"), "Gauge type line missing"); + assertTrue(actual.contains("my-metric{} 123"), "Metric line missing"); + } + + @Test + public void canWriteSampleWithLabels() { + underTest.writeSample("my-other-metric", 123, "cluster", "local"); + underTest.writeSample("my-other-metric", 456, "cluster", "local", "namespace", "my-ns"); + + String actual = writeToString(); + + assertTrue(actual.startsWith("# TYPE my-other-metric gauge"), "Gauge type line missing"); + assertTrue(actual.contains("my-other-metric{cluster=\"local\"} 123"), "Cluster metric line missing"); + assertTrue(actual.contains("my-other-metric{cluster=\"local\",namespace=\"my-ns\"} 456"), + "Cluster and Namespace metric line missing"); + } + + private String writeToString() { + ByteBuf buffer = ByteBufAllocator.DEFAULT.directBuffer(); + try { + SimpleTextOutputStream stream = new SimpleTextOutputStream(buffer); + underTest.flushAllToStream(stream); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int readIndex = buffer.readerIndex(); + int readableBytes = buffer.readableBytes(); + for (int i = 0; i < readableBytes; i++) { + out.write(buffer.getByte(readIndex + i)); + } + return out.toString(); + } finally { + buffer.release(); + } + } +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java index 9fc4b347c854f..dd78b4cfe58b9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java @@ -22,12 +22,11 @@ /** * Format strings and numbers into a ByteBuf without any memory allocation. - * */ public class SimpleTextOutputStream { private final ByteBuf buffer; - private static final char[] hexChars = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', - 'f' }; + private static final char[] hexChars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', + 'f'}; public SimpleTextOutputStream(ByteBuf buffer) { this.buffer = buffer; @@ -131,4 +130,12 @@ public SimpleTextOutputStream write(double d) { write(r); return this; } + + public void write(ByteBuf byteBuf) { + buffer.writeBytes(byteBuf); + } + + public ByteBuf getBuffer() { + return buffer; + } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java index f7a205c7db02c..f5aa273f656ac 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java @@ -37,6 +37,11 @@ public static void write004(Writer writer, Enumeration 0) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java index c8b411cbf5706..2ad407b2e5e3e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java @@ -328,6 +328,11 @@ private void generateLeaderMetrics(StringWriter stream) { } private void writeMetric(String metricName, long value, StringWriter stream) { + stream.write("# TYPE "); + stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); + stream.write(metricName); + stream.write(" gauge \n"); + stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); stream.write(metricName); stream.write("{"); From aa4c677cf9e38e1cec3a0b11051d9c414ee10f3c Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Tue, 13 Sep 2022 19:24:40 -0700 Subject: [PATCH 758/823] [improve][loadbalance] added loadBalancerReportUpdateMinIntervalMillis and ignores memory usage in getMaxResourceUsage() (#17598) (cherry picked from commit de7c586b92fcf4506317b90e1d5eba482ca93626) --- conf/broker.conf | 3 + conf/standalone.conf | 3 + .../pulsar/broker/ServiceConfiguration.java | 6 ++ .../apache/pulsar/broker/PulsarService.java | 3 +- .../loadbalance/impl/LoadManagerShared.java | 3 - .../impl/SimpleLoadManagerImpl.java | 9 +-- .../data/loadbalancer/LocalBrokerData.java | 3 +- .../loadbalancer/LocalBrokerDataTest.java | 62 +++++++++++++++++++ 8 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java diff --git a/conf/broker.conf b/conf/broker.conf index 3c4e4b4524f2f..32ded75096aca 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1041,6 +1041,9 @@ loadBalancerEnabled=true # Percentage of change to trigger load report update loadBalancerReportUpdateThresholdPercentage=10 +# minimum interval to update load report +loadBalancerReportUpdateMinIntervalMillis=5000 + # maximum interval to update load report loadBalancerReportUpdateMaxIntervalMinutes=15 diff --git a/conf/standalone.conf b/conf/standalone.conf index 01030d049d488..08742a43cd5da 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -742,6 +742,9 @@ loadBalancerEnabled=false # Percentage of change to trigger load report update loadBalancerReportUpdateThresholdPercentage=10 +# minimum interval to update load report +loadBalancerReportUpdateMinIntervalMillis=5000 + # maximum interval to update load report loadBalancerReportUpdateMaxIntervalMinutes=15 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 47e47a71d3a4b..cbcac11acf2b2 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1753,6 +1753,12 @@ public class ServiceConfiguration implements PulsarConfiguration { category = CATEGORY_LOAD_BALANCER, doc = "maximum interval to update load report" ) + private int loadBalancerReportUpdateMinIntervalMillis = 5000; + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + dynamic = true, + doc = "Min delay of load report to collect, in milli-seconds" + ) private int loadBalancerReportUpdateMaxIntervalMinutes = 15; @FieldContext( category = CATEGORY_LOAD_BALANCER, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index e47bd1bc78356..c10e6acc2a8f5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -89,7 +89,6 @@ import org.apache.pulsar.broker.loadbalance.LoadReportUpdaterTask; import org.apache.pulsar.broker.loadbalance.LoadResourceQuotaUpdaterTask; import org.apache.pulsar.broker.loadbalance.LoadSheddingTask; -import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.protocol.ProtocolHandlers; import org.apache.pulsar.broker.resourcegroup.ResourceGroupService; @@ -1043,7 +1042,7 @@ protected void startLoadManagementService() throws PulsarServerException { if (config.isLoadBalancerEnabled()) { LOG.info("Starting load balancer"); if (this.loadReportTask == null) { - long loadReportMinInterval = LoadManagerShared.LOAD_REPORT_UPDATE_MINIMUM_INTERVAL; + long loadReportMinInterval = config.getLoadBalancerReportUpdateMinIntervalMillis(); this.loadReportTask = this.loadManagerExecutor.scheduleAtFixedRate( new LoadReportUpdaterTask(loadManager), loadReportMinInterval, loadReportMinInterval, TimeUnit.MILLISECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 20b14a9d2202c..2737e97df1476 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -75,9 +75,6 @@ protected Set initialValue() throws Exception { } }; - // update LoadReport at most every 5 seconds - public static final long LOAD_REPORT_UPDATE_MINIMUM_INTERVAL = TimeUnit.SECONDS.toMillis(5); - private static final String DEFAULT_DOMAIN = "default"; // Don't allow construction: static method namespace only. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index 212d299953e40..ed766b1b93004 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.impl; -import static org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.LOAD_REPORT_UPDATE_MINIMUM_INTERVAL; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; @@ -1141,10 +1140,12 @@ public void setLoadReportForceUpdateFlag() { public void writeLoadReportOnZookeeper() throws Exception { // update average JVM heap usage to average value of the last 120 seconds long realtimeJvmHeapUsage = getRealtimeJvmHeapUsageMBytes(); + int minInterval = pulsar.getConfiguration().getLoadBalancerReportUpdateMinIntervalMillis(); if (this.avgJvmHeapUsageMBytes <= 0) { this.avgJvmHeapUsageMBytes = realtimeJvmHeapUsage; } else { - long weight = Math.max(1, TimeUnit.SECONDS.toMillis(120) / LOAD_REPORT_UPDATE_MINIMUM_INTERVAL); + + long weight = Math.max(1, TimeUnit.SECONDS.toMillis(120) / minInterval); this.avgJvmHeapUsageMBytes = ((weight - 1) * this.avgJvmHeapUsageMBytes + realtimeJvmHeapUsage) / weight; } @@ -1163,7 +1164,7 @@ public void writeLoadReportOnZookeeper() throws Exception { int maxUpdateIntervalInMinutes = pulsar.getConfiguration().getLoadBalancerReportUpdateMaxIntervalMinutes(); if (timeElapsedSinceLastReport > TimeUnit.MINUTES.toMillis(maxUpdateIntervalInMinutes)) { needUpdate = true; - } else if (timeElapsedSinceLastReport > LOAD_REPORT_UPDATE_MINIMUM_INTERVAL) { + } else if (timeElapsedSinceLastReport > minInterval) { // check number of bundles assigned, comparing with last LoadReport long oldBundleCount = lastLoadReport.getNumBundles(); long newBundleCount = pulsar.getBrokerService().getNumberOfNamespaceBundles(); @@ -1232,7 +1233,7 @@ public void writeLoadReportOnZookeeper() throws Exception { */ private boolean isLoadReportGenerationIntervalPassed() { long timeSinceLastGenMillis = System.currentTimeMillis() - lastLoadReport.getTimestamp(); - return timeSinceLastGenMillis > LOAD_REPORT_UPDATE_MINIMUM_INTERVAL; + return timeSinceLastGenMillis > pulsar.getConfiguration().getLoadBalancerReportUpdateMinIntervalMillis(); } // todo: changeme: this can be optimized, we don't have to iterate through everytime diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java index a503a85171517..75f32eaa82e8f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java @@ -235,7 +235,8 @@ private void updateBundleData(final Map bundleStat } public double getMaxResourceUsage() { - return max(cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), + // does not consider memory because it is noisy by gc. + return max(cpu.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), bandwidthOut.percentUsage()) / 100; } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java new file mode 100644 index 0000000000000..69d4a7f4cd198 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.policies.data.loadbalancer; + +import com.google.gson.Gson; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class LocalBrokerDataTest { + + @Test + public void testLocalBrokerDataDeserialization() { + String data = "{\"webServiceUrl\":\"http://10.244.2.23:8080\",\"webServiceUrlTls\":\"https://10.244.2.23:8081\",\"pulsarServiceUrlTls\":\"pulsar+ssl://10.244.2.23:6651\",\"persistentTopicsEnabled\":true,\"nonPersistentTopicsEnabled\":false,\"cpu\":{\"usage\":3.1577712104798255,\"limit\":100.0},\"memory\":{\"usage\":614.0,\"limit\":1228.0},\"directMemory\":{\"usage\":32.0,\"limit\":1228.0},\"bandwidthIn\":{\"usage\":0.0,\"limit\":0.0},\"bandwidthOut\":{\"usage\":0.0,\"limit\":0.0},\"msgThroughputIn\":0.0,\"msgThroughputOut\":0.0,\"msgRateIn\":0.0,\"msgRateOut\":0.0,\"lastUpdate\":1650886425227,\"lastStats\":{\"pulsar/pulsar/10.244.2.23:8080/0x00000000_0xffffffff\":{\"msgRateIn\":0.0,\"msgThroughputIn\":0.0,\"msgRateOut\":0.0,\"msgThroughputOut\":0.0,\"consumerCount\":0,\"producerCount\":0,\"topics\":1,\"cacheSize\":0}},\"numTopics\":1,\"numBundles\":1,\"numConsumers\":0,\"numProducers\":0,\"bundles\":[\"pulsar/pulsar/10.244.2.23:8080/0x00000000_0xffffffff\"],\"lastBundleGains\":[],\"lastBundleLosses\":[],\"brokerVersionString\":\"2.11.0-hw-0.0.4-SNAPSHOT\",\"protocols\":{},\"advertisedListeners\":{},\"bundleStats\":{\"pulsar/pulsar/10.244.2.23:8080/0x00000000_0xffffffff\":{\"msgRateIn\":0.0,\"msgThroughputIn\":0.0,\"msgRateOut\":0.0,\"msgThroughputOut\":0.0,\"consumerCount\":0,\"producerCount\":0,\"topics\":1,\"cacheSize\":0}},\"maxResourceUsage\":0.49645519256591797,\"loadReportType\":\"LocalBrokerData\"}"; + Gson gson = new Gson(); + LocalBrokerData localBrokerData = gson.fromJson(data, LocalBrokerData.class); + Assert.assertEquals(localBrokerData.getMemory().limit, 1228.0d, 0.0001f); + Assert.assertEquals(localBrokerData.getMemory().usage, 614.0d, 0.0001f); + Assert.assertEquals(localBrokerData.getMemory().percentUsage(), ((float) localBrokerData.getMemory().usage) / ((float) localBrokerData.getMemory().limit) * 100, 0.0001f); + } + + @Test + public void testMaxResourceUsage() { + LocalBrokerData data = new LocalBrokerData(); + data.setCpu(new ResourceUsage(1.0, 100.0)); + data.setMemory(new ResourceUsage(800.0, 200.0)); + data.setDirectMemory(new ResourceUsage(2.0, 100.0)); + data.setBandwidthIn(new ResourceUsage(3.0, 100.0)); + data.setBandwidthOut(new ResourceUsage(4.0, 100.0)); + + double epsilon = 0.00001; + double weight = 0.5; + // skips memory usage + assertEquals(data.getMaxResourceUsage(), 0.04, epsilon); + + assertEquals( + data.getMaxResourceUsageWithWeight( + weight, weight, weight, weight, weight), 2.0, epsilon); + + assertEquals( + data.getMaxResourceUsageWithWeightWithinLimit( + weight, weight, weight, weight, weight), 0.02, epsilon); + + } +} From 70453766a50b7215e644e43753cd367cd4dd1d60 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 23 Sep 2022 22:45:08 +0800 Subject: [PATCH 759/823] [improve][broker] Make MessageRedeliveryController work more efficiently (#17804) (cherry picked from commit c60f895cae6aa3a51a69603c6215c92e358a0e47) --- .../server/src/assemble/LICENSE.bin.txt | 2 + pom.xml | 8 + pulsar-broker/pom.xml | 5 + .../MessageRedeliveryController.java | 41 +--- .../ConcurrentBitmapSortedLongPairSet.java | 143 ++++++++++++ .../MessageRedeliveryControllerTest.java | 22 +- ...ConcurrentBitmapSortedLongPairSetTest.java | 211 ++++++++++++++++++ 7 files changed, 391 insertions(+), 41 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 43dfb644027f3..e4a4e5ca20be7 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -533,6 +533,8 @@ The Apache Software License, Version 2.0 - com.google.re2j-re2j-1.5.jar * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar + * RoaringBitmap + - org.roaringbitmap-RoaringBitmap-0.9.15.jar BSD 3-clause "New" or "Revised" License * Google auth library diff --git a/pom.xml b/pom.xml index 5d5bcb966a4a9..f70afeb8f0496 100644 --- a/pom.xml +++ b/pom.xml @@ -255,6 +255,7 @@ flexible messaging model and an intuitive client API. 1.3 0.4 7.1.0 + 0.9.15 rename-netty-native-libs.sh @@ -1254,6 +1255,13 @@ flexible messaging model and an intuitive client API. netty-reactive-streams ${netty-reactive-streams.version} + + + org.roaringbitmap + RoaringBitmap + ${roaringbitmap.version} + + diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index b579d3f987989..b26d74265dba4 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -312,6 +312,11 @@ hppc + + org.roaringbitmap + RoaringBitmap + + ${project.groupId} pulsar-functions-api-examples diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java index 46fa1b2b050a2..7aaba9a4e1023 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java @@ -22,46 +22,43 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; -import org.apache.pulsar.common.util.collections.ConcurrentSortedLongPairSet; -import org.apache.pulsar.common.util.collections.LongPairSet; +import org.apache.pulsar.utils.ConcurrentBitmapSortedLongPairSet; public class MessageRedeliveryController { - private final LongPairSet messagesToRedeliver; + private final ConcurrentBitmapSortedLongPairSet messagesToRedeliver; private final ConcurrentLongLongPairHashMap hashesToBeBlocked; public MessageRedeliveryController(boolean allowOutOfOrderDelivery) { - this.messagesToRedeliver = new ConcurrentSortedLongPairSet(128, 2, true); + this.messagesToRedeliver = new ConcurrentBitmapSortedLongPairSet(); this.hashesToBeBlocked = allowOutOfOrderDelivery ? null : ConcurrentLongLongPairHashMap .newBuilder().concurrencyLevel(2).expectedItems(128).autoShrink(true).build(); } - public boolean add(long ledgerId, long entryId) { - return messagesToRedeliver.add(ledgerId, entryId); + public void add(long ledgerId, long entryId) { + messagesToRedeliver.add(ledgerId, entryId); } - public boolean add(long ledgerId, long entryId, long stickyKeyHash) { + public void add(long ledgerId, long entryId, long stickyKeyHash) { if (hashesToBeBlocked != null) { hashesToBeBlocked.put(ledgerId, entryId, stickyKeyHash, 0); } - return messagesToRedeliver.add(ledgerId, entryId); + messagesToRedeliver.add(ledgerId, entryId); } - public boolean remove(long ledgerId, long entryId) { + public void remove(long ledgerId, long entryId) { if (hashesToBeBlocked != null) { hashesToBeBlocked.remove(ledgerId, entryId); } - return messagesToRedeliver.remove(ledgerId, entryId); + messagesToRedeliver.remove(ledgerId, entryId); } - public int removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { + public void removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { if (hashesToBeBlocked != null) { List keysToRemove = new ArrayList<>(); hashesToBeBlocked.forEach((ledgerId, entryId, stickyKeyHash, none) -> { @@ -73,10 +70,7 @@ public int removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { keysToRemove.forEach(longPair -> hashesToBeBlocked.remove(longPair.first, longPair.second)); keysToRemove.clear(); } - return messagesToRedeliver.removeIf((ledgerId, entryId) -> { - return ComparisonChain.start().compare(ledgerId, markDeleteLedgerId).compare(entryId, markDeleteEntryId) - .result() <= 0; - }); + messagesToRedeliver.removeUpTo(markDeleteLedgerId, markDeleteEntryId + 1); } public boolean isEmpty() { @@ -107,17 +101,6 @@ public boolean containsStickyKeyHashes(Set stickyKeyHashes) { } public Set getMessagesToReplayNow(int maxMessagesToRead) { - if (hashesToBeBlocked != null) { - // allowOutOfOrderDelivery is false - return messagesToRedeliver.items().stream() - .sorted((l1, l2) -> ComparisonChain.start().compare(l1.first, l2.first) - .compare(l1.second, l2.second).result()) - .limit(maxMessagesToRead).map(longPair -> new PositionImpl(longPair.first, longPair.second)) - .collect(Collectors.toCollection(TreeSet::new)); - } else { - // allowOutOfOrderDelivery is true - return messagesToRedeliver.items(maxMessagesToRead, - (ledgerId, entryId) -> new PositionImpl(ledgerId, entryId)); - } + return messagesToRedeliver.items(maxMessagesToRead, PositionImpl::new); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java new file mode 100644 index 0000000000000..c9f1c65daca37 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.pulsar.common.util.collections.LongPairSet; +import org.roaringbitmap.RoaringBitmap; + +public class ConcurrentBitmapSortedLongPairSet { + + private final NavigableMap map = new TreeMap<>(); + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + public void add(long item1, long item2) { + lock.writeLock().lock(); + try { + RoaringBitmap bitSet = map.computeIfAbsent(item1, k -> new RoaringBitmap()); + bitSet.add(item2, item2 + 1); + } finally { + lock.writeLock().unlock(); + } + } + + public void remove(long item1, long item2) { + lock.writeLock().lock(); + try { + RoaringBitmap bitSet = map.get(item1); + if (bitSet != null) { + bitSet.remove(item2, item2 + 1); + if (bitSet.isEmpty()) { + map.remove(item1, bitSet); + } + } + } finally { + lock.writeLock().unlock(); + } + } + + public boolean contains(long item1, long item2) { + lock.readLock().lock(); + try { + RoaringBitmap bitSet = map.get(item1); + return bitSet != null && bitSet.contains(item2, item2 + 1); + } finally { + lock.readLock().unlock(); + } + } + + public void removeUpTo(long item1, long item2) { + lock.writeLock().lock(); + try { + Map.Entry firstEntry = map.firstEntry(); + while (firstEntry != null && firstEntry.getKey() <= item1) { + if (firstEntry.getKey() < item1) { + map.remove(firstEntry.getKey(), firstEntry.getValue()); + } else { + RoaringBitmap bitSet = firstEntry.getValue(); + if (bitSet != null) { + bitSet.remove(0, item2); + if (bitSet.isEmpty()) { + map.remove(firstEntry.getKey(), bitSet); + } + } + break; + } + firstEntry = map.firstEntry(); + } + } finally { + lock.writeLock().unlock(); + } + } + + + public Set items(int numberOfItems, LongPairSet.LongPairFunction longPairConverter) { + NavigableSet items = new TreeSet<>(); + lock.readLock().lock(); + try { + for (Map.Entry entry : map.entrySet()) { + Iterator iterator = entry.getValue().stream().iterator(); + while (iterator.hasNext() && items.size() < numberOfItems) { + items.add(longPairConverter.apply(entry.getKey(), iterator.next())); + } + if (items.size() == numberOfItems) { + break; + } + } + } finally { + lock.readLock().unlock(); + } + return items; + } + + public boolean isEmpty() { + lock.readLock().lock(); + try { + return map.isEmpty() || map.values().stream().allMatch(RoaringBitmap::isEmpty); + } finally { + lock.readLock().unlock(); + } + } + + public void clear() { + lock.writeLock().lock(); + try { + map.clear(); + } finally { + lock.writeLock().unlock(); + } + } + + public int size() { + lock.readLock().lock(); + try { + return map.isEmpty() ? 0 : map.values().stream().mapToInt(RoaringBitmap::getCardinality).sum(); + } finally { + lock.readLock().unlock(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java index 478677a25e42a..3cd6fc23de744 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java @@ -30,8 +30,8 @@ import java.util.Set; import java.util.TreeSet; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.utils.ConcurrentBitmapSortedLongPairSet; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; -import org.apache.pulsar.common.util.collections.LongPairSet; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -48,7 +48,8 @@ public void testAddAndRemove(boolean allowOutOfOrderDelivery) throws Exception { Field messagesToRedeliverField = MessageRedeliveryController.class.getDeclaredField("messagesToRedeliver"); messagesToRedeliverField.setAccessible(true); - LongPairSet messagesToRedeliver = (LongPairSet) messagesToRedeliverField.get(controller); + ConcurrentBitmapSortedLongPairSet messagesToRedeliver = + (ConcurrentBitmapSortedLongPairSet) messagesToRedeliverField.get(controller); Field hashesToBeBlockedField = MessageRedeliveryController.class.getDeclaredField("hashesToBeBlocked"); hashesToBeBlockedField.setAccessible(true); @@ -67,9 +68,8 @@ public void testAddAndRemove(boolean allowOutOfOrderDelivery) throws Exception { assertEquals(hashesToBeBlocked.size(), 0); } - assertTrue(controller.add(1, 1)); - assertTrue(controller.add(1, 2)); - assertFalse(controller.add(1, 1)); + controller.add(1, 1); + controller.add(1, 2); assertFalse(controller.isEmpty()); assertEquals(messagesToRedeliver.size(), 2); @@ -81,9 +81,8 @@ public void testAddAndRemove(boolean allowOutOfOrderDelivery) throws Exception { assertFalse(hashesToBeBlocked.containsKey(1, 2)); } - assertTrue(controller.remove(1, 1)); - assertTrue(controller.remove(1, 2)); - assertFalse(controller.remove(1, 1)); + controller.remove(1, 1); + controller.remove(1, 2); assertTrue(controller.isEmpty()); assertEquals(messagesToRedeliver.size(), 0); @@ -93,10 +92,9 @@ public void testAddAndRemove(boolean allowOutOfOrderDelivery) throws Exception { assertEquals(hashesToBeBlocked.size(), 0); } - assertTrue(controller.add(2, 1, 100)); - assertTrue(controller.add(2, 2, 101)); - assertTrue(controller.add(2, 3, 101)); - assertFalse(controller.add(2, 1, 100)); + controller.add(2, 1, 100); + controller.add(2, 2, 101); + controller.add(2, 3, 101); assertFalse(controller.isEmpty()); assertEquals(messagesToRedeliver.size(), 3); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java new file mode 100644 index 0000000000000..3c53fc159d027 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import lombok.Cleanup; +import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet; +import org.testng.annotations.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@Test(groups = "utils") +public class ConcurrentBitmapSortedLongPairSetTest { + + @Test + public void testAdd() { + ConcurrentBitmapSortedLongPairSet set = new ConcurrentBitmapSortedLongPairSet(); + int items = 10; + for (int i = 0; i < items; i++) { + set.add(1, i); + } + assertEquals(set.size(), items); + + for (int i = 0; i < items; i++) { + set.add(2, i); + } + assertEquals(set.size(), items * 2); + + for (int i = 0; i < items; i++) { + set.add(2, i); + } + assertEquals(set.size(), items * 2); + } + + @Test + public void testRemove() { + ConcurrentBitmapSortedLongPairSet set = new ConcurrentBitmapSortedLongPairSet(); + int items = 10; + for (int i = 0; i < items; i++) { + set.add(1, i); + } + + for (int i = 0; i < items / 2; i++) { + set.remove(1, i); + } + assertEquals(set.size(), items / 2); + + for (int i = 0; i < items / 2; i++) { + set.remove(2, i); + } + assertEquals(set.size(), items / 2); + + for (int i = 0; i < items / 2; i++) { + set.remove(1, i + 10000); + } + assertEquals(set.size(), items / 2); + + for (int i = 0; i < items / 2; i++) { + set.remove(1, i + items / 2); + } + assertEquals(set.size(), 0); + assertTrue(set.isEmpty()); + } + + @Test + public void testContains() { + ConcurrentBitmapSortedLongPairSet set = new ConcurrentBitmapSortedLongPairSet(); + assertFalse(set.contains(1, 1)); + + int items = 10; + for (int i = 0; i < items; i++) { + set.add(1, i); + } + + for (int i = 0; i < items; i++) { + assertTrue(set.contains(1, i)); + } + + assertFalse(set.contains(1, 10000)); + } + + @Test + public void testRemoveUpTo() { + ConcurrentBitmapSortedLongPairSet set = new ConcurrentBitmapSortedLongPairSet(); + set.removeUpTo(0, 1000); + set.removeUpTo(10, 10000); + assertTrue(set.isEmpty()); + + set.add(1, 0); + + int items = 10; + for (int i = 0; i < items; i++) { + set.add(1, i); + } + + set.removeUpTo(1, 5); + assertFalse(set.isEmpty()); + assertEquals(set.size(), 5); + + for (int i = 5; i < items; i++) { + assertTrue(set.contains(1, i)); + } + + set.removeUpTo(2, 0); + assertTrue(set.isEmpty()); + } + + @Test + public void testItems() { + ConcurrentBitmapSortedLongPairSet set = new ConcurrentBitmapSortedLongPairSet(); + Set items = set.items(10, ConcurrentLongPairSet.LongPair::new); + assertEquals(items.size(), 0); + for (int i = 0; i < 100; i++) { + set.add(1, i); + set.add(2, i); + set.add(5, i); + } + for (int i = 0; i < 100; i++) { + set.add(1, i + 1000); + set.add(2, i + 1000); + set.add(5, i + 1000); + } + + for (int i = 0; i < 100; i++) { + set.add(1, i + 500); + set.add(2, i + 500); + set.add(5, i + 500); + } + assertEquals(set.size(), 900); + assertFalse(set.isEmpty()); + items = set.items(10, ConcurrentLongPairSet.LongPair::new); + assertEquals(items.size(), 10); + ConcurrentLongPairSet.LongPair last = null; + for (ConcurrentLongPairSet.LongPair item : items) { + if (last != null) { + assertTrue(item.compareTo(last) > 0); + } + last = item; + } + + items = set.items(900, ConcurrentLongPairSet.LongPair::new); + assertEquals(items.size(), 900); + last = null; + for (ConcurrentLongPairSet.LongPair item : items) { + if (last != null) { + assertTrue(item.compareTo(last) > 0); + } + last = item; + } + + items = set.items(1000, ConcurrentLongPairSet.LongPair::new); + assertEquals(items.size(), 900); + } + + @Test + public void concurrentInsertions() throws Throwable { + ConcurrentBitmapSortedLongPairSet set = new ConcurrentBitmapSortedLongPairSet(); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newCachedThreadPool(); + + final int nThreads = 8; + final int N = 1000; + + List> futures = new ArrayList<>(); + for (int i = 0; i < nThreads; i++) { + final int threadIdx = i; + + futures.add(executor.submit(() -> { + Random random = new Random(); + + for (int j = 0; j < N; j++) { + int key = random.nextInt(); + // Ensure keys are unique + key -= key % (threadIdx + 1); + key = Math.abs(key); + set.add(key, key); + } + })); + } + + for (Future future : futures) { + future.get(); + } + + assertEquals(set.size(), N * nThreads); + } +} From 29420622eb78fec2af9f5e83f5fedacd3b12f9b2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Sep 2022 21:19:05 +0300 Subject: [PATCH 760/823] [doc][proxy] Document how to mitigate CVE-2022-24280 (#17825) (cherry picked from commit 0c0984a81488f6da0cf46576b50345bb08e78be7) (cherry picked from commit 5ffd0b9da7b64a8157222243573c93a2b9d4195c) --- site2/docs/administration-proxy.md | 70 +++++++++++++++---- .../website-next/docs/administration-proxy.md | 47 ++++++++++++- 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/site2/docs/administration-proxy.md b/site2/docs/administration-proxy.md index 9d99c5ff5a87f..ed29e0ace90c2 100644 --- a/site2/docs/administration-proxy.md +++ b/site2/docs/administration-proxy.md @@ -6,21 +6,15 @@ sidebar_label: Pulsar proxy Pulsar proxy is an optional gateway. Pulsar proxy is used when direct connections between clients and Pulsar brokers are either infeasible or undesirable. For example, when you run Pulsar in a cloud environment or on [Kubernetes](https://kubernetes.io) or an analogous platform, you can run Pulsar proxy. -## Configure the proxy - -Before using the proxy, you need to configure it with the brokers addresses in the cluster. You can configure the proxy to connect directly to service discovery, or specify a broker URL in the configuration. +The Pulsar proxy is not intended to be exposed on the public internet. The security considerations in the current design expect network perimeter security. The requirement of network perimeter security can be achieved with private networks. -### Use service discovery +If a proxy deployment cannot be protected with network perimeter security, the alternative would be to use [Pulsar's "Proxy SNI routing" feature](concepts-proxy-sni-routing.md) with a properly secured and audited solution. In that case Pulsar proxy component is not used at all. -Pulsar uses [ZooKeeper](https://zookeeper.apache.org) for service discovery. To connect the proxy to ZooKeeper, specify the following in `conf/proxy.conf`. -```properties -zookeeperServers=zk-0,zk-1,zk-2 -configurationStoreServers=zk-0:2184,zk-remote:2184 -``` +## Configure the proxy -> To use service discovery, you need to open the network ACLs, so the proxy can connects to the ZooKeeper nodes through the ZooKeeper client port (port `2181`) and the configuration store client port (port `2184`). +Before using the proxy, you need to configure it with the brokers addresses in the cluster. You can configure the broker URL in the proxy configuration, or the proxy to connect directly using service discovery. -> However, it is not secure to use service discovery. Because if the network ACL is open, when someone compromises a proxy, they have full access to ZooKeeper. +> In a production environment service discovery is not recommended. ### Use broker URLs @@ -49,13 +43,65 @@ The ports to connect to the brokers (6650 and 8080, or in the case of TLS, 6651 Note that if you do not use functions, you do not need to configure `functionWorkerWebServiceURL`. +### Use service discovery + +Pulsar uses [ZooKeeper](https://zookeeper.apache.org) for service discovery. To connect the proxy to ZooKeeper, specify the following in `conf/proxy.conf`. +```properties +metadataStoreUrl=my-zk-0:2181,my-zk-1:2181,my-zk-2:2181 +configurationMetadataStoreUrl=my-zk-0:2184,my-zk-remote:2184 +``` + +> To use service discovery, you need to open the network ACLs, so the proxy can connects to the ZooKeeper nodes through the ZooKeeper client port (port `2181`) and the configuration store client port (port `2184`). + +> However, it is not secure to use service discovery. Because if the network ACL is open, when someone compromises a proxy, they have full access to ZooKeeper. + +### Restricting target broker addresses to mitigate CVE-2022-24280 + +The Pulsar Proxy trusts clients to provide valid target broker addresses to connect to. +Unless the Pulsar Proxy is explicitly configured to limit access, the Pulsar Proxy is vulnerable as described in the security advisory [Apache Pulsar Proxy target broker address isn't validated (CVE-2022-24280)](https://github.com/apache/pulsar/wiki/CVE-2022-24280). + +It is necessary to limit proxied broker connections to known broker addresses by specifying `brokerProxyAllowedHostNames` and `brokerProxyAllowedIPAddresses` settings. + +When specifying `brokerProxyAllowedHostNames`, it's possible to use a wildcard. +Please notice that `*` is a wildcard that matches any character in the hostname. It also matches dot `.` characters. + +It is recommended to use a pattern that matches only the desired brokers and no other hosts in the local network. Pulsar lookups will use the default host name of the broker by default. This can be overridden with the `advertisedAddress` setting in `broker.conf`. + +To increase security, it is also possible to restrict access with the `brokerProxyAllowedIPAddresses` setting. It is not mandatory to configure `brokerProxyAllowedIPAddresses` when `brokerProxyAllowedHostNames` is properly configured so that the pattern matches only the target brokers. +`brokerProxyAllowedIPAddresses` setting supports a comma separate list of IP address, IP address ranges and IP address networks [(supported format reference)](https://seancfoley.github.io/IPAddress/IPAddress/apidocs/inet/ipaddr/IPAddressString.html). + +Example: limiting by host name in a Kubernetes deployment +```yaml + # example of limiting to Kubernetes statefulset hostnames that contain "broker-" + PULSAR_PREFIX_brokerProxyAllowedHostNames: '*broker-*.*.*.svc.cluster.local' +``` + +Example: limiting by both host name and ip address in a `proxy.conf` file for host deployment. +```properties +# require "broker" in host name +brokerProxyAllowedHostNames=*broker*.localdomain +# limit target ip addresses to a specific network +brokerProxyAllowedIPAddresses=10.0.0.0/8 +``` + +Example: limiting by multiple host name patterns and multiple ip address ranges in a `proxy.conf` file for host deployment. +```properties +# require "broker" in host name +brokerProxyAllowedHostNames=*broker*.localdomain,*broker*.otherdomain +# limit target ip addresses to a specific network or range demonstrating multiple supported formats +brokerProxyAllowedIPAddresses=10.10.0.0/16,192.168.1.100-120,172.16.2.*,10.1.2.3 +``` + + ## Start the proxy To start the proxy: ```bash $ cd /path/to/pulsar/directory -$ bin/pulsar proxy +$ bin/pulsar proxy \ + --metadata-store zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 \ + --configuration-metadata-store zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 ``` > You can run multiple instances of the Pulsar proxy in a cluster. diff --git a/site2/website-next/docs/administration-proxy.md b/site2/website-next/docs/administration-proxy.md index f1c38d71c2981..1371ee2d6059f 100644 --- a/site2/website-next/docs/administration-proxy.md +++ b/site2/website-next/docs/administration-proxy.md @@ -10,9 +10,13 @@ import TabItem from '@theme/TabItem'; Pulsar proxy is an optional gateway. Pulsar proxy is used when direct connections between clients and Pulsar brokers are either infeasible or undesirable. For example, when you run Pulsar in a cloud environment or on [Kubernetes](https://kubernetes.io) or an analogous platform, you can run Pulsar proxy. +The Pulsar proxy is not intended to be exposed on the public internet. The security considerations in the current design expect network perimeter security. The requirement of network perimeter security can be achieved with private networks. + +If a proxy deployment cannot be protected with network perimeter security, the alternative would be to use [Pulsar's "Proxy SNI routing" feature](concepts-proxy-sni-routing.md) with a properly secured and audited solution. In that case Pulsar proxy component is not used at all. + ## Configure the proxy -Before using the proxy, you need to configure it with the brokers addresses in the cluster. You can configure the proxy to connect directly to service discovery, or specify a broker URL in the configuration. +Before using a proxy, you need to configure it with a broker's address in the cluster. You can configure the broker URL in the proxy configuration, or the proxy to connect directly using service discovery. ### Use service discovery @@ -47,12 +51,51 @@ brokerWebServiceURLTLS=https://brokers.example.com:8443 functionWorkerWebServiceURL=https://function-workers.example.com:8443 ``` -The hostname in the URLs provided should be a DNS entry which points to multiple brokers or a virtual IP address, which is backed by multiple broker IP addresses, so that the proxy does not lose connectivity to Pulsar cluster if a single broker becomes unavailable. +The hostname in the URLs provided should be a DNS entry that points to multiple brokers or a virtual IP address, which is backed by multiple broker IP addresses, so that the proxy does not lose connectivity to Pulsar cluster if a single broker becomes unavailable. The ports to connect to the brokers (6650 and 8080, or in the case of TLS, 6651 and 8443) should be open in the network ACLs. Note that if you do not use functions, you do not need to configure `functionWorkerWebServiceURL`. +> However, it is not secure to use service discovery. Because if the network ACL is open, when someone compromises a proxy, they have full access to ZooKeeper. + +### Restricting target broker addresses to mitigate CVE-2022-24280 + +The Pulsar Proxy trusts clients to provide valid target broker addresses to connect to. +Unless the Pulsar Proxy is explicitly configured to limit access, the Pulsar Proxy is vulnerable as described in the security advisory [Apache Pulsar Proxy target broker address isn't validated (CVE-2022-24280)](https://github.com/apache/pulsar/wiki/CVE-2022-24280). + +It is necessary to limit proxied broker connections to known broker addresses by specifying `brokerProxyAllowedHostNames` and `brokerProxyAllowedIPAddresses` settings. + +When specifying `brokerProxyAllowedHostNames`, it's possible to use a wildcard. +Please notice that `*` is a wildcard that matches any character in the hostname. It also matches dot `.` characters. + +It is recommended to use a pattern that matches only the desired brokers and no other hosts in the local network. Pulsar lookups will use the default host name of the broker by default. This can be overridden with the `advertisedAddress` setting in `broker.conf`. + +To increase security, it is also possible to restrict access with the `brokerProxyAllowedIPAddresses` setting. It is not mandatory to configure `brokerProxyAllowedIPAddresses` when `brokerProxyAllowedHostNames` is properly configured so that the pattern matches only the target brokers. +`brokerProxyAllowedIPAddresses` setting supports a comma separate list of IP address, IP address ranges and IP address networks [(supported format reference)](https://seancfoley.github.io/IPAddress/IPAddress/apidocs/inet/ipaddr/IPAddressString.html). + +Example: limiting by host name in a Kubernetes deployment +```yaml + # example of limiting to Kubernetes statefulset hostnames that contain "broker-" + PULSAR_PREFIX_brokerProxyAllowedHostNames: '*broker-*.*.*.svc.cluster.local' +``` + +Example: limiting by both host name and ip address in a `proxy.conf` file for host deployment. +```properties +# require "broker" in host name +brokerProxyAllowedHostNames=*broker*.localdomain +# limit target ip addresses to a specific network +brokerProxyAllowedIPAddresses=10.0.0.0/8 +``` + +Example: limiting by multiple host name patterns and multiple ip address ranges in a `proxy.conf` file for host deployment. +```properties +# require "broker" in host name +brokerProxyAllowedHostNames=*broker*.localdomain,*broker*.otherdomain +# limit target ip addresses to a specific network or range demonstrating multiple supported formats +brokerProxyAllowedIPAddresses=10.10.0.0/16,192.168.1.100-120,172.16.2.*,10.1.2.3 +``` + ## Start the proxy To start the proxy: From 3564f423c4893f6f097d8a9c7e9e3d99687ca776 Mon Sep 17 00:00:00 2001 From: penghui Date: Sat, 24 Sep 2022 11:43:08 +0800 Subject: [PATCH 761/823] Revert "[fix][broker][functions-worker] Ensure prometheus metrics are grouped by type (#8407, #13865) (#17618)" This reverts commit bc69cfbe9d3cf06fc6e407c284eed45f095b41be. --- .../prometheus/AggregatedNamespaceStats.java | 2 +- .../prometheus/NamespaceStatsAggregator.java | 354 +++++------ .../prometheus/PrometheusMetricStreams.java | 75 --- .../PrometheusMetricsGenerator.java | 60 +- .../broker/stats/prometheus/TopicStats.java | 598 ++++++++++-------- .../prometheus/TransactionAggregator.java | 321 ++++++---- .../metrics/PrometheusTextFormatUtil.java | 32 + .../broker/stats/PrometheusMetricsTest.java | 58 -- .../PrometheusMetricStreamsTest.java | 85 --- .../common/util/SimpleTextOutputStream.java | 13 +- .../instance/stats/PrometheusTextFormat.java | 5 - .../functions/worker/WorkerStatsManager.java | 5 - 12 files changed, 730 insertions(+), 878 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 1980af91b7b54..5610dbab218e0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -96,7 +96,7 @@ void updateStats(TopicStats stats) { stats.replicationStats.forEach((n, as) -> { AggregatedReplicationStats replStats = - replicationStats.computeIfAbsent(n, k -> new AggregatedReplicationStats()); + replicationStats.computeIfAbsent(n, k -> new AggregatedReplicationStats()); replStats.msgRateIn += as.msgRateIn; replStats.msgRateOut += as.msgRateOut; replStats.msgThroughputIn += as.msgThroughputIn; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 29915f071c099..16e438e2a2eb3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -19,11 +19,8 @@ package org.apache.pulsar.broker.stats.prometheus; import io.netty.util.concurrent.FastThreadLocal; -import java.util.HashMap; -import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.LongAdder; -import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -35,6 +32,7 @@ import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; +import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.CompactorMXBean; @@ -42,75 +40,72 @@ @Slf4j public class NamespaceStatsAggregator { - private static final FastThreadLocal localNamespaceStats = + private static FastThreadLocal localNamespaceStats = new FastThreadLocal() { @Override - protected AggregatedNamespaceStats initialValue() { + protected AggregatedNamespaceStats initialValue() throws Exception { return new AggregatedNamespaceStats(); } }; - private static final FastThreadLocal localTopicStats = new FastThreadLocal() { + private static FastThreadLocal localTopicStats = new FastThreadLocal() { @Override - protected TopicStats initialValue() { + protected TopicStats initialValue() throws Exception { return new TopicStats(); } }; public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - PrometheusMetricStreams stream) { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, SimpleTextOutputStream stream) { String cluster = pulsar.getConfiguration().getClusterName(); AggregatedNamespaceStats namespaceStats = localNamespaceStats.get(); + TopicStats.resetTypes(); TopicStats topicStats = localTopicStats.get(); - Optional compactorMXBean = getCompactorMXBean(pulsar); - LongAdder topicsCount = new LongAdder(); - Map localNamespaceTopicCount = new HashMap<>(); printDefaultBrokerStats(stream, cluster); + Optional compactorMXBean = getCompactorMXBean(pulsar); + LongAdder topicsCount = new LongAdder(); pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { namespaceStats.reset(); topicsCount.reset(); - bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { - getTopicStats(topic, topicStats, includeConsumerMetrics, includeProducerMetrics, - pulsar.getConfiguration().isExposePreciseBacklogInPrometheus(), - pulsar.getConfiguration().isExposeSubscriptionBacklogSizeInPrometheus(), - compactorMXBean - ); - - if (includeTopicMetrics) { - topicsCount.add(1); - TopicStats.printTopicStats(stream, topicStats, compactorMXBean, cluster, namespace, name, - splitTopicAndPartitionIndexLabel); - } else { - namespaceStats.updateStats(topicStats); - } - })); + bundlesMap.forEach((bundle, topicsMap) -> { + topicsMap.forEach((name, topic) -> { + getTopicStats(topic, topicStats, includeConsumerMetrics, includeProducerMetrics, + pulsar.getConfiguration().isExposePreciseBacklogInPrometheus(), + pulsar.getConfiguration().isExposeSubscriptionBacklogSizeInPrometheus(), + compactorMXBean + ); + + if (includeTopicMetrics) { + topicsCount.add(1); + TopicStats.printTopicStats(stream, cluster, namespace, name, topicStats, compactorMXBean, + splitTopicAndPartitionIndexLabel); + } else { + namespaceStats.updateStats(topicStats); + } + }); + }); if (!includeTopicMetrics) { - // Only include namespace level stats if we don't have the per-topic, otherwise we're going to - // report the same data twice, and it will make the aggregation difficult - printNamespaceStats(stream, namespaceStats, cluster, namespace); + // Only include namespace level stats if we don't have the per-topic, otherwise we're going to report + // the same data twice, and it will make the aggregation difficult + printNamespaceStats(stream, cluster, namespace, namespaceStats); } else { - localNamespaceTopicCount.put(namespace, topicsCount.sum()); + printTopicsCountStats(stream, cluster, namespace, topicsCount); } }); - - if (includeTopicMetrics) { - printTopicsCountStats(stream, localNamespaceTopicCount, cluster); - } } private static Optional getCompactorMXBean(PulsarService pulsar) { Compactor compactor = pulsar.getNullableCompactor(); - return Optional.ofNullable(compactor).map(Compactor::getStats); + return Optional.ofNullable(compactor).map(c -> c.getStats()); } private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean getPreciseBacklog, - boolean subscriptionBacklogSize, Optional compactorMXBean) { + boolean includeProducerMetrics, boolean getPreciseBacklog, boolean subscriptionBacklogSize, + Optional compactorMXBean) { stats.reset(); if (topic instanceof PersistentTopic) { @@ -272,176 +267,161 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include }); } - private static void printDefaultBrokerStats(PrometheusMetricStreams stream, String cluster) { + private static void printDefaultBrokerStats(SimpleTextOutputStream stream, String cluster) { // Print metrics with 0 values. This is necessary to have the available brokers being // reported in the brokers dashboard even if they don't have any topic or traffic - writeMetric(stream, "pulsar_topics_count", 0, cluster); - writeMetric(stream, "pulsar_subscriptions_count", 0, cluster); - writeMetric(stream, "pulsar_producers_count", 0, cluster); - writeMetric(stream, "pulsar_consumers_count", 0, cluster); - writeMetric(stream, "pulsar_rate_in", 0, cluster); - writeMetric(stream, "pulsar_rate_out", 0, cluster); - writeMetric(stream, "pulsar_throughput_in", 0, cluster); - writeMetric(stream, "pulsar_throughput_out", 0, cluster); - writeMetric(stream, "pulsar_storage_size", 0, cluster); - writeMetric(stream, "pulsar_storage_logical_size", 0, cluster); - writeMetric(stream, "pulsar_storage_write_rate", 0, cluster); - writeMetric(stream, "pulsar_storage_read_rate", 0, cluster); - writeMetric(stream, "pulsar_msg_backlog", 0, cluster); + metric(stream, cluster, "pulsar_topics_count", 0); + metric(stream, cluster, "pulsar_subscriptions_count", 0); + metric(stream, cluster, "pulsar_producers_count", 0); + metric(stream, cluster, "pulsar_consumers_count", 0); + metric(stream, cluster, "pulsar_rate_in", 0); + metric(stream, cluster, "pulsar_rate_out", 0); + metric(stream, cluster, "pulsar_throughput_in", 0); + metric(stream, cluster, "pulsar_throughput_out", 0); + metric(stream, cluster, "pulsar_storage_size", 0); + metric(stream, cluster, "pulsar_storage_logical_size", 0); + metric(stream, cluster, "pulsar_storage_write_rate", 0); + metric(stream, cluster, "pulsar_storage_read_rate", 0); + metric(stream, cluster, "pulsar_msg_backlog", 0); } - private static void printTopicsCountStats(PrometheusMetricStreams stream, Map namespaceTopicsCount, - String cluster) { - namespaceTopicsCount.forEach( - (ns, topicCount) -> writeMetric(stream, "pulsar_topics_count", topicCount, cluster, ns) - ); + private static void printTopicsCountStats(SimpleTextOutputStream stream, String cluster, String namespace, + LongAdder topicsCount) { + metric(stream, cluster, namespace, "pulsar_topics_count", topicsCount.sum()); } - private static void printNamespaceStats(PrometheusMetricStreams stream, AggregatedNamespaceStats stats, - String cluster, String namespace) { - writeMetric(stream, "pulsar_topics_count", stats.topicsCount, cluster, namespace); - writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, cluster, - namespace); - writeMetric(stream, "pulsar_producers_count", stats.producersCount, cluster, namespace); - writeMetric(stream, "pulsar_consumers_count", stats.consumersCount, cluster, namespace); - - writeMetric(stream, "pulsar_rate_in", stats.rateIn, cluster, namespace); - writeMetric(stream, "pulsar_rate_out", stats.rateOut, cluster, namespace); - writeMetric(stream, "pulsar_throughput_in", stats.throughputIn, cluster, namespace); - writeMetric(stream, "pulsar_throughput_out", stats.throughputOut, cluster, namespace); - writeMetric(stream, "pulsar_consumer_msg_ack_rate", stats.messageAckRate, cluster, namespace); - - writeMetric(stream, "pulsar_in_bytes_total", stats.bytesInCounter, cluster, namespace); - writeMetric(stream, "pulsar_in_messages_total", stats.msgInCounter, cluster, namespace); - writeMetric(stream, "pulsar_out_bytes_total", stats.bytesOutCounter, cluster, namespace); - writeMetric(stream, "pulsar_out_messages_total", stats.msgOutCounter, cluster, namespace); - - writeMetric(stream, "pulsar_storage_size", stats.managedLedgerStats.storageSize, cluster, - namespace); - writeMetric(stream, "pulsar_storage_logical_size", - stats.managedLedgerStats.storageLogicalSize, cluster, namespace); - writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, cluster, - namespace); - writeMetric(stream, "pulsar_storage_offloaded_size", - stats.managedLedgerStats.offloadedStorageUsed, cluster, namespace); - - writeMetric(stream, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate, - cluster, namespace); - writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, - cluster, namespace); - - writeMetric(stream, "pulsar_subscription_delayed", stats.msgDelayed, cluster, namespace); - - writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace); + private static void printNamespaceStats(SimpleTextOutputStream stream, String cluster, String namespace, + AggregatedNamespaceStats stats) { + metric(stream, cluster, namespace, "pulsar_topics_count", stats.topicsCount); + metric(stream, cluster, namespace, "pulsar_subscriptions_count", stats.subscriptionsCount); + metric(stream, cluster, namespace, "pulsar_producers_count", stats.producersCount); + metric(stream, cluster, namespace, "pulsar_consumers_count", stats.consumersCount); + + metric(stream, cluster, namespace, "pulsar_rate_in", stats.rateIn); + metric(stream, cluster, namespace, "pulsar_rate_out", stats.rateOut); + metric(stream, cluster, namespace, "pulsar_throughput_in", stats.throughputIn); + metric(stream, cluster, namespace, "pulsar_throughput_out", stats.throughputOut); + metric(stream, cluster, namespace, "pulsar_consumer_msg_ack_rate", stats.messageAckRate); + + metric(stream, cluster, namespace, "pulsar_in_bytes_total", stats.bytesInCounter); + metric(stream, cluster, namespace, "pulsar_in_messages_total", stats.msgInCounter); + metric(stream, cluster, namespace, "pulsar_out_bytes_total", stats.bytesOutCounter); + metric(stream, cluster, namespace, "pulsar_out_messages_total", stats.msgOutCounter); + + metric(stream, cluster, namespace, "pulsar_storage_size", stats.managedLedgerStats.storageSize); + metric(stream, cluster, namespace, "pulsar_storage_logical_size", stats.managedLedgerStats.storageLogicalSize); + metric(stream, cluster, namespace, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize); + metric(stream, cluster, namespace, "pulsar_storage_offloaded_size", + stats.managedLedgerStats.offloadedStorageUsed); + + metric(stream, cluster, namespace, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate); + metric(stream, cluster, namespace, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate); + + metric(stream, cluster, namespace, "pulsar_subscription_delayed", stats.msgDelayed); + + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_msg_backlog", "local", stats.msgBacklog); stats.managedLedgerStats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_1", latencyBuckets[1], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_5", latencyBuckets[2], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_10", latencyBuckets[3], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_20", latencyBuckets[4], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_50", latencyBuckets[5], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_100", latencyBuckets[6], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_200", latencyBuckets[7], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_overflow", latencyBuckets[9], cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_count", - stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), cluster, namespace); - writeMetric(stream, "pulsar_storage_write_latency_sum", - stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), cluster, namespace); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_1", latencyBuckets[1]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_5", latencyBuckets[2]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_10", latencyBuckets[3]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_20", latencyBuckets[4]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_50", latencyBuckets[5]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_100", latencyBuckets[6]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_200", latencyBuckets[7]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_1000", latencyBuckets[8]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_overflow", latencyBuckets[9]); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_count", + stats.managedLedgerStats.storageWriteLatencyBuckets.getCount()); + metric(stream, cluster, namespace, "pulsar_storage_write_latency_sum", + stats.managedLedgerStats.storageWriteLatencyBuckets.getSum()); stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.refresh(); - long[] ledgerWriteLatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWriteLatencyBuckets[0], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", ledgerWriteLatencyBuckets[1], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", ledgerWriteLatencyBuckets[2], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", ledgerWriteLatencyBuckets[3], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", ledgerWriteLatencyBuckets[4], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", ledgerWriteLatencyBuckets[5], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", ledgerWriteLatencyBuckets[6], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", ledgerWriteLatencyBuckets[7], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", ledgerWriteLatencyBuckets[8], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", ledgerWriteLatencyBuckets[9], - cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_count", - stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount(), cluster, namespace); - writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", - stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum(), cluster, namespace); + long[] ledgerWritelatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWritelatencyBuckets[0]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_1", ledgerWritelatencyBuckets[1]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_5", ledgerWritelatencyBuckets[2]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_10", ledgerWritelatencyBuckets[3]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_20", ledgerWritelatencyBuckets[4]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_50", ledgerWritelatencyBuckets[5]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_100", ledgerWritelatencyBuckets[6]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_200", ledgerWritelatencyBuckets[7]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_1000", ledgerWritelatencyBuckets[8]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_overflow", + ledgerWritelatencyBuckets[9]); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_count", + stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount()); + metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_sum", + stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum()); stats.managedLedgerStats.entrySizeBuckets.refresh(); long[] entrySizeBuckets = stats.managedLedgerStats.entrySizeBuckets.getBuckets(); - writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace); - writeMetric(stream, "pulsar_entry_size_count", stats.managedLedgerStats.entrySizeBuckets.getCount(), - cluster, namespace); - writeMetric(stream, "pulsar_entry_size_sum", stats.managedLedgerStats.entrySizeBuckets.getSum(), - cluster, namespace); - - writeReplicationStat(stream, "pulsar_replication_rate_in", stats, - replStats -> replStats.msgRateIn, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_rate_out", stats, - replStats -> replStats.msgRateOut, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_throughput_in", stats, - replStats -> replStats.msgThroughputIn, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_throughput_out", stats, - replStats -> replStats.msgThroughputOut, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_backlog", stats, - replStats -> replStats.replicationBacklog, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_connected_count", stats, - replStats -> replStats.connectedCount, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_rate_expired", stats, - replStats -> replStats.msgRateExpired, cluster, namespace); - writeReplicationStat(stream, "pulsar_replication_delay_in_seconds", stats, - replStats -> replStats.replicationDelayInSeconds, cluster, namespace); + metric(stream, cluster, namespace, "pulsar_entry_size_le_128", entrySizeBuckets[0]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_512", entrySizeBuckets[1]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7]); + metric(stream, cluster, namespace, "pulsar_entry_size_le_overflow", entrySizeBuckets[8]); + metric(stream, cluster, namespace, "pulsar_entry_size_count", + stats.managedLedgerStats.entrySizeBuckets.getCount()); + metric(stream, cluster, namespace, "pulsar_entry_size_sum", + stats.managedLedgerStats.entrySizeBuckets.getSum()); + + if (!stats.replicationStats.isEmpty()) { + stats.replicationStats.forEach((remoteCluster, replStats) -> { + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_in", remoteCluster, + replStats.msgRateIn); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_out", remoteCluster, + replStats.msgRateOut); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_throughput_in", remoteCluster, + replStats.msgThroughputIn); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_throughput_out", remoteCluster, + replStats.msgThroughputOut); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_backlog", remoteCluster, + replStats.replicationBacklog); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_connected_count", remoteCluster, + replStats.connectedCount); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_expired", remoteCluster, + replStats.msgRateExpired); + metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_delay_in_seconds", + remoteCluster, replStats.replicationDelayInSeconds); + }); + } } - private static void writePulsarMsgBacklog(PrometheusMetricStreams stream, Number value, - String cluster, String namespace) { - stream.writeSample("pulsar_msg_backlog", value, "cluster", cluster, "namespace", namespace, - "remote_cluster", - "local"); + private static void metric(SimpleTextOutputStream stream, String cluster, String name, + long value) { + TopicStats.metricType(stream, name); + stream.write(name) + .write("{cluster=\"").write(cluster).write("\"} ") + .write(value).write(' ').write(System.currentTimeMillis()) + .write('\n'); } - private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, - String cluster) { - stream.writeSample(metricName, value, "cluster", cluster); + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String name, + long value) { + TopicStats.metricType(stream, name); + stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace).write("\"} "); + stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); } - private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace) { - stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace); + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String name, + double value) { + TopicStats.metricType(stream, name); + stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace).write("\"} "); + stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); } - private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName, - AggregatedNamespaceStats namespaceStats, - Function sampleValueFunction, - String cluster, String namespace) { - if (!namespaceStats.replicationStats.isEmpty()) { - namespaceStats.replicationStats.forEach((remoteCluster, replStats) -> - stream.writeSample(metricName, sampleValueFunction.apply(replStats), - "cluster", cluster, - "namespace", namespace, - "remote_cluster", remoteCluster) - ); - } + private static void metricWithRemoteCluster(SimpleTextOutputStream stream, String cluster, String namespace, + String name, String remoteCluster, double value) { + TopicStats.metricType(stream, name); + stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace); + stream.write("\",remote_cluster=\"").write(remoteCluster).write("\"} "); + stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); } - - } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java deleted file mode 100644 index 6b6b972c175f0..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.stats.prometheus; - -import java.util.HashMap; -import java.util.Map; -import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; -import org.apache.pulsar.common.util.SimpleTextOutputStream; - -/** - * Helper class to ensure that metrics of the same name are grouped together under the same TYPE header when written. - * Those are the requirements of the - * Prometheus Exposition Format. - */ -public class PrometheusMetricStreams { - private final Map metricStreamMap = new HashMap<>(); - - /** - * Write the given metric and sample value to the stream. Will write #TYPE header if metric not seen before. - * @param metricName name of the metric. - * @param value value of the sample - * @param labelsAndValuesArray varargs of label and label value - */ - void writeSample(String metricName, Number value, String... labelsAndValuesArray) { - SimpleTextOutputStream stream = initGaugeType(metricName); - stream.write(metricName).write('{'); - for (int i = 0; i < labelsAndValuesArray.length; i += 2) { - stream.write(labelsAndValuesArray[i]).write("=\"").write(labelsAndValuesArray[i + 1]).write('\"'); - if (i + 2 != labelsAndValuesArray.length) { - stream.write(','); - } - } - stream.write("} ").write(value).write(' ').write(System.currentTimeMillis()).write('\n'); - } - - /** - * Flush all the stored metrics to the supplied stream. - * @param stream the stream to write to. - */ - void flushAllToStream(SimpleTextOutputStream stream) { - metricStreamMap.values().forEach(s -> stream.write(s.getBuffer())); - } - - /** - * Release all the streams to clean up resources. - */ - void releaseAll() { - metricStreamMap.values().forEach(s -> s.getBuffer().release()); - metricStreamMap.clear(); - } - - private SimpleTextOutputStream initGaugeType(String metricName) { - return metricStreamMap.computeIfAbsent(metricName, s -> { - SimpleTextOutputStream stream = new SimpleTextOutputStream(PulsarByteBufAllocator.DEFAULT.directBuffer()); - stream.write("# TYPE ").write(metricName).write(" gauge\n"); - return stream; - }); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index a993d1edf3a3d..cd6afd1535dec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -52,8 +52,7 @@ /** * Generate metrics aggregated at the namespace level and optionally at a topic level and formats them out * in a text format suitable to be consumed by Prometheus. - * Format specification can be found at Exposition Formats + * Format specification can be found at {@link https://prometheus.io/docs/instrumenting/exposition_formats/} */ public class PrometheusMetricsGenerator { @@ -87,44 +86,38 @@ public double get() { } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, OutputStream out) throws IOException { + boolean includeProducerMetrics, OutputStream out) throws IOException { generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, false, out, null); } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - OutputStream out) throws IOException { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + OutputStream out) throws IOException { generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, splitTopicAndPartitionIndexLabel, out, null); } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - OutputStream out, - List metricsProviders) - throws IOException { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, OutputStream out, + List metricsProviders) + throws IOException { ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer(); - boolean exceptionHappens = false; - //Used in namespace/topic and transaction aggregators as share metric names - PrometheusMetricStreams metricStreams = new PrometheusMetricStreams(); try { SimpleTextOutputStream stream = new SimpleTextOutputStream(buf); generateSystemMetrics(stream, pulsar.getConfiguration().getClusterName()); NamespaceStatsAggregator.generate(pulsar, includeTopicMetrics, includeConsumerMetrics, - includeProducerMetrics, splitTopicAndPartitionIndexLabel, metricStreams); + includeProducerMetrics, splitTopicAndPartitionIndexLabel, stream); if (pulsar.getWorkerServiceOpt().isPresent()) { pulsar.getWorkerService().generateFunctionsStats(stream); } if (pulsar.getConfiguration().isTransactionCoordinatorEnabled()) { - TransactionAggregator.generate(pulsar, metricStreams, includeTopicMetrics); + TransactionAggregator.generate(pulsar, stream, includeTopicMetrics); } - metricStreams.flushAllToStream(stream); - generateBrokerBasicMetrics(pulsar, stream); generateManagedLedgerBookieClientMetrics(pulsar, stream); @@ -136,12 +129,7 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b } out.write(buf.array(), buf.arrayOffset(), buf.readableBytes()); } finally { - //release all the metrics buffers - metricStreams.releaseAll(); - //if exception happens, release buffer - if (exceptionHappens) { - buf.release(); - } + buf.release(); } } @@ -154,17 +142,17 @@ private static void generateBrokerBasicMetrics(PulsarService pulsar, SimpleTextO if (pulsar.getConfiguration().isExposeManagedLedgerMetricsInPrometheus()) { // generate managedLedger metrics parseMetricsToPrometheusMetrics(new ManagedLedgerMetrics(pulsar).generate(), - clusterName, Collector.Type.GAUGE, stream); + clusterName, Collector.Type.GAUGE, stream); } if (pulsar.getConfiguration().isExposeManagedCursorMetricsInPrometheus()) { // generate managedCursor metrics parseMetricsToPrometheusMetrics(new ManagedCursorMetrics(pulsar).generate(), - clusterName, Collector.Type.GAUGE, stream); + clusterName, Collector.Type.GAUGE, stream); } parseMetricsToPrometheusMetrics(Collections.singletonList(pulsar.getBrokerService() - .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), + .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), clusterName, Collector.Type.GAUGE, stream); // generate loadBalance metrics @@ -279,17 +267,17 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String static String getTypeStr(Collector.Type type) { switch (type) { - case COUNTER: - return "counter"; - case GAUGE: - return "gauge"; - case SUMMARY: - return "summary"; - case HISTOGRAM: - return "histogram"; - case UNTYPED: - default: - return "untyped"; + case COUNTER: + return "counter"; + case GAUGE: + return "gauge"; + case SUMMARY : + return "summary"; + case HISTOGRAM: + return "histogram"; + case UNTYPED: + default: + return "untyped"; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index e91521aff5511..e6e5883847df2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -23,12 +23,12 @@ import java.util.Map; import java.util.Optional; import org.apache.bookkeeper.mledger.util.StatsBuckets; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.compaction.CompactionRecord; import org.apache.pulsar.compaction.CompactorMXBean; class TopicStats { + int subscriptionsCount; int producersCount; int consumersCount; @@ -43,6 +43,7 @@ class TopicStats { double averageMsgSize; public long msgBacklog; + long publishRateLimitedTimes; long backlogQuotaLimit; @@ -54,6 +55,9 @@ class TopicStats { Map subscriptionStats = new HashMap<>(); Map producerStats = new HashMap<>(); + // Used for tracking duplicate TYPE definitions + static Map metricWithTypeDefinition = new HashMap<>(); + // For compaction long compactionRemovedEventCount; long compactionSucceedCount; @@ -99,340 +103,378 @@ public void reset() { compactionLatencyBuckets.reset(); } - public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats, - Optional compactorMXBean, String cluster, String namespace, - String topic, boolean splitTopicAndPartitionIndexLabel) { - writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_producers_count", stats.producersCount, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_consumers_count", stats.consumersCount, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - - writeMetric(stream, "pulsar_rate_in", stats.rateIn, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_rate_out", stats.rateOut, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_throughput_in", stats.throughputIn, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_throughput_out", stats.throughputOut, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_average_msg_size", stats.averageMsgSize, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - - writeMetric(stream, "pulsar_storage_size", stats.managedLedgerStats.storageSize, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_logical_size", - stats.managedLedgerStats.storageLogicalSize, cluster, namespace, topic, - splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_msg_backlog", stats.msgBacklog, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_offloaded_size", stats.managedLedgerStats - .offloadedStorageUsed, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_backlog_quota_limit_time", stats.backlogQuotaLimitTime, - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + static void resetTypes() { + metricWithTypeDefinition.clear(); + } + + static void printTopicStats(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + TopicStats stats, Optional compactorMXBean, + boolean splitTopicAndPartitionIndexLabel) { + metric(stream, cluster, namespace, topic, "pulsar_subscriptions_count", stats.subscriptionsCount, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_producers_count", stats.producersCount, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_consumers_count", stats.consumersCount, + splitTopicAndPartitionIndexLabel); + + metric(stream, cluster, namespace, topic, "pulsar_rate_in", stats.rateIn, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_rate_out", stats.rateOut, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_throughput_in", stats.throughputIn, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_throughput_out", stats.throughputOut, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_average_msg_size", stats.averageMsgSize, + splitTopicAndPartitionIndexLabel); + + metric(stream, cluster, namespace, topic, "pulsar_storage_size", stats.managedLedgerStats.storageSize, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_logical_size", + stats.managedLedgerStats.storageLogicalSize, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_msg_backlog", stats.msgBacklog, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_rate", + stats.managedLedgerStats.storageWriteRate, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_size", + stats.managedLedgerStats.backlogSize, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_offloaded_size", stats.managedLedgerStats + .offloadedStorageUsed, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit_time", + stats.backlogQuotaLimitTime, splitTopicAndPartitionIndexLabel); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_storage_write_latency_le_0_5", - latencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_1", - latencyBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_5", - latencyBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_10", - latencyBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_20", - latencyBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_50", - latencyBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_100", - latencyBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_200", - latencyBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_le_1000", - latencyBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_overflow", - latencyBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_count", - stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_write_latency_sum", - stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_1", latencyBuckets[1], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_5", latencyBuckets[2], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_10", latencyBuckets[3], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_20", latencyBuckets[4], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_50", latencyBuckets[5], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_100", latencyBuckets[6], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_200", latencyBuckets[7], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_overflow", latencyBuckets[9], + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_count", + stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_sum", + stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), splitTopicAndPartitionIndexLabel); long[] ledgerWriteLatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", - ledgerWriteLatencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", - ledgerWriteLatencyBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", - ledgerWriteLatencyBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", - ledgerWriteLatencyBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", - ledgerWriteLatencyBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", - ledgerWriteLatencyBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", - ledgerWriteLatencyBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", - ledgerWriteLatencyBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", - ledgerWriteLatencyBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", - ledgerWriteLatencyBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_count", + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_0_5", + ledgerWriteLatencyBuckets[0], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_1", + ledgerWriteLatencyBuckets[1], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_5", + ledgerWriteLatencyBuckets[2], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_10", + ledgerWriteLatencyBuckets[3], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_20", + ledgerWriteLatencyBuckets[4], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_50", + ledgerWriteLatencyBuckets[5], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_100", + ledgerWriteLatencyBuckets[6], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_200", + ledgerWriteLatencyBuckets[7], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_1000", + ledgerWriteLatencyBuckets[8], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_overflow", + ledgerWriteLatencyBuckets[9], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_count", stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount(), - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_sum", stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum(), - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + splitTopicAndPartitionIndexLabel); long[] entrySizeBuckets = stats.managedLedgerStats.entrySizeBuckets.getBuckets(); - writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_128", entrySizeBuckets[0], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_512", entrySizeBuckets[1], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_count", stats.managedLedgerStats.entrySizeBuckets.getCount(), - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_entry_size_sum", stats.managedLedgerStats.entrySizeBuckets.getSum(), - cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_entry_size_count", + stats.managedLedgerStats.entrySizeBuckets.getCount(), splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_entry_size_sum", + stats.managedLedgerStats.entrySizeBuckets.getSum(), splitTopicAndPartitionIndexLabel); stats.producerStats.forEach((p, producerStats) -> { - writeProducerMetric(stream, "pulsar_producer_msg_rate_in", producerStats.msgRateIn, - cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); - writeProducerMetric(stream, "pulsar_producer_msg_throughput_in", producerStats.msgThroughputIn, - cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); - writeProducerMetric(stream, "pulsar_producer_msg_average_Size", producerStats.averageMsgSize, - cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_rate_in", + producerStats.msgRateIn, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_throughput_in", + producerStats.msgThroughputIn, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_average_Size", + producerStats.averageMsgSize, splitTopicAndPartitionIndexLabel); }); - stats.subscriptionStats.forEach((sub, subsStats) -> { - writeSubscriptionMetric(stream, "pulsar_subscription_back_log", subsStats.msgBacklog, - cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_back_log_no_delayed", - subsStats.msgBacklogNoDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_delayed", - subsStats.msgDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_redeliver", - subsStats.msgRateRedeliver, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_unacked_messages", - subsStats.unackedMessages, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_blocked_on_unacked_messages", - subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, cluster, namespace, topic, sub, - splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_out", - subsStats.msgRateOut, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_msg_ack_rate", - subsStats.messageAckRate, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_msg_throughput_out", - subsStats.msgThroughputOut, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_out_bytes_total", - subsStats.bytesOutCounter, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_out_messages_total", - subsStats.msgOutCounter, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_last_expire_timestamp", - subsStats.lastExpireTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_last_acked_timestamp", - subsStats.lastAckedTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_last_consumed_flow_timestamp", - subsStats.lastConsumedFlowTimestamp, cluster, namespace, topic, sub, - splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_last_consumed_timestamp", - subsStats.lastConsumedTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_last_mark_delete_advanced_timestamp", - subsStats.lastMarkDeleteAdvancedTimestamp, cluster, namespace, topic, sub, - splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_expired", - subsStats.msgRateExpired, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - writeSubscriptionMetric(stream, "pulsar_subscription_total_msg_expired", - subsStats.totalMsgExpired, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); - + stats.subscriptionStats.forEach((n, subsStats) -> { + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_back_log", + subsStats.msgBacklog, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_back_log_no_delayed", + subsStats.msgBacklogNoDelayed, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_delayed", + subsStats.msgDelayed, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_redeliver", + subsStats.msgRateRedeliver, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_unacked_messages", + subsStats.unackedMessages, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_blocked_on_unacked_messages", + subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_out", + subsStats.msgRateOut, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_ack_rate", + subsStats.messageAckRate, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_throughput_out", + subsStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_out_bytes_total", + subsStats.bytesOutCounter, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_out_messages_total", + subsStats.msgOutCounter, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_expire_timestamp", + subsStats.lastExpireTimestamp, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_acked_timestamp", + subsStats.lastAckedTimestamp, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_consumed_flow_timestamp", + subsStats.lastConsumedFlowTimestamp, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_consumed_timestamp", + subsStats.lastConsumedTimestamp, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_mark_delete_advanced_timestamp", + subsStats.lastMarkDeleteAdvancedTimestamp, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_expired", + subsStats.msgRateExpired, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, "pulsar_subscription_total_msg_expired", + subsStats.totalMsgExpired, splitTopicAndPartitionIndexLabel); subsStats.consumerStat.forEach((c, consumerStats) -> { - writeConsumerMetric(stream, "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - writeConsumerMetric(stream, "pulsar_consumer_unacked_messages", consumerStats.unackedMessages, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - writeConsumerMetric(stream, "pulsar_consumer_blocked_on_unacked_messages", + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_unacked_messages", consumerStats.unackedMessages, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_blocked_on_unacked_messages", consumerStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - writeConsumerMetric(stream, "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - - writeConsumerMetric(stream, "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - - writeConsumerMetric(stream, "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - writeConsumerMetric(stream, "pulsar_consumer_available_permits", consumerStats.availablePermits, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - writeConsumerMetric(stream, "pulsar_out_bytes_total", consumerStats.bytesOutCounter, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); - writeConsumerMetric(stream, "pulsar_out_messages_total", consumerStats.msgOutCounter, - cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_consumer_available_permits", consumerStats.availablePermits, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_out_bytes_total", consumerStats.bytesOutCounter, + splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), + "pulsar_out_messages_total", consumerStats.msgOutCounter, + splitTopicAndPartitionIndexLabel); }); }); if (!stats.replicationStats.isEmpty()) { stats.replicationStats.forEach((remoteCluster, replStats) -> { - writeMetric(stream, "pulsar_replication_rate_in", replStats.msgRateIn, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_rate_out", replStats.msgRateOut, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_throughput_in", replStats.msgThroughputIn, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_throughput_out", replStats.msgThroughputOut, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_backlog", replStats.replicationBacklog, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_connected_count", replStats.connectedCount, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_rate_expired", replStats.msgRateExpired, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_replication_delay_in_seconds", replStats.replicationDelayInSeconds, - cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_in", remoteCluster, + replStats.msgRateIn, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_out", remoteCluster, + replStats.msgRateOut, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_throughput_in", + remoteCluster, replStats.msgThroughputIn, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_throughput_out", + remoteCluster, replStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_backlog", remoteCluster, + replStats.replicationBacklog, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_connected_count", + remoteCluster, replStats.connectedCount, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_expired", + remoteCluster, replStats.msgRateExpired, splitTopicAndPartitionIndexLabel); + metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_delay_in_seconds", + remoteCluster, replStats.replicationDelayInSeconds, splitTopicAndPartitionIndexLabel); }); } - writeMetric(stream, "pulsar_in_bytes_total", stats.bytesInCounter, cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_in_bytes_total", stats.bytesInCounter, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_in_messages_total", stats.msgInCounter, cluster, namespace, topic, + metric(stream, cluster, namespace, topic, "pulsar_in_messages_total", stats.msgInCounter, splitTopicAndPartitionIndexLabel); // Compaction boolean hasCompaction = compactorMXBean.flatMap(mxBean -> mxBean.getCompactionRecordForTopic(topic)) - .isPresent(); + .map(__ -> true).orElse(false); if (hasCompaction) { - writeMetric(stream, "pulsar_compaction_removed_event_count", - stats.compactionRemovedEventCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_succeed_count", - stats.compactionSucceedCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_failed_count", - stats.compactionFailedCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_duration_time_in_mills", - stats.compactionDurationTimeInMills, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_read_throughput", - stats.compactionReadThroughput, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_write_throughput", - stats.compactionWriteThroughput, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_compacted_entries_count", - stats.compactionCompactedEntriesCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_compacted_entries_size", - stats.compactionCompactedEntriesSize, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - - long[] compactionBuckets = stats.compactionLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_compaction_latency_le_0_5", - compactionBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_1", - compactionBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_5", - compactionBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_10", - compactionBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_20", - compactionBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_50", - compactionBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_100", - compactionBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_200", - compactionBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_le_1000", - compactionBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_overflow", - compactionBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_sum", - stats.compactionLatencyBuckets.getSum(), cluster, namespace, topic, - splitTopicAndPartitionIndexLabel); - writeMetric(stream, "pulsar_compaction_latency_count", - stats.compactionLatencyBuckets.getCount(), cluster, namespace, topic, - splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_removed_event_count", + stats.compactionRemovedEventCount, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_succeed_count", + stats.compactionSucceedCount, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_failed_count", + stats.compactionFailedCount, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_duration_time_in_mills", + stats.compactionDurationTimeInMills, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_read_throughput", + stats.compactionReadThroughput, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_write_throughput", + stats.compactionWriteThroughput, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_compacted_entries_count", + stats.compactionCompactedEntriesCount, splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_compacted_entries_size", + stats.compactionCompactedEntriesSize, splitTopicAndPartitionIndexLabel); + long[] compactionLatencyBuckets = stats.compactionLatencyBuckets.getBuckets(); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_0_5", + compactionLatencyBuckets[0], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_1", + compactionLatencyBuckets[1], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_5", + compactionLatencyBuckets[2], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_10", + compactionLatencyBuckets[3], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_20", + compactionLatencyBuckets[4], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_50", + compactionLatencyBuckets[5], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_100", + compactionLatencyBuckets[6], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_200", + compactionLatencyBuckets[7], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_1000", + compactionLatencyBuckets[8], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_overflow", + compactionLatencyBuckets[9], splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_sum", + stats.compactionLatencyBuckets.getSum(), splitTopicAndPartitionIndexLabel); + metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_count", + stats.compactionLatencyBuckets.getCount(), splitTopicAndPartitionIndexLabel); } } - private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace, String topic, boolean splitTopicAndPartitionIndexLabel) { - writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + static void metricType(SimpleTextOutputStream stream, String name) { + + if (!metricWithTypeDefinition.containsKey(name)) { + metricWithTypeDefinition.put(name, "gauge"); + stream.write("# TYPE ").write(name).write(" gauge\n"); + } + + } + + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + String name, double value, boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel).write("\"} "); + stream.write(value); + appendEndings(stream); } - private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace, String topic, String remoteCluster, - boolean splitTopicAndPartitionIndexLabel) { - writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, - "remote_cluster", remoteCluster); + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + String subscription, String name, long value, boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) + .write("\",subscription=\"").write(subscription).write("\"} "); + stream.write(value); + appendEndings(stream); } - private static void writeProducerMetric(PrometheusMetricStreams stream, String metricName, Number value, - String cluster, String namespace, String topic, String producer, - long producerId, boolean splitTopicAndPartitionIndexLabel) { - writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, - "producer_name", producer, "producer_id", String.valueOf(producerId)); + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + String producerName, long produceId, String name, double value, boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) + .write("\",producer_name=\"").write(producerName) + .write("\",producer_id=\"").write(produceId).write("\"} "); + stream.write(value); + appendEndings(stream); } + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + String subscription, String name, double value, boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) + .write("\",subscription=\"").write(subscription).write("\"} "); + stream.write(value); + appendEndings(stream); + } + + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + String subscription, String consumerName, long consumerId, String name, long value, + boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) + .write("\",subscription=\"").write(subscription) + .write("\",consumer_name=\"").write(consumerName).write("\",consumer_id=\"").write(consumerId) + .write("\"} "); + stream.write(value); + appendEndings(stream); + } - private static void writeSubscriptionMetric(PrometheusMetricStreams stream, String metricName, Number value, - String cluster, String namespace, String topic, String subscription, - boolean splitTopicAndPartitionIndexLabel) { - writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, - "subscription", subscription); + private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, + String subscription, String consumerName, long consumerId, String name, double value, + boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) + .write("\",subscription=\"").write(subscription) + .write("\",consumer_name=\"").write(consumerName).write("\",consumer_id=\"") + .write(consumerId).write("\"} "); + stream.write(value); + appendEndings(stream); } - private static void writeConsumerMetric(PrometheusMetricStreams stream, String metricName, Number value, - String cluster, String namespace, String topic, String subscription, - Consumer consumer, boolean splitTopicAndPartitionIndexLabel) { - writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, - "subscription", subscription, "consumer_name", consumer.consumerName(), - "consumer_id", String.valueOf(consumer.consumerId())); + private static void metricWithRemoteCluster(SimpleTextOutputStream stream, String cluster, String namespace, + String topic, String name, String remoteCluster, double value, boolean splitTopicAndPartitionIndexLabel) { + metricType(stream, name); + appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) + .write("\",remote_cluster=\"").write(remoteCluster).write("\"} "); + stream.write(value); + appendEndings(stream); } - static void writeTopicMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace, String topic, boolean splitTopicAndPartitionIndexLabel, - String... extraLabelsAndValues) { - String[] labelsAndValues = new String[splitTopicAndPartitionIndexLabel ? 8 : 6]; - labelsAndValues[0] = "cluster"; - labelsAndValues[1] = cluster; - labelsAndValues[2] = "namespace"; - labelsAndValues[3] = namespace; - labelsAndValues[4] = "topic"; + private static SimpleTextOutputStream appendRequiredLabels(SimpleTextOutputStream stream, String cluster, + String namespace, String topic, String name, boolean splitTopicAndPartitionIndexLabel) { + stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace); if (splitTopicAndPartitionIndexLabel) { int index = topic.indexOf(PARTITIONED_TOPIC_SUFFIX); if (index > 0) { - labelsAndValues[5] = topic.substring(0, index); - labelsAndValues[6] = "partition"; - labelsAndValues[7] = topic.substring(index + PARTITIONED_TOPIC_SUFFIX.length()); + stream.write("\",topic=\"").write(topic.substring(0, index)).write("\",partition=\"") + .write(topic.substring(index + PARTITIONED_TOPIC_SUFFIX.length())); } else { - labelsAndValues[5] = topic; - labelsAndValues[6] = "partition"; - labelsAndValues[7] = "-1"; + stream.write("\",topic=\"").write(topic).write("\",partition=\"").write("-1"); } } else { - labelsAndValues[5] = topic; + stream.write("\",topic=\"").write(topic); } - String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues); - stream.writeSample(metricName, value, labels); + return stream; + } + + private static void appendEndings(SimpleTextOutputStream stream) { + stream.write(' ').write(System.currentTimeMillis()).write('\n'); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java index 8c58b516333f5..e6ac1535f43c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java @@ -20,6 +20,8 @@ import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; @@ -28,6 +30,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.impl.TransactionMetadataStoreStats; @@ -35,10 +38,21 @@ @Slf4j public class TransactionAggregator { + /** + * Used for tracking duplicate TYPE definitions. + */ + private static final FastThreadLocal> threadLocalMetricWithTypeDefinition = + new FastThreadLocal() { + @Override + protected Map initialValue() { + return new HashMap<>(); + } + }; + private static final FastThreadLocal localTransactionCoordinatorStats = new FastThreadLocal() { @Override - protected AggregatedTransactionCoordinatorStats initialValue() { + protected AggregatedTransactionCoordinatorStats initialValue() throws Exception { return new AggregatedTransactionCoordinatorStats(); } }; @@ -46,18 +60,21 @@ protected AggregatedTransactionCoordinatorStats initialValue() { private static final FastThreadLocal localManageLedgerStats = new FastThreadLocal() { @Override - protected ManagedLedgerStats initialValue() { + protected ManagedLedgerStats initialValue() throws Exception { return new ManagedLedgerStats(); } }; - public static void generate(PulsarService pulsar, PrometheusMetricStreams stream, boolean includeTopicMetrics) { + public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, boolean includeTopicMetrics) { String cluster = pulsar.getConfiguration().getClusterName(); + Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); + metricWithTypeDefinition.clear(); if (includeTopicMetrics) { + pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { - pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> - bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { + bundlesMap.forEach((bundle, topicsMap) -> { + topicsMap.forEach((name, topic) -> { if (topic instanceof PersistentTopic) { topic.getSubscriptions().values().forEach(subscription -> { try { @@ -65,8 +82,9 @@ public static void generate(PulsarService pulsar, PrometheusMetricStreams stream if (!checkTopicIsEventsNames(TopicName.get(subscription.getTopic().getName())) && subscription instanceof PersistentSubscription && ((PersistentSubscription) subscription).checkIfPendingAckStoreInit()) { - ManagedLedger managedLedger = ((PersistentSubscription) subscription) - .getPendingAckManageLedger().get(); + ManagedLedger managedLedger = + ((PersistentSubscription) subscription) + .getPendingAckManageLedger().get(); generateManageLedgerStats(managedLedger, stream, cluster, namespace, name, subscription.getName()); } @@ -75,7 +93,9 @@ public static void generate(PulsarService pulsar, PrometheusMetricStreams stream } }); } - }))); + }); + }); + }); } AggregatedTransactionCoordinatorStats transactionCoordinatorStats = localTransactionCoordinatorStats.get(); @@ -104,18 +124,18 @@ public static void generate(PulsarService pulsar, PrometheusMetricStreams stream localManageLedgerStats.get().reset(); if (transactionMetadataStore instanceof MLTransactionMetadataStore) { - ManagedLedger managedLedger = - ((MLTransactionMetadataStore) transactionMetadataStore).getManagedLedger(); + ManagedLedger managedLedger = + ((MLTransactionMetadataStore) transactionMetadataStore).getManagedLedger(); generateManageLedgerStats(managedLedger, stream, cluster, NamespaceName.SYSTEM_NAMESPACE.toString(), MLTransactionLogImpl.TRANSACTION_LOG_PREFIX + transactionCoordinatorID.getId(), MLTransactionLogImpl.TRANSACTION_SUBSCRIPTION_NAME); } - }); + }); } - private static void generateManageLedgerStats(ManagedLedger managedLedger, PrometheusMetricStreams stream, + private static void generateManageLedgerStats(ManagedLedger managedLedger, SimpleTextOutputStream stream, String cluster, String namespace, String topic, String subscription) { ManagedLedgerStats managedLedgerStats = localManageLedgerStats.get(); ManagedLedgerMBeanImpl mlStats = (ManagedLedgerMBeanImpl) managedLedger.getStats(); @@ -137,149 +157,174 @@ private static void generateManageLedgerStats(ManagedLedger managedLedger, Prome managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate(); managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate(); - printManageLedgerStats(stream, cluster, namespace, topic, subscription, managedLedgerStats); + printManageLedgerStats(stream, cluster, namespace, topic, + subscription, managedLedgerStats); + } + + private static void metricType(SimpleTextOutputStream stream, String name) { + Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); + if (!metricWithTypeDefinition.containsKey(name)) { + metricWithTypeDefinition.put(name, "gauge"); + stream.write("# TYPE ").write(name).write(" gauge\n"); + } + + } + + private static void metric(SimpleTextOutputStream stream, String cluster, String name, + double value, long coordinatorId) { + metricType(stream, name); + stream.write(name) + .write("{cluster=\"").write(cluster) + .write("\",coordinator_id=\"").write(coordinatorId).write("\"} ") + .write(value).write(' ').write(System.currentTimeMillis()) + .write('\n'); } - private static void printManageLedgerStats(PrometheusMetricStreams stream, String cluster, String namespace, + private static void metrics(SimpleTextOutputStream stream, String cluster, String namespace, + String topic, String subscription, String name, long value) { + stream.write(name).write("{cluster=\"").write(cluster).write("\", namespace=\"").write(namespace) + .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); + stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + } + + private static void metrics(SimpleTextOutputStream stream, String cluster, String namespace, + String topic, String subscription, String name, double value) { + stream.write(name).write("{cluster=\"").write(cluster).write("\", namespace=\"").write(namespace) + .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); + stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + } + + private static void printManageLedgerStats(SimpleTextOutputStream stream, String cluster, String namespace, String topic, String subscription, ManagedLedgerStats stats) { - writeMetric(stream, "pulsar_storage_size", stats.storageSize, cluster, namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_logical_size", stats.storageLogicalSize, cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_backlog_size", stats.backlogSize, cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_offloaded_size", stats.offloadedStorageUsed, cluster, namespace, topic, - subscription); + metrics(stream, cluster, namespace, topic, subscription, + "pulsar_storage_size", stats.storageSize); + metrics(stream, cluster, namespace, topic, subscription, + "pulsar_storage_logical_size", stats.storageLogicalSize); + metrics(stream, cluster, namespace, topic, subscription, + "pulsar_storage_backlog_size", stats.backlogSize); + metrics(stream, cluster, namespace, topic, subscription, + "pulsar_storage_offloaded_size", stats.offloadedStorageUsed); - writeMetric(stream, "pulsar_storage_write_rate", stats.storageWriteRate, cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_read_rate", stats.storageReadRate, cluster, namespace, topic, - subscription); + metrics(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_rate", stats.storageWriteRate); + metrics(stream, cluster, namespace, topic, subscription, + "pulsar_storage_read_rate", stats.storageReadRate); stats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.storageWriteLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_1", latencyBuckets[1], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_5", latencyBuckets[2], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_10", latencyBuckets[3], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_20", latencyBuckets[4], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_50", latencyBuckets[5], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_100", latencyBuckets[6], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_200", latencyBuckets[7], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_overflow", latencyBuckets[9], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_storage_write_latency_count", stats.storageWriteLatencyBuckets.getCount(), - cluster, namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_write_latency_sum", stats.storageWriteLatencyBuckets.getSum(), cluster, - namespace, topic, subscription); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_0_5", latencyBuckets[0]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_1", latencyBuckets[1]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_5", latencyBuckets[2]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_10", latencyBuckets[3]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_20", latencyBuckets[4]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_50", latencyBuckets[5]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_100", latencyBuckets[6]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_200", latencyBuckets[7]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_le_1000", latencyBuckets[8]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_write_latency_overflow", latencyBuckets[9]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_write_latency_count", + stats.storageWriteLatencyBuckets.getCount()); + metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_write_latency_sum", + stats.storageWriteLatencyBuckets.getSum()); stats.storageLedgerWriteLatencyBuckets.refresh(); - long[] ledgerWriteLatencyBuckets = stats.storageLedgerWriteLatencyBuckets.getBuckets(); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWriteLatencyBuckets[0], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", ledgerWriteLatencyBuckets[1], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", ledgerWriteLatencyBuckets[2], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", ledgerWriteLatencyBuckets[3], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", ledgerWriteLatencyBuckets[4], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", ledgerWriteLatencyBuckets[5], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", ledgerWriteLatencyBuckets[6], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", ledgerWriteLatencyBuckets[7], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", ledgerWriteLatencyBuckets[8], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", ledgerWriteLatencyBuckets[9], cluster, - namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_count", - stats.storageLedgerWriteLatencyBuckets.getCount(), cluster, namespace, topic, subscription); - writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", - stats.storageLedgerWriteLatencyBuckets.getSum(), cluster, namespace, topic, subscription); + long[] ledgerWritelatencyBuckets = stats.storageLedgerWriteLatencyBuckets.getBuckets(); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_0_5", ledgerWritelatencyBuckets[0]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_1", ledgerWritelatencyBuckets[1]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_5", ledgerWritelatencyBuckets[2]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_10", ledgerWritelatencyBuckets[3]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_20", ledgerWritelatencyBuckets[4]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_50", ledgerWritelatencyBuckets[5]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_100", ledgerWritelatencyBuckets[6]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_200", ledgerWritelatencyBuckets[7]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_storage_ledger_write_latency_le_1000", ledgerWritelatencyBuckets[8]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_overflow", + ledgerWritelatencyBuckets[9]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_count", + stats.storageLedgerWriteLatencyBuckets.getCount()); + metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_sum", + stats.storageLedgerWriteLatencyBuckets.getSum()); stats.entrySizeBuckets.refresh(); long[] entrySizeBuckets = stats.entrySizeBuckets.getBuckets(); - writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace, topic, - subscription); - writeMetric(stream, "pulsar_entry_size_count", stats.entrySizeBuckets.getCount(), cluster, namespace, - topic, subscription); - writeMetric(stream, "pulsar_entry_size_sum", stats.entrySizeBuckets.getSum(), cluster, namespace, topic, - subscription); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_128", entrySizeBuckets[0]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_512", entrySizeBuckets[1]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_entry_size_le_100_kb", entrySizeBuckets[6]); + metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_entry_size_le_overflow", entrySizeBuckets[8]); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_entry_size_count", stats.entrySizeBuckets.getCount()); + metric(stream, cluster, namespace, topic, subscription, + "pulsar_entry_size_sum", stats.entrySizeBuckets.getSum()); + } + + private static void metric(SimpleTextOutputStream stream, String cluster, + String namespace, String topic, String subscription, + String name, long value) { + stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace) + .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); + stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); } - static void printTransactionCoordinatorStats(PrometheusMetricStreams stream, String cluster, + static void printTransactionCoordinatorStats(SimpleTextOutputStream stream, String cluster, AggregatedTransactionCoordinatorStats stats, long coordinatorId) { - writeMetric(stream, "pulsar_txn_active_count", stats.actives, cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_committed_count", stats.committedCount, cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_aborted_count", stats.abortedCount, cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_created_count", stats.createdCount, cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_timeout_count", stats.timeoutCount, cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_append_log_count", stats.appendLogCount, cluster, - coordinatorId); + metric(stream, cluster, "pulsar_txn_active_count", + stats.actives, coordinatorId); + metric(stream, cluster, "pulsar_txn_committed_count", + stats.committedCount, coordinatorId); + metric(stream, cluster, "pulsar_txn_aborted_count", + stats.abortedCount, coordinatorId); + metric(stream, cluster, "pulsar_txn_created_count", + stats.createdCount, coordinatorId); + metric(stream, cluster, "pulsar_txn_timeout_count", + stats.timeoutCount, coordinatorId); + metric(stream, cluster, "pulsar_txn_append_log_count", + stats.appendLogCount, coordinatorId); long[] latencyBuckets = stats.executionLatency; - writeMetric(stream, "pulsar_txn_execution_latency_le_10", latencyBuckets[0], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_20", latencyBuckets[1], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_50", latencyBuckets[2], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_100", latencyBuckets[3], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_500", latencyBuckets[4], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_1000", latencyBuckets[5], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_5000", latencyBuckets[6], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_15000", latencyBuckets[7], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_30000", latencyBuckets[8], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_60000", latencyBuckets[9], cluster, coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_300000", latencyBuckets[10], cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_1500000", latencyBuckets[11], cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_3000000", latencyBuckets[12], cluster, - coordinatorId); - writeMetric(stream, "pulsar_txn_execution_latency_le_overflow", latencyBuckets[13], cluster, - coordinatorId); - } - - private static void writeMetric(PrometheusMetricStreams stream, String metricName, double value, String cluster, - long coordinatorId) { - stream.writeSample(metricName, value, "cluster", cluster, "coordinator_id", String.valueOf(coordinatorId)); - } - - private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace, String topic, String subscription) { - stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace, "topic", topic, - "subscription", subscription); + metric(stream, cluster, "pulsar_txn_execution_latency_le_10", latencyBuckets[0], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_20", latencyBuckets[1], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_50", latencyBuckets[2], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_100", latencyBuckets[3], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_500", latencyBuckets[4], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_1000", latencyBuckets[5], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_5000", latencyBuckets[6], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_15000", latencyBuckets[7], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_30000", latencyBuckets[8], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_60000", latencyBuckets[9], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_300000", + latencyBuckets[10], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_1500000", + latencyBuckets[11], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_3000000", + latencyBuckets[12], coordinatorId); + metric(stream, cluster, "pulsar_txn_execution_latency_le_overflow", + latencyBuckets[13], coordinatorId); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java index 8f704b11e764c..7550096c2b584 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java @@ -18,8 +18,13 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import io.prometheus.client.Collector; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.Collector.MetricFamilySamples.Sample; +import io.prometheus.client.CollectorRegistry; import java.io.IOException; import java.io.Writer; +import java.util.Enumeration; import org.apache.bookkeeper.stats.Counter; /** @@ -135,4 +140,31 @@ private static void writeSum(Writer w, DataSketchesOpStatsLogger opStat, String .append(success.toString()).append("\"} ") .append(Double.toString(opStat.getSum(success))).append('\n'); } + + public static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) + throws IOException { + Enumeration metricFamilySamples = registry.metricFamilySamples(); + while (metricFamilySamples.hasMoreElements()) { + MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); + + for (int i = 0; i < metricFamily.samples.size(); i++) { + Sample sample = metricFamily.samples.get(i); + w.write(sample.name); + w.write('{'); + for (int j = 0; j < sample.labelNames.size(); j++) { + if (j != 0) { + w.write(", "); + } + w.write(sample.labelNames.get(j)); + w.write("=\""); + w.write(sample.labelValues.get(j)); + w.write('"'); + } + + w.write("} "); + w.write(Collector.doubleToGoString(sample.value)); + w.write('\n'); + } + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index be10c8dc65e94..f28412ea75160 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -50,7 +50,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.crypto.SecretKey; @@ -1405,63 +1404,6 @@ private void compareCompactionStateCount(List cm, double count) { assertEquals(cm.get(0).value, count); } - @Test - public void testMetricsGroupedByTypeDefinitions() throws Exception { - Producer p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create(); - Producer p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create(); - for (int i = 0; i < 10; i++) { - String message = "my-message-" + i; - p1.send(message.getBytes()); - p2.send(message.getBytes()); - } - - ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); - String metricsStr = statsOut.toString(); - - Pattern typePattern = Pattern.compile("^#\\s+TYPE\\s+(\\w+)\\s+(\\w+)"); - Pattern metricNamePattern = Pattern.compile("^(\\w+)\\{.+"); - - AtomicReference currentMetric = new AtomicReference<>(); - Splitter.on("\n").split(metricsStr).forEach(line -> { - if (line.isEmpty()) { - return; - } - if (line.startsWith("#")) { - // Get the current type definition - Matcher typeMatcher = typePattern.matcher(line); - checkArgument(typeMatcher.matches()); - String metricName = typeMatcher.group(1); - currentMetric.set(metricName); - } else { - Matcher metricMatcher = metricNamePattern.matcher(line); - checkArgument(metricMatcher.matches()); - String metricName = metricMatcher.group(1); - - if (metricName.endsWith("_bucket")) { - metricName = metricName.substring(0, metricName.indexOf("_bucket")); - } else if (metricName.endsWith("_count") && !currentMetric.get().endsWith("_count")) { - metricName = metricName.substring(0, metricName.indexOf("_count")); - } else if (metricName.endsWith("_sum") && !currentMetric.get().endsWith("_sum")) { - metricName = metricName.substring(0, metricName.indexOf("_sum")); - } else if (metricName.endsWith("_total") && !currentMetric.get().endsWith("_total")) { - metricName = metricName.substring(0, metricName.indexOf("_total")); - } else if (metricName.endsWith("_created") && !currentMetric.get().endsWith("_created")) { - metricName = metricName.substring(0, metricName.indexOf("_created")); - } - - if (!metricName.equals(currentMetric.get())) { - System.out.println(metricsStr); - fail("Metric not grouped under its type definition: " + line); - } - - } - }); - - p1.close(); - p2.close(); - } - /** * Hacky parsing of Prometheus text format. Should be good enough for unit tests */ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java deleted file mode 100644 index 15c29a0dc66bb..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.stats.prometheus; - -import static org.testng.Assert.assertTrue; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; -import org.apache.pulsar.common.util.SimpleTextOutputStream; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Test(groups = "broker") -public class PrometheusMetricStreamsTest { - - private PrometheusMetricStreams underTest; - - @BeforeMethod(alwaysRun = true) - protected void setup() throws Exception { - underTest = new PrometheusMetricStreams(); - } - - @AfterMethod(alwaysRun = true) - protected void cleanup() throws Exception { - underTest.releaseAll(); - } - - @Test - public void canWriteSampleWithoutLabels() { - underTest.writeSample("my-metric", 123); - - String actual = writeToString(); - - assertTrue(actual.startsWith("# TYPE my-metric gauge"), "Gauge type line missing"); - assertTrue(actual.contains("my-metric{} 123"), "Metric line missing"); - } - - @Test - public void canWriteSampleWithLabels() { - underTest.writeSample("my-other-metric", 123, "cluster", "local"); - underTest.writeSample("my-other-metric", 456, "cluster", "local", "namespace", "my-ns"); - - String actual = writeToString(); - - assertTrue(actual.startsWith("# TYPE my-other-metric gauge"), "Gauge type line missing"); - assertTrue(actual.contains("my-other-metric{cluster=\"local\"} 123"), "Cluster metric line missing"); - assertTrue(actual.contains("my-other-metric{cluster=\"local\",namespace=\"my-ns\"} 456"), - "Cluster and Namespace metric line missing"); - } - - private String writeToString() { - ByteBuf buffer = ByteBufAllocator.DEFAULT.directBuffer(); - try { - SimpleTextOutputStream stream = new SimpleTextOutputStream(buffer); - underTest.flushAllToStream(stream); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int readIndex = buffer.readerIndex(); - int readableBytes = buffer.readableBytes(); - for (int i = 0; i < readableBytes; i++) { - out.write(buffer.getByte(readIndex + i)); - } - return out.toString(); - } finally { - buffer.release(); - } - } -} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java index dd78b4cfe58b9..9fc4b347c854f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java @@ -22,11 +22,12 @@ /** * Format strings and numbers into a ByteBuf without any memory allocation. + * */ public class SimpleTextOutputStream { private final ByteBuf buffer; - private static final char[] hexChars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', - 'f'}; + private static final char[] hexChars = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', + 'f' }; public SimpleTextOutputStream(ByteBuf buffer) { this.buffer = buffer; @@ -130,12 +131,4 @@ public SimpleTextOutputStream write(double d) { write(r); return this; } - - public void write(ByteBuf byteBuf) { - buffer.writeBytes(byteBuf); - } - - public ByteBuf getBuffer() { - return buffer; - } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java index f5aa273f656ac..f7a205c7db02c 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java @@ -37,11 +37,6 @@ public static void write004(Writer writer, Enumeration 0) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java index 2ad407b2e5e3e..c8b411cbf5706 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java @@ -328,11 +328,6 @@ private void generateLeaderMetrics(StringWriter stream) { } private void writeMetric(String metricName, long value, StringWriter stream) { - stream.write("# TYPE "); - stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); - stream.write(metricName); - stream.write(" gauge \n"); - stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); stream.write(metricName); stream.write("{"); From 976a3181b7e47bcc501294b8489362a1cd3e3e89 Mon Sep 17 00:00:00 2001 From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com> Date: Thu, 29 Sep 2022 08:34:41 +0800 Subject: [PATCH 762/823] Group prometheus metrics (#17852) --- .../prometheus/AggregatedNamespaceStats.java | 2 +- .../prometheus/NamespaceStatsAggregator.java | 352 +++++----- .../prometheus/PrometheusMetricStreams.java | 75 +++ .../PrometheusMetricsGenerator.java | 54 +- .../broker/stats/prometheus/TopicStats.java | 600 ++++++++---------- .../prometheus/TransactionAggregator.java | 321 ++++------ .../metrics/PrometheusTextFormatUtil.java | 32 - .../broker/stats/PrometheusMetricsTest.java | 59 ++ .../PrometheusMetricStreamsTest.java | 85 +++ .../common/util/SimpleTextOutputStream.java | 12 +- .../instance/stats/PrometheusTextFormat.java | 29 +- .../functions/worker/WorkerStatsManager.java | 5 + 12 files changed, 885 insertions(+), 741 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 5610dbab218e0..1980af91b7b54 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -96,7 +96,7 @@ void updateStats(TopicStats stats) { stats.replicationStats.forEach((n, as) -> { AggregatedReplicationStats replStats = - replicationStats.computeIfAbsent(n, k -> new AggregatedReplicationStats()); + replicationStats.computeIfAbsent(n, k -> new AggregatedReplicationStats()); replStats.msgRateIn += as.msgRateIn; replStats.msgRateOut += as.msgRateOut; replStats.msgThroughputIn += as.msgThroughputIn; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 16e438e2a2eb3..3e67d92935020 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -19,8 +19,11 @@ package org.apache.pulsar.broker.stats.prometheus; import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -32,7 +35,6 @@ import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; -import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.CompactorMXBean; @@ -40,72 +42,75 @@ @Slf4j public class NamespaceStatsAggregator { - private static FastThreadLocal localNamespaceStats = + private static final FastThreadLocal localNamespaceStats = new FastThreadLocal() { @Override - protected AggregatedNamespaceStats initialValue() throws Exception { + protected AggregatedNamespaceStats initialValue() { return new AggregatedNamespaceStats(); } }; - private static FastThreadLocal localTopicStats = new FastThreadLocal() { + private static final FastThreadLocal localTopicStats = new FastThreadLocal() { @Override - protected TopicStats initialValue() throws Exception { + protected TopicStats initialValue() { return new TopicStats(); } }; public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, SimpleTextOutputStream stream) { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + PrometheusMetricStreams stream) { String cluster = pulsar.getConfiguration().getClusterName(); AggregatedNamespaceStats namespaceStats = localNamespaceStats.get(); - TopicStats.resetTypes(); TopicStats topicStats = localTopicStats.get(); + Optional compactorMXBean = getCompactorMXBean(pulsar); + LongAdder topicsCount = new LongAdder(); + Map localNamespaceTopicCount = new HashMap<>(); printDefaultBrokerStats(stream, cluster); - Optional compactorMXBean = getCompactorMXBean(pulsar); - LongAdder topicsCount = new LongAdder(); pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { namespaceStats.reset(); topicsCount.reset(); - bundlesMap.forEach((bundle, topicsMap) -> { - topicsMap.forEach((name, topic) -> { - getTopicStats(topic, topicStats, includeConsumerMetrics, includeProducerMetrics, - pulsar.getConfiguration().isExposePreciseBacklogInPrometheus(), - pulsar.getConfiguration().isExposeSubscriptionBacklogSizeInPrometheus(), - compactorMXBean - ); - - if (includeTopicMetrics) { - topicsCount.add(1); - TopicStats.printTopicStats(stream, cluster, namespace, name, topicStats, compactorMXBean, - splitTopicAndPartitionIndexLabel); - } else { - namespaceStats.updateStats(topicStats); - } - }); - }); + bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { + getTopicStats(topic, topicStats, includeConsumerMetrics, includeProducerMetrics, + pulsar.getConfiguration().isExposePreciseBacklogInPrometheus(), + pulsar.getConfiguration().isExposeSubscriptionBacklogSizeInPrometheus(), + compactorMXBean + ); + + if (includeTopicMetrics) { + topicsCount.add(1); + TopicStats.printTopicStats(stream, topicStats, compactorMXBean, cluster, namespace, name, + splitTopicAndPartitionIndexLabel); + } else { + namespaceStats.updateStats(topicStats); + } + })); if (!includeTopicMetrics) { - // Only include namespace level stats if we don't have the per-topic, otherwise we're going to report - // the same data twice, and it will make the aggregation difficult - printNamespaceStats(stream, cluster, namespace, namespaceStats); + // Only include namespace level stats if we don't have the per-topic, otherwise we're going to + // report the same data twice, and it will make the aggregation difficult + printNamespaceStats(stream, namespaceStats, cluster, namespace); } else { - printTopicsCountStats(stream, cluster, namespace, topicsCount); + localNamespaceTopicCount.put(namespace, topicsCount.sum()); } }); + + if (includeTopicMetrics) { + printTopicsCountStats(stream, localNamespaceTopicCount, cluster); + } } private static Optional getCompactorMXBean(PulsarService pulsar) { Compactor compactor = pulsar.getNullableCompactor(); - return Optional.ofNullable(compactor).map(c -> c.getStats()); + return Optional.ofNullable(compactor).map(Compactor::getStats); } private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean getPreciseBacklog, boolean subscriptionBacklogSize, - Optional compactorMXBean) { + boolean includeProducerMetrics, boolean getPreciseBacklog, + boolean subscriptionBacklogSize, Optional compactorMXBean) { stats.reset(); if (topic instanceof PersistentTopic) { @@ -267,161 +272,174 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include }); } - private static void printDefaultBrokerStats(SimpleTextOutputStream stream, String cluster) { + private static void printDefaultBrokerStats(PrometheusMetricStreams stream, String cluster) { // Print metrics with 0 values. This is necessary to have the available brokers being // reported in the brokers dashboard even if they don't have any topic or traffic - metric(stream, cluster, "pulsar_topics_count", 0); - metric(stream, cluster, "pulsar_subscriptions_count", 0); - metric(stream, cluster, "pulsar_producers_count", 0); - metric(stream, cluster, "pulsar_consumers_count", 0); - metric(stream, cluster, "pulsar_rate_in", 0); - metric(stream, cluster, "pulsar_rate_out", 0); - metric(stream, cluster, "pulsar_throughput_in", 0); - metric(stream, cluster, "pulsar_throughput_out", 0); - metric(stream, cluster, "pulsar_storage_size", 0); - metric(stream, cluster, "pulsar_storage_logical_size", 0); - metric(stream, cluster, "pulsar_storage_write_rate", 0); - metric(stream, cluster, "pulsar_storage_read_rate", 0); - metric(stream, cluster, "pulsar_msg_backlog", 0); + writeMetric(stream, "pulsar_topics_count", 0, cluster); + writeMetric(stream, "pulsar_subscriptions_count", 0, cluster); + writeMetric(stream, "pulsar_producers_count", 0, cluster); + writeMetric(stream, "pulsar_consumers_count", 0, cluster); + writeMetric(stream, "pulsar_rate_in", 0, cluster); + writeMetric(stream, "pulsar_rate_out", 0, cluster); + writeMetric(stream, "pulsar_throughput_in", 0, cluster); + writeMetric(stream, "pulsar_throughput_out", 0, cluster); + writeMetric(stream, "pulsar_storage_size", 0, cluster); + writeMetric(stream, "pulsar_storage_logical_size", 0, cluster); + writeMetric(stream, "pulsar_storage_write_rate", 0, cluster); + writeMetric(stream, "pulsar_storage_read_rate", 0, cluster); + writeMetric(stream, "pulsar_msg_backlog", 0, cluster); } - private static void printTopicsCountStats(SimpleTextOutputStream stream, String cluster, String namespace, - LongAdder topicsCount) { - metric(stream, cluster, namespace, "pulsar_topics_count", topicsCount.sum()); + private static void printTopicsCountStats(PrometheusMetricStreams stream, Map namespaceTopicsCount, + String cluster) { + namespaceTopicsCount.forEach( + (ns, topicCount) -> writeMetric(stream, "pulsar_topics_count", topicCount, cluster, ns) + ); } - private static void printNamespaceStats(SimpleTextOutputStream stream, String cluster, String namespace, - AggregatedNamespaceStats stats) { - metric(stream, cluster, namespace, "pulsar_topics_count", stats.topicsCount); - metric(stream, cluster, namespace, "pulsar_subscriptions_count", stats.subscriptionsCount); - metric(stream, cluster, namespace, "pulsar_producers_count", stats.producersCount); - metric(stream, cluster, namespace, "pulsar_consumers_count", stats.consumersCount); - - metric(stream, cluster, namespace, "pulsar_rate_in", stats.rateIn); - metric(stream, cluster, namespace, "pulsar_rate_out", stats.rateOut); - metric(stream, cluster, namespace, "pulsar_throughput_in", stats.throughputIn); - metric(stream, cluster, namespace, "pulsar_throughput_out", stats.throughputOut); - metric(stream, cluster, namespace, "pulsar_consumer_msg_ack_rate", stats.messageAckRate); - - metric(stream, cluster, namespace, "pulsar_in_bytes_total", stats.bytesInCounter); - metric(stream, cluster, namespace, "pulsar_in_messages_total", stats.msgInCounter); - metric(stream, cluster, namespace, "pulsar_out_bytes_total", stats.bytesOutCounter); - metric(stream, cluster, namespace, "pulsar_out_messages_total", stats.msgOutCounter); - - metric(stream, cluster, namespace, "pulsar_storage_size", stats.managedLedgerStats.storageSize); - metric(stream, cluster, namespace, "pulsar_storage_logical_size", stats.managedLedgerStats.storageLogicalSize); - metric(stream, cluster, namespace, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize); - metric(stream, cluster, namespace, "pulsar_storage_offloaded_size", - stats.managedLedgerStats.offloadedStorageUsed); - - metric(stream, cluster, namespace, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate); - metric(stream, cluster, namespace, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate); - - metric(stream, cluster, namespace, "pulsar_subscription_delayed", stats.msgDelayed); - - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_msg_backlog", "local", stats.msgBacklog); + private static void printNamespaceStats(PrometheusMetricStreams stream, AggregatedNamespaceStats stats, + String cluster, String namespace) { + writeMetric(stream, "pulsar_topics_count", stats.topicsCount, cluster, namespace); + writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, cluster, + namespace); + writeMetric(stream, "pulsar_producers_count", stats.producersCount, cluster, namespace); + writeMetric(stream, "pulsar_consumers_count", stats.consumersCount, cluster, namespace); + + writeMetric(stream, "pulsar_rate_in", stats.rateIn, cluster, namespace); + writeMetric(stream, "pulsar_rate_out", stats.rateOut, cluster, namespace); + writeMetric(stream, "pulsar_throughput_in", stats.throughputIn, cluster, namespace); + writeMetric(stream, "pulsar_throughput_out", stats.throughputOut, cluster, namespace); + writeMetric(stream, "pulsar_consumer_msg_ack_rate", stats.messageAckRate, cluster, namespace); + + writeMetric(stream, "pulsar_in_bytes_total", stats.bytesInCounter, cluster, namespace); + writeMetric(stream, "pulsar_in_messages_total", stats.msgInCounter, cluster, namespace); + writeMetric(stream, "pulsar_out_bytes_total", stats.bytesOutCounter, cluster, namespace); + writeMetric(stream, "pulsar_out_messages_total", stats.msgOutCounter, cluster, namespace); + + writeMetric(stream, "pulsar_storage_size", stats.managedLedgerStats.storageSize, cluster, + namespace); + writeMetric(stream, "pulsar_storage_logical_size", + stats.managedLedgerStats.storageLogicalSize, cluster, namespace); + writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, cluster, + namespace); + writeMetric(stream, "pulsar_storage_offloaded_size", + stats.managedLedgerStats.offloadedStorageUsed, cluster, namespace); + + writeMetric(stream, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate, + cluster, namespace); + writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, + cluster, namespace); + + writeMetric(stream, "pulsar_subscription_delayed", stats.msgDelayed, cluster, namespace); + + writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace); stats.managedLedgerStats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_1", latencyBuckets[1]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_5", latencyBuckets[2]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_10", latencyBuckets[3]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_20", latencyBuckets[4]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_50", latencyBuckets[5]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_100", latencyBuckets[6]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_200", latencyBuckets[7]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_le_1000", latencyBuckets[8]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_overflow", latencyBuckets[9]); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_count", - stats.managedLedgerStats.storageWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, "pulsar_storage_write_latency_sum", - stats.managedLedgerStats.storageWriteLatencyBuckets.getSum()); + writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_1", latencyBuckets[1], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_5", latencyBuckets[2], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_10", latencyBuckets[3], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_20", latencyBuckets[4], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_50", latencyBuckets[5], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_100", latencyBuckets[6], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_200", latencyBuckets[7], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_overflow", latencyBuckets[9], cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_count", + stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), cluster, namespace); + writeMetric(stream, "pulsar_storage_write_latency_sum", + stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), cluster, namespace); stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.refresh(); - long[] ledgerWritelatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWritelatencyBuckets[0]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_1", ledgerWritelatencyBuckets[1]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_5", ledgerWritelatencyBuckets[2]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_10", ledgerWritelatencyBuckets[3]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_20", ledgerWritelatencyBuckets[4]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_50", ledgerWritelatencyBuckets[5]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_100", ledgerWritelatencyBuckets[6]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_200", ledgerWritelatencyBuckets[7]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_le_1000", ledgerWritelatencyBuckets[8]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_overflow", - ledgerWritelatencyBuckets[9]); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_count", - stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, "pulsar_storage_ledger_write_latency_sum", - stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum()); + long[] ledgerWriteLatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWriteLatencyBuckets[0], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", ledgerWriteLatencyBuckets[1], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", ledgerWriteLatencyBuckets[2], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", ledgerWriteLatencyBuckets[3], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", ledgerWriteLatencyBuckets[4], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", ledgerWriteLatencyBuckets[5], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", ledgerWriteLatencyBuckets[6], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", ledgerWriteLatencyBuckets[7], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", ledgerWriteLatencyBuckets[8], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", ledgerWriteLatencyBuckets[9], + cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_count", + stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount(), cluster, namespace); + writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", + stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum(), cluster, namespace); stats.managedLedgerStats.entrySizeBuckets.refresh(); long[] entrySizeBuckets = stats.managedLedgerStats.entrySizeBuckets.getBuckets(); - metric(stream, cluster, namespace, "pulsar_entry_size_le_128", entrySizeBuckets[0]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_512", entrySizeBuckets[1]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7]); - metric(stream, cluster, namespace, "pulsar_entry_size_le_overflow", entrySizeBuckets[8]); - metric(stream, cluster, namespace, "pulsar_entry_size_count", - stats.managedLedgerStats.entrySizeBuckets.getCount()); - metric(stream, cluster, namespace, "pulsar_entry_size_sum", - stats.managedLedgerStats.entrySizeBuckets.getSum()); - - if (!stats.replicationStats.isEmpty()) { - stats.replicationStats.forEach((remoteCluster, replStats) -> { - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_in", remoteCluster, - replStats.msgRateIn); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_out", remoteCluster, - replStats.msgRateOut); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_throughput_in", remoteCluster, - replStats.msgThroughputIn); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_throughput_out", remoteCluster, - replStats.msgThroughputOut); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_backlog", remoteCluster, - replStats.replicationBacklog); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_connected_count", remoteCluster, - replStats.connectedCount); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_rate_expired", remoteCluster, - replStats.msgRateExpired); - metricWithRemoteCluster(stream, cluster, namespace, "pulsar_replication_delay_in_seconds", - remoteCluster, replStats.replicationDelayInSeconds); - }); - } + writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace); + writeMetric(stream, "pulsar_entry_size_count", stats.managedLedgerStats.entrySizeBuckets.getCount(), + cluster, namespace); + writeMetric(stream, "pulsar_entry_size_sum", stats.managedLedgerStats.entrySizeBuckets.getSum(), + cluster, namespace); + + writeReplicationStat(stream, "pulsar_replication_rate_in", stats, + replStats -> replStats.msgRateIn, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_rate_out", stats, + replStats -> replStats.msgRateOut, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_throughput_in", stats, + replStats -> replStats.msgThroughputIn, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_throughput_out", stats, + replStats -> replStats.msgThroughputOut, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_backlog", stats, + replStats -> replStats.replicationBacklog, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_connected_count", stats, + replStats -> replStats.connectedCount, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_rate_expired", stats, + replStats -> replStats.msgRateExpired, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_delay_in_seconds", stats, + replStats -> replStats.replicationDelayInSeconds, cluster, namespace); } - private static void metric(SimpleTextOutputStream stream, String cluster, String name, - long value) { - TopicStats.metricType(stream, name); - stream.write(name) - .write("{cluster=\"").write(cluster).write("\"} ") - .write(value).write(' ').write(System.currentTimeMillis()) - .write('\n'); + private static void writePulsarMsgBacklog(PrometheusMetricStreams stream, Number value, + String cluster, String namespace) { + stream.writeSample("pulsar_msg_backlog", value, "cluster", cluster, "namespace", namespace, + "remote_cluster", + "local"); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String name, - long value) { - TopicStats.metricType(stream, name); - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster) { + stream.writeSample(metricName, value, "cluster", cluster); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String name, - double value) { - TopicStats.metricType(stream, name); - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace) { + stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace); } - private static void metricWithRemoteCluster(SimpleTextOutputStream stream, String cluster, String namespace, - String name, String remoteCluster, double value) { - TopicStats.metricType(stream, name); - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace); - stream.write("\",remote_cluster=\"").write(remoteCluster).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName, + AggregatedNamespaceStats namespaceStats, + Function sampleValueFunction, + String cluster, String namespace) { + if (!namespaceStats.replicationStats.isEmpty()) { + namespaceStats.replicationStats.forEach((remoteCluster, replStats) -> + stream.writeSample(metricName, sampleValueFunction.apply(replStats), + "cluster", cluster, + "namespace", namespace, + "remote_cluster", remoteCluster) + ); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java new file mode 100644 index 0000000000000..6b6b972c175f0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import java.util.HashMap; +import java.util.Map; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.util.SimpleTextOutputStream; + +/** + * Helper class to ensure that metrics of the same name are grouped together under the same TYPE header when written. + * Those are the requirements of the + * Prometheus Exposition Format. + */ +public class PrometheusMetricStreams { + private final Map metricStreamMap = new HashMap<>(); + + /** + * Write the given metric and sample value to the stream. Will write #TYPE header if metric not seen before. + * @param metricName name of the metric. + * @param value value of the sample + * @param labelsAndValuesArray varargs of label and label value + */ + void writeSample(String metricName, Number value, String... labelsAndValuesArray) { + SimpleTextOutputStream stream = initGaugeType(metricName); + stream.write(metricName).write('{'); + for (int i = 0; i < labelsAndValuesArray.length; i += 2) { + stream.write(labelsAndValuesArray[i]).write("=\"").write(labelsAndValuesArray[i + 1]).write('\"'); + if (i + 2 != labelsAndValuesArray.length) { + stream.write(','); + } + } + stream.write("} ").write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + } + + /** + * Flush all the stored metrics to the supplied stream. + * @param stream the stream to write to. + */ + void flushAllToStream(SimpleTextOutputStream stream) { + metricStreamMap.values().forEach(s -> stream.write(s.getBuffer())); + } + + /** + * Release all the streams to clean up resources. + */ + void releaseAll() { + metricStreamMap.values().forEach(s -> s.getBuffer().release()); + metricStreamMap.clear(); + } + + private SimpleTextOutputStream initGaugeType(String metricName) { + return metricStreamMap.computeIfAbsent(metricName, s -> { + SimpleTextOutputStream stream = new SimpleTextOutputStream(PulsarByteBufAllocator.DEFAULT.directBuffer()); + stream.write("# TYPE ").write(metricName).write(" gauge\n"); + return stream; + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index cd6afd1535dec..aa5822c826aa5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -52,7 +52,8 @@ /** * Generate metrics aggregated at the namespace level and optionally at a topic level and formats them out * in a text format suitable to be consumed by Prometheus. - * Format specification can be found at {@link https://prometheus.io/docs/instrumenting/exposition_formats/} + * Format specification can be found at Exposition Formats */ public class PrometheusMetricsGenerator { @@ -86,38 +87,43 @@ public double get() { } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, OutputStream out) throws IOException { + boolean includeProducerMetrics, OutputStream out) throws IOException { generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, false, out, null); } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - OutputStream out) throws IOException { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + OutputStream out) throws IOException { generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, splitTopicAndPartitionIndexLabel, out, null); } public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, OutputStream out, - List metricsProviders) - throws IOException { + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + OutputStream out, + List metricsProviders) + throws IOException { ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer(); + //Used in namespace/topic and transaction aggregators as share metric names + PrometheusMetricStreams metricStreams = new PrometheusMetricStreams(); try { SimpleTextOutputStream stream = new SimpleTextOutputStream(buf); generateSystemMetrics(stream, pulsar.getConfiguration().getClusterName()); NamespaceStatsAggregator.generate(pulsar, includeTopicMetrics, includeConsumerMetrics, - includeProducerMetrics, splitTopicAndPartitionIndexLabel, stream); + includeProducerMetrics, splitTopicAndPartitionIndexLabel, metricStreams); if (pulsar.getWorkerServiceOpt().isPresent()) { pulsar.getWorkerService().generateFunctionsStats(stream); } if (pulsar.getConfiguration().isTransactionCoordinatorEnabled()) { - TransactionAggregator.generate(pulsar, stream, includeTopicMetrics); + TransactionAggregator.generate(pulsar, metricStreams, includeTopicMetrics); } + metricStreams.flushAllToStream(stream); + generateBrokerBasicMetrics(pulsar, stream); generateManagedLedgerBookieClientMetrics(pulsar, stream); @@ -129,6 +135,8 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b } out.write(buf.array(), buf.arrayOffset(), buf.readableBytes()); } finally { + //release all the metrics buffers + metricStreams.releaseAll(); buf.release(); } } @@ -142,17 +150,17 @@ private static void generateBrokerBasicMetrics(PulsarService pulsar, SimpleTextO if (pulsar.getConfiguration().isExposeManagedLedgerMetricsInPrometheus()) { // generate managedLedger metrics parseMetricsToPrometheusMetrics(new ManagedLedgerMetrics(pulsar).generate(), - clusterName, Collector.Type.GAUGE, stream); + clusterName, Collector.Type.GAUGE, stream); } if (pulsar.getConfiguration().isExposeManagedCursorMetricsInPrometheus()) { // generate managedCursor metrics parseMetricsToPrometheusMetrics(new ManagedCursorMetrics(pulsar).generate(), - clusterName, Collector.Type.GAUGE, stream); + clusterName, Collector.Type.GAUGE, stream); } parseMetricsToPrometheusMetrics(Collections.singletonList(pulsar.getBrokerService() - .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), + .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), clusterName, Collector.Type.GAUGE, stream); // generate loadBalance metrics @@ -267,17 +275,17 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String static String getTypeStr(Collector.Type type) { switch (type) { - case COUNTER: - return "counter"; - case GAUGE: - return "gauge"; - case SUMMARY : - return "summary"; - case HISTOGRAM: - return "histogram"; - case UNTYPED: - default: - return "untyped"; + case COUNTER: + return "counter"; + case GAUGE: + return "gauge"; + case SUMMARY: + return "summary"; + case HISTOGRAM: + return "histogram"; + case UNTYPED: + default: + return "untyped"; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index e6e5883847df2..9ac2f04eae488 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -23,12 +23,12 @@ import java.util.Map; import java.util.Optional; import org.apache.bookkeeper.mledger.util.StatsBuckets; -import org.apache.pulsar.common.util.SimpleTextOutputStream; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.compaction.CompactionRecord; import org.apache.pulsar.compaction.CompactorMXBean; class TopicStats { - int subscriptionsCount; int producersCount; int consumersCount; @@ -43,7 +43,6 @@ class TopicStats { double averageMsgSize; public long msgBacklog; - long publishRateLimitedTimes; long backlogQuotaLimit; @@ -55,9 +54,6 @@ class TopicStats { Map subscriptionStats = new HashMap<>(); Map producerStats = new HashMap<>(); - // Used for tracking duplicate TYPE definitions - static Map metricWithTypeDefinition = new HashMap<>(); - // For compaction long compactionRemovedEventCount; long compactionSucceedCount; @@ -103,378 +99,340 @@ public void reset() { compactionLatencyBuckets.reset(); } - static void resetTypes() { - metricWithTypeDefinition.clear(); - } - - static void printTopicStats(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - TopicStats stats, Optional compactorMXBean, - boolean splitTopicAndPartitionIndexLabel) { - metric(stream, cluster, namespace, topic, "pulsar_subscriptions_count", stats.subscriptionsCount, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_producers_count", stats.producersCount, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_consumers_count", stats.consumersCount, - splitTopicAndPartitionIndexLabel); - - metric(stream, cluster, namespace, topic, "pulsar_rate_in", stats.rateIn, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_rate_out", stats.rateOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_throughput_in", stats.throughputIn, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_throughput_out", stats.throughputOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_average_msg_size", stats.averageMsgSize, - splitTopicAndPartitionIndexLabel); - - metric(stream, cluster, namespace, topic, "pulsar_storage_size", stats.managedLedgerStats.storageSize, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_logical_size", - stats.managedLedgerStats.storageLogicalSize, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_msg_backlog", stats.msgBacklog, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_rate", - stats.managedLedgerStats.storageWriteRate, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_size", - stats.managedLedgerStats.backlogSize, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_offloaded_size", stats.managedLedgerStats - .offloadedStorageUsed, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_backlog_quota_limit_time", - stats.backlogQuotaLimitTime, splitTopicAndPartitionIndexLabel); + public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats, + Optional compactorMXBean, String cluster, String namespace, + String topic, boolean splitTopicAndPartitionIndexLabel) { + writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_producers_count", stats.producersCount, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_consumers_count", stats.consumersCount, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + writeMetric(stream, "pulsar_rate_in", stats.rateIn, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_rate_out", stats.rateOut, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_throughput_in", stats.throughputIn, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_throughput_out", stats.throughputOut, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_average_msg_size", stats.averageMsgSize, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + writeMetric(stream, "pulsar_storage_size", stats.managedLedgerStats.storageSize, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_logical_size", + stats.managedLedgerStats.storageLogicalSize, cluster, namespace, topic, + splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_msg_backlog", stats.msgBacklog, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_rate", stats.managedLedgerStats.storageWriteRate, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_offloaded_size", stats.managedLedgerStats + .offloadedStorageUsed, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_quota_limit", stats.backlogQuotaLimit, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_quota_limit_time", stats.backlogQuotaLimitTime, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_1", latencyBuckets[1], + writeMetric(stream, "pulsar_storage_write_latency_le_0_5", + latencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_1", + latencyBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_5", + latencyBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_10", + latencyBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_20", + latencyBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_50", + latencyBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_100", + latencyBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_200", + latencyBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_le_1000", + latencyBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_overflow", + latencyBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_count", + stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_write_latency_sum", + stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_5", latencyBuckets[2], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_10", latencyBuckets[3], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_20", latencyBuckets[4], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_50", latencyBuckets[5], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_100", latencyBuckets[6], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_200", latencyBuckets[7], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_overflow", latencyBuckets[9], - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_count", - stats.managedLedgerStats.storageWriteLatencyBuckets.getCount(), splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_write_latency_sum", - stats.managedLedgerStats.storageWriteLatencyBuckets.getSum(), splitTopicAndPartitionIndexLabel); long[] ledgerWriteLatencyBuckets = stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_0_5", - ledgerWriteLatencyBuckets[0], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_1", - ledgerWriteLatencyBuckets[1], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_5", - ledgerWriteLatencyBuckets[2], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_10", - ledgerWriteLatencyBuckets[3], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_20", - ledgerWriteLatencyBuckets[4], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_50", - ledgerWriteLatencyBuckets[5], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_100", - ledgerWriteLatencyBuckets[6], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_200", - ledgerWriteLatencyBuckets[7], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_le_1000", - ledgerWriteLatencyBuckets[8], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_overflow", - ledgerWriteLatencyBuckets[9], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_count", + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", + ledgerWriteLatencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", + ledgerWriteLatencyBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", + ledgerWriteLatencyBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", + ledgerWriteLatencyBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", + ledgerWriteLatencyBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", + ledgerWriteLatencyBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", + ledgerWriteLatencyBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", + ledgerWriteLatencyBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", + ledgerWriteLatencyBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", + ledgerWriteLatencyBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_count", stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getCount(), - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_storage_ledger_write_latency_sum", + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.getSum(), - splitTopicAndPartitionIndexLabel); + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); long[] entrySizeBuckets = stats.managedLedgerStats.entrySizeBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_128", entrySizeBuckets[0], + writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_512", entrySizeBuckets[1], + writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], + writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], + writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], + writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], + writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], + writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], + writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], + writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_count", - stats.managedLedgerStats.entrySizeBuckets.getCount(), splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_entry_size_sum", - stats.managedLedgerStats.entrySizeBuckets.getSum(), splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_entry_size_count", stats.managedLedgerStats.entrySizeBuckets.getCount(), + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_entry_size_sum", stats.managedLedgerStats.entrySizeBuckets.getSum(), + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); stats.producerStats.forEach((p, producerStats) -> { - metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_rate_in", - producerStats.msgRateIn, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_throughput_in", - producerStats.msgThroughputIn, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, p, producerStats.producerId, "pulsar_producer_msg_average_Size", - producerStats.averageMsgSize, splitTopicAndPartitionIndexLabel); + writeProducerMetric(stream, "pulsar_producer_msg_rate_in", producerStats.msgRateIn, + cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); + writeProducerMetric(stream, "pulsar_producer_msg_throughput_in", producerStats.msgThroughputIn, + cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); + writeProducerMetric(stream, "pulsar_producer_msg_average_Size", producerStats.averageMsgSize, + cluster, namespace, topic, p, producerStats.producerId, splitTopicAndPartitionIndexLabel); }); - stats.subscriptionStats.forEach((n, subsStats) -> { - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_back_log", - subsStats.msgBacklog, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_back_log_no_delayed", - subsStats.msgBacklogNoDelayed, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_delayed", - subsStats.msgDelayed, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_redeliver", - subsStats.msgRateRedeliver, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_unacked_messages", - subsStats.unackedMessages, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_blocked_on_unacked_messages", - subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_out", - subsStats.msgRateOut, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_ack_rate", - subsStats.messageAckRate, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_throughput_out", - subsStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_out_bytes_total", - subsStats.bytesOutCounter, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_out_messages_total", - subsStats.msgOutCounter, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_expire_timestamp", - subsStats.lastExpireTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_acked_timestamp", - subsStats.lastAckedTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_consumed_flow_timestamp", - subsStats.lastConsumedFlowTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_consumed_timestamp", - subsStats.lastConsumedTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_last_mark_delete_advanced_timestamp", - subsStats.lastMarkDeleteAdvancedTimestamp, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_msg_rate_expired", - subsStats.msgRateExpired, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, "pulsar_subscription_total_msg_expired", - subsStats.totalMsgExpired, splitTopicAndPartitionIndexLabel); + stats.subscriptionStats.forEach((sub, subsStats) -> { + writeSubscriptionMetric(stream, "pulsar_subscription_back_log", subsStats.msgBacklog, + cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_back_log_no_delayed", + subsStats.msgBacklogNoDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_delayed", + subsStats.msgDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_redeliver", + subsStats.msgRateRedeliver, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_unacked_messages", + subsStats.unackedMessages, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_blocked_on_unacked_messages", + subsStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, cluster, namespace, topic, sub, + splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_out", + subsStats.msgRateOut, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_ack_rate", + subsStats.messageAckRate, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_throughput_out", + subsStats.msgThroughputOut, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_out_bytes_total", + subsStats.bytesOutCounter, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_out_messages_total", + subsStats.msgOutCounter, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_expire_timestamp", + subsStats.lastExpireTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_acked_timestamp", + subsStats.lastAckedTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_consumed_flow_timestamp", + subsStats.lastConsumedFlowTimestamp, cluster, namespace, topic, sub, + splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_consumed_timestamp", + subsStats.lastConsumedTimestamp, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_last_mark_delete_advanced_timestamp", + subsStats.lastMarkDeleteAdvancedTimestamp, cluster, namespace, topic, sub, + splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_expired", + subsStats.msgRateExpired, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_total_msg_expired", + subsStats.totalMsgExpired, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + subsStats.consumerStat.forEach((c, consumerStats) -> { - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_unacked_messages", consumerStats.unackedMessages, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_blocked_on_unacked_messages", + writeConsumerMetric(stream, "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_unacked_messages", consumerStats.unackedMessages, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_blocked_on_unacked_messages", consumerStats.blockedSubscriptionOnUnackedMsgs ? 1 : 0, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_consumer_available_permits", consumerStats.availablePermits, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_out_bytes_total", consumerStats.bytesOutCounter, - splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, n, c.consumerName(), c.consumerId(), - "pulsar_out_messages_total", consumerStats.msgOutCounter, - splitTopicAndPartitionIndexLabel); + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_msg_rate_out", consumerStats.msgRateOut, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + + writeConsumerMetric(stream, "pulsar_consumer_msg_ack_rate", consumerStats.msgAckRate, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + + writeConsumerMetric(stream, "pulsar_consumer_msg_throughput_out", consumerStats.msgThroughputOut, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_consumer_available_permits", consumerStats.availablePermits, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_out_bytes_total", consumerStats.bytesOutCounter, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); + writeConsumerMetric(stream, "pulsar_out_messages_total", consumerStats.msgOutCounter, + cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel); }); }); if (!stats.replicationStats.isEmpty()) { stats.replicationStats.forEach((remoteCluster, replStats) -> { - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_in", remoteCluster, - replStats.msgRateIn, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_out", remoteCluster, - replStats.msgRateOut, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_throughput_in", - remoteCluster, replStats.msgThroughputIn, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_throughput_out", - remoteCluster, replStats.msgThroughputOut, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_backlog", remoteCluster, - replStats.replicationBacklog, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_connected_count", - remoteCluster, replStats.connectedCount, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_rate_expired", - remoteCluster, replStats.msgRateExpired, splitTopicAndPartitionIndexLabel); - metricWithRemoteCluster(stream, cluster, namespace, topic, "pulsar_replication_delay_in_seconds", - remoteCluster, replStats.replicationDelayInSeconds, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_rate_in", replStats.msgRateIn, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_rate_out", replStats.msgRateOut, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_throughput_in", replStats.msgThroughputIn, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_throughput_out", replStats.msgThroughputOut, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_backlog", replStats.replicationBacklog, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_connected_count", replStats.connectedCount, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_rate_expired", replStats.msgRateExpired, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_delay_in_seconds", replStats.replicationDelayInSeconds, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); }); } - metric(stream, cluster, namespace, topic, "pulsar_in_bytes_total", stats.bytesInCounter, + writeMetric(stream, "pulsar_in_bytes_total", stats.bytesInCounter, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_in_messages_total", stats.msgInCounter, + writeMetric(stream, "pulsar_in_messages_total", stats.msgInCounter, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); // Compaction boolean hasCompaction = compactorMXBean.flatMap(mxBean -> mxBean.getCompactionRecordForTopic(topic)) - .map(__ -> true).orElse(false); + .isPresent(); if (hasCompaction) { - metric(stream, cluster, namespace, topic, "pulsar_compaction_removed_event_count", - stats.compactionRemovedEventCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_succeed_count", - stats.compactionSucceedCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_failed_count", - stats.compactionFailedCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_duration_time_in_mills", - stats.compactionDurationTimeInMills, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_read_throughput", - stats.compactionReadThroughput, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_write_throughput", - stats.compactionWriteThroughput, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_compacted_entries_count", - stats.compactionCompactedEntriesCount, splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_compacted_entries_size", - stats.compactionCompactedEntriesSize, splitTopicAndPartitionIndexLabel); - long[] compactionLatencyBuckets = stats.compactionLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_0_5", - compactionLatencyBuckets[0], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_1", - compactionLatencyBuckets[1], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_5", - compactionLatencyBuckets[2], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_10", - compactionLatencyBuckets[3], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_20", - compactionLatencyBuckets[4], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_50", - compactionLatencyBuckets[5], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_100", - compactionLatencyBuckets[6], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_200", - compactionLatencyBuckets[7], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_le_1000", - compactionLatencyBuckets[8], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_overflow", - compactionLatencyBuckets[9], splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_sum", - stats.compactionLatencyBuckets.getSum(), splitTopicAndPartitionIndexLabel); - metric(stream, cluster, namespace, topic, "pulsar_compaction_latency_count", - stats.compactionLatencyBuckets.getCount(), splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_removed_event_count", + stats.compactionRemovedEventCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_succeed_count", + stats.compactionSucceedCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_failed_count", + stats.compactionFailedCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_duration_time_in_mills", + stats.compactionDurationTimeInMills, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_read_throughput", + stats.compactionReadThroughput, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_write_throughput", + stats.compactionWriteThroughput, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_compacted_entries_count", + stats.compactionCompactedEntriesCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_compacted_entries_size", + stats.compactionCompactedEntriesSize, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + + long[] compactionBuckets = stats.compactionLatencyBuckets.getBuckets(); + writeMetric(stream, "pulsar_compaction_latency_le_0_5", + compactionBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_1", + compactionBuckets[1], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_5", + compactionBuckets[2], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_10", + compactionBuckets[3], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_20", + compactionBuckets[4], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_50", + compactionBuckets[5], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_100", + compactionBuckets[6], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_200", + compactionBuckets[7], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_le_1000", + compactionBuckets[8], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_overflow", + compactionBuckets[9], cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_sum", + stats.compactionLatencyBuckets.getSum(), cluster, namespace, topic, + splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_compaction_latency_count", + stats.compactionLatencyBuckets.getCount(), cluster, namespace, topic, + splitTopicAndPartitionIndexLabel); } } - static void metricType(SimpleTextOutputStream stream, String name) { - - if (!metricWithTypeDefinition.containsKey(name)) { - metricWithTypeDefinition.put(name, "gauge"); - stream.write("# TYPE ").write(name).write(" gauge\n"); - } - - } - - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String name, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String name, long value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, String remoteCluster, + boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "remote_cluster", remoteCluster); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String producerName, long produceId, String name, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",producer_name=\"").write(producerName) - .write("\",producer_id=\"").write(produceId).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeProducerMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, String producer, + long producerId, boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "producer_name", producer, "producer_id", String.valueOf(producerId)); } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String name, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value); - appendEndings(stream); - } - - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String consumerName, long consumerId, String name, long value, - boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription) - .write("\",consumer_name=\"").write(consumerName).write("\",consumer_id=\"").write(consumerId) - .write("\"} "); - stream.write(value); - appendEndings(stream); - } - private static void metric(SimpleTextOutputStream stream, String cluster, String namespace, String topic, - String subscription, String consumerName, long consumerId, String name, double value, - boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",subscription=\"").write(subscription) - .write("\",consumer_name=\"").write(consumerName).write("\",consumer_id=\"") - .write(consumerId).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeSubscriptionMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, String subscription, + boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "subscription", subscription); } - private static void metricWithRemoteCluster(SimpleTextOutputStream stream, String cluster, String namespace, - String topic, String name, String remoteCluster, double value, boolean splitTopicAndPartitionIndexLabel) { - metricType(stream, name); - appendRequiredLabels(stream, cluster, namespace, topic, name, splitTopicAndPartitionIndexLabel) - .write("\",remote_cluster=\"").write(remoteCluster).write("\"} "); - stream.write(value); - appendEndings(stream); + private static void writeConsumerMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, String subscription, + Consumer consumer, boolean splitTopicAndPartitionIndexLabel) { + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "subscription", subscription, "consumer_name", consumer.consumerName(), + "consumer_id", String.valueOf(consumer.consumerId())); } - private static SimpleTextOutputStream appendRequiredLabels(SimpleTextOutputStream stream, String cluster, - String namespace, String topic, String name, boolean splitTopicAndPartitionIndexLabel) { - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace); + static void writeTopicMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, boolean splitTopicAndPartitionIndexLabel, + String... extraLabelsAndValues) { + String[] labelsAndValues = new String[splitTopicAndPartitionIndexLabel ? 8 : 6]; + labelsAndValues[0] = "cluster"; + labelsAndValues[1] = cluster; + labelsAndValues[2] = "namespace"; + labelsAndValues[3] = namespace; + labelsAndValues[4] = "topic"; if (splitTopicAndPartitionIndexLabel) { int index = topic.indexOf(PARTITIONED_TOPIC_SUFFIX); if (index > 0) { - stream.write("\",topic=\"").write(topic.substring(0, index)).write("\",partition=\"") - .write(topic.substring(index + PARTITIONED_TOPIC_SUFFIX.length())); + labelsAndValues[5] = topic.substring(0, index); + labelsAndValues[6] = "partition"; + labelsAndValues[7] = topic.substring(index + PARTITIONED_TOPIC_SUFFIX.length()); } else { - stream.write("\",topic=\"").write(topic).write("\",partition=\"").write("-1"); + labelsAndValues[5] = topic; + labelsAndValues[6] = "partition"; + labelsAndValues[7] = "-1"; } } else { - stream.write("\",topic=\"").write(topic); + labelsAndValues[5] = topic; } - return stream; - } - - private static void appendEndings(SimpleTextOutputStream stream) { - stream.write(' ').write(System.currentTimeMillis()).write('\n'); + String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues); + stream.writeSample(metricName, value, labels); } -} +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java index e6ac1535f43c7..8c58b516333f5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java @@ -20,8 +20,6 @@ import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import io.netty.util.concurrent.FastThreadLocal; -import java.util.HashMap; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; @@ -30,7 +28,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.impl.TransactionMetadataStoreStats; @@ -38,21 +35,10 @@ @Slf4j public class TransactionAggregator { - /** - * Used for tracking duplicate TYPE definitions. - */ - private static final FastThreadLocal> threadLocalMetricWithTypeDefinition = - new FastThreadLocal() { - @Override - protected Map initialValue() { - return new HashMap<>(); - } - }; - private static final FastThreadLocal localTransactionCoordinatorStats = new FastThreadLocal() { @Override - protected AggregatedTransactionCoordinatorStats initialValue() throws Exception { + protected AggregatedTransactionCoordinatorStats initialValue() { return new AggregatedTransactionCoordinatorStats(); } }; @@ -60,21 +46,18 @@ protected AggregatedTransactionCoordinatorStats initialValue() throws Exception private static final FastThreadLocal localManageLedgerStats = new FastThreadLocal() { @Override - protected ManagedLedgerStats initialValue() throws Exception { + protected ManagedLedgerStats initialValue() { return new ManagedLedgerStats(); } }; - public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, boolean includeTopicMetrics) { + public static void generate(PulsarService pulsar, PrometheusMetricStreams stream, boolean includeTopicMetrics) { String cluster = pulsar.getConfiguration().getClusterName(); - Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); - metricWithTypeDefinition.clear(); if (includeTopicMetrics) { - pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { - bundlesMap.forEach((bundle, topicsMap) -> { - topicsMap.forEach((name, topic) -> { + pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> + bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { if (topic instanceof PersistentTopic) { topic.getSubscriptions().values().forEach(subscription -> { try { @@ -82,9 +65,8 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, if (!checkTopicIsEventsNames(TopicName.get(subscription.getTopic().getName())) && subscription instanceof PersistentSubscription && ((PersistentSubscription) subscription).checkIfPendingAckStoreInit()) { - ManagedLedger managedLedger = - ((PersistentSubscription) subscription) - .getPendingAckManageLedger().get(); + ManagedLedger managedLedger = ((PersistentSubscription) subscription) + .getPendingAckManageLedger().get(); generateManageLedgerStats(managedLedger, stream, cluster, namespace, name, subscription.getName()); } @@ -93,9 +75,7 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, } }); } - }); - }); - }); + }))); } AggregatedTransactionCoordinatorStats transactionCoordinatorStats = localTransactionCoordinatorStats.get(); @@ -124,18 +104,18 @@ public static void generate(PulsarService pulsar, SimpleTextOutputStream stream, localManageLedgerStats.get().reset(); if (transactionMetadataStore instanceof MLTransactionMetadataStore) { - ManagedLedger managedLedger = - ((MLTransactionMetadataStore) transactionMetadataStore).getManagedLedger(); + ManagedLedger managedLedger = + ((MLTransactionMetadataStore) transactionMetadataStore).getManagedLedger(); generateManageLedgerStats(managedLedger, stream, cluster, NamespaceName.SYSTEM_NAMESPACE.toString(), MLTransactionLogImpl.TRANSACTION_LOG_PREFIX + transactionCoordinatorID.getId(), MLTransactionLogImpl.TRANSACTION_SUBSCRIPTION_NAME); } - }); + }); } - private static void generateManageLedgerStats(ManagedLedger managedLedger, SimpleTextOutputStream stream, + private static void generateManageLedgerStats(ManagedLedger managedLedger, PrometheusMetricStreams stream, String cluster, String namespace, String topic, String subscription) { ManagedLedgerStats managedLedgerStats = localManageLedgerStats.get(); ManagedLedgerMBeanImpl mlStats = (ManagedLedgerMBeanImpl) managedLedger.getStats(); @@ -157,174 +137,149 @@ private static void generateManageLedgerStats(ManagedLedger managedLedger, Simpl managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate(); managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate(); - printManageLedgerStats(stream, cluster, namespace, topic, - subscription, managedLedgerStats); - } - - private static void metricType(SimpleTextOutputStream stream, String name) { - Map metricWithTypeDefinition = threadLocalMetricWithTypeDefinition.get(); - if (!metricWithTypeDefinition.containsKey(name)) { - metricWithTypeDefinition.put(name, "gauge"); - stream.write("# TYPE ").write(name).write(" gauge\n"); - } - - } - - private static void metric(SimpleTextOutputStream stream, String cluster, String name, - double value, long coordinatorId) { - metricType(stream, name); - stream.write(name) - .write("{cluster=\"").write(cluster) - .write("\",coordinator_id=\"").write(coordinatorId).write("\"} ") - .write(value).write(' ').write(System.currentTimeMillis()) - .write('\n'); + printManageLedgerStats(stream, cluster, namespace, topic, subscription, managedLedgerStats); } - private static void metrics(SimpleTextOutputStream stream, String cluster, String namespace, - String topic, String subscription, String name, long value) { - stream.write(name).write("{cluster=\"").write(cluster).write("\", namespace=\"").write(namespace) - .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); - } - - private static void metrics(SimpleTextOutputStream stream, String cluster, String namespace, - String topic, String subscription, String name, double value) { - stream.write(name).write("{cluster=\"").write(cluster).write("\", namespace=\"").write(namespace) - .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); - } - - private static void printManageLedgerStats(SimpleTextOutputStream stream, String cluster, String namespace, + private static void printManageLedgerStats(PrometheusMetricStreams stream, String cluster, String namespace, String topic, String subscription, ManagedLedgerStats stats) { - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_size", stats.storageSize); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_logical_size", stats.storageLogicalSize); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_backlog_size", stats.backlogSize); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_offloaded_size", stats.offloadedStorageUsed); + writeMetric(stream, "pulsar_storage_size", stats.storageSize, cluster, namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_logical_size", stats.storageLogicalSize, cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_backlog_size", stats.backlogSize, cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_offloaded_size", stats.offloadedStorageUsed, cluster, namespace, topic, + subscription); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_rate", stats.storageWriteRate); - metrics(stream, cluster, namespace, topic, subscription, - "pulsar_storage_read_rate", stats.storageReadRate); + writeMetric(stream, "pulsar_storage_write_rate", stats.storageWriteRate, cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_read_rate", stats.storageReadRate, cluster, namespace, topic, + subscription); stats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.storageWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_0_5", latencyBuckets[0]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_1", latencyBuckets[1]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_5", latencyBuckets[2]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_10", latencyBuckets[3]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_20", latencyBuckets[4]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_50", latencyBuckets[5]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_100", latencyBuckets[6]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_200", latencyBuckets[7]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_le_1000", latencyBuckets[8]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_write_latency_overflow", latencyBuckets[9]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_write_latency_count", - stats.storageWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_write_latency_sum", - stats.storageWriteLatencyBuckets.getSum()); + writeMetric(stream, "pulsar_storage_write_latency_le_0_5", latencyBuckets[0], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_1", latencyBuckets[1], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_5", latencyBuckets[2], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_10", latencyBuckets[3], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_20", latencyBuckets[4], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_50", latencyBuckets[5], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_100", latencyBuckets[6], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_200", latencyBuckets[7], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_le_1000", latencyBuckets[8], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_overflow", latencyBuckets[9], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_storage_write_latency_count", stats.storageWriteLatencyBuckets.getCount(), + cluster, namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_write_latency_sum", stats.storageWriteLatencyBuckets.getSum(), cluster, + namespace, topic, subscription); stats.storageLedgerWriteLatencyBuckets.refresh(); - long[] ledgerWritelatencyBuckets = stats.storageLedgerWriteLatencyBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_0_5", ledgerWritelatencyBuckets[0]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_1", ledgerWritelatencyBuckets[1]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_5", ledgerWritelatencyBuckets[2]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_10", ledgerWritelatencyBuckets[3]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_20", ledgerWritelatencyBuckets[4]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_50", ledgerWritelatencyBuckets[5]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_100", ledgerWritelatencyBuckets[6]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_200", ledgerWritelatencyBuckets[7]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_storage_ledger_write_latency_le_1000", ledgerWritelatencyBuckets[8]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_overflow", - ledgerWritelatencyBuckets[9]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_count", - stats.storageLedgerWriteLatencyBuckets.getCount()); - metric(stream, cluster, namespace, topic, subscription, "pulsar_storage_ledger_write_latency_sum", - stats.storageLedgerWriteLatencyBuckets.getSum()); + long[] ledgerWriteLatencyBuckets = stats.storageLedgerWriteLatencyBuckets.getBuckets(); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_0_5", ledgerWriteLatencyBuckets[0], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1", ledgerWriteLatencyBuckets[1], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_5", ledgerWriteLatencyBuckets[2], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_10", ledgerWriteLatencyBuckets[3], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_20", ledgerWriteLatencyBuckets[4], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_50", ledgerWriteLatencyBuckets[5], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_100", ledgerWriteLatencyBuckets[6], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_200", ledgerWriteLatencyBuckets[7], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_le_1000", ledgerWriteLatencyBuckets[8], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_overflow", ledgerWriteLatencyBuckets[9], cluster, + namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_count", + stats.storageLedgerWriteLatencyBuckets.getCount(), cluster, namespace, topic, subscription); + writeMetric(stream, "pulsar_storage_ledger_write_latency_sum", + stats.storageLedgerWriteLatencyBuckets.getSum(), cluster, namespace, topic, subscription); stats.entrySizeBuckets.refresh(); long[] entrySizeBuckets = stats.entrySizeBuckets.getBuckets(); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_128", entrySizeBuckets[0]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_512", entrySizeBuckets[1]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_le_100_kb", entrySizeBuckets[6]); - metric(stream, cluster, namespace, topic, subscription, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_le_overflow", entrySizeBuckets[8]); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_count", stats.entrySizeBuckets.getCount()); - metric(stream, cluster, namespace, topic, subscription, - "pulsar_entry_size_sum", stats.entrySizeBuckets.getSum()); - } - - private static void metric(SimpleTextOutputStream stream, String cluster, - String namespace, String topic, String subscription, - String name, long value) { - stream.write(name).write("{cluster=\"").write(cluster).write("\",namespace=\"").write(namespace) - .write("\",topic=\"").write(topic).write("\",subscription=\"").write(subscription).write("\"} "); - stream.write(value).write(' ').write(System.currentTimeMillis()).write('\n'); + writeMetric(stream, "pulsar_entry_size_le_128", entrySizeBuckets[0], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_512", entrySizeBuckets[1], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_1_kb", entrySizeBuckets[2], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_2_kb", entrySizeBuckets[3], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_4_kb", entrySizeBuckets[4], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_16_kb", entrySizeBuckets[5], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_100_kb", entrySizeBuckets[6], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_1_mb", entrySizeBuckets[7], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_le_overflow", entrySizeBuckets[8], cluster, namespace, topic, + subscription); + writeMetric(stream, "pulsar_entry_size_count", stats.entrySizeBuckets.getCount(), cluster, namespace, + topic, subscription); + writeMetric(stream, "pulsar_entry_size_sum", stats.entrySizeBuckets.getSum(), cluster, namespace, topic, + subscription); } - static void printTransactionCoordinatorStats(SimpleTextOutputStream stream, String cluster, + static void printTransactionCoordinatorStats(PrometheusMetricStreams stream, String cluster, AggregatedTransactionCoordinatorStats stats, long coordinatorId) { - metric(stream, cluster, "pulsar_txn_active_count", - stats.actives, coordinatorId); - metric(stream, cluster, "pulsar_txn_committed_count", - stats.committedCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_aborted_count", - stats.abortedCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_created_count", - stats.createdCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_timeout_count", - stats.timeoutCount, coordinatorId); - metric(stream, cluster, "pulsar_txn_append_log_count", - stats.appendLogCount, coordinatorId); + writeMetric(stream, "pulsar_txn_active_count", stats.actives, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_committed_count", stats.committedCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_aborted_count", stats.abortedCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_created_count", stats.createdCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_timeout_count", stats.timeoutCount, cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_append_log_count", stats.appendLogCount, cluster, + coordinatorId); long[] latencyBuckets = stats.executionLatency; - metric(stream, cluster, "pulsar_txn_execution_latency_le_10", latencyBuckets[0], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_20", latencyBuckets[1], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_50", latencyBuckets[2], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_100", latencyBuckets[3], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_500", latencyBuckets[4], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_1000", latencyBuckets[5], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_5000", latencyBuckets[6], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_15000", latencyBuckets[7], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_30000", latencyBuckets[8], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_60000", latencyBuckets[9], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_300000", - latencyBuckets[10], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_1500000", - latencyBuckets[11], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_3000000", - latencyBuckets[12], coordinatorId); - metric(stream, cluster, "pulsar_txn_execution_latency_le_overflow", - latencyBuckets[13], coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_10", latencyBuckets[0], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_20", latencyBuckets[1], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_50", latencyBuckets[2], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_100", latencyBuckets[3], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_500", latencyBuckets[4], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_1000", latencyBuckets[5], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_5000", latencyBuckets[6], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_15000", latencyBuckets[7], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_30000", latencyBuckets[8], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_60000", latencyBuckets[9], cluster, coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_300000", latencyBuckets[10], cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_1500000", latencyBuckets[11], cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_3000000", latencyBuckets[12], cluster, + coordinatorId); + writeMetric(stream, "pulsar_txn_execution_latency_le_overflow", latencyBuckets[13], cluster, + coordinatorId); + } + + private static void writeMetric(PrometheusMetricStreams stream, String metricName, double value, String cluster, + long coordinatorId) { + stream.writeSample(metricName, value, "cluster", cluster, "coordinator_id", String.valueOf(coordinatorId)); + } + + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, + String namespace, String topic, String subscription) { + stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace, "topic", topic, + "subscription", subscription); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java index 7550096c2b584..8f704b11e764c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java @@ -18,13 +18,8 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; -import io.prometheus.client.Collector; -import io.prometheus.client.Collector.MetricFamilySamples; -import io.prometheus.client.Collector.MetricFamilySamples.Sample; -import io.prometheus.client.CollectorRegistry; import java.io.IOException; import java.io.Writer; -import java.util.Enumeration; import org.apache.bookkeeper.stats.Counter; /** @@ -140,31 +135,4 @@ private static void writeSum(Writer w, DataSketchesOpStatsLogger opStat, String .append(success.toString()).append("\"} ") .append(Double.toString(opStat.getSum(success))).append('\n'); } - - public static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) - throws IOException { - Enumeration metricFamilySamples = registry.metricFamilySamples(); - while (metricFamilySamples.hasMoreElements()) { - MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); - - for (int i = 0; i < metricFamily.samples.size(); i++) { - Sample sample = metricFamily.samples.get(i); - w.write(sample.name); - w.write('{'); - for (int j = 0; j < sample.labelNames.size(); j++) { - if (j != 0) { - w.write(", "); - } - w.write(sample.labelNames.get(j)); - w.write("=\""); - w.write(sample.labelValues.get(j)); - w.write('"'); - } - - w.write("} "); - w.write(Collector.doubleToGoString(sample.value)); - w.write('\n'); - } - } - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index f28412ea75160..18f7597207e30 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -50,6 +50,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.crypto.SecretKey; @@ -1397,6 +1398,64 @@ public void testSplitTopicAndPartitionLabel() throws Exception { consumer2.close(); } + + @Test + public void testMetricsGroupedByTypeDefinitions() throws Exception { + Producer p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create(); + Producer p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create(); + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + p1.send(message.getBytes()); + p2.send(message.getBytes()); + } + + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + String metricsStr = statsOut.toString(); + + Pattern typePattern = Pattern.compile("^#\\s+TYPE\\s+(\\w+)\\s+(\\w+)"); + Pattern metricNamePattern = Pattern.compile("^(\\w+)\\{.+"); + + AtomicReference currentMetric = new AtomicReference<>(); + Splitter.on("\n").split(metricsStr).forEach(line -> { + if (line.isEmpty()) { + return; + } + if (line.startsWith("#")) { + // Get the current type definition + Matcher typeMatcher = typePattern.matcher(line); + checkArgument(typeMatcher.matches()); + String metricName = typeMatcher.group(1); + currentMetric.set(metricName); + } else { + Matcher metricMatcher = metricNamePattern.matcher(line); + checkArgument(metricMatcher.matches()); + String metricName = metricMatcher.group(1); + + if (metricName.endsWith("_bucket")) { + metricName = metricName.substring(0, metricName.indexOf("_bucket")); + } else if (metricName.endsWith("_count") && !currentMetric.get().endsWith("_count")) { + metricName = metricName.substring(0, metricName.indexOf("_count")); + } else if (metricName.endsWith("_sum") && !currentMetric.get().endsWith("_sum")) { + metricName = metricName.substring(0, metricName.indexOf("_sum")); + } else if (metricName.endsWith("_total") && !currentMetric.get().endsWith("_total")) { + metricName = metricName.substring(0, metricName.indexOf("_total")); + } else if (metricName.endsWith("_created") && !currentMetric.get().endsWith("_created")) { + metricName = metricName.substring(0, metricName.indexOf("_created")); + } + + if (!metricName.equals(currentMetric.get())) { + System.out.println(metricsStr); + fail("Metric not grouped under its type definition: " + line); + } + + } + }); + + p1.close(); + p2.close(); + } + private void compareCompactionStateCount(List cm, double count) { assertEquals(cm.size(), 1); assertEquals(cm.get(0).tags.get("cluster"), "test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java new file mode 100644 index 0000000000000..15c29a0dc66bb --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreamsTest.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import org.apache.pulsar.common.util.SimpleTextOutputStream; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class PrometheusMetricStreamsTest { + + private PrometheusMetricStreams underTest; + + @BeforeMethod(alwaysRun = true) + protected void setup() throws Exception { + underTest = new PrometheusMetricStreams(); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + underTest.releaseAll(); + } + + @Test + public void canWriteSampleWithoutLabels() { + underTest.writeSample("my-metric", 123); + + String actual = writeToString(); + + assertTrue(actual.startsWith("# TYPE my-metric gauge"), "Gauge type line missing"); + assertTrue(actual.contains("my-metric{} 123"), "Metric line missing"); + } + + @Test + public void canWriteSampleWithLabels() { + underTest.writeSample("my-other-metric", 123, "cluster", "local"); + underTest.writeSample("my-other-metric", 456, "cluster", "local", "namespace", "my-ns"); + + String actual = writeToString(); + + assertTrue(actual.startsWith("# TYPE my-other-metric gauge"), "Gauge type line missing"); + assertTrue(actual.contains("my-other-metric{cluster=\"local\"} 123"), "Cluster metric line missing"); + assertTrue(actual.contains("my-other-metric{cluster=\"local\",namespace=\"my-ns\"} 456"), + "Cluster and Namespace metric line missing"); + } + + private String writeToString() { + ByteBuf buffer = ByteBufAllocator.DEFAULT.directBuffer(); + try { + SimpleTextOutputStream stream = new SimpleTextOutputStream(buffer); + underTest.flushAllToStream(stream); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int readIndex = buffer.readerIndex(); + int readableBytes = buffer.readableBytes(); + for (int i = 0; i < readableBytes; i++) { + out.write(buffer.getByte(readIndex + i)); + } + return out.toString(); + } finally { + buffer.release(); + } + } +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java index 9fc4b347c854f..9bfbbff0211b2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java @@ -26,8 +26,8 @@ */ public class SimpleTextOutputStream { private final ByteBuf buffer; - private static final char[] hexChars = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', - 'f' }; + private static final char[] hexChars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', + 'f'}; public SimpleTextOutputStream(ByteBuf buffer) { this.buffer = buffer; @@ -131,4 +131,12 @@ public SimpleTextOutputStream write(double d) { write(r); return this; } + + public void write(ByteBuf byteBuf) { + buffer.writeBytes(byteBuf); + } + + public ByteBuf getBuffer() { + return buffer; + } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java index f7a205c7db02c..46d232da3e73d 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/PrometheusTextFormat.java @@ -37,6 +37,11 @@ public static void write004(Writer writer, Enumeration 0) { @@ -64,19 +69,19 @@ private static void writeEscapedLabelValue(Writer writer, String s) throws IOExc for (int i = 0; i < s.length(); i++) { char c = s.charAt(i); switch (c) { - case '\\': - writer.append("\\\\"); - break; - case '\"': - writer.append("\\\""); - break; - case '\n': - writer.append("\\n"); - break; - default: - writer.append(c); + case '\\': + writer.append("\\\\"); + break; + case '\"': + writer.append("\\\""); + break; + case '\n': + writer.append("\\n"); + break; + default: + writer.append(c); } } } -} +} \ No newline at end of file diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java index c8b411cbf5706..2ad407b2e5e3e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java @@ -328,6 +328,11 @@ private void generateLeaderMetrics(StringWriter stream) { } private void writeMetric(String metricName, long value, StringWriter stream) { + stream.write("# TYPE "); + stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); + stream.write(metricName); + stream.write(" gauge \n"); + stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); stream.write(metricName); stream.write("{"); From 39ce3240b4eacc7e2da5d338ffda8c0b10f2fdb7 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 29 Sep 2022 18:48:38 +0800 Subject: [PATCH 763/823] [fix][proxy] Fix refresh client auth (#17831) * [fix][proxy] Fix refresh client auth Signed-off-by: Zixuan Liu * Fix style Signed-off-by: Zixuan Liu (cherry picked from commit c952f3c9f891f85ff4b6cee6e28b6f68db3b5bcd) Signed-off-by: Zixuan Liu --- .../apache/pulsar/client/impl/ClientCnx.java | 24 ++- .../pulsar/client/impl/ConnectionPool.java | 8 + pulsar-proxy/pom.xml | 5 + .../pulsar/proxy/server/ProxyClientCnx.java | 82 ++++++-- .../pulsar/proxy/server/ProxyConnection.java | 89 +++++++-- .../proxy/server/ProxyRefreshAuthTest.java | 186 ++++++++++++++++++ 6 files changed, 349 insertions(+), 45 deletions(-) create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 03a72fe16b470..506a0d36ca7b7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -55,12 +55,9 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.ConnectException; import org.apache.pulsar.client.api.PulsarClientException.TimeoutException; -import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; -import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.BinaryProtoLookupService.LookupDataResult; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.common.api.proto.CommandTcClientConnectResponse; -import org.apache.pulsar.common.tls.TlsHostnameVerifier; +import org.apache.pulsar.client.impl.schema.SchemaInfoUtil; import org.apache.pulsar.client.impl.transaction.TransactionBufferHandler; import org.apache.pulsar.client.util.TimedCompletableFuture; import org.apache.pulsar.common.api.AuthData; @@ -89,10 +86,10 @@ import org.apache.pulsar.common.api.proto.CommandSendError; import org.apache.pulsar.common.api.proto.CommandSendReceipt; import org.apache.pulsar.common.api.proto.CommandSuccess; +import org.apache.pulsar.common.api.proto.CommandTcClientConnectResponse; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarHandler; -import org.apache.pulsar.client.impl.schema.SchemaInfoUtil; import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.util.FutureUtil; @@ -104,7 +101,7 @@ public class ClientCnx extends PulsarHandler { protected final Authentication authentication; - private State state; + protected State state; @Getter private final ConcurrentLongHashMap> pendingRequests = @@ -149,7 +146,7 @@ public class ClientCnx extends PulsarHandler { private final int maxNumberOfRejectedRequestPerConnection; private final int rejectedRequestResetTimeSec = 60; - private final int protocolVersion; + protected final int protocolVersion; private final long operationTimeoutMs; protected String proxyToTargetBrokerAddress = null; @@ -165,7 +162,10 @@ public class ClientCnx extends PulsarHandler { protected AuthenticationDataProvider authenticationDataProvider; private TransactionBufferHandler transactionBufferHandler; - enum State { + @Getter + private long lastDisconnectedTimestamp; + + protected enum State { None, SentConnectFrame, Ready, Failed, Connecting } @@ -268,6 +268,7 @@ protected ByteBuf newConnectCommand() throws Exception { @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); + lastDisconnectedTimestamp = System.currentTimeMillis(); log.info("{} Disconnected", ctx.channel()); if (!connectionFuture.isDone()) { connectionFuture.completeExceptionally(new PulsarClientException("Connection already closed")); @@ -1163,6 +1164,13 @@ public void close() { } } + protected void closeWithException(Throwable e) { + if (ctx != null) { + connectionFuture.completeExceptionally(e); + ctx.close(); + } + } + private void checkRequestTimeout() { while (!requestTimeoutQueue.isEmpty()) { RequestTime request = requestTimeoutQueue.peek(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 15517e45ba3d7..433f20b3dafb2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -33,15 +33,18 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.InvalidServiceURL; @@ -345,6 +348,11 @@ private void cleanupConnection(InetSocketAddress address, int connectionKey, } } + public Set> getConnections() { + return Collections.unmodifiableSet( + pool.values().stream().flatMap(n -> n.values().stream()).collect(Collectors.toSet())); + } + @VisibleForTesting int getPoolSize() { return pool.values().stream().mapToInt(Map::size).sum(); diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 983bc627cbc52..f0687580ee827 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -185,6 +185,11 @@ ipaddress ${seancfoley.ipaddress.version} + + org.awaitility + awaitility + test + diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java index 665b9f83fd604..283b835fff54f 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java @@ -18,47 +18,95 @@ */ package org.apache.pulsar.proxy.server; +import static com.google.common.base.Preconditions.checkArgument; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; - +import java.util.Arrays; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.api.AuthData; +import org.apache.pulsar.common.api.proto.CommandAuthChallenge; import org.apache.pulsar.common.protocol.Commands; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Slf4j public class ProxyClientCnx extends ClientCnx { - - String clientAuthRole; - AuthData clientAuthData; - String clientAuthMethod; - int protocolVersion; + private final boolean forwardClientAuthData; + private final String clientAuthMethod; + private final String clientAuthRole; + private final AuthData clientAuthData; + private final ProxyConnection proxyConnection; public ProxyClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, String clientAuthRole, - AuthData clientAuthData, String clientAuthMethod, int protocolVersion) { - super(conf, eventLoopGroup); + AuthData clientAuthData, String clientAuthMethod, int protocolVersion, + boolean forwardClientAuthData, ProxyConnection proxyConnection) { + super(conf, eventLoopGroup, protocolVersion); this.clientAuthRole = clientAuthRole; this.clientAuthData = clientAuthData; this.clientAuthMethod = clientAuthMethod; - this.protocolVersion = protocolVersion; + this.forwardClientAuthData = forwardClientAuthData; + this.proxyConnection = proxyConnection; } @Override protected ByteBuf newConnectCommand() throws Exception { if (log.isDebugEnabled()) { - log.debug("New Connection opened via ProxyClientCnx with params clientAuthRole = {}," + - " clientAuthData = {}, clientAuthMethod = {}", + log.debug("New Connection opened via ProxyClientCnx with params clientAuthRole = {}," + + " clientAuthData = {}, clientAuthMethod = {}", clientAuthRole, clientAuthData, clientAuthMethod); } authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); - return Commands.newConnect(authentication.getAuthMethodName(), authData, this.protocolVersion, - PulsarVersion.getVersion(), proxyToTargetBrokerAddress, clientAuthRole, clientAuthData, - clientAuthMethod); + return Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion, + PulsarVersion.getVersion(), proxyToTargetBrokerAddress, clientAuthRole, clientAuthData, + clientAuthMethod); } - private static final Logger log = LoggerFactory.getLogger(ProxyClientCnx.class); + @Override + protected void handleAuthChallenge(CommandAuthChallenge authChallenge) { + checkArgument(authChallenge.hasChallenge()); + checkArgument(authChallenge.getChallenge().hasAuthData()); + + boolean isRefresh = Arrays.equals(AuthData.REFRESH_AUTH_DATA_BYTES, authChallenge.getChallenge().getAuthData()); + if (!forwardClientAuthData || !isRefresh) { + super.handleAuthChallenge(authChallenge); + return; + } + + try { + if (log.isDebugEnabled()) { + log.debug("Proxy {} request to refresh the original client authentication data for " + + "the proxy client {}", proxyConnection.ctx().channel(), ctx.channel()); + } + + proxyConnection.ctx().writeAndFlush(Commands.newAuthChallenge(clientAuthMethod, AuthData.REFRESH_AUTH_DATA, + protocolVersion)) + .addListener(writeFuture -> { + if (writeFuture.isSuccess()) { + if (log.isDebugEnabled()) { + log.debug("Proxy {} sent the auth challenge to original client to refresh credentials " + + "with method {} for the proxy client {}", + proxyConnection.ctx().channel(), clientAuthMethod, ctx.channel()); + } + } else { + log.error("Failed to send the auth challenge to original client by the proxy {} " + + "for the proxy client {}", + proxyConnection.ctx().channel(), + ctx.channel(), + writeFuture.cause()); + closeWithException(writeFuture.cause()); + } + }); + + if (state == State.SentConnectFrame) { + state = State.Connecting; + } + } catch (Exception e) { + log.error("Failed to send the auth challenge to origin client by the proxy {} for the proxy client {}", + proxyConnection.ctx().channel(), ctx.channel(), e); + closeWithException(e); + } + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 42b4d0f08f77c..bb8f26787455a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -20,7 +20,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.handler.ssl.SslHandler; import io.netty.resolver.dns.DnsAddressResolverGroup; @@ -37,7 +40,8 @@ import java.util.function.Supplier; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; - +import lombok.Getter; +import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; @@ -51,26 +55,21 @@ import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.api.AuthData; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.api.proto.CommandAuthResponse; import org.apache.pulsar.common.api.proto.CommandConnect; import org.apache.pulsar.common.api.proto.CommandConnected; +import org.apache.pulsar.common.api.proto.CommandGetSchema; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.api.proto.CommandLookupTopic; -import org.apache.pulsar.common.api.proto.CommandGetSchema; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.api.proto.ServerError; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import lombok.Getter; - /** * Handles incoming discovery request from client and sends appropriate response back to client * @@ -268,12 +267,11 @@ private synchronized void completeConnect(AuthData clientData) throws PulsarClie this.clientAuthData = clientData; this.clientAuthMethod = authMethod; } - clientCnxSupplier = - () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, clientAuthData, - clientAuthMethod, protocolVersionToAdvertise); + clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, + clientAuthData, clientAuthMethod, protocolVersionToAdvertise, + service.getConfiguration().isForwardAuthorizationCredentials(), this); } else { - clientCnxSupplier = - () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise); + clientCnxSupplier = () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise); } if (this.connectionPool == null) { @@ -374,16 +372,22 @@ public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnec } // According to auth result, send newConnected or newAuthChallenge command. - private void doAuthentication(AuthData clientData) throws Exception { + private void doAuthentication(AuthData clientData) + throws Exception { AuthData brokerData = authState.authenticate(clientData); // authentication has completed, will send newConnected command. if (authState.isComplete()) { clientAuthRole = authState.getAuthRole(); if (LOG.isDebugEnabled()) { LOG.debug("[{}] Client successfully authenticated with {} role {}", - remoteAddress, authMethod, clientAuthRole); + remoteAddress, authMethod, clientAuthRole); + } + + // First connection + if (this.connectionPool == null || state == State.Connecting) { + // authentication has completed, will send newConnected command. + completeConnect(clientData); } - completeConnect(clientData); return; } @@ -392,7 +396,7 @@ private void doAuthentication(AuthData clientData) throws Exception { .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); if (LOG.isDebugEnabled()) { LOG.debug("[{}] Authentication in progress client by method {}.", - remoteAddress, authMethod); + remoteAddress, authMethod); } state = State.Connecting; } @@ -474,18 +478,63 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), @Override protected void handleAuthResponse(CommandAuthResponse authResponse) { - checkArgument(state == State.Connecting); checkArgument(authResponse.hasResponse()); checkArgument(authResponse.getResponse().hasAuthData() && authResponse.getResponse().hasAuthMethodName()); if (LOG.isDebugEnabled()) { LOG.debug("Received AuthResponse from {}, auth method: {}", - remoteAddress, authResponse.getResponse().getAuthMethodName()); + remoteAddress, authResponse.getResponse().getAuthMethodName()); } try { AuthData clientData = AuthData.of(authResponse.getResponse().getAuthData()); doAuthentication(clientData); + if (service.getConfiguration().isForwardAuthorizationCredentials() + && connectionPool != null && state == State.ProxyLookupRequests) { + connectionPool.getConnections().forEach(toBrokerCnxFuture -> { + String clientVersion; + if (authResponse.hasClientVersion()) { + clientVersion = authResponse.getClientVersion(); + } else { + clientVersion = PulsarVersion.getVersion(); + } + int protocolVersion; + if (authResponse.hasProtocolVersion()) { + protocolVersion = authResponse.getProtocolVersion(); + } else { + protocolVersion = Commands.getCurrentProtocolVersion(); + } + + ByteBuf cmd = + Commands.newAuthResponse(clientAuthMethod, clientData, protocolVersion, clientVersion); + toBrokerCnxFuture.thenAccept(toBrokerCnx -> toBrokerCnx.ctx().writeAndFlush(cmd) + .addListener(writeFuture -> { + if (writeFuture.isSuccess()) { + if (LOG.isDebugEnabled()) { + LOG.debug("{} authentication is refreshed successfully by {}, " + + "auth method: {} ", + toBrokerCnx.ctx().channel(), ctx.channel(), clientAuthMethod); + } + } else { + LOG.error("Failed to forward the auth response " + + "from the proxy to the broker through the proxy client, " + + "proxy: {}, proxy client: {}", + ctx.channel(), + toBrokerCnx.ctx().channel(), + writeFuture.cause()); + toBrokerCnx.ctx().channel().pipeline() + .fireExceptionCaught(writeFuture.cause()); + } + })) + .whenComplete((__, ex) -> { + if (ex != null) { + LOG.error("Failed to forward the auth response from the proxy to " + + "the broker through the proxy client, proxy: {}", + ctx().channel(), ex); + } + }); + }); + } } catch (Exception e) { String msg = "Unable to handleAuthResponse"; LOG.warn("[{}] {} ", remoteAddress, msg, e); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java new file mode 100644 index 0000000000000..9ccb067adbf10 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.Sets; +import io.jsonwebtoken.SignatureAlgorithm; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import javax.crypto.SecretKey; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +public class ProxyRefreshAuthTest extends ProducerConsumerBase { + private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + private ProxyService proxyService; + private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + + // enable tls and auth&auth at broker + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(false); + conf.setTopicLevelPoliciesEnabled(false); + conf.setProxyRoles(Collections.singleton("Proxy")); + conf.setAdvertisedAddress(null); + conf.setAuthenticateOriginalAuthData(true); + conf.setBrokerServicePort(Optional.of(0)); + conf.setWebServicePort(Optional.of(0)); + + Set superUserRoles = new HashSet<>(); + superUserRoles.add("superUser"); + conf.setSuperUserRoles(superUserRoles); + + conf.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + Properties properties = new Properties(); + properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); + conf.setProperties(properties); + + conf.setClusterName("proxy-authorization"); + conf.setNumExecutorThreadPoolSize(5); + + conf.setAuthenticationRefreshCheckSeconds(1); + } + + @BeforeClass + @Override + protected void setup() throws Exception { + super.init(); + + admin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()) + .authentication(new AuthenticationToken( + () -> AuthTokenUtils.createToken(SECRET_KEY, "client", Optional.empty()))).build(); + String namespaceName = "my-tenant/my-ns"; + admin.clusters().createCluster("proxy-authorization", + ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); + admin.tenants().createTenant("my-tenant", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); + admin.namespaces().createNamespace(namespaceName); + + // start proxy service + proxyConfig.setAuthenticationEnabled(true); + proxyConfig.setAuthorizationEnabled(false); + proxyConfig.setForwardAuthorizationCredentials(true); + proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setAdvertisedAddress(null); + + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setWebServicePort(Optional.of(0)); + + proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + proxyConfig.setBrokerClientAuthenticationParameters( + AuthTokenUtils.createToken(SECRET_KEY, "Proxy", Optional.empty())); + proxyConfig.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + Properties properties = new Properties(); + properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); + proxyConfig.setProperties(properties); + + proxyService = Mockito.spy(new ProxyService(proxyConfig, + new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)))); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + proxyService.close(); + } + + private void startProxy(boolean forwardAuthData) throws Exception { + pulsar.getConfiguration().setAuthenticateOriginalAuthData(forwardAuthData); + proxyConfig.setForwardAuthorizationCredentials(forwardAuthData); + proxyService.start(); + } + + @DataProvider + Object[] forwardAuthDataProvider() { + return new Object[]{true, false}; + } + + @Test(dataProvider = "forwardAuthDataProvider") + public void testAuthDataRefresh(boolean forwardAuthData) throws Exception { + log.info("-- Starting {} test --", methodName); + + startProxy(forwardAuthData); + + AuthenticationToken authenticationToken = new AuthenticationToken(() -> { + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.SECOND, 1); + return AuthTokenUtils.createToken(SECRET_KEY, "client", Optional.of(calendar.getTime())); + }); + + pulsarClient = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()) + .authentication(authenticationToken) + .build(); + + String topic = "persistent://my-tenant/my-ns/my-topic1"; + @Cleanup + Producer ignored = spy(pulsarClient.newProducer() + .topic(topic).create()); + + PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + Set> connections = pulsarClientImpl.getCnxPool().getConnections(); + + Awaitility.await().during(4, SECONDS).untilAsserted(() -> { + pulsarClient.getPartitionsForTopic(topic).get(); + assertTrue(connections.stream().allMatch(n -> { + try { + ClientCnx clientCnx = n.get(); + long timestamp = clientCnx.getLastDisconnectedTimestamp(); + return timestamp == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + })); + }); + } +} From b07d31bf12ffa3dfb8d28ca064c06922d174c3a8 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 30 Sep 2022 09:57:15 +0800 Subject: [PATCH 764/823] [branch-2.9] Fix cherry-pick issue (#17894) * Verify branch-2.9 CI * [branch-2.9] Fix cherry-pick issue --- .../org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java index 9ccb067adbf10..907865d76c579 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java @@ -78,7 +78,7 @@ protected void doInitConf() throws Exception { superUserRoles.add("superUser"); conf.setSuperUserRoles(superUserRoles); - conf.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + conf.setAuthenticationProviders(Collections.singleton(AuthenticationProviderToken.class.getName())); Properties properties = new Properties(); properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); conf.setProperties(properties); @@ -118,7 +118,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( AuthTokenUtils.createToken(SECRET_KEY, "Proxy", Optional.empty())); - proxyConfig.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + proxyConfig.setAuthenticationProviders(Collections.singleton(AuthenticationProviderToken.class.getName())); Properties properties = new Properties(); properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); proxyConfig.setProperties(properties); From 9de48f570a5e3a8a856c3f47d1cc39f5811dd08f Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Sun, 2 Oct 2022 08:46:09 +0800 Subject: [PATCH 765/823] [fix][broker] Fix the broker shutdown issue after Zookeeper node crashed (#17909) (cherry picked from commit e26060a1e15a3488fc93cdff6bb0e95e7ec52fed) --- .../java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java index 2cee1c7a627f6..c59ea0a940bb2 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java @@ -75,7 +75,7 @@ public ZKMetadataStore(String metadataURL, MetadataStoreConfig metadataStoreConf .sessionTimeoutMs(metadataStoreConfig.getSessionTimeoutMillis()) .watchers(Collections.singleton(event -> { if (sessionWatcher != null) { - sessionWatcher.ifPresent(sw -> sw.process(event)); + sessionWatcher.ifPresent(sw -> executor.execute(() -> sw.process(event))); } })) .build(); From 1f8cef3d8e741143971ad89f67fe4507a9191769 Mon Sep 17 00:00:00 2001 From: penghui Date: Sun, 2 Oct 2022 08:57:23 +0800 Subject: [PATCH 766/823] Make executor of AbstractMetadataStore protected --- .../org/apache/pulsar/metadata/impl/AbstractMetadataStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 4346d8ad4088a..40c209f4d3e87 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -62,7 +62,7 @@ public abstract class AbstractMetadataStore implements MetadataStoreExtended, Co private final CopyOnWriteArrayList> listeners = new CopyOnWriteArrayList<>(); private final CopyOnWriteArrayList> sessionListeners = new CopyOnWriteArrayList<>(); - private final ExecutorService executor; + protected final ExecutorService executor; private final AsyncLoadingCache> childrenCache; private final AsyncLoadingCache existsCache; private final CopyOnWriteArrayList> metadataCaches = new CopyOnWriteArrayList<>(); From 4abd4d23e8ab498e0c72c4e062063d6409f66da9 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 1 Oct 2022 08:06:13 -0700 Subject: [PATCH 767/823] Allow to configure and disable the size of lookahead for detecting fixed delays in messages (#17907) --- conf/broker.conf | 6 ++++ .../pulsar/broker/ServiceConfiguration.java | 6 ++++ .../InMemoryDelayedDeliveryTracker.java | 18 ++++++---- ...InMemoryDelayedDeliveryTrackerFactory.java | 5 ++- .../delayed/InMemoryDeliveryTrackerTest.java | 34 +++++++++++-------- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 32ded75096aca..0fd9bbaba8df2 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -500,6 +500,12 @@ delayedDeliveryTickTimeMillis=1000 # delayedDeliveryTickTimeMillis. isDelayedDeliveryDeliverAtTimeStrict=false +# Size of the lookahead window to use when detecting if all the messages in the topic +# have a fixed delay. +# Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle +# fixed delays in messages in a different way. +delayedDeliveryFixedDelayDetectionLookahead=50000 + # Whether to enable acknowledge of batch local index. acknowledgmentAtBatchIndexLevelEnabled=false diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index cbcac11acf2b2..aed35bdad41f3 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -288,6 +288,12 @@ public class ServiceConfiguration implements PulsarConfiguration { + "delayedDeliveryTickTimeMillis.") private boolean isDelayedDeliveryDeliverAtTimeStrict = false; + @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use " + + "when detecting if all the messages in the topic have a fixed delay. " + + "Default is 50,000. Setting the lookahead window to 0 will disable the " + + "logic to handle fixed delays in messages in a different way.") + private long delayedDeliveryFixedDelayDetectionLookahead = 50_000; + @FieldContext(category = CATEGORY_SERVER, doc = "Whether to enable the acknowledge of batch local index") private boolean acknowledgmentAtBatchIndexLevelEnabled = false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 38ca138d0337b..4a8842b15c14e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -59,7 +59,7 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // always going to be in FIFO order, then we can avoid pulling all the messages in // tracker. Instead, we use the lookahead for detection and pause the read from // the cursor if the delays are fixed. - public static final long DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES = 50_000; + private final long fixedDelayDetectionLookahead; // This is the timestamp of the message with the highest delivery time // If new added messages are lower than this, it means the delivery is requested @@ -70,17 +70,22 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T private boolean messagesHaveFixedDelay = true; InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, - boolean isDelayedDeliveryDeliverAtTimeStrict) { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); + boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead) { + this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, + fixedDelayDetectionLookahead); } InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, - long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict) { + long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead) { this.dispatcher = dispatcher; this.timer = timer; this.tickTimeMillis = tickTimeMillis; this.clock = clock; this.isDelayedDeliveryDeliverAtTimeStrict = isDelayedDeliveryDeliverAtTimeStrict; + this.fixedDelayDetectionLookahead = fixedDelayDetectionLookahead; } /** @@ -277,8 +282,9 @@ public void close() { @Override public boolean shouldPauseAllDeliveries() { // Pause deliveries if we know all delays are fixed within the lookahead window - return messagesHaveFixedDelay - && priorityQueue.size() >= DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES + return fixedDelayDetectionLookahead > 0 + && messagesHaveFixedDelay + && priorityQueue.size() >= fixedDelayDetectionLookahead && !hasMessageAvailable(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java index 5c04a6d53b257..7bf0ca87c40c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java @@ -33,18 +33,21 @@ public class InMemoryDelayedDeliveryTrackerFactory implements DelayedDeliveryTra private boolean isDelayedDeliveryDeliverAtTimeStrict; + private long fixedDelayDetectionLookahead; + @Override public void initialize(ServiceConfiguration config) { this.timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-delayed-delivery"), config.getDelayedDeliveryTickTimeMillis(), TimeUnit.MILLISECONDS); this.tickTimeMillis = config.getDelayedDeliveryTickTimeMillis(); this.isDelayedDeliveryDeliverAtTimeStrict = config.isDelayedDeliveryDeliverAtTimeStrict(); + this.fixedDelayDetectionLookahead = config.getDelayedDeliveryFixedDelayDetectionLookahead(); } @Override public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { return new InMemoryDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, - isDelayedDeliveryDeliverAtTimeStrict); + isDelayedDeliveryDeliverAtTimeStrict, fixedDelayDetectionLookahead); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index db2db6cc1dbb0..1ff47a4ca5065 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -74,7 +74,7 @@ public void test() throws Exception { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false); + false, 0); assertFalse(tracker.hasMessageAvailable()); @@ -146,7 +146,7 @@ public void testWithTimer() throws Exception { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false); + false, 0); assertTrue(tasks.isEmpty()); assertTrue(tracker.addMessage(2, 2, 20)); @@ -187,7 +187,7 @@ public void testAddWithinTickTime() { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, - false); + false, 0); clockTime.set(0); @@ -209,7 +209,7 @@ public void testAddMessageWithStrictDelay() { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, - true); + true, 0); clockTime.set(10); @@ -236,7 +236,7 @@ public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithSt // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 1000, clock, true); + 1000, clock, true, 0); // Set clock time, then run tracker to inherit clock time as the last tick time. clockTime.set(10000); @@ -274,7 +274,7 @@ public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStr // a previous tick run. @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 100000, clock, true); + 100000, clock, true, 0); clockTime.set(500000); @@ -299,7 +299,7 @@ public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict() throws // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 500, clock, true); + 500, clock, true, 0); clockTime.set(0); @@ -323,9 +323,11 @@ public void testWithFixedDelays() throws Exception { Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); + final long fixedDelayLookahead = 100; + @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true); + true, fixedDelayLookahead); assertFalse(tracker.hasMessageAvailable()); @@ -339,13 +341,13 @@ public void testWithFixedDelays() throws Exception { assertEquals(tracker.getNumberOfDelayedMessages(), 5); assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + for (int i = 6; i <= fixedDelayLookahead; i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } assertTrue(tracker.shouldPauseAllDeliveries()); - clockTime.set(InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES * 10); + clockTime.set(fixedDelayLookahead * 10); tracker.getScheduledMessages(100); assertFalse(tracker.shouldPauseAllDeliveries()); @@ -367,9 +369,11 @@ public void testWithMixedDelays() throws Exception { Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); + long fixedDelayLookahead = 100; + @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true); + true, fixedDelayLookahead); assertFalse(tracker.hasMessageAvailable()); @@ -381,7 +385,7 @@ public void testWithMixedDelays() throws Exception { assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + for (int i = 6; i <= fixedDelayLookahead; i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } @@ -401,9 +405,11 @@ public void testWithNoDelays() throws Exception { Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); + long fixedDelayLookahead = 100; + @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true); + true, fixedDelayLookahead); assertFalse(tracker.hasMessageAvailable()); @@ -415,7 +421,7 @@ public void testWithNoDelays() throws Exception { assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + for (int i = 6; i <= fixedDelayLookahead; i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } From 717dd1d4cbcb81442de7914cdd86abef8c849813 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Wed, 12 Oct 2022 09:44:19 +0800 Subject: [PATCH 768/823] [fix][broker] Fix system topic schema not compatible bug (#17986) --- .../apache/pulsar/broker/service/AbstractTopic.java | 4 ++++ .../NamespaceEventsSystemTopicServiceTest.java | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 1307001a72068..7c8d3b2422372 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -158,6 +158,10 @@ public AbstractTopic(String topic, BrokerService brokerService) { this.preciseTopicPublishRateLimitingEnable = brokerService.pulsar().getConfiguration().isPreciseTopicPublishRateLimiterEnable(); updatePublishDispatcher(Optional.empty()); + if (isSystemTopic()) { + schemaCompatibilityStrategy = + brokerService.pulsar().getConfig().getSystemTopicSchemaCompatibilityStrategy(); + } } protected boolean isProducersExceeded() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java index d81c89d859f5a..12f9ea26029d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.broker.systopic; +import static org.mockito.Mockito.mock; import com.google.common.collect.Sets; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -90,6 +93,16 @@ public void testSchemaCompatibility() throws Exception { Assert.assertEquals(SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE, topic.getSchemaCompatibilityStrategy()); } + @Test + public void testSystemTopicSchemaCompatibility() throws Exception { + TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory + .createTopicPoliciesSystemTopicClient(NamespaceName.get(NAMESPACE1)); + String topicName = systemTopicClientForNamespace1.getTopicName().toString(); + SystemTopic topic = new SystemTopic(topicName, mock(ManagedLedger.class), pulsar.getBrokerService()); + + Assert.assertEquals(SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE, topic.getSchemaCompatibilityStrategy()); + } + @Test public void testSendAndReceiveNamespaceEvents() throws Exception { TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory From 88d7cf2969a48914ea48159766693478cfb24bf8 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Sun, 16 Oct 2022 23:11:06 +0800 Subject: [PATCH 769/823] [improve][authentication] Improve get the basic authentication config (#16944) Signed-off-by: Zixuan Liu --- conf/broker.conf | 7 ++ conf/proxy.conf | 7 ++ conf/standalone.conf | 6 + .../AuthenticationProviderBasic.java | 57 +++++++--- .../AuthenticationProviderBasicTest.java | 104 ++++++++++++++++++ .../resources/authentication/basic/.htpasswd | 2 + .../org/apache/pulsar/client/api/url/URL.java | 11 ++ 7 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java create mode 100644 pulsar-broker-common/src/test/resources/authentication/basic/.htpasswd diff --git a/conf/broker.conf b/conf/broker.conf index 0fd9bbaba8df2..46413ad824f4d 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -715,6 +715,13 @@ athenzDomainNames= # When this parameter is not empty, unauthenticated users perform as anonymousUserRole anonymousUserRole= +## Configure the datasource of basic authenticate, supports the file and Base64 format. +# file: +# basicAuthConf=/path/my/.htpasswd +# use Base64 to encode the contents of .htpasswd: +# basicAuthConf=YOUR-BASE64-DATA +basicAuthConf= + ### --- Token Authentication Provider --- ### ## Symmetric key diff --git a/conf/proxy.conf b/conf/proxy.conf index 2454b9bf20c00..77ab31b80cf4e 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -238,6 +238,13 @@ httpRequestsLimitEnabled=false httpRequestsMaxPerSecond=100.0 +## Configure the datasource of basic authenticate, supports the file and Base64 format. +# file: +# basicAuthConf=/path/my/.htpasswd +# use Base64 to encode the contents of .htpasswd: +# basicAuthConf=YOUR-BASE64-DATA +basicAuthConf= + ### --- Token Authentication Provider --- ### ## Symmetric key diff --git a/conf/standalone.conf b/conf/standalone.conf index 08742a43cd5da..67c0c6b89fa2d 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -469,6 +469,12 @@ athenzDomainNames= # When this parameter is not empty, unauthenticated users perform as anonymousUserRole anonymousUserRole= +## Configure the datasource of basic authenticate, supports the file and Base64 format. +# file: +# basicAuthConf=/path/my/.htpasswd +# use Base64 to encode the contents of .htpasswd: +# basicAuthConf=YOUR-BASE64-DATA +basicAuthConf= ### --- Token Authentication Provider --- ### diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java index 46c1e3a36de6d..9f6bacf729891 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java @@ -19,28 +19,32 @@ package org.apache.pulsar.broker.authentication; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.naming.AuthenticationException; +import lombok.Cleanup; import org.apache.commons.codec.digest.Crypt; import org.apache.commons.codec.digest.Md5Crypt; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; - -import lombok.Cleanup; import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; - -import javax.naming.AuthenticationException; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; +import org.apache.pulsar.client.api.url.URL; public class AuthenticationProviderBasic implements AuthenticationProvider { private static final String HTTP_HEADER_NAME = "Authorization"; private static final String CONF_SYSTEM_PROPERTY_KEY = "pulsar.auth.basic.conf"; + private static final String CONF_PULSAR_PROPERTY_KEY = "basicAuthConf"; private Map users; @Override @@ -48,16 +52,38 @@ public void close() throws IOException { // noop } + public static byte[] readData(String data) + throws IOException, URISyntaxException, InstantiationException, IllegalAccessException { + if (data.startsWith("data:") || data.startsWith("file:")) { + return IOUtils.toByteArray(URL.createURL(data)); + } else if (Files.exists(Paths.get(data))) { + return Files.readAllBytes(Paths.get(data)); + } else if (org.apache.commons.codec.binary.Base64.isBase64(data)) { + return Base64.getDecoder().decode(data); + } else { + String msg = "Not supported config"; + throw new IllegalArgumentException(msg); + } + } + @Override public void initialize(ServiceConfiguration config) throws IOException { - File confFile = new File(System.getProperty(CONF_SYSTEM_PROPERTY_KEY)); - if (!confFile.exists()) { - throw new IOException("The password auth conf file does not exist"); - } else if (!confFile.isFile()) { - throw new IOException("The path is not a file"); + String data = config.getProperties().getProperty(CONF_PULSAR_PROPERTY_KEY); + if (StringUtils.isEmpty(data)) { + data = System.getProperty(CONF_SYSTEM_PROPERTY_KEY); + } + if (StringUtils.isEmpty(data)) { + throw new IOException("No basic authentication config provided"); + } + + @Cleanup BufferedReader reader = null; + try { + byte[] bytes = readData(data); + reader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(bytes))); + } catch (Exception e) { + throw new IllegalArgumentException(e); } - @Cleanup BufferedReader reader = new BufferedReader(new FileReader(confFile)); users = new HashMap<>(); for (String line : reader.lines().toArray(s -> new String[s])) { List splitLine = Arrays.asList(line.split(":")); @@ -99,7 +125,8 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat throw new AuthenticationException(msg); } } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), exception.getMessage()); + AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), + exception.getMessage()); throw exception; } AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java new file mode 100644 index 0000000000000..ef7dca23e550f --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication; + +import static org.testng.Assert.assertEquals; +import com.google.common.io.Resources; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Base64; +import java.util.Properties; +import javax.naming.AuthenticationException; +import lombok.Cleanup; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.api.AuthData; +import org.testng.annotations.Test; + +public class AuthenticationProviderBasicTest { + private final String basicAuthConf = Resources.getResource("authentication/basic/.htpasswd").getPath(); + private final String basicAuthConfBase64 = Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(basicAuthConf))); + + public AuthenticationProviderBasicTest() throws IOException { + } + + private void testAuthenticate(AuthenticationProviderBasic provider) throws AuthenticationException { + AuthData authData = AuthData.of("superUser2:superpassword".getBytes(StandardCharsets.UTF_8)); + provider.newAuthState(authData, null, null); + } + + @Test + public void testLoadFileFromPulsarProperties() throws Exception { + @Cleanup + AuthenticationProviderBasic provider = new AuthenticationProviderBasic(); + ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + Properties properties = new Properties(); + properties.setProperty("basicAuthConf", basicAuthConf); + serviceConfiguration.setProperties(properties); + provider.initialize(serviceConfiguration); + testAuthenticate(provider); + } + + @Test + public void testLoadBase64FromPulsarProperties() throws Exception { + @Cleanup + AuthenticationProviderBasic provider = new AuthenticationProviderBasic(); + ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + Properties properties = new Properties(); + properties.setProperty("basicAuthConf", basicAuthConfBase64); + serviceConfiguration.setProperties(properties); + provider.initialize(serviceConfiguration); + testAuthenticate(provider); + } + + @Test + public void testLoadFileFromSystemProperties() throws Exception { + @Cleanup + AuthenticationProviderBasic provider = new AuthenticationProviderBasic(); + ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + System.setProperty("pulsar.auth.basic.conf", basicAuthConf); + provider.initialize(serviceConfiguration); + testAuthenticate(provider); + } + + @Test + public void testLoadBase64FromSystemProperties() throws Exception { + @Cleanup + AuthenticationProviderBasic provider = new AuthenticationProviderBasic(); + ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + System.setProperty("pulsar.auth.basic.conf", basicAuthConfBase64); + provider.initialize(serviceConfiguration); + testAuthenticate(provider); + } + + @Test + public void testReadData() throws Exception { + byte[] data = Files.readAllBytes(Paths.get(basicAuthConf)); + String base64Data = Base64.getEncoder().encodeToString(data); + + // base64 format + assertEquals(AuthenticationProviderBasic.readData("data:;base64," + base64Data), data); + assertEquals(AuthenticationProviderBasic.readData(base64Data), data); + + // file format + assertEquals(AuthenticationProviderBasic.readData("file://" + basicAuthConf), data); + assertEquals(AuthenticationProviderBasic.readData(basicAuthConf), data); + } +} diff --git a/pulsar-broker-common/src/test/resources/authentication/basic/.htpasswd b/pulsar-broker-common/src/test/resources/authentication/basic/.htpasswd new file mode 100644 index 0000000000000..b1a099a5f0ecb --- /dev/null +++ b/pulsar-broker-common/src/test/resources/authentication/basic/.htpasswd @@ -0,0 +1,2 @@ +superUser:mQQQIsyvvKRtU +superUser2:$apr1$foobarmq$kuSZlLgOITksCkRgl57ie/ diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/api/url/URL.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/url/URL.java index b2037377d2f41..3286900ecb792 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/api/url/URL.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/api/url/URL.java @@ -42,6 +42,17 @@ public URL(String spec) } } + /** + * Creates java.net.URL with data protocol support. + * + * @param spec the input URL as String + * @return java.net.URL instance + */ + public static final java.net.URL createURL(String spec) + throws MalformedURLException, URISyntaxException, InstantiationException, IllegalAccessException { + return new URL(spec).url; + } + public URLConnection openConnection() throws IOException { return this.url.openConnection(); } From 44d4183a1e69a632f06203ee25d561f1bbd85c6b Mon Sep 17 00:00:00 2001 From: WJL3333 Date: Mon, 17 Oct 2022 12:19:53 +0800 Subject: [PATCH 770/823] [fix][client][txn] Use PulsarClient HashWheelTimer to schedule producer batch trigger task (#18058) * Use PulsarClient HashWheelTimer to schedule producer batch trigger task fix when create a lot of producer (> 10000) in the same jvm which will use eventLoop to do task check * fix batchTimerTask change fix when create a lot of producer (> 10000) in the same jvm which will use eventLoop to do task check Fixes #18055 ### Motivation when broker own a lot of topic ( > 10000 ) and enable txn. the producer create for `TransactionBufferSystemTopicClient` enable batch for current code. which will use broker workerGroup thread to schedule batch check task. which will cause high cpu usage. ### Modifications use client HashWheelTimer for schedule batch check task and when trigger task execute on pulsarClient internalExecutorPool ### Verifying this change - [ ] Make sure that the change passes the CI checks. This change is already covered by existing tests, such as *(please describe tests)*. ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [x] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [ ] The admin CLI options - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` --- .../pulsar/client/impl/ProducerImpl.java | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 9c77d8b4ae6e0..f015c23ce23b5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -56,6 +56,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.BatcherBuilder; @@ -147,7 +148,7 @@ public class ProducerImpl extends ProducerBase implements TimerTask, Conne private final ConnectionHandler connectionHandler; - private ScheduledFuture batchTimerTask; + private final AtomicReference batchTimerTask; private Optional topicEpoch = Optional.empty(); private final List previousExceptions = new CopyOnWriteArrayList(); @@ -238,8 +239,10 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration } this.batchMessageContainer = (BatchMessageContainerBase)containerBuilder.build(); this.batchMessageContainer.setProducer(this); + this.batchTimerTask = new AtomicReference<>(); } else { this.batchMessageContainer = null; + this.batchTimerTask = null; } if (client.getConfiguration().getStatsIntervalSeconds() > 0) { stats = new ProducerStatsRecorderImpl(client, conf, this); @@ -1520,26 +1523,11 @@ public void connectionOpened(final ClientCnx cnx) { if (!producerCreatedFuture.isDone() && isBatchMessagingEnabled()) { // schedule the first batch message task - batchTimerTask = cnx.ctx().executor() - .scheduleWithFixedDelay(catchingAndLoggingThrowables(() -> { - if (log.isTraceEnabled()) { - log.trace( - "[{}] [{}] Batching the messages from the batch container from " - + "timer thread", - topic, - producerName); - } - // semaphore acquired when message was enqueued to container - synchronized (ProducerImpl.this) { - // If it's closing/closed we need to ignore the send batch timer and not - // schedule next timeout. - if (getState() == State.Closing || getState() == State.Closed) { - return; - } - - batchMessageAndSend(); - } - }), 0, conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS); + Timeout task = client.timer() + .newTimeout(this::triggerBatchMessageAndSend, + conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS); + + batchTimerTask.set(task); } resendMessages(cnx, epoch); } @@ -1641,6 +1629,31 @@ public void connectionOpened(final ClientCnx cnx) { }); } + private void triggerBatchMessageAndSend(Timeout timeout) { + client.getInternalExecutorService().execute(catchingAndLoggingThrowables(() -> { + if (log.isTraceEnabled()) { + log.trace("[{}] [{}] Batching the messages from the batch container from " + "timer thread", topic, producerName); + } + // semaphore acquired when message was enqueued to container + synchronized (ProducerImpl.this) { + // If it's closing/closed we need to ignore the send batch timer and not + // schedule next timeout. + if (getState() == State.Closing || getState() == State.Closed) { + return; + } + + batchMessageAndSend(); + } + + Timeout task = client.timer() + .newTimeout(this::triggerBatchMessageAndSend, + conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS); + + batchTimerTask.set(task); + + })); + } + @Override public void connectionFailed(PulsarClientException exception) { boolean nonRetriableError = !PulsarClientException.isRetriableError(exception); @@ -1669,10 +1682,11 @@ private void closeProducerTasks() { sendTimeout = null; } - ScheduledFuture batchTimerTask = this.batchTimerTask; if (batchTimerTask != null) { - batchTimerTask.cancel(false); - this.batchTimerTask = null; + Timeout batchTimerTask = this.batchTimerTask.getAndSet(null); + if (batchTimerTask != null) { + batchTimerTask.cancel(); + } } if (keyGeneratorTask != null && !keyGeneratorTask.isCancelled()) { From 10f6f07f5f2c457fb12beb48a4e3cac6ca214e6a Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 11 Jul 2022 14:42:42 +0800 Subject: [PATCH 771/823] Bump jackson version from 2.13.2 to 2.13.3 (#16508) ### Modifications - Bump jackson version from 2.13.2 to 2.13.3 https://github.com/FasterXML/jackson/wiki/Jackson-Release-2.13.3 - jackson always release bom when components update, so we don't need to list `jackson-databind` version separately ### Documentation Check the box below or label this PR directly. Need to update docs? - [x] `doc-not-needed` (cherry picked from commit 7db1af31ac6a4fc6c7dff40e51c77cb8479218b0) (cherry picked from commit d8cec8ef84767beba357b5f120ca3733710b0190) --- .../server/src/assemble/LICENSE.bin.txt | 16 ++--- pom.xml | 8 +-- pulsar-sql/pom.xml | 66 ------------------- pulsar-sql/presto-distribution/LICENSE | 28 ++++---- pulsar-sql/presto-distribution/pom.xml | 9 --- .../docker-images/java-test-functions/pom.xml | 1 - 6 files changed, 23 insertions(+), 105 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index e4a4e5ca20be7..0903f478571ba 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -312,14 +312,14 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.78.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.7.3.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.13.2.jar - - com.fasterxml.jackson.core-jackson-core-2.13.2.jar - - com.fasterxml.jackson.core-jackson-databind-2.13.2.1.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.2.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.2.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.2.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.2.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.2.jar + - com.fasterxml.jackson.core-jackson-annotations-2.13.3.jar + - com.fasterxml.jackson.core-jackson-core-2.13.3.jar + - com.fasterxml.jackson.core-jackson-databind-2.13.3.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.3.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.3.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.3.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.3.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.3.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar diff --git a/pom.xml b/pom.xml index f70afeb8f0496..334f565db9bc3 100644 --- a/pom.xml +++ b/pom.xml @@ -123,8 +123,7 @@ flexible messaging model and an intuitive client API. 2.18.0 1.69 1.0.2 - 2.13.2 - 2.13.2.1 + 2.13.3 0.9.11 1.6.2 8.37 @@ -798,11 +797,6 @@ flexible messaging model and an intuitive client API. pom import - - com.fasterxml.jackson.core - jackson-databind - ${jackson.databind.version} - org.hdrhistogram diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 480cec5679ee3..14648f39852a9 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -47,72 +47,6 @@ - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.databind.version} - - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-base - ${jackson.version} - - - - com.fasterxml.jackson.datatype - jackson-datatype-joda - ${jackson.version} - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - - com.fasterxml.jackson.module - jackson-module-jsonSchema - ${jackson.version} - - - - com.fasterxml.jackson.datatype - jackson-datatype-guava - ${jackson.version} - - - - com.fasterxml.jackson.datatype - jackson-datatype-jdk8 - ${jackson.version} - - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - ${jackson.version} - - com.squareup.okhttp3 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 18aed92a7b9ca..7fbdc4761de53 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.13.2.jar - - jackson-core-2.13.2.jar - - jackson-databind-2.13.2.1.jar - - jackson-dataformat-smile-2.13.2.jar - - jackson-datatype-guava-2.13.2.jar - - jackson-datatype-jdk8-2.13.2.jar - - jackson-datatype-joda-2.13.2.jar - - jackson-datatype-jsr310-2.13.2.jar - - jackson-dataformat-yaml-2.13.2.jar - - jackson-jaxrs-base-2.13.2.jar - - jackson-jaxrs-json-provider-2.13.2.jar - - jackson-module-jaxb-annotations-2.13.2.jar - - jackson-module-jsonSchema-2.13.2.jar + - jackson-annotations-2.13.3.jar + - jackson-core-2.13.3.jar + - jackson-databind-2.13.3.jar + - jackson-dataformat-smile-2.13.3.jar + - jackson-datatype-guava-2.13.3.jar + - jackson-datatype-jdk8-2.13.3.jar + - jackson-datatype-joda-2.13.3.jar + - jackson-datatype-jsr310-2.13.3.jar + - jackson-dataformat-yaml-2.13.3.jar + - jackson-jaxrs-base-2.13.3.jar + - jackson-jaxrs-json-provider-2.13.3.jar + - jackson-module-jaxb-annotations-2.13.3.jar + - jackson-module-jsonSchema-2.13.3.jar * Guava - guava-30.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -444,7 +444,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.7.jar * Jackson - - jackson-module-parameter-names-2.13.2.jar + - jackson-module-parameter-names-2.13.3.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 87e112599825a..b1b3d4281d409 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -145,55 +145,46 @@ com.fasterxml.jackson.core jackson-core - ${jackson.version} com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} com.fasterxml.jackson.core jackson-annotations - ${jackson.version} com.fasterxml.jackson.datatype jackson-datatype-joda - ${jackson.version} com.fasterxml.jackson.dataformat jackson-dataformat-yaml - ${jackson.version} com.fasterxml.jackson.datatype jackson-datatype-guava - ${jackson.version} com.fasterxml.jackson.datatype jackson-datatype-jdk8 - ${jackson.version} com.fasterxml.jackson.datatype jackson-datatype-jsr310 - ${jackson.version} com.fasterxml.jackson.dataformat jackson-dataformat-smile - ${jackson.version} diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 185102bb6f128..7461043fd7a32 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -49,7 +49,6 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} provided From 5cb46ccbc8c367318b9a8aa0bd2b3d5b3e716159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 17 Oct 2022 09:02:56 +0200 Subject: [PATCH 772/823] [fix][sec] Upgrade JacksonXML to 2.13.4 (#18020) (cherry picked from commit 44215012ae74671674aee8b83c0a4522c309f421) --- .../server/src/assemble/LICENSE.bin.txt | 16 +++++------ pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 28 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 0903f478571ba..d837bf2de1684 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -312,14 +312,14 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.78.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.7.3.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.13.3.jar - - com.fasterxml.jackson.core-jackson-core-2.13.3.jar - - com.fasterxml.jackson.core-jackson-databind-2.13.3.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.3.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.3.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.3.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.3.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.3.jar + - com.fasterxml.jackson.core-jackson-annotations-2.13.4.jar + - com.fasterxml.jackson.core-jackson-core-2.13.4.jar + - com.fasterxml.jackson.core-jackson-databind-2.13.4.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.4.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.4.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.4.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.4.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.4.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar diff --git a/pom.xml b/pom.xml index 334f565db9bc3..3754b2b29d92e 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ flexible messaging model and an intuitive client API. 2.18.0 1.69 1.0.2 - 2.13.3 + 2.13.4 0.9.11 1.6.2 8.37 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 7fbdc4761de53..eea9c04d48c35 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.13.3.jar - - jackson-core-2.13.3.jar - - jackson-databind-2.13.3.jar - - jackson-dataformat-smile-2.13.3.jar - - jackson-datatype-guava-2.13.3.jar - - jackson-datatype-jdk8-2.13.3.jar - - jackson-datatype-joda-2.13.3.jar - - jackson-datatype-jsr310-2.13.3.jar - - jackson-dataformat-yaml-2.13.3.jar - - jackson-jaxrs-base-2.13.3.jar - - jackson-jaxrs-json-provider-2.13.3.jar - - jackson-module-jaxb-annotations-2.13.3.jar - - jackson-module-jsonSchema-2.13.3.jar + - jackson-annotations-2.13.4.jar + - jackson-core-2.13.4.jar + - jackson-databind-2.13.4.jar + - jackson-dataformat-smile-2.13.4.jar + - jackson-datatype-guava-2.13.4.jar + - jackson-datatype-jdk8-2.13.4.jar + - jackson-datatype-joda-2.13.4.jar + - jackson-datatype-jsr310-2.13.4.jar + - jackson-dataformat-yaml-2.13.4.jar + - jackson-jaxrs-base-2.13.4.jar + - jackson-jaxrs-json-provider-2.13.4.jar + - jackson-module-jaxb-annotations-2.13.4.jar + - jackson-module-jsonSchema-2.13.4.jar * Guava - guava-30.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -444,7 +444,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.7.jar * Jackson - - jackson-module-parameter-names-2.13.3.jar + - jackson-module-parameter-names-2.13.4.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access From b53bc51027b192ae42b68308f8b03950e2ec2a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 17 Oct 2022 23:09:50 +0200 Subject: [PATCH 773/823] [fix][sec] File tiered storage: upgrade jettison to get rid of CVE-2022-40149 (#18022) * [fix][sec] File tiered storage: upgrade jettison to get rid of CVE-2022-40149 * fix (cherry picked from commit 9acafc9b9084cbb127ce5669236bdd5dd8e85a0f) --- pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3754b2b29d92e..8f25966ae56de 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ flexible messaging model and an intuitive client API. 2.3.1 1.5.0 3.1 - 4.0.3 + 1.5.1 0.6.1 @@ -798,6 +798,13 @@ flexible messaging model and an intuitive client API. import + + org.codehaus.jettison + jettison + ${jettison.version} + + + org.hdrhistogram HdrHistogram From 84644ec85f05c2ee37275962d60b3157d848f2f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 18 Oct 2022 21:33:34 +0200 Subject: [PATCH 774/823] [fix][sec] Upgrade protobuf to 3.19.6 to get rid of CVE-2022-3171 (#18086) (cherry picked from commit 3da7b9f8bfb55bf4698dc68a8b05b4daa3277faf) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index d837bf2de1684..f5b58f8146c77 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -558,8 +558,8 @@ MIT License Protocol Buffers License * Protocol Buffers - - com.google.protobuf-protobuf-java-3.19.2.jar -- licenses/LICENSE-protobuf.txt - - com.google.protobuf-protobuf-java-util-3.19.2.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-3.19.6.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-util-3.19.6.jar -- licenses/LICENSE-protobuf.txt CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt * Java Annotations API diff --git a/pom.xml b/pom.xml index 8f25966ae56de..ad9d464792fe9 100644 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ flexible messaging model and an intuitive client API. 8.37 1.4.13 0.5.0 - 3.19.2 + 3.19.6 ${protobuf3.version} 1.45.1 1.41.0 From 68bfd13f5c3e7171146386558f3a52dd2d51c12c Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 18 Oct 2022 17:38:17 -0700 Subject: [PATCH 775/823] [broker] Fixed delayed delivery after read operation error (#18098) --- ...PersistentDispatcherMultipleConsumers.java | 17 +++++-- .../persistent/DelayedDeliveryTest.java | 45 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 0b8e7e18338ca..2fdb03cb1941a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -93,8 +93,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul "totalAvailablePermits"); protected volatile int totalAvailablePermits = 0; protected volatile int readBatchSize; - protected final Backoff readFailureBackoff = new Backoff(15, TimeUnit.SECONDS, - 1, TimeUnit.MINUTES, 0, TimeUnit.MILLISECONDS); + protected final Backoff readFailureBackoff; private static final AtomicIntegerFieldUpdater TOTAL_UNACKED_MESSAGES_UPDATER = AtomicIntegerFieldUpdater.newUpdater(PersistentDispatcherMultipleConsumers.class, @@ -129,6 +128,10 @@ public PersistentDispatcherMultipleConsumers(PersistentTopic topic, ManagedCurso : RedeliveryTrackerDisabled.REDELIVERY_TRACKER_DISABLED; this.readBatchSize = serviceConfig.getDispatcherMaxReadBatchSize(); this.initializeDispatchRateLimiterIfNeeded(Optional.empty()); + this.readFailureBackoff = new Backoff( + topic.getBrokerService().pulsar().getConfiguration().getDispatcherReadFailureBackoffInitialTimeInMs(), + TimeUnit.MILLISECONDS, + 1, TimeUnit.MINUTES, 0, TimeUnit.MILLISECONDS); } @Override @@ -659,7 +662,10 @@ public synchronized void readEntriesFailed(ManagedLedgerException exception, Obj topic.getBrokerService().executor().schedule(() -> { synchronized (PersistentDispatcherMultipleConsumers.this) { - if (!havePendingRead) { + // If it's a replay read we need to retry even if there's already + // another scheduled read, otherwise we'd be stuck until + // more messages are published. + if (!havePendingRead || readType == ReadType.Replay) { log.info("[{}] Retrying read operation", name); readMoreEntries(); } else { @@ -869,7 +875,10 @@ protected synchronized Set getMessagesToReplayNow(int maxMessagesT return redeliveryMessages.getMessagesToReplayNow(maxMessagesToRead); } else if (delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().hasMessageAvailable()) { delayedDeliveryTracker.get().resetTickTime(topic.getDelayedDeliveryTickTimeMillis()); - return delayedDeliveryTracker.get().getScheduledMessages(maxMessagesToRead); + Set messagesAvailableNow = + delayedDeliveryTracker.get().getScheduledMessages(maxMessagesToRead); + messagesAvailableNow.forEach(p -> redeliveryMessages.add(p.getLedgerId(), p.getEntryId())); + return messagesAvailableNow; } else { return Collections.emptySet(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java index 480da2f5b94a8..041818850eda4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java @@ -34,6 +34,7 @@ import lombok.Cleanup; +import org.apache.bookkeeper.client.BKException; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -61,6 +62,7 @@ public void setup() throws Exception { conf.setSystemTopicEnabled(true); conf.setTopicLevelPoliciesEnabled(true); conf.setDelayedDeliveryTickTimeMillis(1024); + conf.setDispatcherReadFailureBackoffInitialTimeInMs(1000); super.internalSetup(); super.producerBaseSetup(); } @@ -539,4 +541,47 @@ public void testDelayedDeliveryWithAllConsumersDisconnecting() throws Exception Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 0)); } + + @Test + public void testDispatcherReadFailure() throws Exception { + String topic = BrokerTestUtil.newUniqueName("testDispatcherReadFailure"); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("shared-sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + for (int i = 0; i < 10; i++) { + producer.newMessage() + .value("msg-" + i) + .deliverAfter(5, TimeUnit.SECONDS) + .sendAsync(); + } + + producer.flush(); + + Message msg = consumer.receive(100, TimeUnit.MILLISECONDS); + assertNull(msg); + + // Inject failure in BK read + this.mockBookKeeper.failNow(BKException.Code.ReadException); + + Set receivedMsgs = new TreeSet<>(); + for (int i = 0; i < 10; i++) { + msg = consumer.receive(10, TimeUnit.SECONDS); + receivedMsgs.add(msg.getValue()); + } + + assertEquals(receivedMsgs.size(), 10); + for (int i = 0; i < 10; i++) { + assertTrue(receivedMsgs.contains("msg-" + i)); + } + } } From 36a9d1324764b50dbbc14aaf1be27a732d9f548f Mon Sep 17 00:00:00 2001 From: mattisonchao Date: Wed, 19 Oct 2022 14:41:53 +0800 Subject: [PATCH 776/823] Revert "[fix][sec] File tiered storage: upgrade jettison to get rid of CVE-2022-40149 (#18022)" This reverts commit b53bc51027b192ae42b68308f8b03950e2ec2a6d. --- pom.xml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index ad9d464792fe9..adc6c1b7415c8 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ flexible messaging model and an intuitive client API. 2.3.1 1.5.0 3.1 - 1.5.1 + 4.0.3 0.6.1 @@ -798,13 +798,6 @@ flexible messaging model and an intuitive client API. import - - org.codehaus.jettison - jettison - ${jettison.version} - - - org.hdrhistogram HdrHistogram From 469d44daaa69edf2c60cca71c22206a5c7e7884f Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 19 Oct 2022 18:11:06 +0800 Subject: [PATCH 777/823] [fix][sec] File tiered storage: upgrade jettison to get rid of CVE-2022-40149 (#18105) --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index adc6c1b7415c8..2cf20885df0d2 100644 --- a/pom.xml +++ b/pom.xml @@ -224,6 +224,7 @@ flexible messaging model and an intuitive client API. 1.5.0 3.1 4.0.3 + 1.5.1 0.6.1 @@ -798,6 +799,12 @@ flexible messaging model and an intuitive client API. import + + org.codehaus.jettison + jettison + ${jettison.version} + + org.hdrhistogram HdrHistogram From 3f2ebad22b2f439d376d584100651adf90dbf471 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Wed, 19 Oct 2022 20:02:24 +0800 Subject: [PATCH 778/823] [branch-2.9] Fixed key-shared delivery of messages with interleaved delays. (#18108) --- .../InMemoryDelayedDeliveryTracker.java | 2 +- ...PersistentDispatcherMultipleConsumers.java | 6 ++- ...tStickyKeyDispatcherMultipleConsumers.java | 51 +++++++++++-------- .../persistent/DelayedDeliveryTest.java | 43 ++++++++++++++++ 4 files changed, 78 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 4a8842b15c14e..7e4155c061375 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -266,7 +266,7 @@ public void run(Timeout timeout) throws Exception { synchronized (dispatcher) { lastTickRun = clock.millis(); currentTimeoutTarget = -1; - timeout = null; + this.timeout = null; dispatcher.readMoreEntries(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 2fdb03cb1941a..a36b1eded3dc5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -277,7 +277,11 @@ public synchronized void readMoreEntries() { consumerList.size()); } havePendingRead = true; - minReplayedPosition = getMessagesToReplayNow(1).stream().findFirst().orElse(null); + Set toReplay = getMessagesToReplayNow(1); + minReplayedPosition = toReplay.stream().findFirst().orElse(null); + if (minReplayedPosition != null) { + redeliveryMessages.add(minReplayedPosition.getLedgerId(), minReplayedPosition.getEntryId()); + } cursor.asyncReadEntriesOrWait(messagesToRead, bytesToRead, this, ReadType.Normal, topic.getMaxReadPosition()); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 73eb031d60c87..85b2f5163ce9b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -173,29 +173,36 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { // This may happen when consumer closed. See issue #12885 for details. if (!allowOutOfOrderDelivery) { Set messagesToReplayNow = this.getMessagesToReplayNow(1); - if (messagesToReplayNow != null && !messagesToReplayNow.isEmpty() && this.minReplayedPosition != null) { - PositionImpl relayPosition = messagesToReplayNow.stream().findFirst().get(); - // If relayPosition is a new entry wither smaller position is inserted for redelivery during this async - // read, it is possible that this relayPosition should dispatch to consumer first. So in order to - // preserver order delivery, we need to discard this read result, and try to trigger a replay read, - // that containing "relayPosition", by calling readMoreEntries. - if (relayPosition.compareTo(minReplayedPosition) < 0) { - if (log.isDebugEnabled()) { - log.debug("[{}] Position {} (<{}) is inserted for relay during current {} read, discard this " - + "read and retry with readMoreEntries.", - name, relayPosition, minReplayedPosition, readType); - } - if (readType == ReadType.Normal) { - entries.forEach(entry -> { - long stickyKeyHash = getStickyKeyHash(entry); - addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); - entry.release(); - }); - } else if (readType == ReadType.Replay) { - entries.forEach(Entry::release); + if (messagesToReplayNow != null && !messagesToReplayNow.isEmpty()) { + PositionImpl replayPosition = messagesToReplayNow.stream().findFirst().get(); + // We have received a message potentially from the delayed tracker and, since we're not using it + // right now, it needs to be added to the redelivery tracker or we won't attempt anymore to + // resend it (until we disconnect consumer). + redeliveryMessages.add(replayPosition.getLedgerId(), replayPosition.getEntryId()); + + if (this.minReplayedPosition != null) { + // If relayPosition is a new entry wither smaller position is inserted for redelivery during this + // async read, it is possible that this relayPosition should dispatch to consumer first. So in + // order to preserver order delivery, we need to discard this read result, and try to trigger a + // replay read, that containing "relayPosition", by calling readMoreEntries. + if (replayPosition.compareTo(minReplayedPosition) < 0) { + if (log.isDebugEnabled()) { + log.debug("[{}] Position {} (<{}) is inserted for relay during current {} read, " + + "discard this read and retry with readMoreEntries.", + name, replayPosition, minReplayedPosition, readType); + } + if (readType == ReadType.Normal) { + entries.forEach(entry -> { + long stickyKeyHash = getStickyKeyHash(entry); + addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); + entry.release(); + }); + } else if (readType == ReadType.Replay) { + entries.forEach(Entry::release); + } + readMoreEntries(); + return; } - readMoreEntries(); - return; } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java index 041818850eda4..fc94f4d72c063 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -542,6 +543,48 @@ public void testDelayedDeliveryWithAllConsumersDisconnecting() throws Exception Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 0)); } + @Test + public void testInterleavedMessagesOnKeySharedSubscription() throws Exception { + String topic = BrokerTestUtil.newUniqueName("testInterleavedMessagesOnKeySharedSubscription"); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("key-shared-sub") + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + Random random = new Random(0); + for (int i = 0; i < 10; i++) { + // Publish 1 message without delay and 1 with delay + producer.newMessage() + .value("immediate-msg-" + i) + .sendAsync(); + + int delayMillis = 1000 + random.nextInt(1000); + producer.newMessage() + .value("delayed-msg-" + i) + .deliverAfter(delayMillis, TimeUnit.MILLISECONDS) + .sendAsync(); + Thread.sleep(1000); + } + + producer.flush(); + + Set receivedMessages = new HashSet<>(); + + while (receivedMessages.size() < 20) { + Message msg = consumer.receive(3, TimeUnit.SECONDS); + receivedMessages.add(msg.getValue()); + consumer.acknowledge(msg); + } + } + @Test public void testDispatcherReadFailure() throws Exception { String topic = BrokerTestUtil.newUniqueName("testDispatcherReadFailure"); From dced66cb0559106e1437c20e9deffa9e8823c47f Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 20 Oct 2022 00:05:19 +0800 Subject: [PATCH 779/823] [branch-2.9] Fix license check issue (#18117) --- .github/workflows/ci-license.yaml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 8 +++++++- pulsar-sql/presto-distribution/LICENSE | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml index 3c9756748804c..8e1225c328385 100644 --- a/.github/workflows/ci-license.yaml +++ b/.github/workflows/ci-license.yaml @@ -88,4 +88,4 @@ jobs: - name: license check if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: src/check-binary-license ./distribution/server/target/apache-pulsar-*-bin.tar.gz + run: src/check-binary-license --no-presto ./distribution/server/target/apache-pulsar-*-bin.tar.gz diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index f5b58f8146c77..5296c8002838f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -449,7 +449,7 @@ The Apache Software License, Version 2.0 - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar - * SnakeYaml -- org.yaml-snakeyaml-1.30.jar + * SnakeYaml -- org.yaml-snakeyaml-1.31.jar * RocksDB - org.rocksdb-rocksdbjni-6.16.4.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar diff --git a/pom.xml b/pom.xml index 2cf20885df0d2..3b1aac8cc03c5 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ flexible messaging model and an intuitive client API. 1.21 - + 1.31 4.14.5 3.6.3 1.5.0 @@ -1263,6 +1263,12 @@ flexible messaging model and an intuitive client API. ${roaringbitmap.version} + + org.yaml + snakeyaml + ${snakeyaml.version} + + diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index eea9c04d48c35..39ff98aff3b2c 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -398,7 +398,7 @@ The Apache Software License, Version 2.0 * RocksDB JNI - rocksdbjni-6.16.4.jar * SnakeYAML - - snakeyaml-1.30.jar + - snakeyaml-1.31.jar * Bean Validation API - validation-api-2.0.1.Final.jar * Objectsize From 92da0bb3a6de7da6726eb5eb74c34c59d33cf21f Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Thu, 20 Oct 2022 12:21:45 +0800 Subject: [PATCH 780/823] [fix][cpp][branch-2.9] Fix zlib dowload path. (#18127) --- pulsar-client-cpp/docker/alpine/Dockerfile | 8 ++++---- pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 | 8 ++++---- pulsar-client-cpp/docker/manylinux1/Dockerfile | 8 ++++---- pulsar-client-cpp/docker/manylinux2014/Dockerfile | 8 ++++---- pulsar-client-cpp/docker/manylinux_musl/Dockerfile | 8 ++++---- pulsar-client-cpp/python/build-mac-wheels.sh | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pulsar-client-cpp/docker/alpine/Dockerfile b/pulsar-client-cpp/docker/alpine/Dockerfile index 12d1e2f9f9742..d7c266646c8cd 100644 --- a/pulsar-client-cpp/docker/alpine/Dockerfile +++ b/pulsar-client-cpp/docker/alpine/Dockerfile @@ -43,12 +43,12 @@ RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/ rm -rf /boost_1_72_0.tar.gz /boost_1_72_0 # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ - tar xfz zlib-1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN curl -O -L https://zlib.net/fossils/zlib-1.2.13.tar.gz && \ + tar xfz zlib-1.2.13.tar.gz && \ + cd zlib-1.2.13 && \ CFLAGS="-fPIC -O3" ./configure && \ make -j4 && make install && \ - rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 + rm -rf /zlib-1.2.13.tar.gz /zlib-1.2.13 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ diff --git a/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 b/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 index 862785e901566..6b19326b9ddb5 100644 --- a/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 +++ b/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 @@ -43,12 +43,12 @@ RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/ rm -rf /boost_1_72_0.tar.gz /boost_1_72_0 # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ - tar xfz zlib-1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN curl -O -L https://zlib.net/fossils/zlib-1.2.13.tar.gz && \ + tar xfz zlib-1.2.13.tar.gz && \ + cd zlib-1.2.13 && \ CFLAGS="-fPIC -O3" ./configure && \ make -j4 && make install && \ - rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 + rm -rf /zlib-1.2.13.tar.gz /zlib-1.2.13 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ diff --git a/pulsar-client-cpp/docker/manylinux1/Dockerfile b/pulsar-client-cpp/docker/manylinux1/Dockerfile index 502c6f6fe774d..57c7c06d7ee14 100644 --- a/pulsar-client-cpp/docker/manylinux1/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux1/Dockerfile @@ -46,12 +46,12 @@ RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \ #################################### # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ - tar xvfz zlib-1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN curl -O -L https://zlib.net/fossils/zlib-1.2.13.tar.gz && \ + tar xvfz zlib-1.2.13.tar.gz && \ + cd zlib-1.2.13 && \ CFLAGS="-fPIC -O3" ./configure && \ make && make install && \ - rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 + rm -rf /zlib-1.2.13.tar.gz /zlib-1.2.13 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \ diff --git a/pulsar-client-cpp/docker/manylinux2014/Dockerfile b/pulsar-client-cpp/docker/manylinux2014/Dockerfile index 4d1cfeef92ba3..31cc9a45a60d4 100644 --- a/pulsar-client-cpp/docker/manylinux2014/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux2014/Dockerfile @@ -48,12 +48,12 @@ RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \ #################################### # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ - tar xvfz zlib-1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN curl -O -L https://zlib.net/fossils/zlib-1.2.13.tar.gz && \ + tar xvfz zlib-1.2.13.tar.gz && \ + cd zlib-1.2.13 && \ CFLAGS="-fPIC -O3" ./configure && \ make && make install && \ - rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 + rm -rf /zlib-1.2.13.tar.gz /zlib-1.2.13 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ diff --git a/pulsar-client-cpp/docker/manylinux_musl/Dockerfile b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile index 89dd2d107558c..40417bcd79097 100644 --- a/pulsar-client-cpp/docker/manylinux_musl/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile @@ -48,12 +48,12 @@ RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \ #################################### # ZLib -RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \ - tar xvfz zlib-1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN curl -O -L https://zlib.net/fossils/zlib-1.2.13.tar.gz && \ + tar xvfz zlib-1.2.13.tar.gz && \ + cd zlib-1.2.13 && \ CFLAGS="-fPIC -O3" ./configure && \ make -j8 && make install && \ - rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12 + rm -rf /zlib-1.2.13.tar.gz /zlib-1.2.13 # Compile OpenSSL RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ diff --git a/pulsar-client-cpp/python/build-mac-wheels.sh b/pulsar-client-cpp/python/build-mac-wheels.sh index 0236eb44f79c5..0a8823d1875ea 100755 --- a/pulsar-client-cpp/python/build-mac-wheels.sh +++ b/pulsar-client-cpp/python/build-mac-wheels.sh @@ -34,7 +34,7 @@ PYTHON_VERSIONS=( export MACOSX_DEPLOYMENT_TARGET=11.0 MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*} -ZLIB_VERSION=1.2.12 +ZLIB_VERSION=1.2.13 OPENSSL_VERSION=1_1_1n BOOST_VERSION=1.78.0 PROTOBUF_VERSION=3.20.0 @@ -85,7 +85,7 @@ done ############################################################################### if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then echo "Building ZLib" - curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz + curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz tar xvfz zlib-$ZLIB_VERSION.tar.gz pushd zlib-$ZLIB_VERSION CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX From 6245518056084de4a98a61a30b17791d6ce9bc37 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 24 Oct 2022 11:17:12 -0700 Subject: [PATCH 781/823] [fix][storage] Autorecovery default reppDnsResolverClass to ZkBookieRackAffinityMapping (#15640) * [Autorecovery] Default reppDnsResolverClass to ZkBookieRackAffinityMapping * Improve documentation Fixes: https://github.com/apache/pulsar/issues/18012 ### Motivation The current Bookkeeper configuration defaults to using `org.apache.bookkeeper.net.ScriptBasedMapping` for the `DNSToSwitchMapping` implementation. However, this default configuration does not align with the Broker's default configuration, which is `org.apache.pulsar.zookeeper.ZkBookieRackAffinityMapping`. As such, the default configuration for a Pulsar cluster does not lead to ideal rack awareness when ledgers need to be recovered. The result is that a user can configure a cluster for rack awareness and the brokers will honor that configuration, but the autorecovery process will not because it does not have the correct bookkeeper cluster topology view. I propose we configure bookkeeper to use the broker's `ZkBookieRackAffinityMapping` class. That way, autorecovery will honor the operator's configured rack awareness policies out of the box. ### Modifications * Add default value for `reppDnsResolverClass` to the `conf/bookkeeper.conf` configuration. This change effectively switches the default from `org.apache.bookkeeper.net.ScriptBasedMapping` to `org.apache.pulsar.zookeeper.ZkBookieRackAffinityMapping`. ### Verifying this change I manually verified that the `ZkBookieRackAffinityMapping` works by running some tests in a minikube cluster deployed with the DataStax helm chart. I set up 3 racks, 4 bookies, and a topic with a E=2, Qw=2, and Qa=2. I then verified that the autorecovery pod correctly discovered the racks and then identified when an ensemble was not following the rack placement policy after two bookies were removed. I documented my testing a bit more here: https://github.com/datastax/pulsar-helm-chart/pull/214. ### Does this pull request potentially affect one of the following parts: It changes a default value. The tradeoff is that a user relying on the `ScriptBasedMapping` default might accidentally get switched to using the `ZkBookieRackAffinityMapping` implementation. Given that `ScriptBasedMapping` doesn't work out of the box, and that the broker's default to `ZkBookieRackAffinityMapping`, I think this is an acceptable tradeoff. - [x] `doc` (cherry picked from commit 9812297b4c1353e097c64e6c9981d17c22eff5bf) --- conf/bookkeeper.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf index f26abe086e4f0..7c9b627bbc783 100644 --- a/conf/bookkeeper.conf +++ b/conf/bookkeeper.conf @@ -277,6 +277,17 @@ useV2WireProtocol=true # # ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +# The DNS resolver class used for resolving the network location of each bookie. All bookkeeper clients +# should be configured with the same value to ensure that each bookkeeper client builds +# the same network topology in order to then place ensembles consistently. The setting is used +# when using either RackawareEnsemblePlacementPolicy and RegionAwareEnsemblePlacementPolicy. +# The setting in this file is used to configure the bookkeeper client used in the bookkeeper, which +# is used by the autorecovery process. +# Some available options: +# - org.apache.pulsar.zookeeper.ZkBookieRackAffinityMapping (Pulsar default) +# - org.apache.bookkeeper.net.ScriptBasedMapping (Bookkeeper default) +reppDnsResolverClass=org.apache.pulsar.zookeeper.ZkBookieRackAffinityMapping + ############################################################################# ## Netty server settings ############################################################################# From a280fd2a6893170daeecbc9a1d1d10b0bd86a432 Mon Sep 17 00:00:00 2001 From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com> Date: Mon, 24 Oct 2022 20:24:36 +0800 Subject: [PATCH 782/823] [fix][monitor] fix metrics string encoding (#18138) (cherry picked from commit 031e37c33b11b7513f458404c5e2fdfc4ae61a30) --- .../prometheus/PrometheusMetricsServlet.java | 3 +-- .../utils/SimpleTextOutputStreamTest.java | 23 +++++++++++++++++++ .../common/util/SimpleTextOutputStream.java | 8 +++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java index 145f7a744cd9e..3c9498cd11399 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java @@ -73,12 +73,11 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) HttpServletResponse res = (HttpServletResponse) context.getResponse(); try { res.setStatus(HttpStatus.OK_200); - res.setContentType("text/plain"); + res.setContentType("text/plain;charset=utf-8"); PrometheusMetricsGenerator.generate(pulsar, shouldExportTopicMetrics, shouldExportConsumerMetrics, shouldExportProducerMetrics, splitTopicAndPartitionLabel, res.getOutputStream(), metricsProviders); context.complete(); - } catch (Exception e) { log.error("Failed to generate prometheus stats", e); res.setStatus(HttpStatus.INTERNAL_SERVER_ERROR_500); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleTextOutputStreamTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleTextOutputStreamTest.java index e463c8d45ad4d..1722c4bbc131e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleTextOutputStreamTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleTextOutputStreamTest.java @@ -118,4 +118,27 @@ public String str() { reset(); return s; } + + @Test + public void testWriteString() { + String str = "persistence://test/test/test_¬¬¬¬¬¬¬aabbcc"; + stream.write(str); + assertEquals(str, str()); + } + + + @Test + public void testWriteChar() { + String str = "persistence://test/test/test_¬¬¬¬¬¬¬aabbcc\"\n"; + for (char c : str.toCharArray()) { + stream.write(c); + } + assertEquals(str, str()); + + buf.clear(); + + stream.write('\n').write('"').write('A').write('Z').write('a').write('z').write(' ').write(',').write('{') + .write('}').write('[').write(']').write('¬'); + assertEquals(str(), "\n\"AZaz ,{}[]¬"); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java index 9bfbbff0211b2..ecfbc10d482e1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java @@ -19,6 +19,7 @@ package org.apache.pulsar.common.util; import io.netty.buffer.ByteBuf; +import io.netty.util.CharsetUtil; /** * Format strings and numbers into a ByteBuf without any memory allocation. @@ -44,7 +45,7 @@ public SimpleTextOutputStream write(byte[] a, int offset, int len) { } public SimpleTextOutputStream write(char c) { - buffer.writeByte((byte) c); + write(String.valueOf(c)); return this; } @@ -52,11 +53,8 @@ public SimpleTextOutputStream write(String s) { if (s == null) { return this; } - int len = s.length(); - for (int i = 0; i < len; i++) { - buffer.writeByte((byte) s.charAt(i)); - } + buffer.writeCharSequence(s, CharsetUtil.UTF_8); return this; } From 56125f9dcbcc773f2911e72e575b8b785351d71e Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Mon, 7 Feb 2022 03:56:43 +0800 Subject: [PATCH 783/823] Prevent StackOverFlowException in KEY_SHARED subscription (#14121) (cherry picked from commit f97a76e80a6966c601f73e24fe4cfb2af0114df4) --- ...ntStickyKeyDispatcherMultipleConsumers.java | 2 +- ...ickyKeyDispatcherMultipleConsumersTest.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 85b2f5163ce9b..844b72607dfa6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -328,7 +328,7 @@ protected void sendMessagesToConsumers(ReadType readType, List entries) { } // readMoreEntries should run regardless whether or not stuck is caused by // stuckConsumers for avoid stopping dispatch. - readMoreEntries(); + topic.getBrokerService().executor().execute(() -> readMoreEntries()); } else if (currentThreadKeyNumber == 0) { topic.getBrokerService().executor().schedule(() -> { synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 3cb5bfbdc4456..6d32a83ffc5c2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -21,6 +21,17 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelPromise; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import io.netty.channel.EventLoopGroup; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.EntryImpl; @@ -118,6 +129,13 @@ public void setup() throws Exception { brokerMock = mock(BrokerService.class); doReturn(pulsarMock).when(brokerMock).pulsar(); + EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class); + doReturn(eventLoopGroup).when(brokerMock).executor(); + doAnswer(invocation -> { + ((Runnable)invocation.getArguments()[0]).run(); + return null; + }).when(eventLoopGroup).execute(any(Runnable.class)); + topicMock = mock(PersistentTopic.class); doReturn(brokerMock).when(topicMock).getBrokerService(); doReturn(topicName).when(topicMock).getName(); From 7923d544df946bb9f5ce7ba8a62ae16aa2131138 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 2 Nov 2022 17:29:01 +0800 Subject: [PATCH 784/823] Revert "Clean up C++ client curl configuration (#16064)" This reverts commit 3211f91ae91239fb58bc1c709b66f41dcc9023ed. --- pulsar-client-cpp/lib/auth/AuthOauth2.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.cc b/pulsar-client-cpp/lib/auth/AuthOauth2.cc index 31225b15a71c5..438239a46d69e 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.cc +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.cc @@ -186,6 +186,8 @@ void ClientCredentialFlow::initialize() { curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); char errorBuffer[CURL_ERROR_SIZE]; curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, errorBuffer); @@ -309,6 +311,8 @@ Oauth2TokenResultPtr ClientCredentialFlow::authenticate() { curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(handle, CURLOPT_POSTFIELDS, postData.c_str()); From b25b09d6a21207799c4dac0d3ce58b4df95139c2 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 2 Nov 2022 22:44:07 +0800 Subject: [PATCH 785/823] Revert "Revert "Clean up C++ client curl configuration (#16064)"" This reverts commit 7923d544df946bb9f5ce7ba8a62ae16aa2131138. --- pulsar-client-cpp/lib/auth/AuthOauth2.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-client-cpp/lib/auth/AuthOauth2.cc b/pulsar-client-cpp/lib/auth/AuthOauth2.cc index 438239a46d69e..31225b15a71c5 100644 --- a/pulsar-client-cpp/lib/auth/AuthOauth2.cc +++ b/pulsar-client-cpp/lib/auth/AuthOauth2.cc @@ -186,8 +186,6 @@ void ClientCredentialFlow::initialize() { curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); char errorBuffer[CURL_ERROR_SIZE]; curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, errorBuffer); @@ -311,8 +309,6 @@ Oauth2TokenResultPtr ClientCredentialFlow::authenticate() { curl_easy_setopt(handle, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(handle, CURLOPT_POSTFIELDS, postData.c_str()); From 94de85fc6d84a3210ccb5456017500e044fce090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Preu=C3=9F?= Date: Thu, 3 Nov 2022 02:05:15 +0100 Subject: [PATCH 786/823] [fix][sec] Updating dependencies to get rid of CVEs brought in with kafka and log4j-1.2 libs (#13726) (#18304) CVE-2020-27218, CVE-2021-38153, CVE-2021-44228, CVE-2021-44832, CVE-2021-45046, CVE-2021-45105, CVE-2020-9488, CVE-2019-17571, CVE-2021-4104 (cherry picked from commit 3acdbfe4eac79040be140b423342c28a07dc0327) --- pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 6 ++++++ pulsar-io/hbase/pom.xml | 10 ++++++++++ pulsar-io/hdfs2/pom.xml | 10 ++++++++++ pulsar-io/hdfs3/pom.xml | 8 ++++++++ pulsar-io/kafka-connect-adaptor/pom.xml | 6 ++++++ pulsar-io/kafka/pom.xml | 10 ++++++++++ tiered-storage/file-system/pom.xml | 10 ++++++++++ 8 files changed, 61 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3b1aac8cc03c5..bbccabc820c58 100644 --- a/pom.xml +++ b/pom.xml @@ -141,7 +141,7 @@ flexible messaging model and an intuitive client API. 2.2.0 3.6.0 4.4.20 - 2.7.0 + 2.7.2 5.1.1 1.11.774 1.10.2 diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index e084d43a8a16a..139416214a442 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -66,6 +66,12 @@ org.apache.kafka connect-runtime ${kafka-client.version} + + + org.apache.kafka + kafka-log4j-appender + + diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 60d02110748fc..4d782fb15fe03 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -68,6 +68,16 @@ org.apache.hbase hbase-client ${hbase.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 2505b0b6a56b4..5f038bf810e1a 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -49,6 +49,16 @@ org.apache.hadoop hadoop-client 2.8.5 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + org.apache.commons diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index ba53000940b1d..be94f18de041c 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -54,6 +54,14 @@ jakarta.activation jakarta.activation-api + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 334ba9255c489..6cf8d7a155b3f 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -54,6 +54,12 @@ org.apache.kafka connect-runtime ${kafka-client.version} + + + org.apache.kafka + kafka-log4j-appender + + diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 0fb41f92e7c8b..6d6c61fb55828 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -70,6 +70,16 @@ io.confluent kafka-schema-registry ${kafka.confluent.schemaregistryclient.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index e5f1c880cca83..25c6d1d0f4f1d 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -46,6 +46,16 @@ org.apache.hadoop hadoop-common ${hdfs-offload-version3} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + com.google.protobuf From 5ed247de3a5e0ff67f181b4805fd6140d8174994 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 2 Nov 2022 18:19:10 +0800 Subject: [PATCH 787/823] [fix][client] Fix NPE of MultiTopicsConsumerImpl due to race condition (#18287) (cherry picked from commit 516db5176b5f5ea8a9fe8072b71482abc880faee) --- .../pulsar/client/impl/MultiTopicsConsumerImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index aad4c62273492..71fef6f83f056 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -250,6 +250,10 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { log.debug("[{}] [{}] Receive message from sub consumer:{}", topic, subscription, consumer.getTopic()); } + // Stop to process the remaining message after the consumer is closed. + if (getState() == State.Closed) { + return; + } // Process the message, add to the queue and trigger listener or async callback messageReceived(consumer, message); @@ -533,14 +537,14 @@ public CompletableFuture unsubscribeAsync() { .map(ConsumerImpl::unsubscribeAsync).collect(Collectors.toList()); FutureUtil.waitForAll(futureList) - .thenCompose((r) -> { + .thenComposeAsync((r) -> { setState(State.Closed); cleanupMultiConsumer(); log.info("[{}] [{}] [{}] Unsubscribed Topics Consumer", topic, subscription, consumerName); // fail all pending-receive futures to notify application return failPendingReceive(); - }) + }, internalPinnedExecutor) .whenComplete((r, ex) -> { if (ex == null) { unsubscribeFuture.complete(null); @@ -575,13 +579,13 @@ public CompletableFuture closeAsync() { .map(ConsumerImpl::closeAsync).collect(Collectors.toList()); FutureUtil.waitForAll(futureList) - .thenCompose((r) -> { + .thenComposeAsync((r) -> { setState(State.Closed); cleanupMultiConsumer(); log.info("[{}] [{}] Closed Topics Consumer", topic, subscription); // fail all pending-receive futures to notify application return failPendingReceive(); - }) + }, internalPinnedExecutor) .whenComplete((r, ex) -> { if (ex == null) { closeFuture.complete(null); From 604c8720b6461e8f9209182e9631d1902b5ec850 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Wed, 19 Oct 2022 15:54:10 +0800 Subject: [PATCH 788/823] [improve][schema] Change update schema auth from tenant to produce (#18074) (cherry picked from commit 26b47ffbcdc7f91425ed1ff1cc6cd4d7644a2451) --- .../pulsar/broker/admin/impl/SchemasResourceBase.java | 2 +- .../pulsar/broker/admin/AdminApiSchemaWithAuthTest.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 304b311cbea54..b94b8a2d9623c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -292,7 +292,7 @@ private static void handleGetAllSchemasResponse(AsyncResponse response, List adminWithConsumePermission.schemas().getSchemaInfo(topicName)); assertThrows(PulsarAdminException.class, () -> adminWithoutPermission.schemas().createSchema(topicName, si)); + adminWithProducePermission.schemas().createSchema(topicName, si); adminWithAdminPermission.schemas().createSchema(topicName, si); assertThrows(PulsarAdminException.class, () -> adminWithoutPermission.schemas().getSchemaInfo(topicName)); From 48a99e963517a1a3676167c8ad3ea01aad0d982c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Jan 2022 08:48:02 +0200 Subject: [PATCH 789/823] [Client] Support passing existing executor providers to the client (#12037) In load and performance testing, there's a need to simulate production use cases and production workloads. For this purpose, it would be useful to be able to share the thread pools used by Pulsar client instances in order to be able to run a large amount of Pulsar clients in a single JVM without the overhead of a lot of threads. In the current solution, it's already possible to share the EventLoopGroup and HashedWheelTimer instances. The solution for sharing the thread pools for the external / internal executors was missing. This PR adds support for that. Example usage: ```java // shared thread pool related resources ExecutorProvider internalExecutorProvider = new ExecutorProvider(8, "shared-internal-executor"); ExecutorProvider externalExecutorProvider = new ExecutorProvider(8, "shared-external-executor"); Timer sharedTimer = new HashedWheelTimer(getThreadFactory("shared-pulsar-timer"), 1, TimeUnit.MILLISECONDS); EventLoopGroup sharedEventLoopGroup = new EpollEventLoopGroup(); // example of creating a client which uses the shared thread pools PulsarClientImpl client = PulsarClientImpl.builder().conf(conf) .internalExecutorProvider(internalExecutorProvider) .externalExecutorProvider(externalExecutorProvider) .timer(sharedTimer) .eventLoopGroup(sharedEventLoopGroup) .build(); ``` It seems that this would also improve the performance of the Pulsar Proxy since new thread pools for every client connection. That happens in the Pulsar Proxy currently: https://github.com/apache/pulsar/blob/af63e96d4aaa0ae4c4086583aa4f9b1edd72279b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java#L445-L451 An optimization was added in #9802 for sharing the timer, but it would be useful to also share the internal / external executors. (cherry picked from commit 4591a210a8d924d99811edfdfeb7433452e5eeb4) --- .../pulsar/client/impl/PulsarClientImpl.java | 170 ++++++++++-------- .../client/impl/PulsarClientImplTest.java | 32 ++++ 2 files changed, 130 insertions(+), 72 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 7c318f25a8ea5..3f14558a7ed92 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -19,18 +19,15 @@ package org.apache.pulsar.client.impl; import static org.apache.commons.lang3.StringUtils.isBlank; - import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.Lists; - import io.netty.channel.EventLoopGroup; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; -import java.io.IOException; import java.net.InetSocketAddress; import java.time.Clock; import java.util.ArrayList; @@ -50,7 +47,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import java.util.stream.Collectors; - +import lombok.Builder; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.AuthenticationFactory; @@ -94,13 +91,14 @@ public class PulsarClientImpl implements PulsarClient { private static final Logger log = LoggerFactory.getLogger(PulsarClientImpl.class); protected final ClientConfigurationData conf; + private final boolean createdExecutorProviders; private LookupService lookup; private final ConnectionPool cnxPool; @Getter private final Timer timer; private boolean needStopTimer; private final ExecutorProvider externalExecutorProvider; - private final ExecutorProvider internalExecutorService; + private final ExecutorProvider internalExecutorProvider; private final boolean createdEventLoopGroup; private final boolean createdCnxPool; @@ -115,20 +113,22 @@ public enum State { private final AtomicLong producerIdGenerator = new AtomicLong(); private final AtomicLong consumerIdGenerator = new AtomicLong(); - private final AtomicLong requestIdGenerator - = new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE/2)); + private final AtomicLong requestIdGenerator = + new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2)); protected final EventLoopGroup eventLoopGroup; private final MemoryLimitController memoryLimitController; - private final LoadingCache schemaProviderLoadingCache = CacheBuilder.newBuilder().maximumSize(100000) - .expireAfterAccess(30, TimeUnit.MINUTES).build(new CacheLoader() { + private final LoadingCache schemaProviderLoadingCache = + CacheBuilder.newBuilder().maximumSize(100000) + .expireAfterAccess(30, TimeUnit.MINUTES) + .build(new CacheLoader() { - @Override - public SchemaInfoProvider load(String topicName) { - return newSchemaProvider(topicName); - } - }); + @Override + public SchemaInfoProvider load(String topicName) { + return newSchemaProvider(topicName); + } + }); private final Clock clientClock; @@ -136,48 +136,59 @@ public SchemaInfoProvider load(String topicName) { private TransactionCoordinatorClientImpl tcClient; public PulsarClientImpl(ClientConfigurationData conf) throws PulsarClientException { - this(conf, getEventLoopGroup(conf), true); + this(conf, null, null, null, null, null); } public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { - this(conf, eventLoopGroup, new ConnectionPool(conf, eventLoopGroup), null, false, true); + this(conf, eventLoopGroup, null, null, null, null); } public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool cnxPool) throws PulsarClientException { - this(conf, eventLoopGroup, cnxPool, null, false, false); + this(conf, eventLoopGroup, cnxPool, null, null, null); } - public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool cnxPool, Timer timer) + public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool cnxPool, + Timer timer) throws PulsarClientException { - this(conf, eventLoopGroup, cnxPool, timer, false, false); + this(conf, eventLoopGroup, cnxPool, timer, null, null); } - private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, boolean createdEventLoopGroup) - throws PulsarClientException { - this(conf, eventLoopGroup, new ConnectionPool(conf, eventLoopGroup), null, createdEventLoopGroup, true); - } - - private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool cnxPool, Timer timer, - boolean createdEventLoopGroup, boolean createdCnxPool) throws PulsarClientException { + @Builder(builderClassName = "PulsarClientImplBuilder") + private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool connectionPool, + Timer timer, ExecutorProvider externalExecutorProvider, + ExecutorProvider internalExecutorProvider) throws PulsarClientException { + EventLoopGroup eventLoopGroupReference = null; + ConnectionPool connectionPoolReference = null; try { - this.createdEventLoopGroup = createdEventLoopGroup; - this.createdCnxPool = createdCnxPool; - if (conf == null || isBlank(conf.getServiceUrl()) || eventLoopGroup == null) { + this.createdEventLoopGroup = eventLoopGroup == null; + this.createdCnxPool = connectionPool == null; + if ((externalExecutorProvider == null) != (internalExecutorProvider == null)) { + throw new IllegalArgumentException( + "Both externalExecutorProvider and internalExecutorProvider must be specified or unspecified."); + } + this.createdExecutorProviders = externalExecutorProvider == null; + eventLoopGroupReference = eventLoopGroup != null ? eventLoopGroup : getEventLoopGroup(conf); + this.eventLoopGroup = eventLoopGroupReference; + if (conf == null || isBlank(conf.getServiceUrl()) || this.eventLoopGroup == null) { throw new PulsarClientException.InvalidConfigurationException("Invalid client configuration"); } - this.eventLoopGroup = eventLoopGroup; setAuth(conf); this.conf = conf; clientClock = conf.getClock(); conf.getAuthentication().start(); - this.cnxPool = cnxPool; - externalExecutorProvider = new ExecutorProvider(conf.getNumListenerThreads(), "pulsar-external-listener"); - internalExecutorService = new ExecutorProvider(conf.getNumIoThreads(), "pulsar-client-internal"); + connectionPoolReference = + connectionPool != null ? connectionPool : new ConnectionPool(conf, this.eventLoopGroup); + this.cnxPool = connectionPoolReference; + this.externalExecutorProvider = externalExecutorProvider != null ? externalExecutorProvider : + new ExecutorProvider(conf.getNumListenerThreads(), "pulsar-external-listener"); + this.internalExecutorProvider = internalExecutorProvider != null ? internalExecutorProvider : + new ExecutorProvider(conf.getNumIoThreads(), "pulsar-client-internal"); if (conf.getServiceUrl().startsWith("http")) { - lookup = new HttpLookupService(conf, eventLoopGroup); + lookup = new HttpLookupService(conf, this.eventLoopGroup); } else { - lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(), conf.isUseTls(), externalExecutorProvider.getExecutor()); + lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(), + conf.isUseTls(), this.externalExecutorProvider.getExecutor()); } if (timer == null) { this.timer = new HashedWheelTimer(getThreadFactory("pulsar-timer"), 1, TimeUnit.MILLISECONDS); @@ -200,8 +211,8 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG state.set(State.Open); } catch (Throwable t) { shutdown(); - shutdownEventLoopGroup(eventLoopGroup); - closeCnxPool(cnxPool); + shutdownEventLoopGroup(eventLoopGroupReference); + closeCnxPool(connectionPoolReference); throw t; } } @@ -279,11 +290,13 @@ public CompletableFuture> createProducerAsync(ProducerConfigurat if (schema instanceof AutoConsumeSchema) { return FutureUtil.failedFuture( - new PulsarClientException.InvalidConfigurationException("AutoConsumeSchema is only used by consumers to detect schemas automatically")); + new PulsarClientException.InvalidConfigurationException( + "AutoConsumeSchema is only used by consumers to detect schemas automatically")); } if (state.get() != State.Open) { - return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Client already closed : state = " + state.get())); + return FutureUtil.failedFuture( + new PulsarClientException.AlreadyClosedException("Client already closed : state = " + state.get())); } String topic = conf.getTopicName(); @@ -360,7 +373,8 @@ protected PartitionedProducerImpl newPartitionedProducerImpl(String topic ProducerConfigurationData conf, Schema schema, ProducerInterceptors interceptors, - CompletableFuture> producerCreatedFuture, + CompletableFuture> + producerCreatedFuture, PartitionedTopicMetadata metadata) { return new PartitionedProducerImpl<>(PulsarClientImpl.this, topic, conf, metadata.partitions, producerCreatedFuture, schema, interceptors); @@ -394,7 +408,8 @@ public CompletableFuture> subscribeAsync(ConsumerConfigurationD return subscribeAsync(conf, Schema.BYTES, null); } - public CompletableFuture> subscribeAsync(ConsumerConfigurationData conf, Schema schema, ConsumerInterceptors interceptors) { + public CompletableFuture> subscribeAsync(ConsumerConfigurationData conf, Schema schema, + ConsumerInterceptors interceptors) { if (state.get() != State.Open) { return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Client already closed")); } @@ -406,7 +421,8 @@ public CompletableFuture> subscribeAsync(ConsumerConfigurationDa for (String topic : conf.getTopicNames()) { if (!TopicName.isValid(topic)) { - return FutureUtil.failedFuture(new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'")); + return FutureUtil.failedFuture( + new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'")); } } @@ -442,12 +458,16 @@ public CompletableFuture> subscribeAsync(ConsumerConfigurationDa } } - private CompletableFuture> singleTopicSubscribeAsync(ConsumerConfigurationData conf, Schema schema, ConsumerInterceptors interceptors) { + private CompletableFuture> singleTopicSubscribeAsync(ConsumerConfigurationData conf, + Schema schema, + ConsumerInterceptors interceptors) { return preProcessSchemaBeforeSubscribe(this, schema, conf.getSingleTopic()) - .thenCompose(schemaClone -> doSingleTopicSubscribeAsync(conf, schemaClone, interceptors)); + .thenCompose(schemaClone -> doSingleTopicSubscribeAsync(conf, schemaClone, interceptors)); } - private CompletableFuture> doSingleTopicSubscribeAsync(ConsumerConfigurationData conf, Schema schema, ConsumerInterceptors interceptors) { + private CompletableFuture> doSingleTopicSubscribeAsync(ConsumerConfigurationData conf, + Schema schema, + ConsumerInterceptors interceptors) { CompletableFuture> consumerSubscribedFuture = new CompletableFuture<>(); String topic = conf.getSingleTopic(); @@ -477,7 +497,9 @@ private CompletableFuture> doSingleTopicSubscribeAsync(ConsumerC return consumerSubscribedFuture; } - private CompletableFuture> multiTopicSubscribeAsync(ConsumerConfigurationData conf, Schema schema, ConsumerInterceptors interceptors) { + private CompletableFuture> multiTopicSubscribeAsync(ConsumerConfigurationData conf, + Schema schema, + ConsumerInterceptors interceptors) { CompletableFuture> consumerSubscribedFuture = new CompletableFuture<>(); ConsumerBase consumer = new MultiTopicsConsumerImpl<>(PulsarClientImpl.this, conf, @@ -504,9 +526,9 @@ private CompletableFuture> patternTopicSubscribeAsync(ConsumerCo lookup.getTopicsUnderNamespace(namespaceName, subscriptionMode) .thenAccept(topics -> { if (log.isDebugEnabled()) { - log.debug("Get topics under namespace {}, topics.size: {}", namespaceName.toString(), topics.size()); + log.debug("Get topics under namespace {}, topics.size: {}", namespaceName, topics.size()); topics.forEach(topicName -> - log.debug("Get topics under namespace {}, topic: {}", namespaceName.toString(), topicName)); + log.debug("Get topics under namespace {}, topic: {}", namespaceName, topicName)); } List topicsList = topicsPatternFilter(topics, conf.getTopicsPattern()); @@ -600,7 +622,8 @@ protected CompletableFuture> createSingleTopicReaderAsync( if (log.isDebugEnabled()) { log.debug("[{}] Received topic metadata. partitions: {}", topic, metadata.partitions); } - if (metadata.partitions > 0 && MultiTopicsConsumerImpl.isIllegalMultiTopicsMessageId(conf.getStartMessageId())) { + if (metadata.partitions > 0 && + MultiTopicsConsumerImpl.isIllegalMultiTopicsMessageId(conf.getStartMessageId())) { readerFuture.completeExceptionally( new PulsarClientException("The partitioned topic startMessageId is illegal")); return; @@ -613,7 +636,8 @@ protected CompletableFuture> createSingleTopicReaderAsync( conf, externalExecutorProvider, consumerSubscribedFuture, schema); consumer = ((MultiTopicsReaderImpl) reader).getMultiTopicsConsumer(); } else { - reader = new ReaderImpl<>(PulsarClientImpl.this, conf, externalExecutorProvider, consumerSubscribedFuture, schema); + reader = new ReaderImpl<>(PulsarClientImpl.this, conf, externalExecutorProvider, + consumerSubscribedFuture, schema); consumer = ((ReaderImpl) reader).getConsumer(); } @@ -644,7 +668,8 @@ public CompletableFuture> getSchema(String topic) { topicName = TopicName.get(topic); } catch (Throwable t) { return FutureUtil - .failedFuture(new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'")); + .failedFuture( + new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'")); } return lookup.getSchema(topicName); @@ -787,27 +812,29 @@ private void shutdownEventLoopGroup(EventLoopGroup eventLoopGroup) throws Pulsar } private void shutdownExecutors() throws PulsarClientException { - PulsarClientException pulsarClientException = null; + if (createdExecutorProviders) { + PulsarClientException pulsarClientException = null; - if (externalExecutorProvider != null && !externalExecutorProvider.isShutdown()) { - try { - externalExecutorProvider.shutdownNow(); - } catch (Throwable t) { - log.warn("Failed to shutdown externalExecutorProvider", t); - pulsarClientException = PulsarClientException.unwrap(t); + if (externalExecutorProvider != null && !externalExecutorProvider.isShutdown()) { + try { + externalExecutorProvider.shutdownNow(); + } catch (Throwable t) { + log.warn("Failed to shutdown externalExecutorProvider", t); + pulsarClientException = PulsarClientException.unwrap(t); + } } - } - if (internalExecutorService != null && !internalExecutorService.isShutdown()) { - try { - internalExecutorService.shutdownNow(); - } catch (Throwable t) { - log.warn("Failed to shutdown internalExecutorService", t); - pulsarClientException = PulsarClientException.unwrap(t); + if (internalExecutorProvider != null && !internalExecutorProvider.isShutdown()) { + try { + internalExecutorProvider.shutdownNow(); + } catch (Throwable t) { + log.warn("Failed to shutdown internalExecutorService", t); + pulsarClientException = PulsarClientException.unwrap(t); + } } - } - if (pulsarClientException != null) { - throw pulsarClientException; + if (pulsarClientException != null) { + throw pulsarClientException; + } } } @@ -923,8 +950,8 @@ private void getPartitionedTopicMetadata(TopicName topicName, previousExceptions.add(e); ((ScheduledExecutorService) externalExecutorProvider.getExecutor()).schedule(() -> { - log.warn("[topic: {}] Could not get connection while getPartitionedTopicMetadata -- Will try again in {} ms", - topicName, nextDelay); + log.warn("[topic: {}] Could not get connection while getPartitionedTopicMetadata -- " + + "Will try again in {} ms", topicName, nextDelay); remainingTime.addAndGet(-nextDelay); getPartitionedTopicMetadata(topicName, backoff, remainingTime, future, previousExceptions); }, nextDelay, TimeUnit.MILLISECONDS); @@ -1042,7 +1069,7 @@ protected CompletableFuture> preProcessSchemaBeforeSubscribe(Pulsa } public ExecutorService getInternalExecutorService() { - return internalExecutorService.getExecutor(); + return internalExecutorProvider.getExecutor(); } // // Transaction related API @@ -1057,5 +1084,4 @@ public TransactionBuilder newTransaction() throws PulsarClientException { } return new TransactionBuilderImpl(this, tcClient); } - } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java index 3f1e667517ee9..386294e24b784 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java @@ -48,10 +48,12 @@ import java.util.concurrent.ThreadFactory; import java.util.regex.Pattern; +import lombok.Cleanup; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -217,4 +219,34 @@ public void testResourceCleanup() throws PulsarClientException { assertFalse(eventLoopGroup.isShutdown()); } } + + @Test + public void testInitializingWithExecutorProviders() throws PulsarClientException { + ClientConfigurationData conf = clientImpl.conf; + @Cleanup("shutdownNow") + ExecutorProvider executorProvider = new ExecutorProvider(2, "shared-executor"); + @Cleanup + PulsarClientImpl client2 = PulsarClientImpl.builder().conf(conf) + .internalExecutorProvider(executorProvider) + .externalExecutorProvider(executorProvider) + .build(); + @Cleanup + PulsarClientImpl client3 = PulsarClientImpl.builder().conf(conf) + .internalExecutorProvider(executorProvider) + .externalExecutorProvider(executorProvider) + .build(); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "Both externalExecutorProvider and internalExecutorProvider must be " + + "specified or unspecified.") + public void testBothExecutorProvidersMustBeSpecified() throws PulsarClientException { + ClientConfigurationData conf = clientImpl.conf; + @Cleanup("shutdownNow") + ExecutorProvider executorProvider = new ExecutorProvider(2, "shared-executor"); + @Cleanup + PulsarClientImpl client2 = PulsarClientImpl.builder().conf(conf) + .internalExecutorProvider(executorProvider) + .build(); + } } From d9211c97776fe61baddb4d9b3c4049dfd37b48c7 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 28 Jun 2022 11:29:32 +0800 Subject: [PATCH 790/823] [improve][java-client] Replace ScheduledExecutor to improve performance of message consumption (#16236) The Scheduled Executor doesn't work very efficiently because each task will add to a DelayedQueue(A priority queue) first even if using the `.execute()` method without any schedule delay. image image Profile result: [perf_consumer_0.html.txt](https://github.com/apache/pulsar/files/8989093/perf_consumer_0.html.txt) Running a performance test for single topic max message read rate test: ``` bin/pulsar-perf consume test -q 1000000 -p 100000000 bin/pulsar-perf produce test -r 1000000 -s 1 -mk random -o 10000 -threads 2 ``` Without this PR (2.10.1): ``` Profiling started 2022-06-27T13:44:01,183+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 23919664 msg --- 265702.851 msg/s --- 2.027 Mbit/s --- Latency: mean: 49430.572 ms - med: 49406 - 95pct: 52853 - 99pct: 53024 - 99.9pct: 53053 - 99.99pct: 53056 - Max: 53057 2022-06-27T13:44:11,196+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 26690802 msg --- 276759.125 msg/s --- 2.112 Mbit/s --- Latency: mean: 56106.186 ms - med: 56000 - 95pct: 59289 - 99pct: 59985 - 99.9pct: 60037 - 99.99pct: 60042 - Max: 60042 2022-06-27T13:44:21,216+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 28788693 msg --- 209467.861 msg/s --- 1.598 Mbit/s --- Latency: mean: 63523.143 ms - med: 63580 - 95pct: 67202 - 99pct: 67523 - 99.9pct: 67547 - 99.99pct: 67548 - Max: 67548 2022-06-27T13:44:31,233+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 31255365 msg --- 246190.932 msg/s --- 1.878 Mbit/s --- Latency: mean: 71152.370 ms - med: 71058 - 95pct: 74555 - 99pct: 74806 - 99.9pct: 74842 - 99.99pct: 74847 - Max: 74847 2022-06-27T13:44:41,247+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 33606630 msg --- 234769.313 msg/s --- 1.791 Mbit/s --- Latency: mean: 78636.478 ms - med: 78724 - 95pct: 81694 - 99pct: 82090 - 99.9pct: 82279 - 99.99pct: 82285 - Max: 82286 ``` With this PR: ``` Profiling started 2022-06-27T13:56:20,426+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 431272207 msg --- 1079360.516 msg/s --- 8.235 Mbit/s --- Latency: mean: 272.645 ms - med: 334 - 95pct: 470 - 99pct: 510 - 99.9pct: 522 - 99.99pct: 523 - Max: 524 2022-06-27T13:56:30,438+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 441292346 msg --- 1000645.852 msg/s --- 7.634 Mbit/s --- Latency: mean: 15.512 ms - med: 14 - 95pct: 29 - 99pct: 39 - 99.9pct: 54 - 99.99pct: 55 - Max: 55 2022-06-27T13:56:40,450+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 451303308 msg --- 999973.040 msg/s --- 7.629 Mbit/s --- Latency: mean: 18.265 ms - med: 14 - 95pct: 53 - 99pct: 98 - 99.9pct: 174 - 99.99pct: 176 - Max: 177 2022-06-27T13:56:50,462+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 461308082 msg --- 999309.458 msg/s --- 7.624 Mbit/s --- Latency: mean: 14.728 ms - med: 14 - 95pct: 18 - 99pct: 41 - 99.9pct: 50 - 99.99pct: 51 - Max: 52 2022-06-27T13:57:00,475+0800 [main] INFO org.apache.pulsar.testclient.PerformanceConsumer - Throughput received: 471327606 msg --- 1000738.584 msg/s --- 7.635 Mbit/s --- Latency: mean: 21.291 ms - med: 16 - 95pct: 52 - 99pct: 61 - 99.9pct: 65 - 99.99pct: 66 - Max: 66 ``` Profile result with this PR: [perf_consumer_1.html.txt](https://github.com/apache/pulsar/files/8989095/perf_consumer_1.html.txt) - Change internal executor and external executor to normal executor service - Added a new ScheduledExecutorProvider to handle the scheduled tasks. (cherry picked from commit 96237a9615fefa2bed247b416bf1a12d8bc4b201) --- .../pendingack/PendingAckStore.java | 4 +- .../impl/InMemoryPendingAckStore.java | 4 +- .../pendingack/impl/MLPendingAckStore.java | 4 +- .../pendingack/impl/PendingAckHandleImpl.java | 4 +- .../PersistentSubscriptionTest.java | 4 +- .../client/api/MultiTopicsConsumerTest.java | 2 +- .../pulsar/client/impl/ConsumerBase.java | 9 ++-- .../pulsar/client/impl/ConsumerImpl.java | 13 +++-- .../client/impl/MultiTopicsConsumerImpl.java | 47 ++++++++++--------- .../pulsar/client/impl/PulsarClientImpl.java | 12 ++++- .../pulsar/client/util/ExecutorProvider.java | 10 ++-- .../util/ScheduledExecutorProvider.java | 36 ++++++++++++++ 12 files changed, 99 insertions(+), 50 deletions(-) create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/util/ScheduledExecutorProvider.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java index 3da676eb827d0..2f85d2430dbbd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; @@ -38,7 +38,7 @@ public interface PendingAckStore { * @param pendingAckHandle the handle of pending ack * @param executorService the replay executor service */ - void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService); + void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService); /** * Close the transaction pending ack store. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java index d882c80c47863..44c9fbe039b0b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; @@ -33,7 +33,7 @@ public class InMemoryPendingAckStore implements PendingAckStore { @Override - public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService scheduledExecutorService) { + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService scheduledExecutorService) { pendingAckHandle.changeToReadyState(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index e6d16fb7eae90..af4e664b1e33d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -26,7 +26,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -110,7 +110,7 @@ public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, } @Override - public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService transactionReplayExecutor) { + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService transactionReplayExecutor) { transactionReplayExecutor .execute(new PendingAckReplay(new MLPendingAckReplyCallBack(pendingAckHandle))); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 5b808f1dedbca..41ef25b3e4d65 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -33,7 +33,6 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -156,8 +155,7 @@ private void initPendingAckStore() { this.pendingAckStoreFuture = pendingAckStoreProvider.newPendingAckStore(persistentSubscription); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { - pendingAckStore.replayAsync(this, - (ScheduledExecutorService) internalPinnedExecutor); + pendingAckStore.replayAsync(this, internalPinnedExecutor); }).exceptionally(e -> { acceptQueue.clear(); changeToErrorState(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index b9304cb5fb8ea..946f90a1ddd3c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -40,7 +40,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -127,7 +127,7 @@ public void setup() throws Exception { public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { return CompletableFuture.completedFuture(new PendingAckStore() { @Override - public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService) { + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) { try { Field field = PendingAckHandleState.class.getDeclaredField("state"); field.setAccessible(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java index d8c8bd657f8cc..29ecb39853a2e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java @@ -72,7 +72,7 @@ protected PulsarClient createNewPulsarClient(ClientBuilder clientBuilder) throws return new PulsarClientImpl(conf) { { ScheduledExecutorService internalExecutorService = - (ScheduledExecutorService) super.getInternalExecutorService(); + (ScheduledExecutorService) super.getScheduledExecutorProvider().getExecutor(); internalExecutorServiceDelegate = mock(ScheduledExecutorService.class, // a spy isn't used since that doesn't work for private classes, instead // the mock delegatesTo an existing instance. A delegate is sufficient for verifying diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 71fb2d6275610..c53d49ad4bdb5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -32,7 +32,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.locks.Lock; @@ -69,8 +68,8 @@ public abstract class ConsumerBase extends HandlerState implements Consumer listener; protected final ConsumerEventListener consumerEventListener; protected final ExecutorProvider executorProvider; - protected final ScheduledExecutorService externalPinnedExecutor; - protected final ScheduledExecutorService internalPinnedExecutor; + protected final ExecutorService externalPinnedExecutor; + protected final ExecutorService internalPinnedExecutor; final BlockingQueue> incomingMessages; protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap; protected final ConcurrentLinkedQueue>> pendingReceives; @@ -102,8 +101,8 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat this.unAckedChunkedMessageIdSequenceMap = ConcurrentOpenHashMap.newBuilder().build(); this.executorProvider = executorProvider; - this.externalPinnedExecutor = (ScheduledExecutorService) executorProvider.getExecutor(); - this.internalPinnedExecutor = (ScheduledExecutorService) client.getInternalExecutorService(); + this.externalPinnedExecutor = executorProvider.getExecutor(); + this.internalPinnedExecutor = client.getInternalExecutorService(); this.pendingReceives = Queues.newConcurrentLinkedQueue(); this.pendingBatchReceives = Queues.newConcurrentLinkedQueue(); this.schema = schema; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 3bcf95e7813fd..1a185d4c17d5c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -1267,10 +1268,12 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // Lazy task scheduling to expire incomplete chunk message if (!expireChunkMessageTaskScheduled && expireTimeOfIncompleteChunkedMessageMillis > 0) { - internalPinnedExecutor - .scheduleAtFixedRate(catchingAndLoggingThrowables(this::removeExpireIncompleteChunkedMessages), - expireTimeOfIncompleteChunkedMessageMillis, expireTimeOfIncompleteChunkedMessageMillis, - TimeUnit.MILLISECONDS); + ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).scheduleAtFixedRate( + () -> internalPinnedExecutor + .execute(catchingAndLoggingThrowables(this::removeExpireIncompleteChunkedMessages)), + expireTimeOfIncompleteChunkedMessageMillis, expireTimeOfIncompleteChunkedMessageMillis, + TimeUnit.MILLISECONDS + ); expireChunkMessageTaskScheduled = true; } @@ -2236,7 +2239,7 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, return; } - internalPinnedExecutor.schedule(() -> { + ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).schedule(() -> { log.warn("[{}] [{}] Could not get connection while getLastMessageId -- Will try again in {} ms", topic, getHandlerName(), nextDelay); remainingTime.addAndGet(-nextDelay); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 71fef6f83f056..2c3d8cb03a639 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -24,28 +24,6 @@ import com.google.common.collect.Lists; import io.netty.util.Timeout; import io.netty.util.TimerTask; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.ConsumerStats; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.Messages; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.PulsarClientException.NotSupportedException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; -import org.apache.pulsar.client.impl.transaction.TransactionImpl; -import org.apache.pulsar.client.util.ConsumerName; -import org.apache.pulsar.client.util.ExecutorProvider; -import org.apache.pulsar.common.api.proto.CommandAck.AckType; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; -import org.apache.pulsar.common.util.CompletableFutureCancellationHandler; -import org.apache.pulsar.common.util.FutureUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -61,6 +39,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -69,6 +48,27 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerStats; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Messages; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.PulsarClientException.NotSupportedException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.transaction.TransactionImpl; +import org.apache.pulsar.client.util.ConsumerName; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.util.CompletableFutureCancellationHandler; +import org.apache.pulsar.common.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -280,7 +280,8 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { return null; } log.error("Receive operation failed on consumer {} - Retrying later", consumer, ex); - internalPinnedExecutor.schedule(() -> receiveMessageFromConsumer(consumer), 10, TimeUnit.SECONDS); + ((ScheduledExecutorService) client.getScheduledExecutorProvider()) + .schedule(() -> receiveMessageFromConsumer(consumer), 10, TimeUnit.SECONDS); return null; }); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 3f14558a7ed92..9a4bada327899 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -75,6 +75,7 @@ import org.apache.pulsar.client.impl.transaction.TransactionBuilderImpl; import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl; import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.client.util.ScheduledExecutorProvider; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; @@ -99,6 +100,8 @@ public class PulsarClientImpl implements PulsarClient { private boolean needStopTimer; private final ExecutorProvider externalExecutorProvider; private final ExecutorProvider internalExecutorProvider; + + private final ScheduledExecutorProvider scheduledExecutorProvider; private final boolean createdEventLoopGroup; private final boolean createdCnxPool; @@ -184,6 +187,8 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG new ExecutorProvider(conf.getNumListenerThreads(), "pulsar-external-listener"); this.internalExecutorProvider = internalExecutorProvider != null ? internalExecutorProvider : new ExecutorProvider(conf.getNumIoThreads(), "pulsar-client-internal"); + this.scheduledExecutorProvider = new ScheduledExecutorProvider(conf.getNumIoThreads(), + "pulsar-client-scheduled"); if (conf.getServiceUrl().startsWith("http")) { lookup = new HttpLookupService(conf, this.eventLoopGroup); } else { @@ -949,7 +954,7 @@ private void getPartitionedTopicMetadata(TopicName topicName, } previousExceptions.add(e); - ((ScheduledExecutorService) externalExecutorProvider.getExecutor()).schedule(() -> { + ((ScheduledExecutorService) scheduledExecutorProvider.getExecutor()).schedule(() -> { log.warn("[topic: {}] Could not get connection while getPartitionedTopicMetadata -- " + "Will try again in {} ms", topicName, nextDelay); remainingTime.addAndGet(-nextDelay); @@ -1071,6 +1076,11 @@ protected CompletableFuture> preProcessSchemaBeforeSubscribe(Pulsa public ExecutorService getInternalExecutorService() { return internalExecutorProvider.getExecutor(); } + + public ScheduledExecutorProvider getScheduledExecutorProvider() { + return scheduledExecutorProvider; + } + // // Transaction related API // diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java index 1318d5665ae83..db11358057f78 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -43,7 +42,7 @@ public class ExecutorProvider { private final String poolName; private volatile boolean isShutdown; - private static class ExtendedThreadFactory extends DefaultThreadFactory { + protected static class ExtendedThreadFactory extends DefaultThreadFactory { @Getter private Thread thread; @@ -58,7 +57,6 @@ public Thread newThread(Runnable r) { } } - public ExecutorProvider(int numThreads, String poolName) { checkArgument(numThreads > 0); this.numThreads = numThreads; @@ -67,13 +65,17 @@ public ExecutorProvider(int numThreads, String poolName) { for (int i = 0; i < numThreads; i++) { ExtendedThreadFactory threadFactory = new ExtendedThreadFactory( poolName, Thread.currentThread().isDaemon()); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(threadFactory); + ExecutorService executor = createExecutor(threadFactory); executors.add(Pair.of(executor, threadFactory)); } isShutdown = false; this.poolName = poolName; } + protected ExecutorService createExecutor(ExtendedThreadFactory threadFactory) { + return Executors.newSingleThreadExecutor(threadFactory); + } + public ExecutorService getExecutor() { return executors.get((currentThread.getAndIncrement() & Integer.MAX_VALUE) % numThreads).getKey(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ScheduledExecutorProvider.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ScheduledExecutorProvider.java new file mode 100644 index 0000000000000..887ae3bb7fff4 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ScheduledExecutorProvider.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.util; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ScheduledExecutorProvider extends ExecutorProvider { + + public ScheduledExecutorProvider(int numThreads, String poolName) { + super(numThreads, poolName); + } + + @Override + protected ExecutorService createExecutor(ExtendedThreadFactory threadFactory) { + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } +} From 8fa9ab160d7faff94e7c9591474f491f6e9ceb26 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 23 Mar 2022 10:23:31 +0800 Subject: [PATCH 791/823] Optimize conusmer pause (#14566) (cherry picked from commit a32edc7aac67804195c69736277c826bd16c5268) --- .../pulsar/client/impl/MultiTopicsConsumerImpl.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 2c3d8cb03a639..057eda176bd50 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -978,6 +978,7 @@ private void doSubscribeTopicPartitions(Schema schema, partitionIndex -> { String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString(); CompletableFuture> subFuture = new CompletableFuture<>(); + configurationData.setStartPaused(paused); ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, partitionName, configurationData, client.externalExecutorProvider(), partitionIndex, true, listener != null, subFuture, @@ -986,6 +987,8 @@ private void doSubscribeTopicPartitions(Schema schema, synchronized (pauseMutex) { if (paused) { newConsumer.pause(); + } else { + newConsumer.resume(); } consumers.putIfAbsent(newConsumer.getTopic(), newConsumer); } @@ -1005,6 +1008,7 @@ private void doSubscribeTopicPartitions(Schema schema, subscribeResult.completeExceptionally(new PulsarClientException(errorMessage)); return existingValue; } else { + internalConfig.setStartPaused(paused); ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, topicName, internalConfig, client.externalExecutorProvider(), -1, true, listener != null, subFuture, startMessageId, schema, interceptors, @@ -1013,6 +1017,8 @@ private void doSubscribeTopicPartitions(Schema schema, synchronized (pauseMutex) { if (paused) { newConsumer.pause(); + } else { + newConsumer.resume(); } } return newConsumer; @@ -1304,6 +1310,7 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa int partitionIndex = TopicName.getPartitionIndex(partitionName); CompletableFuture> subFuture = new CompletableFuture<>(); ConsumerConfigurationData configurationData = getInternalConsumerConfig(); + configurationData.setStartPaused(paused); ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl( client, partitionName, configurationData, client.externalExecutorProvider(), @@ -1312,6 +1319,8 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa synchronized (pauseMutex) { if (paused) { newConsumer.pause(); + } else { + newConsumer.resume(); } consumers.putIfAbsent(newConsumer.getTopic(), newConsumer); } From 1ac258746e542283626f7b4c7a02f0e1aad95abe Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 5 Jul 2022 14:31:30 +0800 Subject: [PATCH 792/823] [improve][java-client] Improve performance of multi-topic consumer with more than one IO thread (#16336) (cherry picked from commit bdda1ebd6843be6ffd9dff67adcd85ee367f4e93) --- .../pulsar/client/impl/MessagesImpl.java | 6 +- .../client/impl/MultiTopicsConsumerImpl.java | 59 ++++++++++++------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java index bb0335b1f158b..532d152a8318c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java @@ -30,7 +30,7 @@ @NotThreadSafe public class MessagesImpl implements Messages { - private List> messageList; + private final List> messageList; private final int maxNumberOfMessages; private final long maxSizeOfMessages; @@ -81,6 +81,10 @@ public void clear() { this.messageList.clear(); } + List> getMessageList() { + return messageList; + } + @Override public Iterator> iterator() { return messageList.iterator(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 057eda176bd50..e5cc32af8fe9d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -49,6 +49,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerStats; import org.apache.pulsar.client.api.Message; @@ -239,13 +240,19 @@ private void startReceivingMessages(List> newConsumers) { if (getState() == State.Ready) { newConsumers.forEach(consumer -> { consumer.increaseAvailablePermits(consumer.getConnectionHandler().cnx(), conf.getReceiverQueueSize()); - internalPinnedExecutor.execute(() -> receiveMessageFromConsumer(consumer)); + internalPinnedExecutor.execute(() -> receiveMessageFromConsumer(consumer, true)); }); } } - private void receiveMessageFromConsumer(ConsumerImpl consumer) { - consumer.receiveAsync().thenAcceptAsync(message -> { + private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchReceive) { + CompletableFuture>> messagesFuture; + if (batchReceive) { + messagesFuture = consumer.batchReceiveAsync().thenApply(msgs -> ((MessagesImpl) msgs).getMessageList()); + } else { + messagesFuture = consumer.receiveAsync().thenApply(Collections::singletonList); + } + messagesFuture.thenAcceptAsync(messages -> { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Receive message from sub consumer:{}", topic, subscription, consumer.getTopic()); @@ -255,7 +262,7 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { return; } // Process the message, add to the queue and trigger listener or async callback - messageReceived(consumer, message); + messages.forEach(msg -> messageReceived(consumer, msg)); int size = incomingMessages.size(); if (size >= maxReceiverQueueSize @@ -271,7 +278,7 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { } else { // Call receiveAsync() if the incoming queue is not full. Because this block is run with // thenAcceptAsync, there is no chance for recursion that would lead to stack overflow. - receiveMessageFromConsumer(consumer); + receiveMessageFromConsumer(consumer, messages.size() > 0); } }, internalPinnedExecutor).exceptionally(ex -> { if (ex instanceof PulsarClientException.AlreadyClosedException @@ -280,8 +287,8 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer) { return null; } log.error("Receive operation failed on consumer {} - Retrying later", consumer, ex); - ((ScheduledExecutorService) client.getScheduledExecutorProvider()) - .schedule(() -> receiveMessageFromConsumer(consumer), 10, TimeUnit.SECONDS); + ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()) + .schedule(() -> receiveMessageFromConsumer(consumer, true), 10, TimeUnit.SECONDS); return null; }); } @@ -324,7 +331,7 @@ private void resumeReceivingFromPausedConsumersIfNeeded() { } internalPinnedExecutor.execute(() -> { - receiveMessageFromConsumer(consumer); + receiveMessageFromConsumer(consumer, true); }); } } @@ -979,11 +986,8 @@ private void doSubscribeTopicPartitions(Schema schema, String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString(); CompletableFuture> subFuture = new CompletableFuture<>(); configurationData.setStartPaused(paused); - ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, partitionName, - configurationData, client.externalExecutorProvider(), - partitionIndex, true, listener != null, subFuture, - startMessageId, schema, interceptors, - createIfDoesNotExist, startMessageRollbackDurationInSec); + ConsumerImpl newConsumer = createInternalConsumer(configurationData, partitionName, + partitionIndex, subFuture, createIfDoesNotExist, schema); synchronized (pauseMutex) { if (paused) { newConsumer.pause(); @@ -1009,10 +1013,8 @@ private void doSubscribeTopicPartitions(Schema schema, return existingValue; } else { internalConfig.setStartPaused(paused); - ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl(client, topicName, internalConfig, - client.externalExecutorProvider(), -1, - true, listener != null, subFuture, startMessageId, schema, interceptors, - createIfDoesNotExist, startMessageRollbackDurationInSec); + ConsumerImpl newConsumer = createInternalConsumer(internalConfig, topicName, + -1, subFuture, createIfDoesNotExist, schema); synchronized (pauseMutex) { if (paused) { @@ -1054,6 +1056,22 @@ private void doSubscribeTopicPartitions(Schema schema, }); } + private ConsumerImpl createInternalConsumer(ConsumerConfigurationData configurationData, String partitionName, + int partitionIndex, CompletableFuture> subFuture, + boolean createIfDoesNotExist, Schema schema) { + BatchReceivePolicy internalBatchReceivePolicy = BatchReceivePolicy.builder() + .maxNumMessages(Math.max(configurationData.getReceiverQueueSize() / 2, 1)) + .maxNumBytes(-1) + .timeout(1, TimeUnit.MILLISECONDS) + .build(); + configurationData.setBatchReceivePolicy(internalBatchReceivePolicy); + return ConsumerImpl.newConsumerImpl(client, partitionName, + configurationData, client.externalExecutorProvider(), + partitionIndex, true, listener != null, subFuture, + startMessageId, schema, interceptors, + createIfDoesNotExist, startMessageRollbackDurationInSec); + } + // handling failure during subscribe new topic, unsubscribe success created partitions private void handleSubscribeOneTopicError(String topicName, Throwable error, CompletableFuture subscribeFuture) { log.warn("[{}] Failed to subscribe for topic [{}] in topics consumer {}", topic, topicName, error.getMessage()); @@ -1311,11 +1329,8 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa CompletableFuture> subFuture = new CompletableFuture<>(); ConsumerConfigurationData configurationData = getInternalConsumerConfig(); configurationData.setStartPaused(paused); - ConsumerImpl newConsumer = ConsumerImpl.newConsumerImpl( - client, partitionName, configurationData, - client.externalExecutorProvider(), - partitionIndex, true, listener != null, subFuture, startMessageId, schema, interceptors, - true /* createTopicIfDoesNotExist */, startMessageRollbackDurationInSec); + ConsumerImpl newConsumer = createInternalConsumer(configurationData, partitionName, + partitionIndex, subFuture, true, schema); synchronized (pauseMutex) { if (paused) { newConsumer.pause(); From 1de291560524218ad35be6bbbc30e3acc95b53ee Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 15 Nov 2022 11:37:16 +0800 Subject: [PATCH 793/823] [fix][client] Fix failover/exclusive consumer with batch cumulate ack issue. (#18454) (cherry picked from commit 7712aa396bfca55a79a45761e0a405e90185e0f8) --- .../impl/ConsumerDedupPermitsUpdateTest.java | 21 ++++- .../pulsar/client/impl/NegativeAcksTest.java | 77 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 9 ++- .../client/impl/MultiTopicsConsumerImpl.java | 3 - 4 files changed, 100 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerDedupPermitsUpdateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerDedupPermitsUpdateTest.java index 4c9922acbec06..ceb7d7fd4844c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerDedupPermitsUpdateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerDedupPermitsUpdateTest.java @@ -116,10 +116,23 @@ public void testConsumerDedup(boolean batchingEnabled, int receiverQueueSize) th } producer.flush(); - for (int i = 0; i < 30; i++) { - Message msg = consumer.receive(); - assertEquals(msg.getValue(), "new-message-" + i); - consumer.acknowledge(msg); + if (batchingEnabled) { + for (int i = 0; i < 30; i++) { + Message msg = consumer.receive(); + assertEquals(msg.getValue(), "hello-" + i); + consumer.acknowledge(msg); + } + for (int i = 0; i < 30; i++) { + Message msg = consumer.receive(); + assertEquals(msg.getValue(), "new-message-" + i); + consumer.acknowledge(msg); + } + } else { + for (int i = 0; i < 30; i++) { + Message msg = consumer.receive(); + assertEquals(msg.getValue(), "new-message-" + i); + consumer.acknowledge(msg); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java index 5eb43af38f771..de130b7827078 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.Cleanup; @@ -35,6 +36,7 @@ import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -154,4 +156,79 @@ public void testNegativeAcks(boolean batching, boolean usePartitions, Subscripti consumer.close(); producer.close(); } + + @Test + public void testFailoverConsumerBatchCumulateAck() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("my-topic"); + admin.topics().createPartitionedTopic(topic, 2); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Failover) + .enableBatchIndexAcknowledgment(true) + .acknowledgmentGroupTime(100, TimeUnit.MILLISECONDS) + .receiverQueueSize(10) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .batchingMaxMessages(10) + .batchingMaxPublishDelay(3, TimeUnit.SECONDS) + .blockIfQueueFull(true) + .create(); + + int count = 0; + Set datas = new HashSet<>(); + CountDownLatch producerLatch = new CountDownLatch(10); + while (count < 10) { + datas.add(count); + producer.sendAsync(count).whenComplete((m, e) -> { + producerLatch.countDown(); + }); + count++; + } + producerLatch.await(); + CountDownLatch consumerLatch = new CountDownLatch(1); + new Thread(new Runnable() { + @Override + public void run() { + consumer.receiveAsync() + .thenCompose(m -> { + log.info("received one msg : {}", m.getMessageId()); + datas.remove(m.getValue()); + return consumer.acknowledgeCumulativeAsync(m); + }) + .thenAccept(ignore -> { + try { + Thread.sleep(500); + consumer.redeliverUnacknowledgedMessages(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .whenComplete((r, e) -> { + consumerLatch.countDown(); + }); + } + }).start(); + consumerLatch.await(); + Thread.sleep(500); + count = 0; + while(true) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + if (msg == null) { + break; + } + consumer.acknowledgeCumulative(msg); + Thread.sleep(200); + datas.remove(msg.getValue()); + log.info("received msg : {}", msg.getMessageId()); + count++; + } + Assert.assertEquals(count, 9); + Assert.assertEquals(0, datas.size()); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1a185d4c17d5c..5381b28033085 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1174,9 +1174,9 @@ void messageReceived(MessageIdData messageId, int redeliveryCount, List ac final int numMessages = msgMetadata.getNumMessagesInBatch(); final int numChunks = msgMetadata.hasNumChunksFromMsg() ? msgMetadata.getNumChunksFromMsg() : 0; final boolean isChunkedMessage = numChunks > 1 && conf.getSubscriptionType() != SubscriptionType.Shared; - MessageIdImpl msgId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), getPartitionIndex()); - if (acknowledgmentsGroupingTracker.isDuplicate(msgId)) { + if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch() + && acknowledgmentsGroupingTracker.isDuplicate(msgId)) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Ignoring message as it was already being acked earlier by same consumer {}/{}", topic, subscription, consumerName, msgId); @@ -1429,7 +1429,10 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, skippedMessages++; continue; } - + } + if (acknowledgmentsGroupingTracker.isDuplicate(message.getMessageId())) { + skippedMessages++; + continue; } executeNotifyCallback(message); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index e5cc32af8fe9d..b2de0e3b92c38 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -985,7 +985,6 @@ private void doSubscribeTopicPartitions(Schema schema, partitionIndex -> { String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString(); CompletableFuture> subFuture = new CompletableFuture<>(); - configurationData.setStartPaused(paused); ConsumerImpl newConsumer = createInternalConsumer(configurationData, partitionName, partitionIndex, subFuture, createIfDoesNotExist, schema); synchronized (pauseMutex) { @@ -1012,7 +1011,6 @@ private void doSubscribeTopicPartitions(Schema schema, subscribeResult.completeExceptionally(new PulsarClientException(errorMessage)); return existingValue; } else { - internalConfig.setStartPaused(paused); ConsumerImpl newConsumer = createInternalConsumer(internalConfig, topicName, -1, subFuture, createIfDoesNotExist, schema); @@ -1328,7 +1326,6 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa int partitionIndex = TopicName.getPartitionIndex(partitionName); CompletableFuture> subFuture = new CompletableFuture<>(); ConsumerConfigurationData configurationData = getInternalConsumerConfig(); - configurationData.setStartPaused(paused); ConsumerImpl newConsumer = createInternalConsumer(configurationData, partitionName, partitionIndex, subFuture, true, schema); synchronized (pauseMutex) { From dc1ce0814208993f821ddb96e38a9ef83a425e0d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 16 Nov 2021 16:15:07 -0800 Subject: [PATCH 794/823] Ensure cache is refreshed (and not just invalidated) after a store write (#12788) ### Motivation When we're doing a write to the store from outside the `MetadataCache`, we are immediately invalidating the cache to ensure read-after-write consistency through the cache. The only issue is that the invalidation, will not trigger a reloading of the value. Instead it is relying on the next call to `cache.get()` which will see the cache miss and it will load the new value into the cache. This means that calls `cache.getIfCached()`, which is not triggering a cache load, will keep seeing the key as missing. ### Modification Ensure we're calling refresh on the cache to get the value automatically reloaded in background and make sure the `getIfCached()` will eventually return the new value, even if there are no calls to `cache.get()`. (cherry picked from commit 2bc449933f72f28dfae24ca8a6fb022be7c55a44) --- .../pulsar/metadata/api/MetadataCache.java | 7 ++++ .../cache/impl/MetadataCacheImpl.java | 17 ++++++---- .../metadata/impl/AbstractMetadataStore.java | 2 +- .../pulsar/metadata/MetadataCacheTest.java | 34 +++++++++++++++++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java index 360c092b6f1b4..1272130eb76f2 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java @@ -148,4 +148,11 @@ public interface MetadataCache { * @param path the path of the object in the metadata store */ void invalidate(String path); + + /** + * Invalidate and reload an object in the metadata cache. + * + * @param path the path of the object in the metadata store + */ + void refresh(String path); } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java index 16b419fe9ed91..0ab56bb1bc18d 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java @@ -168,8 +168,7 @@ public CompletableFuture readModifyUpdateOrCreate(String path, Function { - objCache.synchronous().invalidate(path); - objCache.synchronous().refresh(path); + refresh(path); }).thenApply(__ -> newValueObj); }), path); } @@ -198,8 +197,7 @@ public CompletableFuture readModifyUpdate(String path, Function modifyF } return store.put(path, newValue, Optional.of(expectedVersion)).thenAccept(__ -> { - objCache.synchronous().invalidate(path); - objCache.synchronous().refresh(path); + refresh(path); }).thenApply(__ -> newValueObj); }), path); } @@ -220,7 +218,7 @@ public CompletableFuture create(String path, T value) { // In addition to caching the value, we need to add a watch on the path, // so when/if it changes on any other node, we are notified and we can // update the cache - objCache.get(path).whenComplete( (stat2, ex) -> { + objCache.get(path).whenComplete((stat2, ex) -> { if (ex == null) { future.complete(null); } else { @@ -261,6 +259,12 @@ public void invalidate(String path) { objCache.synchronous().invalidate(path); } + @Override + public void refresh(String path) { + objCache.synchronous().invalidate(path); + objCache.synchronous().refresh(path); + } + @VisibleForTesting public void invalidateAll() { objCache.synchronous().invalidateAll(); @@ -275,8 +279,7 @@ public void accept(Notification t) { if (objCache.synchronous().getIfPresent(path) != null) { // Trigger background refresh of the cached item, but before make sure // to invalidate the entry so that we won't serve a stale cached version - objCache.synchronous().invalidate(path); - objCache.synchronous().refresh(path); + refresh(path); } break; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 40c209f4d3e87..ac7feb4747f67 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -252,7 +252,7 @@ public final CompletableFuture put(String path, byte[] data, Optional c.invalidate(path)); + metadataCaches.forEach(c -> c.refresh(path)); return stat; }); } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java index 322c9bb779bde..e768029656371 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java @@ -278,6 +278,34 @@ public void insertionDeletion(String provider, Supplier urlSupplier) thr assertEquals(objCache.get(key1).join(), Optional.empty()); } + @Test(dataProvider = "impl") + public void insertionWithInvalidation(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataCache objCache = store.getMetadataCache(MyClass.class); + + String key1 = newKey(); + + assertEquals(objCache.getIfCached(key1), Optional.empty()); + assertEquals(objCache.get(key1).join(), Optional.empty()); + + MyClass value1 = new MyClass("a", 1); + store.put(key1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(value1), Optional.of(-1L)).join(); + + Awaitility.await().untilAsserted(() -> { + assertEquals(objCache.getIfCached(key1), Optional.of(value1)); + assertEquals(objCache.get(key1).join(), Optional.of(value1)); + }); + + MyClass value2 = new MyClass("a", 2); + store.put(key1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(value2), Optional.of(0L)).join(); + + Awaitility.await().untilAsserted(() -> { + assertEquals(objCache.getIfCached(key1), Optional.of(value2)); + assertEquals(objCache.get(key1).join(), Optional.of(value2)); + }); + } + @Test(dataProvider = "impl") public void insertionOutsideCache(String provider, Supplier urlSupplier) throws Exception { @Cleanup @@ -310,8 +338,10 @@ public void insertionOutsideCacheWithGenericType(String provider, Supplier { + assertEquals(objCache.getIfCached(key1), Optional.of(v)); + assertEquals(objCache.get(key1).join(), Optional.of(v)); + }); } @Test(dataProvider = "impl") From 2bbeace8f99abe8e16c4d2db4f61ea3994a1f359 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 1 Dec 2021 05:09:08 +0800 Subject: [PATCH 795/823] fix-12894 (#12896) Co-authored-by: Jiang Haiting (cherry picked from commit 2b939b7a8b0bf9ed2fd31353067033d049ea5350) --- .../pulsar/metadata/cache/impl/MetadataCacheImpl.java | 7 +++++-- .../java/org/apache/pulsar/metadata/MetadataCacheTest.java | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java index 0ab56bb1bc18d..fccbe2ca38a6c 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java @@ -261,8 +261,11 @@ public void invalidate(String path) { @Override public void refresh(String path) { - objCache.synchronous().invalidate(path); - objCache.synchronous().refresh(path); + // Refresh object of path if only it is cached before. + if (objCache.getIfPresent(path) != null) { + objCache.synchronous().invalidate(path); + objCache.synchronous().refresh(path); + } } @VisibleForTesting diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java index e768029656371..a4ba88852b0e3 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java @@ -325,13 +325,14 @@ public void insertionOutsideCache(String provider, Supplier urlSupplier) } @Test(dataProvider = "impl") - public void insertionOutsideCacheWithGenericType(String provider, Supplier urlSupplier) throws Exception { + public void updateOutsideCacheWithGenericType(String provider, Supplier urlSupplier) throws Exception { @Cleanup MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); MetadataCache> objCache = store.getMetadataCache(new TypeReference>() { }); String key1 = newKey(); + objCache.get(key1); Map v = new TreeMap<>(); v.put("a", "1"); From c3bf4525ae0ec521695fd20f0674ee3b6623e185 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 14 Feb 2022 11:22:05 +0800 Subject: [PATCH 796/823] [Broker] Fix NPE of internalExpireMessagesByTimestamp (#14243) This issue was triggered by #13880, it doesn't handle the NPE in [here](https://github.com/apache/pulsar/pull/13880/files#diff-66aeb65a64cbe7c541f013ae807c5bcbeab567bef77706c7ff2e0cbfe0d77eb1R3502), we can check the [diff code](https://github.com/apache/pulsar/pull/13880/files#diff-66aeb65a64cbe7c541f013ae807c5bcbeab567bef77706c7ff2e0cbfe0d77eb1L3489) in #13880. (cherry picked from commit a3c8525c93edce681f3d99a0b8828624fb05919a) --- .../admin/impl/PersistentTopicsBase.java | 8 ++- .../admin/AdminApiSubscriptionTest.java | 72 +++++++++++++++++++ .../auth/MockedPulsarServiceBaseTest.java | 20 ++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSubscriptionTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 4ff236af5ac27..7725f1cd98c5d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3302,11 +3302,15 @@ private void internalExpireMessagesByTimestampForSinglePartition(String subName, if (subName.startsWith(topic.getReplicatorPrefix())) { String remoteCluster = PersistentReplicator.getRemoteCluster(subName); PersistentReplicator repl = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); - checkNotNull(repl); + if (repl == null) { + throw new RestException(Status.NOT_FOUND, "Replicator not found"); + } issued = repl.expireMessages(expireTimeInSeconds); } else { PersistentSubscription sub = topic.getSubscription(subName); - checkNotNull(sub); + if (sub == null) { + throw new RestException(Status.NOT_FOUND, "Subscription not found"); + } issued = sub.expireMessages(expireTimeInSeconds); } if (issued) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSubscriptionTest.java new file mode 100644 index 0000000000000..6f38ccd8b5cf5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSubscriptionTest.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.expectThrows; +import javax.ws.rs.core.Response; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageId; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-admin") +public class AdminApiSubscriptionTest extends MockedPulsarServiceBaseTest { + @BeforeMethod + @Override + public void setup() throws Exception { + super.internalSetup(); + super.setupDefaultTenantAndNamespace(); + } + + @AfterMethod(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testExpireNonExistTopic() throws Exception { + String topic = "test-expire-messages-topic"; + String subscriptionName = "test-expire-messages-sub"; + admin.topics().createSubscription(topic, subscriptionName, MessageId.latest); + assertEquals(expectThrows(PulsarAdminException.class, + () -> admin.topics().expireMessages(topic, subscriptionName, 1)).getStatusCode(), + Response.Status.CONFLICT.getStatusCode()); + assertEquals(expectThrows(PulsarAdminException.class, + () -> admin.topics().expireMessagesForAllSubscriptions(topic, 1)).getStatusCode(), + Response.Status.CONFLICT.getStatusCode()); + } + + @Test + public void TestExpireNonExistTopicAndNonExistSub() { + String topic = "test-expire-messages-topic"; + String subscriptionName = "test-expire-messages-sub"; + assertEquals(expectThrows(PulsarAdminException.class, + () -> admin.topics().expireMessages(topic, subscriptionName, 1)).getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + assertEquals(expectThrows(PulsarAdminException.class, + () -> admin.topics().expireMessagesForAllSubscriptions(topic, 1)).getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index c731d310d9941..27ac9cb869dca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -60,6 +60,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -499,5 +500,24 @@ protected static ServiceConfiguration getDefaultConf() { return configuration; } + protected void setupDefaultTenantAndNamespace() throws Exception { + final String tenant = "public"; + final String namespace = tenant + "/default"; + + if (!admin.clusters().getClusters().contains(configClusterName)) { + admin.clusters().createCluster(configClusterName, + ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + } + + if (!admin.tenants().getTenants().contains(tenant)) { + admin.tenants().createTenant(tenant, TenantInfo.builder().allowedClusters( + Sets.newHashSet(configClusterName)).build()); + } + + if (!admin.namespaces().getNamespaces(tenant).contains(namespace)) { + admin.namespaces().createNamespace(namespace); + } + } + private static final Logger log = LoggerFactory.getLogger(MockedPulsarServiceBaseTest.class); } From 977a81ce9b8fd9bf9f7f82bb1f3d001f49dba083 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 15 Feb 2022 18:42:13 +0800 Subject: [PATCH 797/823] Fix metadata cache inconsistency on do refresh (#14283) (cherry picked from commit 2e16b4341682083a329b63ffaae6d2df2eeec3c5) --- .../pulsar/metadata/cache/impl/MetadataCacheImpl.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java index fccbe2ca38a6c..8bf5a729b7694 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java @@ -262,10 +262,7 @@ public void invalidate(String path) { @Override public void refresh(String path) { // Refresh object of path if only it is cached before. - if (objCache.getIfPresent(path) != null) { - objCache.synchronous().invalidate(path); - objCache.synchronous().refresh(path); - } + objCache.asMap().computeIfPresent(path, (oldKey, oldValue) -> readValueFromStore(path)); } @VisibleForTesting @@ -279,11 +276,7 @@ public void accept(Notification t) { switch (t.getType()) { case Created: case Modified: - if (objCache.synchronous().getIfPresent(path) != null) { - // Trigger background refresh of the cached item, but before make sure - // to invalidate the entry so that we won't serve a stale cached version - refresh(path); - } + refresh(path); break; case Deleted: From 06954d131c63d9136ab7d895ed83ad67d857463e Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 22 Aug 2022 16:44:31 +0800 Subject: [PATCH 798/823] [fix][broker] Make `deleteTopicPolicies` serialized is executed when close topic. (#15811) (cherry picked from commit e8ee996dd0c7a3a742117aee399b31e89e6e2d9d) --- .../pulsar/broker/service/BrokerService.java | 15 ++++++++++++--- .../service/persistent/PersistentTopic.java | 2 +- .../api/AuthenticatedProducerConsumerTest.java | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 1fbc19657f51b..aed023ee5f52d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2773,11 +2773,20 @@ public Optional getTopicPolicies(TopicName topicName) { } public CompletableFuture deleteTopicPolicies(TopicName topicName) { - if (!pulsar().getConfig().isTopicLevelPoliciesEnabled()) { + final PulsarService pulsarService = pulsar(); + if (!pulsarService.getConfig().isTopicLevelPoliciesEnabled()) { return CompletableFuture.completedFuture(null); } - TopicName cloneTopicName = TopicName.get(topicName.getPartitionedTopicName()); - return pulsar.getTopicPoliciesService().deleteTopicPoliciesAsync(cloneTopicName); + return pulsarService.getPulsarResources().getNamespaceResources() + .getPoliciesAsync(topicName.getNamespaceObject()) + .thenComposeAsync(optPolicies -> { + if (optPolicies.isPresent() && optPolicies.get().deleted) { + // We can return the completed future directly if the namespace is already deleted. + return CompletableFuture.completedFuture(null); + } + TopicName cloneTopicName = TopicName.get(topicName.getPartitionedTopicName()); + return pulsar.getTopicPoliciesService().deleteTopicPoliciesAsync(cloneTopicName); + }); } private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, int numPartitions) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 99f12d1b1a92e..b3d611150324d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1163,7 +1163,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, deleteTopicAuthenticationFuture.thenCompose( __ -> deleteSchema ? deleteSchema() : CompletableFuture.completedFuture(null)) - .thenAccept(__ -> deleteTopicPolicies()) + .thenCompose(__ -> deleteTopicPolicies()) .thenCompose(__ -> transactionBufferCleanupAndClose()) .whenComplete((v, ex) -> { if (ex != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java index 046b26846e2d3..e0cc980991ef7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java @@ -80,6 +80,7 @@ protected void setup() throws Exception { conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); conf.setTlsAllowInsecureConnection(true); + conf.setTopicLevelPoliciesEnabled(false); Set superUserRoles = new HashSet<>(); superUserRoles.add("localhost"); From 0844f3153d22546780cd36a5c6610b7a382bdcbc Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 28 Jun 2022 09:19:18 +0800 Subject: [PATCH 799/823] [improve][broker] Avoid go through all the consumers to get the message ack owner (#16245) ### Motivation The broker don't need to go through all the consumers to get the ack owner consumer. Instead, it should check the current consumer first. If the pending acks of current consumer don't have the ack position, go through all the consumers to find the owner consumer. (cherry picked from commit 68484f9162bc768816cfd039140fb78196485d19) --- .../org/apache/pulsar/broker/service/Consumer.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 297aefee6e857..25dd3b908e519 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -612,10 +612,12 @@ private void checkCanRemovePendingAcksAndHandle(PositionImpl position, MessageId private Consumer getAckOwnerConsumer(long ledgerId, long entryId) { Consumer ackOwnerConsumer = this; if (Subscription.isIndividualAckMode(subType)) { - for (Consumer consumer : subscription.getConsumers()) { - if (consumer != this && consumer.getPendingAcks().containsKey(ledgerId, entryId)) { - ackOwnerConsumer = consumer; - break; + if (!getPendingAcks().containsKey(ledgerId, entryId)) { + for (Consumer consumer : subscription.getConsumers()) { + if (consumer != this && consumer.getPendingAcks().containsKey(ledgerId, entryId)) { + ackOwnerConsumer = consumer; + break; + } } } } From 26c4b95ce1ad34ff741712483c454647479e4d40 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 28 Jun 2022 09:11:47 +0800 Subject: [PATCH 800/823] [improve][broker] Reduce the consumers list sort by priority level (#16243) ### Motivation While create many consumers (> 10000), the IO thread run into BLOCK state for long time which will affect the message publish and subsequent consumer creation. ``` "pulsar-io-15-24" #195 prio=5 os_prio=31 cpu=15744.67ms elapsed=272.18s tid=0x00007faaa7183400 nid=0x19c03 waiting for monitor entry [0x0000700019642000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.pulsar.broker.service.persistent.PersistentSubscription.lambda$addConsumer$2(PersistentSubscription.java:207) - waiting to lock <0x0000100015823488> (a org.apache.pulsar.broker.service.persistent.PersistentSubscription) at org.apache.pulsar.broker.service.persistent.PersistentSubscription$$Lambda$984/0x000000080136d898.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.persistent.PersistentSubscription.addConsumer(PersistentSubscription.java:206) at org.apache.pulsar.broker.service.AbstractTopic.addConsumerToSubscription(AbstractTopic.java:513) at org.apache.pulsar.broker.service.persistent.PersistentTopic.lambda$internalSubscribe$15(PersistentTopic.java:782) at org.apache.pulsar.broker.service.persistent.PersistentTopic$$Lambda$983/0x000000080136cd28.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.persistent.PersistentTopic.lambda$internalSubscribe$17(PersistentTopic.java:777) at org.apache.pulsar.broker.service.persistent.PersistentTopic$$Lambda$982/0x000000080136cae0.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.persistent.PersistentTopic.internalSubscribe(PersistentTopic.java:698) at org.apache.pulsar.broker.service.persistent.PersistentTopic.subscribe(PersistentTopic.java:674) at org.apache.pulsar.broker.service.ServerCnx.lambda$handleSubscribe$12(ServerCnx.java:1078) at org.apache.pulsar.broker.service.ServerCnx$$Lambda$869/0x0000000801316630.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.ServerCnx.lambda$handleSubscribe$15(ServerCnx.java:1042) at org.apache.pulsar.broker.service.ServerCnx$$Lambda$860/0x000000080130f970.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniApplyNow(java.base@17.0.3/CompletableFuture.java:684) at java.util.concurrent.CompletableFuture.uniApplyStage(java.base@17.0.3/CompletableFuture.java:662) at java.util.concurrent.CompletableFuture.thenApply(java.base@17.0.3/CompletableFuture.java:2168) at org.apache.pulsar.broker.service.ServerCnx.handleSubscribe(ServerCnx.java:984) at org.apache.pulsar.common.protocol.PulsarDecoder.channelRead(PulsarDecoder.java:229) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.handler.flow.FlowControlHandler.dequeue(FlowControlHandler.java:200) at io.netty.handler.flow.FlowControlHandler.channelRead(FlowControlHandler.java:162) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:314) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:435) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@17.0.3/Thread.java:833) ``` ``` "pulsar-io-15-8" #157 prio=5 os_prio=31 cpu=10573.05ms elapsed=314.91s tid=0x00007faa9bf6e800 nid=0x17507 runnable [0x00007000171d5000] java.lang.Thread.State: RUNNABLE at java.util.TimSort.countRunAndMakeAscending(java.base@17.0.3/TimSort.java:360) at java.util.TimSort.sort(java.base@17.0.3/TimSort.java:234) at java.util.Arrays.sort(java.base@17.0.3/Arrays.java:1307) at java.util.concurrent.CopyOnWriteArrayList.sortRange(java.base@17.0.3/CopyOnWriteArrayList.java:896) at java.util.concurrent.CopyOnWriteArrayList.sort(java.base@17.0.3/CopyOnWriteArrayList.java:888) - locked <0x00001000158237d8> (a java.lang.Object) at org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers.addConsumer(PersistentDispatcherMultipleConsumers.java:159) - locked <0x0000100015830888> (a org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) at org.apache.pulsar.broker.service.persistent.PersistentSubscription.lambda$addConsumer$2(PersistentSubscription.java:287) - locked <0x0000100015823488> (a org.apache.pulsar.broker.service.persistent.PersistentSubscription) at org.apache.pulsar.broker.service.persistent.PersistentSubscription$$Lambda$984/0x000000080136d898.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.persistent.PersistentSubscription.addConsumer(PersistentSubscription.java:206) at org.apache.pulsar.broker.service.AbstractTopic.addConsumerToSubscription(AbstractTopic.java:513) at org.apache.pulsar.broker.service.persistent.PersistentTopic.lambda$internalSubscribe$15(PersistentTopic.java:782) at org.apache.pulsar.broker.service.persistent.PersistentTopic$$Lambda$983/0x000000080136cd28.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.persistent.PersistentTopic.lambda$internalSubscribe$17(PersistentTopic.java:777) at org.apache.pulsar.broker.service.persistent.PersistentTopic$$Lambda$982/0x000000080136cae0.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.persistent.PersistentTopic.internalSubscribe(PersistentTopic.java:698) at org.apache.pulsar.broker.service.persistent.PersistentTopic.subscribe(PersistentTopic.java:674) at org.apache.pulsar.broker.service.ServerCnx.lambda$handleSubscribe$12(ServerCnx.java:1078) at org.apache.pulsar.broker.service.ServerCnx$$Lambda$869/0x0000000801316630.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.3/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.3/CompletableFuture.java:2309) at org.apache.pulsar.broker.service.ServerCnx.lambda$handleSubscribe$15(ServerCnx.java:1042) at org.apache.pulsar.broker.service.ServerCnx$$Lambda$860/0x000000080130f970.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniApplyNow(java.base@17.0.3/CompletableFuture.java:684) at java.util.concurrent.CompletableFuture.uniApplyStage(java.base@17.0.3/CompletableFuture.java:662) at java.util.concurrent.CompletableFuture.thenApply(java.base@17.0.3/CompletableFuture.java:2168) at org.apache.pulsar.broker.service.ServerCnx.handleSubscribe(ServerCnx.java:984) at org.apache.pulsar.common.protocol.PulsarDecoder.channelRead(PulsarDecoder.java:229) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.handler.flow.FlowControlHandler.dequeue(FlowControlHandler.java:200) at io.netty.handler.flow.FlowControlHandler.channelRead(FlowControlHandler.java:162) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:314) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:435) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@17.0.3/Thread.java:833) ``` ### Modification - Sort the consumer list only if the new consumer with high priority than the last element in the consumer list, this can avoid the sort operation for all the consumers without priority level (the client-side always pass 0 if priority level absent). (cherry picked from commit 291fedcd2dc2781ef5a7ea2a6d3d653aa882eb3a) --- .../persistent/PersistentDispatcherMultipleConsumers.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index a36b1eded3dc5..02fc80507636f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -159,7 +159,10 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce } consumerList.add(consumer); - consumerList.sort(Comparator.comparingInt(Consumer::getPriorityLevel)); + if (consumerList.size() > 1 + && consumer.getPriorityLevel() < consumerList.get(consumerList.size() - 2).getPriorityLevel()) { + consumerList.sort(Comparator.comparingInt(Consumer::getPriorityLevel)); + } consumerSet.add(consumer); } From 34f14fc1b859c787a40186deee81664d1a15d41e Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Wed, 14 Sep 2022 10:41:13 +0800 Subject: [PATCH 801/823] [fix][tiered-storage] Don't cleanup data when offload met Metastore exception (#17512) * [fix][tiered-storage] Don't cleanup data when offload met BadVersion --- *Motivation* There have two ways that will cause the offload data cleanup. One is met offload conflict exception, and another is completeLedgerInfoForOffloaded reaches max retry time and throws zookeeper exceptions. We retry the zookeeper operation on connection loss exception. We should be careful about this exception, because we may loss data if the metadata update successfully. When a MetaStore exception happens, we can not make sure the metadata update is failed or not. Because we have a retry on the connection loss, it is possible to get a BadVersion or other exception after retrying. So we don't clean up the data if this happens. *Modification* - don't delete data if has meta store exception * log error when skip deleting * improve logs (cherry picked from commit c2588ba6a07c05b2c1ce9cc8a4cf33e5b4a2755d) --- .../mledger/impl/ManagedLedgerImpl.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 09a6fad22430e..7b8d6ff6ea50d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2902,8 +2902,21 @@ private void offloadLoop(CompletableFuture promise, Queue { if (exception != null) { - log.error("[{}] Failed to offload data for the ledgerId {}", + Throwable e = FutureUtil.unwrapCompletionException(exception); + if (e instanceof MetaStoreException) { + // When a MetaStore exception happens, we can not make sure the metadata + // update is failed or not. Because we have a retry on the connection loss, + // it is possible to get a BadVersion or other exception after retrying. + // So we don't clean up the data if it has metadata operation exception. + log.error("[{}] Failed to update offloaded metadata for the ledgerId {}, " + + "the offloaded data will not be cleaned up", name, ledgerId, exception); + return; + } else { + log.error("[{}] Failed to offload data for the ledgerId {}, " + + "clean up the offloaded data", + name, ledgerId, exception); + } cleanupOffloaded( ledgerId, uuid, driverName, driverMetadata, From 78e7dde912fb9ebea58fa226b4bbdb6b513ed1c4 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Thu, 15 Sep 2022 10:14:29 +0800 Subject: [PATCH 802/823] [improve][txn] Implementation of Delayed Transaction Messages (#17548) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit link https://github.com/apache/pulsar/pull/17548 ### Motivation now delayed features and transaction messages cannot be used together. When sending a transaction message with a delayed time and commit this transaction, the message will be immediately received by consumers. Code, eg. ``` @Test public void testDelayedTransactionMessages() throws Exception { String topic = NAMESPACE1 + "/testDelayedTransactionMessages"; @Cleanup Consumer sharedConsumer = pulsarClient.newConsumer(Schema.STRING) .topic(topic) .subscriptionName("shared-sub") .subscriptionType(SubscriptionType.Shared) .subscribe(); @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) .topic(topic) .enableBatching(false) .create(); Transaction transaction = pulsarClient.newTransaction() .withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); // send delayed messages for (int i = 0; i < 10; i++) { producer.newMessage(transaction) .value("msg-" + i) .deliverAfter(5, TimeUnit.SECONDS) .sendAsync(); } producer.flush(); transaction.commit().get(); Message msg = sharedConsumer.receive(1, TimeUnit.SECONDS); // the msg now is not null assertNull(msg); } ``` This PR will implement clients to send delayed messages with transactions. ### Modifications make transaction message can be put in `trackDelayedDelivery` to implement client send delayed messages with the transaction. It is worth noting that the dispatcher sends transaction messages to consumers and should follow the `MaxReadPosition` change—(something about `MaxReadPosition` https://github.com/streamnative/community/blob/master/rfc/rfcs/0003-transaction-buffer-design.md). Because of the existence of maxReadPosition, the distribution of transaction messages depends on whether the previous transaction message is completed. This will cause delay time extended, but not shortened ### Verifying this change add the test (cherry picked from commit 1246d793e85cedc9b3fd251f23f1b74492d22120) --- .../service/AbstractBaseDispatcher.java | 4 +- .../client/impl/TransactionEndToEndTest.java | 61 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 3a3b542b4659f..3f5932a2d34bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -142,7 +142,9 @@ public void filterEntriesForConsumer(Optional entryWrapper, int entry.release(); continue; } - } else if (msgMetadata == null || Markers.isServerOnlyMarker(msgMetadata)) { + } + + if (msgMetadata == null || Markers.isServerOnlyMarker(msgMetadata)) { PositionImpl pos = (PositionImpl) entry.getPosition(); // Message metadata was corrupted or the messages was a server-only marker diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index f5788e8d4b6aa..2d174b9b1fb56 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -32,6 +32,8 @@ import java.lang.reflect.Field; import java.util.Collection; import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.ArrayList; import java.util.List; @@ -1230,4 +1232,63 @@ public void testSendTxnAckBatchMessageToDLQ() throws Exception { assertEquals(value1, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); assertEquals(value2, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); } + + @Test + public void testDelayedTransactionMessages() throws Exception { + String topic = NAMESPACE1 + "/testDelayedTransactionMessages"; + + @Cleanup + Consumer failoverConsumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("failover-sub") + .subscriptionType(SubscriptionType.Failover) + .subscribe(); + + @Cleanup + Consumer sharedConsumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("shared-sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .enableBatching(false) + .create(); + + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); + for (int i = 0; i < 10; i++) { + producer.newMessage(transaction) + .value("msg-" + i) + .deliverAfter(5, TimeUnit.SECONDS) + .sendAsync(); + } + + producer.flush(); + + transaction.commit().get(); + + // Failover consumer will receive the messages immediately while + // the shared consumer will get them after the delay + Message msg = sharedConsumer.receive(1, TimeUnit.SECONDS); + assertNull(msg); + + for (int i = 0; i < 10; i++) { + msg = failoverConsumer.receive(100, TimeUnit.MILLISECONDS); + assertEquals(msg.getValue(), "msg-" + i); + } + + Set receivedMsgs = new TreeSet<>(); + for (int i = 0; i < 10; i++) { + msg = sharedConsumer.receive(10, TimeUnit.SECONDS); + receivedMsgs.add(msg.getValue()); + } + + assertEquals(receivedMsgs.size(), 10); + for (int i = 0; i < 10; i++) { + assertTrue(receivedMsgs.contains("msg-" + i)); + } + } } From 6c0fc323dcb7c1be915bfa2e2160fe1fd9648804 Mon Sep 17 00:00:00 2001 From: LinChen <1572139390@qq.com> Date: Thu, 3 Nov 2022 20:51:18 +0800 Subject: [PATCH 803/823] [fix][broker] In the trimDeletedEntries method, release the removed entry (#18305) (cherry picked from commit 79a3f850c560bc2a8e27940c81ce5b492bf8f2c8) --- .../bookkeeper/mledger/impl/ManagedCursorImpl.java | 10 ++++++++-- .../bookkeeper/mledger/impl/ManagedCursorTest.java | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 8deb70c6a19e5..e31d4eff88730 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -3105,8 +3105,14 @@ public Range getLastIndividualDeletedRange() { @Override public void trimDeletedEntries(List entries) { - entries.removeIf(entry -> ((PositionImpl) entry.getPosition()).compareTo(markDeletePosition) <= 0 - || individualDeletedMessages.contains(entry.getLedgerId(), entry.getEntryId())); + entries.removeIf(entry -> { + boolean isDeleted = ((PositionImpl) entry.getPosition()).compareTo(markDeletePosition) <= 0 + || individualDeletedMessages.contains(entry.getLedgerId(), entry.getEntryId()); + if (isDeleted) { + entry.release(); + } + return isDeleted; + }); } private ManagedCursorImpl cursorImpl() { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 5f1ed11836b54..4c469c1989358 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -2539,6 +2539,11 @@ void testTrimDeletedEntries() throws ManagedLedgerException, InterruptedExceptio assertEquals(entries.size(), 1); assertEquals(entries.get(0).getPosition(), PositionImpl.get(markDeletedPosition.getLedgerId() , markDeletedPosition.getEntryId() + 7)); + + assertEquals(entry1.refCnt(), 0); + assertEquals(entry2.refCnt(), 0); + assertEquals(entry3.refCnt(), 0); + assertEquals(entry4.refCnt(), 0); } @Test(timeOut = 20000) From f934f373a43a029f6fdea66311173edaf32801b2 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 28 Oct 2022 21:06:20 +0800 Subject: [PATCH 804/823] [fix][cli] Fix CLI client produce don't able to use multiple -m send multiple messages (#18238) (cherry picked from commit 67a3de716d5cbf959fe793fde1705e6577106513) --- .../client/cli/PulsarClientToolTest.java | 22 +++++++++++++++++++ .../apache/pulsar/client/cli/CmdProduce.java | 9 ++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java index b1e067035d273..df373f3e288d8 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java @@ -270,6 +270,28 @@ public void testDisableBatching() throws Exception { } } + @Test + public void testSendMultipleMessage() throws Exception { + Properties properties = new Properties(); + properties.setProperty("serviceUrl", brokerUrl.toString()); + properties.setProperty("useTls", "false"); + + final String topicName = getTopicWithRandomSuffix("test-multiple-msg"); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe(); + + PulsarClientTool pulsarClientTool = new PulsarClientTool(properties); + String[] args1 = {"produce", "-m", "msg0", "-m", "msg1,msg2", topicName}; + Assert.assertEquals(pulsarClientTool.run(args1), 0); + + for (int i = 0; i < 3; i++) { + Message msg = consumer.receive(10, TimeUnit.SECONDS); + Assert.assertNotNull(msg); + Assert.assertEquals(new String(msg.getData()), "msg" + i); + } + } + private static String getTopicWithRandomSuffix(String localNameBase) { return String.format("persistent://prop/ns-abc/test/%s-%s", localNameBase, UUID.randomUUID().toString()); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java index 0ec22ba2e7b48..e0fc6327a0f5e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java @@ -34,17 +34,15 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; -import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; - +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.ClientBuilder; @@ -204,7 +202,8 @@ public int run() throws PulsarClientException { } if (messages.size() > 0){ - messages = Collections.unmodifiableList(Arrays.asList(messages.get(0).split(separator))); + messages = messages.stream().map(str -> str.split(separator)) + .flatMap(Stream::of).collect(Collectors.toList()); } if (messages.size() == 0 && messageFileNames.size() == 0) { From c0ff7de75d90170e65288b9d07b15eba1852d5fb Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 13 Oct 2022 14:46:14 +0800 Subject: [PATCH 805/823] [fix][broker] Fix `getPositionAfterN` infinite loop. (#17971) (cherry picked from commit c73285205dafaf5bed827ad99f7fb8edd3ddb7f2) --- .../mledger/impl/ManagedLedgerImpl.java | 17 +++++------------ .../mledger/impl/ManagedLedgerTest.java | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 7b8d6ff6ea50d..6280888174c0e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3185,20 +3185,16 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, long entriesToSkip = n; long currentLedgerId; long currentEntryId; - if (startRange == PositionBound.startIncluded) { currentLedgerId = startPosition.getLedgerId(); currentEntryId = startPosition.getEntryId(); } else { - // e.g. a mark-delete position PositionImpl nextValidPosition = getNextValidPosition(startPosition); currentLedgerId = nextValidPosition.getLedgerId(); currentEntryId = nextValidPosition.getEntryId(); } - boolean lastLedger = false; long totalEntriesInCurrentLedger; - while (entriesToSkip >= 0) { // for the current ledger, the number of entries written is deduced from the lastConfirmedEntry // for previous ledgers, LedgerInfo in ZK has the number of entries @@ -3213,10 +3209,8 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, LedgerInfo ledgerInfo = ledgers.get(currentLedgerId); totalEntriesInCurrentLedger = ledgerInfo != null ? ledgerInfo.getEntries() : 0; } - - - long unreadEntriesInCurrentLedger = totalEntriesInCurrentLedger - currentEntryId; - + long unreadEntriesInCurrentLedger = totalEntriesInCurrentLedger > 0 + ? totalEntriesInCurrentLedger - currentEntryId : 0; if (unreadEntriesInCurrentLedger >= entriesToSkip) { // if the current ledger has more entries than what we need to skip // then the return position is in the same ledger @@ -3229,11 +3223,10 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, // there are no more ledgers, return the last position currentEntryId = totalEntriesInCurrentLedger; break; - } else { - Long lid = ledgers.ceilingKey(currentLedgerId + 1); - currentLedgerId = lid != null ? lid : (ledgers.lastKey() + 1); - currentEntryId = 0; } + Long lid = ledgers.ceilingKey(currentLedgerId + 1); + currentLedgerId = lid != null ? lid : ledgers.lastKey(); + currentEntryId = 0; } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index c0b3d25af0c45..faad15933f8fd 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -2270,6 +2270,23 @@ public void testGetPositionAfterN() throws Exception { log.info("Target position is {}", targetPosition); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); + + // test for n > NumberOfEntriesInStorage + searchPosition = new PositionImpl(secondLedger, 0); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 100, ManagedLedgerImpl.PositionBound.startIncluded); + assertEquals(targetPosition.getLedgerId(), secondLedger); + assertEquals(targetPosition.getEntryId(), 4); + + // test for startPosition > current ledger + searchPosition = new PositionImpl(999, 0); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 0, ManagedLedgerImpl.PositionBound.startIncluded); + assertEquals(targetPosition.getLedgerId(), secondLedger); + assertEquals(targetPosition.getEntryId(), 4); + + searchPosition = new PositionImpl(999, 0); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 10, ManagedLedgerImpl.PositionBound.startExcluded); + assertEquals(targetPosition.getLedgerId(), secondLedger); + assertEquals(targetPosition.getEntryId(), 4); } @Test From 937f2ab001e9232161bec168b16002ce6d8d4a81 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 24 Oct 2022 13:11:24 +0800 Subject: [PATCH 806/823] [fix][function] Fix invalid metric type `gauge ` (#18129) (cherry picked from commit 39270f0439aeb73d65e90dac999a7cfcfbc257ab) --- .../org/apache/pulsar/functions/worker/WorkerStatsManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java index 2ad407b2e5e3e..0076a2f20e914 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerStatsManager.java @@ -331,7 +331,8 @@ private void writeMetric(String metricName, long value, StringWriter stream) { stream.write("# TYPE "); stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); stream.write(metricName); - stream.write(" gauge \n"); + stream.write(" gauge"); + stream.write("\n"); stream.write(PULSAR_FUNCTION_WORKER_METRICS_PREFIX); stream.write(metricName); From e197eaa8b98992d3404c8a438dd2083aad6b5b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Tue, 11 Oct 2022 10:37:23 +0800 Subject: [PATCH 807/823] [fix][broker] Fix incorrect bundle split count metric (#17970) (cherry picked from commit 882fcfba87a14e1027b3660fd00bd177f6bb4120) --- .../loadbalance/impl/ModularLoadManagerImpl.java | 10 ++++++---- site2/docs/reference-metrics.md | 4 ++++ .../versioned_docs/version-2.6.0/reference-metrics.md | 2 +- .../versioned_docs/version-2.6.1/reference-metrics.md | 2 +- .../versioned_docs/version-2.6.2/reference-metrics.md | 2 +- .../versioned_docs/version-2.6.3/reference-metrics.md | 2 +- .../versioned_docs/version-2.6.4/reference-metrics.md | 2 +- .../versioned_docs/version-2.7.0/reference-metrics.md | 2 +- .../versioned_docs/version-2.7.1/reference-metrics.md | 2 +- .../versioned_docs/version-2.7.2/reference-metrics.md | 2 +- .../versioned_docs/version-2.7.3/reference-metrics.md | 2 +- 11 files changed, 19 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 6191078569c07..d821e3399446c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -691,6 +691,7 @@ public void checkNamespaceBundleSplit() { synchronized (bundleSplitStrategy) { final Set bundlesToBeSplit = bundleSplitStrategy.findBundlesToSplit(loadData, pulsar); NamespaceBundleFactory namespaceBundleFactory = pulsar.getNamespaceService().getNamespaceBundleFactory(); + int splitCount = 0; for (String bundleName : bundlesToBeSplit) { try { final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundleName); @@ -712,13 +713,14 @@ public void checkNamespaceBundleSplit() { pulsar.getAdminClient().namespaces().splitNamespaceBundle(namespaceName, bundleRange, unloadSplitBundles, null); + splitCount++; log.info("Successfully split namespace bundle {}", bundleName); } catch (Exception e) { log.error("Failed to split namespace bundle {}", bundleName, e); } } - updateBundleSplitMetrics(bundlesToBeSplit); + updateBundleSplitMetrics(splitCount); } } @@ -726,10 +728,10 @@ public void checkNamespaceBundleSplit() { /** * As leader broker, update bundle split metrics. * - * @param bundlesToBeSplit + * @param bundlesSplit the number of bundles splits */ - private void updateBundleSplitMetrics(Set bundlesToBeSplit) { - bundleSplitCount += bundlesToBeSplit.size(); + private void updateBundleSplitMetrics(int bundlesSplit) { + bundleSplitCount += bundlesSplit; List metrics = Lists.newArrayList(); Map dimensions = new HashMap<>(); diff --git a/site2/docs/reference-metrics.md b/site2/docs/reference-metrics.md index 3467931e86b98..720d077d1e97e 100644 --- a/site2/docs/reference-metrics.md +++ b/site2/docs/reference-metrics.md @@ -328,6 +328,10 @@ All the bundleUnloading metrics are labelled with the following labels: - cluster: cluster=${pulsar_cluster}. ${pulsar_cluster} is the cluster name that you have configured in the `broker.conf` file. - metric: metric="bundlesSplit". +| Name | Type | Description | +|-------------------------------|---------|------------------------------------------------------------| +| pulsar_lb_bundles_split_total | Counter | The total count of bundle split in this leader broker | + | Name | Type | Description | | --- | --- | --- | | pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | diff --git a/site2/website/versioned_docs/version-2.6.0/reference-metrics.md b/site2/website/versioned_docs/version-2.6.0/reference-metrics.md index d3f60bd415f89..71a26f5fba988 100644 --- a/site2/website/versioned_docs/version-2.6.0/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.6.0/reference-metrics.md @@ -280,7 +280,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.6.1/reference-metrics.md b/site2/website/versioned_docs/version-2.6.1/reference-metrics.md index ebd6e4616b55f..d0ba807652c2e 100644 --- a/site2/website/versioned_docs/version-2.6.1/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.6.1/reference-metrics.md @@ -280,7 +280,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.6.2/reference-metrics.md b/site2/website/versioned_docs/version-2.6.2/reference-metrics.md index e1f58788b35ed..8bb6d1fdb3039 100644 --- a/site2/website/versioned_docs/version-2.6.2/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.6.2/reference-metrics.md @@ -280,7 +280,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.6.3/reference-metrics.md b/site2/website/versioned_docs/version-2.6.3/reference-metrics.md index f0feb88e39041..076c08ef24a72 100644 --- a/site2/website/versioned_docs/version-2.6.3/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.6.3/reference-metrics.md @@ -280,7 +280,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.6.4/reference-metrics.md b/site2/website/versioned_docs/version-2.6.4/reference-metrics.md index 9804a138ccf47..24ffea239d360 100644 --- a/site2/website/versioned_docs/version-2.6.4/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.6.4/reference-metrics.md @@ -280,7 +280,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.7.0/reference-metrics.md b/site2/website/versioned_docs/version-2.7.0/reference-metrics.md index cd9731e7804d8..74684fd73d59a 100644 --- a/site2/website/versioned_docs/version-2.7.0/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.7.0/reference-metrics.md @@ -276,7 +276,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.7.1/reference-metrics.md b/site2/website/versioned_docs/version-2.7.1/reference-metrics.md index 0aa3ca92e08fe..fee9402726479 100644 --- a/site2/website/versioned_docs/version-2.7.1/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.7.1/reference-metrics.md @@ -278,7 +278,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.7.2/reference-metrics.md b/site2/website/versioned_docs/version-2.7.2/reference-metrics.md index 0b2dada20c2eb..d2c90a841a39e 100644 --- a/site2/website/versioned_docs/version-2.7.2/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.7.2/reference-metrics.md @@ -278,7 +278,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics diff --git a/site2/website/versioned_docs/version-2.7.3/reference-metrics.md b/site2/website/versioned_docs/version-2.7.3/reference-metrics.md index 60c26903f9847..408acb161ed7f 100644 --- a/site2/website/versioned_docs/version-2.7.3/reference-metrics.md +++ b/site2/website/versioned_docs/version-2.7.3/reference-metrics.md @@ -298,7 +298,7 @@ All the bundleUnloading metrics are labelled with the following labels: | Name | Type | Description | | --- | --- | --- | -| pulsar_lb_bundles_split_count | Counter | bundle split count in this bundle splitting check interval | +| pulsar_lb_bundles_split_count | Counter | The total count of bundle split in this leader broker | ### Subscription metrics From 38f05889d05ac22f3ee694e93cea09de25fc8f1e Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 11 Oct 2022 10:14:58 +0800 Subject: [PATCH 808/823] [fix][broker] Fix executeWithRetry result is null (#17694) (cherry picked from commit 628e7607dd837e3201aa9d3f96970f7dc182fc1a) --- .../metadata/cache/impl/MetadataCacheImpl.java | 4 ++-- .../pulsar/metadata/MetadataCacheTest.java | 16 +++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java index 8bf5a729b7694..52b1272efddea 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java @@ -290,12 +290,12 @@ public void accept(Notification t) { private CompletableFuture executeWithRetry(Supplier> op, String key) { CompletableFuture result = new CompletableFuture<>(); - op.get().thenAccept(r -> result.complete(r)).exceptionally((ex) -> { + op.get().thenAccept(result::complete).exceptionally((ex) -> { if (ex.getCause() instanceof BadVersionException) { // if resource is updated by other than metadata-cache then metadata-cache will get bad-version // exception. so, try to invalidate the cache and try one more time. objCache.synchronous().invalidate(key); - op.get().thenAccept((c) -> result.complete(null)).exceptionally((ex1) -> { + op.get().thenAccept(result::complete).exceptionally((ex1) -> { result.completeExceptionally(ex1.getCause()); return null; }); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java index a4ba88852b0e3..a2b69cd4757e1 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java @@ -492,15 +492,21 @@ public void readModifyUpdateBadVersionRetry() throws Exception { MyClass value1 = new MyClass("a", 1); objCache1.create(key1, value1).join(); - objCache1.get(key1).join(); + assertEquals(objCache1.get(key1).join().get().b, 1); - objCache2.readModifyUpdate(key1, v -> { + CompletableFuture future1 = objCache1.readModifyUpdate(key1, v -> { return new MyClass(v.a, v.b + 1); - }).join(); + }); - objCache1.readModifyUpdate(key1, v -> { + CompletableFuture future2 = objCache2.readModifyUpdate(key1, v -> { return new MyClass(v.a, v.b + 1); - }).join(); + }); + + MyClass myClass1 = future1.join(); + assertEquals(myClass1.b, 2); + + MyClass myClass2 = future2.join(); + assertEquals(myClass2.b, 3); } @Test(dataProvider = "impl") From 668f6d46872f7f12ee8b48a7878895b98ae2f394 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 15 Sep 2022 10:35:44 +0800 Subject: [PATCH 809/823] [fix][broker] Unregister topic policy listener if managed ledger close failed (#17652) (cherry picked from commit dc5499794e0567ce87fc8ecdba5611a6d1fdbc26) --- .../apache/pulsar/broker/service/persistent/PersistentTopic.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index b3d611150324d..42d876d475525 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1306,6 +1306,7 @@ public void closeComplete(Object ctx) { public void closeFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}] Failed to close managed ledger, proceeding anyway.", topic, exception); brokerService.removeTopicFromCache(topic); + unregisterTopicPolicyListener(); closeFuture.complete(null); } }, null); From b8125d9ac5d8d85fd92ec62c96c3fcdf49ea0ff9 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 16 Sep 2022 23:56:03 +0800 Subject: [PATCH 810/823] [fix][common] Fix parsing partitionedKey with Base64 encode issue. (#17687) * Fix parsing partitionedKey with Base64 encode issue. * release the buf * fix checkstyle issue. (cherry picked from commit f3cc1071eb1f32b065d7893cee0c149895d843fa) --- .../pulsar/common/protocol/Commands.java | 4 ++ .../common/compression/CommandsTest.java | 41 ++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 091c9b8b7c6a2..934e331bd6b47 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; @@ -1731,6 +1732,9 @@ public static byte[] peekStickyKey(ByteBuf metadataAndPayload, String topic, Str if (metadata.hasOrderingKey()) { return metadata.getOrderingKey(); } else if (metadata.hasPartitionKey()) { + if (metadata.isPartitionKeyB64Encoded()) { + return Base64.getDecoder().decode(metadata.getPartitionKey()); + } return metadata.getPartitionKey().getBytes(StandardCharsets.UTF_8); } } catch (Throwable t) { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java index 24d34ac547fb7..207c6202426c8 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java @@ -18,22 +18,23 @@ */ package org.apache.pulsar.common.compression; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; - import com.scurrilous.circe.checksum.Crc32cIntChecksum; - import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; - import java.io.IOException; - +import java.util.Base64; +import io.netty.util.ReferenceCountUtil; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Commands.ChecksumType; +import org.testng.Assert; import org.testng.annotations.Test; public class CommandsTest { @@ -93,5 +94,35 @@ private int computeChecksum(MessageMetadata msgMetadata, ByteBuf compressedPaylo return computedChecksum; } - + @Test + public void testPeekStickyKey() { + String message = "msg-1"; + String partitionedKey = "key1"; + MessageMetadata messageMetadata2 = new MessageMetadata() + .setSequenceId(1) + .setProducerName("testProducer") + .setPartitionKey(partitionedKey) + .setPartitionKeyB64Encoded(false) + .setPublishTime(System.currentTimeMillis()); + ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata2, + Unpooled.copiedBuffer(message.getBytes(UTF_8))); + byte[] bytes = Commands.peekStickyKey(byteBuf, "topic-1", "sub-1"); + String key = new String(bytes); + Assert.assertEquals(partitionedKey, key); + ReferenceCountUtil.safeRelease(byteBuf); + // test 64 encoded + String partitionedKey2 = Base64.getEncoder().encodeToString("key2".getBytes(UTF_8)); + MessageMetadata messageMetadata = new MessageMetadata() + .setSequenceId(1) + .setProducerName("testProducer") + .setPartitionKey(partitionedKey2) + .setPartitionKeyB64Encoded(true) + .setPublishTime(System.currentTimeMillis()); + ByteBuf byteBuf2 = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, + Unpooled.copiedBuffer(message.getBytes(UTF_8))); + byte[] bytes2 = Commands.peekStickyKey(byteBuf2, "topic-2", "sub-2"); + String key2 = Base64.getEncoder().encodeToString(bytes2);; + Assert.assertEquals(partitionedKey2, key2); + ReferenceCountUtil.safeRelease(byteBuf2); + } } From 7cf681eeeb954029452ea405578d46517a4b795b Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Fri, 14 Oct 2022 19:14:56 +0800 Subject: [PATCH 811/823] [fix] [pulsar-client] Fix pendingLookupRequestSemaphore leak when channel inactive (#17856) ### Motivation https://github.com/apache/pulsar/blob/b89c1451551a6bbe681465726906a2e61c9d8a69/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java#L282-L297 The `pendingLookupRequestSemaphore` will leak when channel inactive. There are `LookUpRequestSemaphore` not released when removing it from `pendingRequests` ### Modifications We can't easily release the semaphore in `channelInactive`, because there are not only `LookUpRequest`. So release the semaphore when connectionException ### Verifying this change Add unit test case to cover this change ### Documentation - [ ] `doc-required` (Your PR needs to update docs and you will update later) - [x] `doc-not-needed` bug fixs, no need doc - [ ] `doc` (Your PR contains doc changes) - [ ] `doc-complete` (Docs have been already added) (cherry picked from commit b4518802f0dfad857abf3575758a1f69aa9457f8) --- .../apache/pulsar/client/impl/ClientCnx.java | 8 +++ .../pulsar/client/impl/ClientCnxTest.java | 66 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 506a0d36ca7b7..7598da17bd5c5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -132,6 +132,9 @@ public class ClientCnx extends PulsarHandler { private final CompletableFuture connectionFuture = new CompletableFuture(); private final ConcurrentLinkedQueue requestTimeoutQueue = new ConcurrentLinkedQueue<>(); + + @VisibleForTesting + @Getter(AccessLevel.PACKAGE) private final Semaphore pendingLookupRequestSemaphore; private final Semaphore maxLookupRequestSemaphore; private final EventLoopGroup eventLoopGroup; @@ -760,6 +763,11 @@ public CompletableFuture newLookup(ByteBuf request, long reque TimedCompletableFuture future = new TimedCompletableFuture<>(); if (pendingLookupRequestSemaphore.tryAcquire()) { + future.whenComplete((lookupDataResult, throwable) -> { + if (throwable instanceof ConnectException) { + pendingLookupRequestSemaphore.release(); + } + }); addPendingLookupRequests(requestId, future); ctx.writeAndFlush(request).addListener(writeFuture -> { if (!writeFuture.isSuccess()) { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index 6ce4afecd02bd..c46101fd47fc5 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -32,6 +32,7 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Field; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadFactory; import org.apache.pulsar.client.api.PulsarClientException; @@ -44,6 +45,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; import org.testng.annotations.Test; public class ClientCnxTest { @@ -74,6 +76,70 @@ public void testClientCnxTimeout() throws Exception { eventLoop.shutdownGracefully(); } + @Test + public void testPendingLookupRequestSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + when(listenerFuture.addListener(any())).thenReturn(listenerFuture); + when(ctx.writeAndFlush(any())).thenReturn(listenerFuture); + cnx.channelActive(ctx); + CountDownLatch countDownLatch = new CountDownLatch(1); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(1_000); + CompletableFuture future = + cnx.newLookup(null, 123); + countDownLatch.countDown(); + future.get(); + } catch (Exception e) { + completableFuture.complete(e); + } + }).start(); + countDownLatch.await(); + cnx.channelInactive(ctx); + assertTrue(completableFuture.get().getCause() instanceof PulsarClientException.ConnectException); + // wait for subsequent calls over + Awaitility.await().untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), conf.getConcurrentLookupRequest()); + }); + eventLoop.shutdownGracefully(); + } + + @Test + public void testPendingWaitingLookupRequestSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + when(listenerFuture.addListener(any())).thenReturn(listenerFuture); + when(ctx.writeAndFlush(any())).thenReturn(listenerFuture); + cnx.channelActive(ctx); + for (int i = 0; i < 5001; i++) { + cnx.newLookup(null, i); + } + cnx.channelInactive(ctx); + // wait for subsequent calls over + Awaitility.await().untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), conf.getConcurrentLookupRequest()); + }); + eventLoop.shutdownGracefully(); + } + @Test public void testReceiveErrorAtSendConnectFrameState() throws Exception { ThreadFactory threadFactory = new DefaultThreadFactory("testReceiveErrorAtSendConnectFrameState"); From 8c179d449f10f68d88a3df0406cfec0bd9a346dc Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Thu, 27 Oct 2022 23:01:37 +0800 Subject: [PATCH 812/823] =?UTF-8?q?[fix]=20[pulsar-client]=20Fix=20pending?= =?UTF-8?q?LookupRequestSemaphore=20leak=20when=20Ser=E2=80=A6=20(#18219)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation https://github.com/apache/pulsar/blob/b061c6ac5833c21e483368febebd0d30679a35e1/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java#L748-L774 The `pendingLookupRequestSemaphore` will leak when handleError. There are `LookUpRequestSemaphore` not released when removing it from `pendingRequests` related PR: #17856 ### Modifications We can't easily release the semaphore in `handleError`, because there are not only `LookUpRequest`. So release the semaphore when LookupException ### Verifying this change Add unit test case to cover this change ### Documentation - [ ] `doc-required` (Your PR needs to update docs and you will update later) - [x] `doc-not-needed` bug fixs, no need doc - [ ] `doc` (Your PR contains doc changes) - [ ] `doc-complete` (Docs have been already added) (cherry picked from commit fad3cccf87480a7a8c3a938cf5ca539b9a033106) --- .../apache/pulsar/client/impl/ClientCnx.java | 3 +- .../pulsar/client/impl/ClientCnxTest.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 7598da17bd5c5..2dd4cf334bfa4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -764,7 +764,8 @@ public CompletableFuture newLookup(ByteBuf request, long reque if (pendingLookupRequestSemaphore.tryAcquire()) { future.whenComplete((lookupDataResult, throwable) -> { - if (throwable instanceof ConnectException) { + if (throwable instanceof ConnectException + || throwable instanceof PulsarClientException.LookupException) { pendingLookupRequestSemaphore.release(); } }); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index c46101fd47fc5..a33d338fa2249 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -114,6 +114,49 @@ public void testPendingLookupRequestSemaphore() throws Exception { eventLoop.shutdownGracefully(); } + @Test + public void testPendingLookupRequestSemaphoreServiceNotReady() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + when(listenerFuture.addListener(any())).thenReturn(listenerFuture); + when(ctx.writeAndFlush(any())).thenReturn(listenerFuture); + cnx.channelActive(ctx); + cnx.state = ClientCnx.State.Ready; + CountDownLatch countDownLatch = new CountDownLatch(1); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(1_000); + CompletableFuture future = + cnx.newLookup(null, 123); + countDownLatch.countDown(); + future.get(); + } catch (Exception e) { + completableFuture.complete(e); + } + }).start(); + countDownLatch.await(); + CommandError commandError = new CommandError(); + commandError.setRequestId(123L); + commandError.setError(ServerError.ServiceNotReady); + commandError.setMessage("Service not ready"); + cnx.handleError(commandError); + assertTrue(completableFuture.get().getCause() instanceof PulsarClientException.LookupException); + // wait for subsequent calls over + Awaitility.await().untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), conf.getConcurrentLookupRequest()); + }); + eventLoop.shutdownGracefully(); + } + @Test public void testPendingWaitingLookupRequestSemaphore() throws Exception { EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); From 68f7cde9fc2a5889b20e122be618c5f505ee0642 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 9 Nov 2022 17:02:07 +0800 Subject: [PATCH 813/823] [fix][client] Support LocalDateTime Conversion (#18334) * Support LocalDateTime Conversion * move `TimestampMicrosConversion` to correct line (cherry picked from commit b31c5a6a325728b5dc5faebd1a33386952d733d5) --- .../pulsar/client/impl/schema/AvroSchema.java | 4 +++- .../client/impl/schema/AvroSchemaTest.java | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index a5dda082e7c45..8bb2ebeee0ae1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -120,6 +120,8 @@ public static void addLogicalTypeConversions(ReflectData reflectData, boolean js reflectData.addLogicalTypeConversion(new TimeConversions.DateConversion()); reflectData.addLogicalTypeConversion(new TimeConversions.TimeMillisConversion()); reflectData.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); + reflectData.addLogicalTypeConversion(new TimeConversions.LocalTimestampMillisConversion()); + reflectData.addLogicalTypeConversion(new TimeConversions.LocalTimestampMicrosConversion()); if (jsr310ConversionEnabled) { // The conversion that is registered first is higher priority than the registered later. reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMillisConversion()); @@ -130,8 +132,8 @@ public static void addLogicalTypeConversions(ReflectData reflectData, boolean js } catch (ClassNotFoundException e) { // Skip if have not provide joda-time dependency. } - reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); } + reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); reflectData.addLogicalTypeConversion(new Conversions.UUIDConversion()); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 2a5040d7815d3..ed2c8597ded09 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -32,6 +32,7 @@ import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; import java.time.temporal.ChronoUnit; import java.util.Arrays; @@ -549,4 +550,24 @@ public void testTimestampWithJsonDefAndJSR310ConversionEnabled(){ Assert.assertNotEquals(Instant.class, decodeWithJsonNoClassLoader.getValue().getClass()); } + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class LocalDateTimePojo { + LocalDateTime value; + } + + @Test + public void testLocalDateTime() { + SchemaDefinition schemaDefinition = + SchemaDefinition.builder().withPojo(LocalDateTimePojo.class) + .withJSR310ConversionEnabled(true).build(); + + AvroSchema avroSchema = AvroSchema.of(schemaDefinition); + LocalDateTime now = LocalDateTime.now(); + byte[] bytes = avroSchema.encode(new LocalDateTimePojo(now)); + + LocalDateTimePojo pojo = avroSchema.decode(bytes); + assertEquals(pojo.getValue().truncatedTo(ChronoUnit.MILLIS), now.truncatedTo(ChronoUnit.MILLIS)); + } } From 63983307d9a632afc18604dd9032ea19c87fc792 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 20 Sep 2022 11:21:54 +0800 Subject: [PATCH 814/823] [fix][schema]ledger handle leak when update schema (#17283) in the schema update, will create a `ledgerHandle` and write data to BK, after that `ledgerHandle` is no longer useful and no other object holds references to it. `ledgerHandle` will be recycled with GC, but `ledgerHandle` also hold external connections, which will cause leakage. https://github.com/apache/pulsar/blob/40b9d7ea50cef54becb09f2543193e08375abe0b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java#L452-L456 after the schema is updated, close the `ledgerHandle`, just like schema-read: https://github.com/apache/pulsar/blob/40b9d7ea50cef54becb09f2543193e08375abe0b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java#L519-L525 (cherry picked from commit 26204503494871db3818b4d2f35071c6ee1b5b96) --- .../schema/BookkeeperSchemaStorage.java | 13 +++++---- .../SchemaCompatibilityCheckTest.java | 28 +++++++++++++++++++ .../apache/pulsar/client/impl/ClientCnx.java | 2 ++ .../client/PulsarMockLedgerHandle.java | 6 ++-- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 9cac21da67ddf..53157c7b1b2d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -452,11 +452,14 @@ private CompletableFuture addNewSchemaEntryToS byte[] data ) { SchemaStorageFormat.SchemaEntry schemaEntry = newSchemaEntry(index, data); - return createLedger(schemaId).thenCompose(ledgerHandle -> - addEntry(ledgerHandle, schemaEntry).thenApply(entryId -> - Functions.newPositionInfo(ledgerHandle.getId(), entryId) - ) - ); + return createLedger(schemaId).thenCompose(ledgerHandle -> { + final long ledgerId = ledgerHandle.getId(); + return addEntry(ledgerHandle, schemaEntry) + .thenApply(entryId -> { + ledgerHandle.closeAsync(); + return Functions.newPositionInfo(ledgerId, entryId); + }); + }); } @NotNull diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 1b5e4d67232a7..9123ea33066dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Sets; import java.util.Collections; import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.Consumer; @@ -478,6 +479,33 @@ public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityS consumerOne.close(); producerOne.close(); + } + + @Test + public void testSchemaLedgerAutoRelease() throws Exception { + String namespaceName = PUBLIC_TENANT + "/default"; + String topicName = "persistent://" + namespaceName + "/tp"; + admin.namespaces().createNamespace(namespaceName, Sets.newHashSet(CLUSTER_NAME)); + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName, SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE); + // Update schema 100 times. + for (int i = 0; i < 100; i++){ + Schema schema = Schema.JSON(SchemaDefinition.builder() + .withJsonDef(String.format("{\"type\": \"record\",\"name\": " + + "\"Test_Pojo\",\"namespace\": \"org.apache.pulsar.schema.compatibility\"," + + "\"fields\": [{\"name\": \"prop_%s\",\"type\": " + + "[\"null\", \"string\"],\"default\": null}]}", i)) + .build()); + Producer producer = pulsarClient + .newProducer(schema) + .topic(topicName) + .create(); + producer.close(); + } + // The other ledgers are about 5. + Assert.assertTrue(mockBookKeeper.getLedgerMap().values().stream() + .filter(ledger -> !ledger.isFenced()) + .collect(Collectors.toList()).size() < 20); + admin.topics().delete(topicName, true); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 2dd4cf334bfa4..9dda7860e3352 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -46,6 +46,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +import lombok.AccessLevel; import lombok.Getter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Pair; diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java index 8a62e42e05a6f..3f9b17b312f69 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.client; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; @@ -31,7 +32,7 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.CompletableFuture; - +import lombok.Getter; import org.apache.bookkeeper.client.AsyncCallback.AddCallback; import org.apache.bookkeeper.client.AsyncCallback.CloseCallback; import org.apache.bookkeeper.client.AsyncCallback.ReadCallback; @@ -44,7 +45,6 @@ import org.apache.bookkeeper.client.impl.LedgerEntryImpl; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.net.BookieId; -import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.versioning.LongVersion; import org.apache.bookkeeper.versioning.Versioned; import org.slf4j.Logger; @@ -62,6 +62,8 @@ public class PulsarMockLedgerHandle extends LedgerHandle { final byte[] passwd; final ReadHandle readHandle; long lastEntry = -1; + @VisibleForTesting + @Getter boolean fenced = false; public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id, From 26b4747a03a6fbc3d13599b12e41c2426f71d491 Mon Sep 17 00:00:00 2001 From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com> Date: Sat, 13 Aug 2022 10:14:51 +0800 Subject: [PATCH 815/823] [fix][security] Bump PostgreSQL version to 42.4.1(#17066) (cherry picked from commit 16adb61c9c49f48fa5b045afc785dedb2c2e106d) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index bbccabc820c58..7fdfd201e7c7d 100644 --- a/pom.xml +++ b/pom.xml @@ -150,7 +150,7 @@ flexible messaging model and an intuitive client API. 5.1.0 3.8.11.2 8.0.11 - 42.2.25 + 42.4.1 0.3.2 2.6.0 3.3.3 @@ -159,7 +159,7 @@ flexible messaging model and an intuitive client API. 2.13 2.13.6 1.7.1.Final - 42.2.25 + 42.4.1 0.11.1 0.28.0 2.3.0 From ae42b48fa8746c0ffcdd59367ca6b70c6cab673f Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 21 Sep 2022 21:28:38 +0800 Subject: [PATCH 816/823] [fix][broker]Consumer can't consume messages because there has two sames topics in one broker (#17526) (cherry picked from commit 260f5c65e9937ede345d84777debdc8f7b571e1f) --- .../pulsar/broker/service/BrokerService.java | 56 +++++++++++++++++-- .../nonpersistent/NonPersistentTopic.java | 4 +- .../service/persistent/PersistentTopic.java | 6 +- 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index aed023ee5f52d..9e0ff2cca77cc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1835,7 +1835,7 @@ public void cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit) { TopicName topicName = TopicName.get(topic); if (serviceUnit.includes(topicName) && getTopicReference(topic).isPresent()) { log.info("[{}][{}] Clean unloaded topic from cache.", serviceUnit.toString(), topic); - pulsar.getBrokerService().removeTopicFromCache(topicName.toString(), serviceUnit); + pulsar.getBrokerService().removeTopicFromCache(topicName.toString(), serviceUnit, null); } } } @@ -1844,15 +1844,56 @@ public AuthorizationService getAuthorizationService() { return authorizationService; } - public CompletableFuture removeTopicFromCache(String topic) { + public CompletableFuture removeTopicFromCache(String topicName) { + return removeTopicFutureFromCache(topicName, null); + } + + public CompletableFuture removeTopicFromCache(Topic topic) { + Optional>> createTopicFuture = findTopicFutureInCache(topic); + if (createTopicFuture.isEmpty()){ + return CompletableFuture.completedFuture(null); + } + return removeTopicFutureFromCache(topic.getName(), createTopicFuture.get()); + } + + private Optional>> findTopicFutureInCache(Topic topic){ + if (topic == null){ + return Optional.empty(); + } + final CompletableFuture> createTopicFuture = topics.get(topic.getName()); + // If not exists in cache, do nothing. + if (createTopicFuture == null){ + return Optional.empty(); + } + // If the future in cache is not yet complete, the topic instance in the cache is not the same with the topic. + if (!createTopicFuture.isDone()){ + return Optional.empty(); + } + // If the future in cache has exception complete, + // the topic instance in the cache is not the same with the topic. + if (createTopicFuture.isCompletedExceptionally()){ + return Optional.empty(); + } + Optional optionalTopic = createTopicFuture.join(); + Topic topicInCache = optionalTopic.orElse(null); + if (topicInCache == null || topicInCache != topic){ + return Optional.empty(); + } else { + return Optional.of(createTopicFuture); + } + } + + private CompletableFuture removeTopicFutureFromCache(String topic, + CompletableFuture> createTopicFuture) { TopicName topicName = TopicName.get(topic); return pulsar.getNamespaceService().getBundleAsync(topicName) .thenAccept(namespaceBundle -> { - removeTopicFromCache(topic, namespaceBundle); + removeTopicFromCache(topic, namespaceBundle, createTopicFuture); }); } - public void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle) { + private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, + CompletableFuture> createTopicFuture) { String bundleName = namespaceBundle.toString(); String namespaceName = TopicName.get(topic).getNamespaceObject().toString(); @@ -1879,7 +1920,12 @@ public void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle) } } } - topics.remove(topic); + + if (createTopicFuture == null) { + topics.remove(topic); + } else { + topics.remove(topic, createTopicFuture); + } Compactor compactor = pulsar.getNullableCompactor(); if (compactor != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 0c2823a131a8f..35b900c297cba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -408,7 +408,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c // topic GC iterates over topics map and removing from the map with the same thread creates // deadlock. so, execute it in different thread brokerService.executor().execute(() -> { - brokerService.removeTopicFromCache(topic); + brokerService.removeTopicFromCache(NonPersistentTopic.this); log.info("[{}] Topic deleted", topic); deleteFuture.complete(null); }); @@ -474,7 +474,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect // unload topic iterates over topics map and removing from the map with the same thread creates deadlock. // so, execute it in different thread brokerService.executor().execute(() -> { - brokerService.removeTopicFromCache(topic); + brokerService.removeTopicFromCache(NonPersistentTopic.this); closeFuture.complete(null); }); }).exceptionally(exception -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 42d876d475525..eec19dfa1b838 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1183,7 +1183,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, ledger.asyncDelete(new AsyncCallbacks.DeleteLedgerCallback() { @Override public void deleteLedgerComplete(Object ctx) { - brokerService.removeTopicFromCache(topic); + brokerService.removeTopicFromCache(PersistentTopic.this); dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); @@ -1283,7 +1283,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect @Override public void closeComplete(Object ctx) { // Everything is now closed, remove the topic from map - brokerService.removeTopicFromCache(topic) + brokerService.removeTopicFromCache(PersistentTopic.this) .thenRun(() -> { replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close); @@ -1305,7 +1305,7 @@ public void closeComplete(Object ctx) { @Override public void closeFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}] Failed to close managed ledger, proceeding anyway.", topic, exception); - brokerService.removeTopicFromCache(topic); + brokerService.removeTopicFromCache(PersistentTopic.this); unregisterTopicPolicyListener(); closeFuture.complete(null); } From 4dcac0f200f37cc943ddde05913006384678beec Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 13 Oct 2022 12:20:12 +0800 Subject: [PATCH 817/823] [fix][broker] Fix the order of resource close in the InMemoryDelayedDeliveryTracker (#18000) (cherry picked from commit 44ae3487cc2aaaebdc5ce892d05eef76ca0384c6) --- .../InMemoryDelayedDeliveryTracker.java | 9 ++-- .../delayed/InMemoryDeliveryTrackerTest.java | 49 +++++++++++++++++-- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 7e4155c061375..dca5a0b7ee9c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -33,7 +33,7 @@ @Slf4j public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, TimerTask { - private final TripleLongPriorityQueue priorityQueue = new TripleLongPriorityQueue(); + protected final TripleLongPriorityQueue priorityQueue = new TripleLongPriorityQueue(); private final PersistentDispatcherMultipleConsumers dispatcher; @@ -41,7 +41,7 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T private final Timer timer; // Current timeout or null if not set - private Timeout timeout; + protected Timeout timeout; // Timestamp at which the timeout is currently set private long currentTimeoutTarget; @@ -259,7 +259,7 @@ public void run(Timeout timeout) throws Exception { if (log.isDebugEnabled()) { log.debug("[{}] Timer triggered", dispatcher.getName()); } - if (timeout.isCancelled()) { + if (timeout == null || timeout.isCancelled()) { return; } @@ -273,10 +273,11 @@ public void run(Timeout timeout) throws Exception { @Override public void close() { - priorityQueue.close(); if (timeout != null) { timeout.cancel(); + timeout = null; } + priorityQueue.close(); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 1ff47a4ca5065..11b681d80a640 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -28,13 +28,13 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; - +import io.netty.util.concurrent.DefaultThreadFactory; import java.time.Clock; import java.util.Collections; import java.util.NavigableMap; @@ -42,10 +42,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; - -import io.netty.util.concurrent.DefaultThreadFactory; import lombok.Cleanup; - import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.awaitility.Awaitility; @@ -433,4 +430,46 @@ public void testWithNoDelays() throws Exception { assertFalse(tracker.shouldPauseAllDeliveries()); } + @Test + public void testClose() throws Exception { + Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-in-memory-delayed-delivery-test"), + 1, TimeUnit.MILLISECONDS); + + PersistentDispatcherMultipleConsumers dispatcher = mock(PersistentDispatcherMultipleConsumers.class); + + AtomicLong clockTime = new AtomicLong(); + Clock clock = mock(Clock.class); + when(clock.millis()).then(x -> clockTime.get()); + + final Exception[] exceptions = new Exception[1]; + + InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true, 0) { + @Override + public void run(Timeout timeout) throws Exception { + super.timeout = timer.newTimeout(this, 1, TimeUnit.MILLISECONDS); + if (timeout == null || timeout.isCancelled()) { + return; + } + try { + this.priorityQueue.peekN1(); + } catch (Exception e) { + e.printStackTrace(); + exceptions[0] = e; + } + } + }; + + tracker.addMessage(1, 1, 10); + clockTime.set(10); + + Thread.sleep(300); + + tracker.close(); + + assertNull(exceptions[0]); + + timer.stop(); + } + } From 11130e1745b8553a91a8fa33be4a34e2c519468e Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Wed, 16 Nov 2022 13:44:43 +0100 Subject: [PATCH 818/823] [fix][offload] Fix memory leak while Offloading ledgers (#18500) (cherry picked from commit 6ff7d459697c2496de29ef077eb0f574632ebe6d) --- .../impl/BlockAwareSegmentInputStreamImpl.java | 2 +- .../impl/BlockAwareSegmentInputStreamTest.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java index b69f9f5e78544..e5dbcb6434709 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java @@ -261,7 +261,7 @@ public void close() throws IOException { // And through debug, writeBlobStore.uploadMultipartPart in the offload method also will trigger // the close method. // So we add the close variable to avoid release paddingBuf twice. - if (!close.compareAndSet(false, true)) { + if (close.compareAndSet(false, true)) { super.close(); dataBlockHeaderStream.close(); if (!entriesByteBuf.isEmpty()) { diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java index 0cd4bbd70a9e6..fff1ce8b7aa9a 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamTest.java @@ -30,6 +30,7 @@ import io.netty.buffer.Unpooled; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Iterator; @@ -798,4 +799,18 @@ public void testReadTillLacWithSmallBuffer() throws Exception { inputStream.close(); } + + @Test + public void testCloseReleaseResources() throws Exception { + ReadHandle readHandle = new MockReadHandle(1, 10, 10); + + BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0, 1024); + inputStream.read(); + Field field = BlockAwareSegmentInputStreamImpl.class.getDeclaredField("paddingBuf"); + field.setAccessible(true); + ByteBuf paddingBuf = (ByteBuf) field.get(inputStream); + assertEquals(1, paddingBuf.refCnt()); + inputStream.close(); + assertEquals(0, paddingBuf.refCnt()); + } } From 9cf76fdcb4fa054783a38873bc89dccc2ad90fc1 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 22 Nov 2022 17:27:46 +0800 Subject: [PATCH 819/823] [fix][broker] DnsResolverUtil.TTL should be greater than zero (#18565) (cherry picked from commit 67f94613f3ccceb89d756b097d756b1439a9b36f) --- .../common/util/netty/DnsResolverUtil.java | 8 +--- .../common/util/netty/DnsResolverTest.java | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java index 8b06dbf36eca3..5f1fe5a1ea6e1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java @@ -50,12 +50,8 @@ public class DnsResolverUtil { | IllegalAccessException e) { log.warn("Cannot get DNS TTL settings from sun.net.InetAddressCachePolicy class", e); } - TTL = useDefaultTTLWhenSetToForever(ttl, DEFAULT_TTL); - NEGATIVE_TTL = useDefaultTTLWhenSetToForever(negativeTtl, DEFAULT_NEGATIVE_TTL); - } - - private static int useDefaultTTLWhenSetToForever(int ttl, int defaultTtl) { - return ttl < 0 ? defaultTtl : ttl; + TTL = ttl <= 0 ? DEFAULT_TTL : ttl; + NEGATIVE_TTL = negativeTtl < 0 ? DEFAULT_NEGATIVE_TTL : negativeTtl; } private DnsResolverUtil() { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java new file mode 100644 index 0000000000000..0ccb960e79887 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.netty; + +import io.netty.channel.EventLoop; +import io.netty.resolver.dns.DnsNameResolverBuilder; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class DnsResolverTest { + + @Test + public void testMaxTtl() { + EventLoop eventLoop = Mockito.mock(EventLoop.class); + DnsNameResolverBuilder dnsNameResolverBuilder = new DnsNameResolverBuilder(eventLoop); + DnsResolverUtil.applyJdkDnsCacheSettings(dnsNameResolverBuilder); + // If the maxTtl is <=0, it will throw IllegalArgumentException. + try { + dnsNameResolverBuilder.build(); + } catch (Exception ex) { + Assert.assertFalse(ex instanceof IllegalArgumentException); + } + } +} From db5489479abd1f1e9e6b2aa473ab5ea5d21872f4 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 11 Nov 2022 14:25:32 +0800 Subject: [PATCH 820/823] [fix][fn] fix function failed to start if no `typeClassName` provided in `FunctionDetails` (#18111) (cherry picked from commit 8ad7157c1d22195720d256391a32773ca0108b80) --- .../pulsar/broker/service/BrokerService.java | 2 +- .../runtime/JavaInstanceStarter.java | 84 ++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 9e0ff2cca77cc..3418d9f08e391 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1850,7 +1850,7 @@ public CompletableFuture removeTopicFromCache(String topicName) { public CompletableFuture removeTopicFromCache(Topic topic) { Optional>> createTopicFuture = findTopicFutureInCache(topic); - if (createTopicFuture.isEmpty()){ + if (createTopicFuture.isPresent()){ return CompletableFuture.completedFuture(null); } return removeTopicFutureFromCache(topic.getName(), createTopicFuture.get()); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index 5995b87b705e3..36d063f8f558f 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -19,6 +19,8 @@ package org.apache.pulsar.functions.runtime; +import static org.apache.pulsar.functions.utils.FunctionCommon.getSinkType; +import static org.apache.pulsar.functions.utils.FunctionCommon.getSourceType; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.converters.StringConverter; @@ -33,6 +35,7 @@ import io.prometheus.client.exporter.HTTPServer; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceCache; @@ -45,13 +48,13 @@ import org.apache.pulsar.functions.secretsprovider.ClearTextSecretsProvider; import org.apache.pulsar.functions.secretsprovider.SecretsProvider; import org.apache.pulsar.common.util.Reflections; - import java.lang.reflect.Type; import java.net.InetSocketAddress; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.functions.utils.FunctionCommon; @Slf4j @@ -165,6 +168,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL functionDetailsJsonString = functionDetailsJsonString.substring(0, functionDetailsJsonString.length() - 1); } JsonFormat.parser().merge(functionDetailsJsonString, functionDetailsBuilder); + inferringMissingTypeClassName(functionDetailsBuilder, functionInstanceClassLoader); Function.FunctionDetails functionDetails = functionDetailsBuilder.build(); instanceConfig.setFunctionDetails(functionDetails); instanceConfig.setPort(port); @@ -287,6 +291,84 @@ public void close() { } } + private void inferringMissingTypeClassName(Function.FunctionDetails.Builder functionDetailsBuilder, + ClassLoader classLoader) throws ClassNotFoundException { + switch (functionDetailsBuilder.getComponentType()) { + case FUNCTION: + if ((functionDetailsBuilder.hasSource() + && functionDetailsBuilder.getSource().getTypeClassName().isEmpty()) + || (functionDetailsBuilder.hasSink() + && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) { + Map userConfigs = new Gson().fromJson(functionDetailsBuilder.getUserConfig(), + new TypeToken>() { + }.getType()); + boolean isWindowConfigPresent = userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY); + String className = functionDetailsBuilder.getClassName(); + if (isWindowConfigPresent) { + WindowConfig windowConfig = new Gson().fromJson( + (new Gson().toJson(userConfigs.get(WindowConfig.WINDOW_CONFIG_KEY))), + WindowConfig.class); + className = windowConfig.getActualWindowFunctionClassName(); + } + + Class[] typeArgs = FunctionCommon.getFunctionTypes(classLoader.loadClass(className), + isWindowConfigPresent); + if (functionDetailsBuilder.hasSource() + && functionDetailsBuilder.getSource().getTypeClassName().isEmpty() + && typeArgs[0] != null) { + Function.SourceSpec.Builder sourceBuilder = functionDetailsBuilder.getSource().toBuilder(); + sourceBuilder.setTypeClassName(typeArgs[0].getName()); + functionDetailsBuilder.setSource(sourceBuilder.build()); + } + + if (functionDetailsBuilder.hasSink() + && functionDetailsBuilder.getSink().getTypeClassName().isEmpty() + && typeArgs[1] != null) { + Function.SinkSpec.Builder sinkBuilder = functionDetailsBuilder.getSink().toBuilder(); + sinkBuilder.setTypeClassName(typeArgs[1].getName()); + functionDetailsBuilder.setSink(sinkBuilder.build()); + } + } + break; + case SINK: + if ((functionDetailsBuilder.hasSink() + && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) { + String typeArg = getSinkType(functionDetailsBuilder.getClassName(), classLoader).getName(); + + Function.SinkSpec.Builder sinkBuilder = + Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink()); + sinkBuilder.setTypeClassName(typeArg); + functionDetailsBuilder.setSink(sinkBuilder); + + Function.SourceSpec sourceSpec = functionDetailsBuilder.getSource(); + if (null == sourceSpec || StringUtils.isEmpty(sourceSpec.getTypeClassName())) { + Function.SourceSpec.Builder sourceBuilder = Function.SourceSpec.newBuilder(sourceSpec); + sourceBuilder.setTypeClassName(typeArg); + functionDetailsBuilder.setSource(sourceBuilder); + } + } + break; + case SOURCE: + if ((functionDetailsBuilder.hasSource() + && functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) { + String typeArg = getSourceType(functionDetailsBuilder.getClassName(), classLoader).getName(); + + Function.SourceSpec.Builder sourceBuilder = + Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource()); + sourceBuilder.setTypeClassName(typeArg); + functionDetailsBuilder.setSource(sourceBuilder); + + Function.SinkSpec sinkSpec = functionDetailsBuilder.getSink(); + if (null == sinkSpec || StringUtils.isEmpty(sinkSpec.getTypeClassName())) { + Function.SinkSpec.Builder sinkBuilder = Function.SinkSpec.newBuilder(sinkSpec); + sinkBuilder.setTypeClassName(typeArg); + functionDetailsBuilder.setSink(sinkBuilder); + } + } + break; + } + } + class InstanceControlImpl extends InstanceControlGrpc.InstanceControlImplBase { private RuntimeSpawner runtimeSpawner; From 9dd4149da8d91f9194a974c426a1fb3e6f75c686 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Thu, 22 Sep 2022 15:50:12 +0200 Subject: [PATCH 821/823] ManagedLedger: move to FENCED state in case of BadVersionException (#17736) (cherry picked from commit 63d4cf20e7b9c9bd24d3fcd5ba7397f0d185ce57) --- .../mledger/ManagedLedgerException.java | 4 + .../mledger/impl/ManagedLedgerImpl.java | 54 +++++-- .../mledger/impl/ManagedLedgerErrorsTest.java | 70 +++++++++ .../mledger/impl/OffloadPrefixTest.java | 48 ++++++ .../service/persistent/PersistentTopic.java | 42 +++--- .../service/BrokerBkEnsemblesTests.java | 142 ++++++++++++++++++ 6 files changed, 331 insertions(+), 29 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java index 347a380d7eb5c..0dc820ec46d72 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java @@ -80,6 +80,10 @@ public ManagedLedgerFencedException() { super(new Exception("Attempted to use a fenced managed ledger")); } + public ManagedLedgerFencedException(String message) { + super(message); + } + public ManagedLedgerFencedException(Exception e) { super(e); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 6280888174c0e..752f2fb960be0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -402,6 +402,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { @Override public void operationFailed(MetaStoreException e) { + handleBadVersion(e); if (e instanceof MetadataNotFoundException) { callback.initializeFailed(new ManagedLedgerNotFoundException(e)); } else { @@ -452,6 +453,7 @@ public void operationComplete(Void v, Stat stat) { @Override public void operationFailed(MetaStoreException e) { + handleBadVersion(e); callback.initializeFailed(new ManagedLedgerException(e)); } }; @@ -986,6 +988,7 @@ public void operationComplete(Void result, Stat stat) { @Override public void operationFailed(MetaStoreException e) { + handleBadVersion(e); callback.deleteCursorFailed(e, ctx); } @@ -1236,6 +1239,7 @@ public void operationComplete(Void result, Stat stat) { @Override public void operationFailed(MetaStoreException e) { log.error("[{}] Failed to terminate managed ledger: {}", name, e.getMessage()); + handleBadVersion(e); callback.terminateFailed(new ManagedLedgerException(e), ctx); } }); @@ -1320,6 +1324,7 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { public synchronized void asyncClose(final CloseCallback callback, final Object ctx) { State state = STATE_UPDATER.get(this); if (state == State.Fenced) { + cancelScheduledTasks(); factory.close(this); callback.closeFailed(new ManagedLedgerFencedException(), ctx); return; @@ -1443,6 +1448,7 @@ public void operationComplete(Void v, Stat stat) { @Override public void operationFailed(MetaStoreException e) { log.warn("[{}] Error updating meta data with the new list of ledgers: {}", name, e.getMessage()); + handleBadVersion(e); mbean.startDataLedgerDeleteOp(); bookKeeper.asyncDeleteLedger(lh.getId(), (rc1, ctx1) -> { mbean.endDataLedgerDeleteOp(); @@ -1451,14 +1457,12 @@ public void operationFailed(MetaStoreException e) { BKException.getMessage(rc1)); } }, null); - if (e instanceof BadVersionException) { synchronized (ManagedLedgerImpl.this) { log.error( "[{}] Failed to update ledger list. z-node version mismatch. Closing managed ledger", name); lastLedgerCreationFailureTimestamp = clock.millis(); - STATE_UPDATER.set(ManagedLedgerImpl.this, State.Fenced); // Return ManagedLedgerFencedException to addFailed callback // to indicate that the ledger is now fenced and topic needs to be closed clearPendingAddEntries(new ManagedLedgerFencedException(e)); @@ -1480,6 +1484,12 @@ public void operationFailed(MetaStoreException e) { updateLedgersListAfterRollover(cb, newLedger); } } + + private void handleBadVersion(Throwable e) { + if (e instanceof BadVersionException) { + setFenced(); + } + } private void updateLedgersListAfterRollover(MetaStoreCallback callback, LedgerInfo newLedger) { if (!metadataMutex.tryLock()) { // Defer update for later @@ -2362,12 +2372,19 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(), TOTAL_SIZE_UPDATER.get(this)); } - if (STATE_UPDATER.get(this) == State.Closed) { + State currentState = STATE_UPDATER.get(this); + if (currentState == State.Closed) { log.debug("[{}] Ignoring trimming request since the managed ledger was already closed", name); trimmerMutex.unlock(); promise.completeExceptionally(new ManagedLedgerAlreadyClosedException("Can't trim closed ledger")); return; } + if (currentState == State.Fenced) { + log.debug("[{}] Ignoring trimming request since the managed ledger was already fenced", name); + trimmerMutex.unlock(); + promise.completeExceptionally(new ManagedLedgerFencedException("Can't trim fenced ledger")); + return; + } long slowestReaderLedgerId = -1; if (!cursors.hasDurableCursors()) { @@ -2457,7 +2474,7 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { return; } - if (STATE_UPDATER.get(this) == State.CreatingLedger // Give up now and schedule a new trimming + if (currentState == State.CreatingLedger // Give up now and schedule a new trimming || !metadataMutex.tryLock()) { // Avoid deadlocks with other operations updating the ledgers list scheduleDeferredTrimming(isTruncate, promise); trimmerMutex.unlock(); @@ -2524,6 +2541,7 @@ public void operationFailed(MetaStoreException e) { log.warn("[{}] Failed to update the list of ledgers after trimming", name, e); metadataMutex.unlock(); trimmerMutex.unlock(); + handleBadVersion(e); promise.completeExceptionally(e); } @@ -2610,7 +2628,7 @@ public void deleteLedgerFailed(ManagedLedgerException e, Object ctx) { public void asyncDelete(final DeleteLedgerCallback callback, final Object ctx) { // Delete the managed ledger without closing, since we are not interested in gracefully closing cursors and // ledgers - STATE_UPDATER.set(this, State.Fenced); + setFenced(); cancelScheduledTasks(); List cursors = Lists.newArrayList(this.cursors); @@ -2857,7 +2875,7 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct promise.whenComplete((result, exception) -> { offloadMutex.unlock(); if (exception != null) { - callback.offloadFailed(new ManagedLedgerException(exception), ctx); + callback.offloadFailed(ManagedLedgerException.getManagedLedgerException(exception), ctx); } else { callback.offloadComplete(result, ctx); } @@ -2871,11 +2889,17 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct private void offloadLoop(CompletableFuture promise, Queue ledgersToOffload, PositionImpl firstUnoffloaded, Optional firstError) { - if (getState() == State.Closed) { + State currentState = getState(); + if (currentState == State.Closed) { promise.completeExceptionally(new ManagedLedgerAlreadyClosedException( String.format("managed ledger [%s] has already closed", name))); return; } + if (currentState == State.Fenced) { + promise.completeExceptionally(new ManagedLedgerFencedException( + String.format("managed ledger [%s] is fenced", name))); + return; + } LedgerInfo info = ledgersToOffload.poll(); if (info == null) { if (firstError.isPresent()) { @@ -3015,6 +3039,7 @@ public void operationComplete(Void result, Stat stat) { @Override public void operationFailed(MetaStoreException e) { + handleBadVersion(e); unlockingPromise.completeExceptionally(e); } }); @@ -3539,6 +3564,7 @@ private void checkManagedLedgerIsOpen() throws ManagedLedgerException { } synchronized void setFenced() { + log.info("{} Moving to Fenced state", name); STATE_UPDATER.set(this, State.Fenced); } @@ -3747,12 +3773,21 @@ private void scheduleTimeoutTask() { ? Math.max(config.getAddEntryTimeoutSeconds(), config.getReadEntryTimeoutSeconds()) : timeoutSec; this.timeoutTask = this.scheduledExecutor.scheduleAtFixedRate(safeRun(() -> { - checkAddTimeout(); - checkReadTimeout(); + checkTimeouts(); }), timeoutSec, timeoutSec, TimeUnit.SECONDS); } } + private void checkTimeouts() { + final State state = STATE_UPDATER.get(this); + if (state == State.Closed + || state == State.Fenced) { + return; + } + checkAddTimeout(); + checkReadTimeout(); + } + private void checkAddTimeout() { long timeoutSec = config.getAddEntryTimeoutSeconds(); if (timeoutSec < 1) { @@ -3908,6 +3943,7 @@ public void operationComplete(Void result, Stat version) { @Override public void operationFailed(MetaStoreException e) { log.error("[{}] Update managedLedger's properties failed", name, e); + handleBadVersion(e); callback.updatePropertiesFailed(e, ctx); metadataMutex.unlock(); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java index 3c09a2a23875b..a093bac48b95c 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java @@ -18,15 +18,19 @@ */ package org.apache.bookkeeper.mledger.impl; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import static org.testng.Assert.fail; import io.netty.buffer.ByteBuf; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.bookkeeper.client.BKException; @@ -387,6 +391,72 @@ public void recoverAfterZnodeVersionError() throws Exception { } } + @Test + public void recoverAfterZnodeVersionErrorWhileTrimming() throws Exception { + ManagedLedger ledger = factory.open("my_test_ledger_trim", + new ManagedLedgerConfig() + .setMaxEntriesPerLedger(2)); + ledger.addEntry("test".getBytes()); + ledger.addEntry("test".getBytes()); + ledger.addEntry("test".getBytes()); + + metadataStore.failConditional(new MetadataStoreException.BadVersionException("err"), (op, path) -> + path.equals("/managed-ledgers/my_test_ledger_trim") + && op == FaultInjectionMetadataStore.OperationType.PUT + ); + + CompletableFuture handle = new CompletableFuture<>(); + ledger.trimConsumedLedgersInBackground(handle); + assertThat(expectThrows(ExecutionException.class, () -> handle.get()).getCause(), + instanceOf(ManagedLedgerException.BadVersionException.class)); + + assertEquals(ManagedLedgerImpl.State.Fenced, ((ManagedLedgerImpl) ledger).getState()); + + // if the task started after the ML moved to Fenced state, it must fail + CompletableFuture handleAlreadyFenced = new CompletableFuture<>(); + ledger.trimConsumedLedgersInBackground(handleAlreadyFenced); + assertThat(expectThrows(ExecutionException.class, () -> handleAlreadyFenced.get()).getCause(), + instanceOf(ManagedLedgerException.ManagedLedgerFencedException.class)); + + try { + ledger.addEntry("entry".getBytes()); + fail("should fail"); + } catch (ManagedLedgerFencedException e) { + assertEquals("Attempted to use a fenced managed ledger", e.getCause().getMessage()); + } + + assertFalse(factory.ledgers.isEmpty()); + try { + ledger.close(); + } catch (ManagedLedgerFencedException e) { + assertEquals("Attempted to use a fenced managed ledger", e.getCause().getMessage()); + } + + // verify that the ManagedLedger has been unregistered even if it was fenced + assertTrue(factory.ledgers.isEmpty()); + } + + @Test + public void badVersionErrorDuringTruncateLedger() throws Exception { + ManagedLedger ledger = factory.open("my_test_ledger_trim", + new ManagedLedgerConfig() + .setMaxEntriesPerLedger(2)); + ledger.addEntry("test".getBytes()); + ledger.addEntry("test".getBytes()); + ledger.addEntry("test".getBytes()); + + metadataStore.failConditional(new MetadataStoreException.BadVersionException("err"), (op, path) -> + path.equals("/managed-ledgers/my_test_ledger_trim") + && op == FaultInjectionMetadataStore.OperationType.PUT + ); + + CompletableFuture handle = ledger.asyncTruncate(); + assertThat(expectThrows(ExecutionException.class, () -> handle.get()).getCause(), + instanceOf(ManagedLedgerException.BadVersionException.class)); + + assertEquals(ManagedLedgerImpl.State.Fenced, ((ManagedLedgerImpl) ledger).getState()); + } + @Test public void recoverAfterWriteError() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index a0930e4244215..8d0312980a096 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.ImmutableSet; @@ -49,6 +50,8 @@ import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.Test; @@ -126,6 +129,51 @@ public void testOffload() throws Exception { .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); + + // ledgers should be marked as offloaded + ledger.getLedgersInfoAsList().stream().allMatch(l -> l.hasOffloadContext()); + } + + @Test + public void testOffloadFenced() throws Exception { + MockLedgerOffloader offloader = new MockLedgerOffloader(); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setMinimumRolloverTime(0, TimeUnit.SECONDS); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + config.setLedgerOffloader(offloader); + ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); + + int i = 0; + for (; i < 25; i++) { + String content = "entry-" + i; + ledger.addEntry(content.getBytes()); + } + assertEquals(ledger.getLedgersInfoAsList().size(), 3); + + metadataStore.failConditional(new MetadataStoreException.BadVersionException("err"), (op, path) -> + path.equals("/managed-ledgers/my_test_ledger") + && op == FaultInjectionMetadataStore.OperationType.PUT + ); + + assertThrows(ManagedLedgerException.ManagedLedgerFencedException.class, () -> + ledger.offloadPrefix(ledger.getLastConfirmedEntry())); + + assertEquals(ledger.getLedgersInfoAsList().size(), 3); + + // the offloader actually wrote the data on the storage + assertEquals(ledger.getLedgersInfoAsList().stream() + .filter(e -> e.getOffloadContext().getComplete()) + .map(e -> e.getLedgerId()).collect(Collectors.toSet()), + offloader.offloadedLedgers()); + + // but the ledgers should not be marked as offloaded in local memory, as the write to metadata failed + ledger.getLedgersInfoAsList().stream().allMatch(l -> !l.hasOffloadContext()); + + // check that the ledger is fenced + assertEquals(ManagedLedgerImpl.State.Fenced, ledger.getState()); + } @Test diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index eec19dfa1b838..2d154bc54f8bc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1283,31 +1283,13 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect @Override public void closeComplete(Object ctx) { // Everything is now closed, remove the topic from map - brokerService.removeTopicFromCache(PersistentTopic.this) - .thenRun(() -> { - replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close); - - dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - - subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close); - - unregisterTopicPolicyListener(); - log.info("[{}] Topic closed", topic); - cancelFencedTopicMonitoringTask(); - closeFuture.complete(null); - }) - .exceptionally(ex -> { - closeFuture.completeExceptionally(ex); - return null; - }); + disposeTopic(closeFuture); } @Override public void closeFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}] Failed to close managed ledger, proceeding anyway.", topic, exception); - brokerService.removeTopicFromCache(PersistentTopic.this); - unregisterTopicPolicyListener(); - closeFuture.complete(null); + disposeTopic(closeFuture); } }, null); }).exceptionally(exception -> { @@ -1320,6 +1302,26 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { return closeFuture; } + private void disposeTopic(CompletableFuture closeFuture) { + brokerService.removeTopicFromCache(topic) + .thenRun(() -> { + replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close); + + dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); + + subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close); + + unregisterTopicPolicyListener(); + log.info("[{}] Topic closed", topic); + cancelFencedTopicMonitoringTask(); + closeFuture.complete(null); + }) + .exceptionally(ex -> { + closeFuture.completeExceptionally(ex); + return null; + }); + } + @VisibleForTesting CompletableFuture checkReplicationAndRetryOnFailure() { CompletableFuture result = new CompletableFuture(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java index aa63b224a9d1e..d5480e84a0a77 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java @@ -276,6 +276,148 @@ public void testSkipCorruptDataLedger() throws Exception { consumer.close(); } +<<<<<<< HEAD +======= + @Test + public void testTruncateCorruptDataLedger() throws Exception { + // Ensure intended state for autoSkipNonRecoverableData + admin.brokers().updateDynamicConfiguration("autoSkipNonRecoverableData", "false"); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getWebServiceAddress()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + + final int totalMessages = 100; + final int totalDataLedgers = 5; + final int entriesPerLedger = totalMessages / totalDataLedgers; + + final String tenant = "prop"; + try { + admin.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("role1", "role2"), + Sets.newHashSet(config.getClusterName()))); + } catch (Exception e) { + + } + final String ns1 = tenant + "/crash-broker"; + try { + admin.namespaces().createNamespace(ns1, Sets.newHashSet(config.getClusterName())); + } catch (Exception e) { + + } + + final String topic1 = "persistent://" + ns1 + "/my-topic-" + System.currentTimeMillis(); + + // Create subscription + Consumer consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name") + .receiverQueueSize(5).subscribe(); + + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topic1).get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); + ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next(); + Field configField = ManagedCursorImpl.class.getDeclaredField("config"); + configField.setAccessible(true); + // Create multiple data-ledger + ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor); + config.setMaxEntriesPerLedger(entriesPerLedger); + config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS); + // bookkeeper client + Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper"); + bookKeeperField.setAccessible(true); + // Create multiple data-ledger + BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml); + + // (1) publish messages in 10 data-ledgers each with 20 entries under managed-ledger + Producer producer = client.newProducer().topic(topic1).create(); + for (int i = 0; i < totalMessages; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + // validate: consumer is able to consume msg and close consumer after reading 1 entry + Assert.assertNotNull(consumer.receive(1, TimeUnit.SECONDS)); + consumer.close(); + + NavigableMap ledgerInfo = ml.getLedgersInfo(); + Assert.assertEquals(ledgerInfo.size(), totalDataLedgers); + Entry lastLedger = ledgerInfo.lastEntry(); + long firstLedgerToDelete = lastLedger.getKey(); + + // (2) delete first 4 data-ledgers + ledgerInfo.entrySet().forEach(entry -> { + if (!entry.equals(lastLedger)) { + assertEquals(entry.getValue().getEntries(), entriesPerLedger); + try { + bookKeeper.deleteLedger(entry.getKey()); + } catch (Exception e) { + log.warn("failed to delete ledger {}", entry.getKey(), e); + } + } + }); + + // create 5 more ledgers + for (int i = 0; i < totalMessages; i++) { + String message = "my-message2-" + i; + producer.send(message.getBytes()); + } + + // Admin should be able to truncate the topic + admin.topics().truncate(topic1); + + ledgerInfo.entrySet().forEach(entry -> { + log.warn("found ledger: {}", entry.getKey()); + assertNotEquals(firstLedgerToDelete, entry.getKey()); + }); + + // Currently, ledger deletion is async and failed deletion + // does not actually fail truncation but logs an exception + // and creates scheduled task to retry + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + LedgerMetadata meta = bookKeeper + .getLedgerMetadata(firstLedgerToDelete) + .exceptionally(e -> null) + .get(); + assertEquals(null, meta, "ledger should be deleted " + firstLedgerToDelete); + }); + + // Should not throw, deleting absent ledger must be a noop + // unless PulsarManager returned a wrong error which + // got translated to BKUnexpectedConditionException + try { + bookKeeper.deleteLedger(firstLedgerToDelete); + } catch (BKException.BKNoSuchLedgerExistsOnMetadataServerException bke) { + // pass + } + + producer.close(); + consumer.close(); + } + + @Test + public void testDeleteLedgerFactoryCorruptLedger() throws Exception { + ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("test"); + + // bookkeeper client + Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper"); + bookKeeperField.setAccessible(true); + // Create multiple data-ledger + BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml); + + ml.addEntry("dummy-entry-1".getBytes()); + + NavigableMap ledgerInfo = ml.getLedgersInfo(); + long lastLedger = ledgerInfo.lastEntry().getKey(); + + ml.close(); + bookKeeper.deleteLedger(lastLedger); + + // BK ledger is deleted, factory should not throw on delete + factory.delete("test"); + } + +>>>>>>> 63d4cf20e7... ManagedLedger: move to FENCED state in case of BadVersionException (#17736) @Test(timeOut = 20000) public void testTopicWithWildCardChar() throws Exception { @Cleanup From 91385337a33d442b5cfb6f61318736627fd1f8bc Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 23 Jun 2022 09:04:25 +0800 Subject: [PATCH 822/823] [improve][java-client] Only trigger the batch receive timeout when having pending batch receives requests (#16160) The consumer will apply the default batch receive policy even if the user will not use the batch receive API. https://github.com/apache/pulsar/blob/6704f12104219611164aa2bb5bbdfc929613f1bf/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchReceivePolicy.java#L60-L61 This will consume lots of CPU if the client have many consumers (100k consumers) The Pulsar perf tool can also reproduce the problem if run the test with many consumers If there is no pending batch receive operation for a consumer, no need to trigger the batch timeout task periodically. We can only start the timeout check after adding batch receive request to pending request queue. Remove the lock in MultiTopicsConsumerImpl as #10352 does Added new test to verify the batch receive timeout task will not start if no batch receive request (cherry picked from commit a0ccdc96bb05d19651f3778c23b89425d516d77a) --- .../client/api/ConsumerBatchReceiveTest.java | 47 +++++++++++++++++++ .../pulsar/client/impl/ConsumerBase.java | 22 +++++++-- .../pulsar/client/impl/ConsumerImpl.java | 1 + .../client/impl/MultiTopicsConsumerImpl.java | 20 ++------ .../impl/MultiTopicsConsumerImplTest.java | 2 +- 5 files changed, 71 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java index 1473f28075316..19cb25664b2fb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.client.api; import lombok.Cleanup; +import org.apache.pulsar.client.impl.ConsumerBase; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -48,6 +50,14 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @DataProvider(name = "partitioned") + public Object[][] partitionedTopicProvider() { + return new Object[][] { + { true }, + { false } + }; + } + @DataProvider(name = "batchReceivePolicy") public Object[][] batchReceivePolicyProvider() { return new Object[][] { @@ -425,6 +435,43 @@ public void verifyNumBytesSmallerThanMessageSize() throws Exception { latch.await(); } + @Test(dataProvider = "partitioned") + public void testBatchReceiveTimeoutTask(boolean partitioned) throws Exception { + final String topic = "persistent://my-property/my-ns/batch-receive-" + UUID.randomUUID(); + + if (partitioned) { + admin.topics().createPartitionedTopic(topic, 3); + } + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .receiverQueueSize(1) + .batchReceivePolicy(BatchReceivePolicy.builder() + .maxNumBytes(1024 * 1024) + .maxNumMessages(1) + .timeout(5, TimeUnit.SECONDS) + .build()) + .subscribe(); + Assert.assertFalse(((ConsumerBase)consumer).hasBatchReceiveTimeout()); + final int messagesToSend = 500; + sendMessagesAsyncAndWait(producer, messagesToSend); + for (int i = 0; i < 100; i++) { + Assert.assertNotNull(consumer.receive()); + } + Assert.assertFalse(((ConsumerBase)consumer).hasBatchReceiveTimeout()); + for (int i = 0; i < 400; i++) { + Messages batchReceived = consumer.batchReceive(); + Assert.assertEquals(batchReceived.size(), 1); + } + Awaitility.await().untilAsserted(() -> Assert.assertFalse(((ConsumerBase)consumer).hasBatchReceiveTimeout())); + Assert.assertEquals(consumer.batchReceive().size(), 0); + Awaitility.await().untilAsserted(() -> Assert.assertFalse(((ConsumerBase)consumer).hasBatchReceiveTimeout())); + } + private void receiveAllBatchesAndVerifyBatchSizeIsEqualToMaxNumMessages(Consumer consumer, BatchReceivePolicy batchReceivePolicy, diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index c53d49ad4bdb5..49e9acac05643 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -134,9 +134,12 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat } else { this.batchReceivePolicy = BatchReceivePolicy.DEFAULT_POLICY; } + } - if (batchReceivePolicy.getTimeoutMs() > 0) { - batchReceiveTimeout = client.timer().newTimeout(this::pendingBatchReceiveTask, batchReceivePolicy.getTimeoutMs(), TimeUnit.MILLISECONDS); + protected void triggerBatchReceiveTimeoutTask() { + if (!hasBatchReceiveTimeout() && batchReceivePolicy.getTimeoutMs() > 0) { + batchReceiveTimeout = client.timer().newTimeout(this::pendingBatchReceiveTask, + batchReceivePolicy.getTimeoutMs(), TimeUnit.MILLISECONDS); } } @@ -865,7 +868,7 @@ private void doPendingBatchReceiveTask(Timeout timeout) { } long timeToWaitMs; - + boolean hasPendingReceives = false; synchronized (this) { // If it's closing/closed we need to ignore this timeout and not schedule next timeout. if (getState() == State.Closing || getState() == State.Closed) { @@ -902,13 +905,18 @@ private void doPendingBatchReceiveTask(Timeout timeout) { } else { // The diff is greater than zero, set the timeout to the diff value timeToWaitMs = diff; + hasPendingReceives = true; break; } opBatchReceive = pendingBatchReceives.peek(); } - batchReceiveTimeout = client.timer().newTimeout(this::pendingBatchReceiveTask, - timeToWaitMs, TimeUnit.MILLISECONDS); + if (hasPendingReceives) { + batchReceiveTimeout = client.timer().newTimeout(this::pendingBatchReceiveTask, + timeToWaitMs, TimeUnit.MILLISECONDS); + } else { + batchReceiveTimeout = null; + } } } @@ -1034,5 +1042,9 @@ private ExecutorService getInternalExecutor(Message msg) { return executor; } + public boolean hasBatchReceiveTimeout() { + return batchReceiveTimeout != null; + } + private static final Logger log = LoggerFactory.getLogger(ConsumerBase.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 5381b28033085..89e434d41d459 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -500,6 +500,7 @@ protected CompletableFuture> internalBatchReceiveAsync() { } else { OpBatchReceive opBatchReceive = OpBatchReceive.of(result); pendingBatchReceives.add(opBatchReceive); + triggerBatchReceiveTimeoutTask(); cancellationHandler.setCancelAction(() -> pendingBatchReceives.remove(opBatchReceive)); } }); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index b2de0e3b92c38..2dd6bb9e304df 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -43,8 +43,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -101,8 +99,6 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { private volatile Timeout partitionsAutoUpdateTimeout = null; TopicsPartitionChangedListener topicsPartitionChangedListener; CompletableFuture partitionsAutoUpdateFuture = null; - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - private final ConsumerStatsRecorder stats; private UnAckedMessageTracker unAckedMessageTracker; private final ConsumerConfigurationData internalConfig; @@ -388,8 +384,7 @@ protected Messages internalBatchReceive() throws PulsarClientException { protected CompletableFuture> internalBatchReceiveAsync() { CompletableFutureCancellationHandler cancellationHandler = new CompletableFutureCancellationHandler(); CompletableFuture> result = cancellationHandler.createFuture(); - try { - lock.writeLock().lock(); + internalPinnedExecutor.execute(() -> { if (hasEnoughMessagesForBatchReceive()) { MessagesImpl messages = getNewMessagesImpl(); Message msgPeeked = incomingMessages.peek(); @@ -406,13 +401,11 @@ protected CompletableFuture> internalBatchReceiveAsync() { } else { OpBatchReceive opBatchReceive = OpBatchReceive.of(result); pendingBatchReceives.add(opBatchReceive); + triggerBatchReceiveTimeoutTask(); cancellationHandler.setCancelAction(() -> pendingBatchReceives.remove(opBatchReceive)); } resumeReceivingFromPausedConsumersIfNeeded(); - } finally { - lock.writeLock().unlock(); - } - + }); return result; } @@ -640,17 +633,14 @@ private ConsumerConfigurationData getInternalConsumerConfig() { @Override public void redeliverUnacknowledgedMessages() { - lock.writeLock().lock(); - try { + internalPinnedExecutor.execute(() -> { consumers.values().stream().forEach(consumer -> { consumer.redeliverUnacknowledgedMessages(); consumer.unAckedChunkedMessageIdSequenceMap.clear(); }); clearIncomingMessages(); unAckedMessageTracker.clear(); - } finally { - lock.writeLock().unlock(); - } + }); resumeReceivingFromPausedConsumersIfNeeded(); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index 38e668072281f..fe8180694083c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -178,7 +178,7 @@ public void testBatchReceiveAsyncCanBeCancelled() { // given MultiTopicsConsumerImpl consumer = createMultiTopicsConsumer(); CompletableFuture> future = consumer.batchReceiveAsync(); - assertTrue(consumer.hasPendingBatchReceive()); + Awaitility.await().untilAsserted(() -> assertTrue(consumer.hasPendingBatchReceive())); // when future.cancel(true); // then From 52fd58a69322d1675beebc4e94d39fdeb9dbbb40 Mon Sep 17 00:00:00 2001 From: congbobo184 Date: Sat, 26 Nov 2022 20:51:40 +0800 Subject: [PATCH 823/823] [cherry-pick][branch-2.9] cherry-pick #17736 problem --- .../service/BrokerBkEnsemblesTests.java | 142 ------------------ 1 file changed, 142 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java index d5480e84a0a77..aa63b224a9d1e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java @@ -276,148 +276,6 @@ public void testSkipCorruptDataLedger() throws Exception { consumer.close(); } -<<<<<<< HEAD -======= - @Test - public void testTruncateCorruptDataLedger() throws Exception { - // Ensure intended state for autoSkipNonRecoverableData - admin.brokers().updateDynamicConfiguration("autoSkipNonRecoverableData", "false"); - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsar.getWebServiceAddress()) - .statsInterval(0, TimeUnit.SECONDS) - .build(); - - final int totalMessages = 100; - final int totalDataLedgers = 5; - final int entriesPerLedger = totalMessages / totalDataLedgers; - - final String tenant = "prop"; - try { - admin.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("role1", "role2"), - Sets.newHashSet(config.getClusterName()))); - } catch (Exception e) { - - } - final String ns1 = tenant + "/crash-broker"; - try { - admin.namespaces().createNamespace(ns1, Sets.newHashSet(config.getClusterName())); - } catch (Exception e) { - - } - - final String topic1 = "persistent://" + ns1 + "/my-topic-" + System.currentTimeMillis(); - - // Create subscription - Consumer consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name") - .receiverQueueSize(5).subscribe(); - - PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topic1).get(); - ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); - ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next(); - Field configField = ManagedCursorImpl.class.getDeclaredField("config"); - configField.setAccessible(true); - // Create multiple data-ledger - ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor); - config.setMaxEntriesPerLedger(entriesPerLedger); - config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS); - // bookkeeper client - Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper"); - bookKeeperField.setAccessible(true); - // Create multiple data-ledger - BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml); - - // (1) publish messages in 10 data-ledgers each with 20 entries under managed-ledger - Producer producer = client.newProducer().topic(topic1).create(); - for (int i = 0; i < totalMessages; i++) { - String message = "my-message-" + i; - producer.send(message.getBytes()); - } - - // validate: consumer is able to consume msg and close consumer after reading 1 entry - Assert.assertNotNull(consumer.receive(1, TimeUnit.SECONDS)); - consumer.close(); - - NavigableMap ledgerInfo = ml.getLedgersInfo(); - Assert.assertEquals(ledgerInfo.size(), totalDataLedgers); - Entry lastLedger = ledgerInfo.lastEntry(); - long firstLedgerToDelete = lastLedger.getKey(); - - // (2) delete first 4 data-ledgers - ledgerInfo.entrySet().forEach(entry -> { - if (!entry.equals(lastLedger)) { - assertEquals(entry.getValue().getEntries(), entriesPerLedger); - try { - bookKeeper.deleteLedger(entry.getKey()); - } catch (Exception e) { - log.warn("failed to delete ledger {}", entry.getKey(), e); - } - } - }); - - // create 5 more ledgers - for (int i = 0; i < totalMessages; i++) { - String message = "my-message2-" + i; - producer.send(message.getBytes()); - } - - // Admin should be able to truncate the topic - admin.topics().truncate(topic1); - - ledgerInfo.entrySet().forEach(entry -> { - log.warn("found ledger: {}", entry.getKey()); - assertNotEquals(firstLedgerToDelete, entry.getKey()); - }); - - // Currently, ledger deletion is async and failed deletion - // does not actually fail truncation but logs an exception - // and creates scheduled task to retry - Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - LedgerMetadata meta = bookKeeper - .getLedgerMetadata(firstLedgerToDelete) - .exceptionally(e -> null) - .get(); - assertEquals(null, meta, "ledger should be deleted " + firstLedgerToDelete); - }); - - // Should not throw, deleting absent ledger must be a noop - // unless PulsarManager returned a wrong error which - // got translated to BKUnexpectedConditionException - try { - bookKeeper.deleteLedger(firstLedgerToDelete); - } catch (BKException.BKNoSuchLedgerExistsOnMetadataServerException bke) { - // pass - } - - producer.close(); - consumer.close(); - } - - @Test - public void testDeleteLedgerFactoryCorruptLedger() throws Exception { - ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); - ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("test"); - - // bookkeeper client - Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper"); - bookKeeperField.setAccessible(true); - // Create multiple data-ledger - BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml); - - ml.addEntry("dummy-entry-1".getBytes()); - - NavigableMap ledgerInfo = ml.getLedgersInfo(); - long lastLedger = ledgerInfo.lastEntry().getKey(); - - ml.close(); - bookKeeper.deleteLedger(lastLedger); - - // BK ledger is deleted, factory should not throw on delete - factory.delete("test"); - } - ->>>>>>> 63d4cf20e7... ManagedLedger: move to FENCED state in case of BadVersionException (#17736) @Test(timeOut = 20000) public void testTopicWithWildCardChar() throws Exception { @Cleanup